From 9899ee46769fcdb212ee28d74d657e619a129a72 Mon Sep 17 00:00:00 2001 From: gytall Date: Tue, 10 Nov 2026 09:52:42 +0300 Subject: [PATCH] formatted project by python-black --- app/core/background.py | 9 +- app/core/cleanup.py | 4 +- app/core/image_processing.py | 4 +- app/core/job_admin.py | 27 +- app/core/job_worker.py | 51 +- app/core/jobs.py | 4 +- app/core/outbox_publisher.py | 30 +- app/core/redis.py | 3 +- app/core/s3.py | 20 +- app/core/thumbnail_worker.py | 8 +- app/domain/attachment_jobs.py | 123 +++-- app/domain/attachments.py | 14 +- app/domain/health.py | 11 +- app/domain/idempotency.py | 12 +- app/domain/image_pipeline.py | 48 +- app/domain/job_admin.py | 90 +++- app/domain/job_files.py | 21 +- app/domain/job_retry.py | 27 +- app/domain/messages.py | 13 +- app/endpoints/attachments.py | 56 ++- app/endpoints/health.py | 11 +- app/endpoints/messages.py | 8 +- app/errors/exceptions.py | 15 +- app/errors/handlers.py | 21 +- app/main.py | 4 +- app/models/__init__.py | 15 +- app/models/attachment.py | 13 +- app/models/job.py | 7 +- app/schemas/attachment.py | 4 +- app/schemas/job.py | 4 +- app/storage/broker.py | 30 +- app/storage/health.py | 2 +- app/storage/idempotency.py | 17 +- app/storage/job_admin.py | 19 +- app/storage/jobs.py | 53 +- app/storage/outbox.py | 4 +- infra/minio/bootstrap.py | 5 +- .../49d2c947a3f1_add_idempotency_keys.py | 13 +- .../versions/8f31b94a2d10_add_attachments.py | 59 ++- .../versions/abd273076068_add_counters.py | 17 +- .../c35e91a2f706_async_attachment_jobs.py | 83 +++- .../e7048b6da913_attachment_crop_webp.py | 10 +- .../versions/f9183d27a640_job_replay.py | 27 +- scripts/check_contract.py | 88 ++-- scripts/check_jobs.py | 462 +++++++++++++----- scripts/check_outbox_recovery.py | 114 ++++- scripts/check_s3.py | 91 ++-- scripts/deployment.py | 21 +- scripts/init_local.py | 10 +- scripts/validate_openapi.py | 20 +- student-tests/test_attachments.py | 234 ++++++--- student-tests/test_concurrency.py | 9 +- student-tests/test_history_plan.py | 6 +- student-tests/test_idempotency.py | 72 ++- student-tests/test_image_pipeline.py | 122 +++-- student-tests/test_instance.py | 8 +- student-tests/test_jobs.py | 185 +++++-- student-tests/test_lifecycle.py | 12 +- student-tests/test_limits.py | 38 +- student-tests/test_migrations.py | 20 +- student-tests/test_s3_stream.py | 11 +- tests/client.py | 372 +++++++++----- tests/image_probe.py | 31 +- tests/load.py | 85 +++- tests/resilience.py | 206 +++++--- tests/smoke.py | 228 ++++++--- 66 files changed, 2517 insertions(+), 944 deletions(-) diff --git a/app/core/background.py b/app/core/background.py index 06a4d9a4..a0cfb7bb 100644 --- a/app/core/background.py +++ b/app/core/background.py @@ -8,7 +8,14 @@ from aio_pika.exceptions import AMQPException, ChannelInvalidStateError from sqlalchemy.exc import InterfaceError, OperationalError from sqlalchemy.exc import TimeoutError as PoolTimeoutError -TRANSIENT_JOB_ERRORS = (AMQPException, ChannelInvalidStateError, InterfaceError, OperationalError, PoolTimeoutError, OSError) +TRANSIENT_JOB_ERRORS = ( + AMQPException, + ChannelInvalidStateError, + InterfaceError, + OperationalError, + PoolTimeoutError, + OSError, +) logger = logging.getLogger(__name__) diff --git a/app/core/cleanup.py b/app/core/cleanup.py index d5201397..722e1a0d 100644 --- a/app/core/cleanup.py +++ b/app/core/cleanup.py @@ -21,7 +21,9 @@ async def cleanup_once(factory, storage, apply: bool, jobs: JobSettings) -> bool async with factory() as session: result = await reconcile_attachments(session, storage, dry_run=not apply) staging = await cleanup_staging(session, storage, jobs, apply=apply) - print(json.dumps({**asdict(result), "terminal_staging": staging}), flush=True) + print( + json.dumps({**asdict(result), "terminal_staging": staging}), flush=True + ) except RECOVERABLE_STORAGE_ERRORS as error: log_recoverable_error("Reconciliation", error) completed = False diff --git a/app/core/image_processing.py b/app/core/image_processing.py index 9eb46def..107dc42f 100644 --- a/app/core/image_processing.py +++ b/app/core/image_processing.py @@ -5,7 +5,9 @@ from pydantic_settings import BaseSettings, SettingsConfigDict class ImageSettings(BaseSettings): - model_config = SettingsConfigDict(env_prefix="IMAGE_", env_file=".env", extra="ignore") + model_config = SettingsConfigDict( + env_prefix="IMAGE_", env_file=".env", extra="ignore" + ) jpeg_quality: int = Field(default=85, ge=1, le=95) webp_quality: int = Field(default=80, ge=1, le=100) diff --git a/app/core/job_admin.py b/app/core/job_admin.py index 369b9f21..f75efa67 100644 --- a/app/core/job_admin.py +++ b/app/core/job_admin.py @@ -9,7 +9,13 @@ from app.core.background import log_background_error from app.core.config import Settings from app.core.database import create_db_engine, create_session_factory from app.core.jobs import JobSettings -from app.domain.job_admin import REPLAY_REASONS, history, list_failed, replay_failed_job, show_job +from app.domain.job_admin import ( + REPLAY_REASONS, + history, + list_failed, + replay_failed_job, + show_job, +) from app.errors.exceptions import JobOperationError from app.storage.s3 import create_s3_storage @@ -29,8 +35,15 @@ async def execute(args): result = await history(session, args.job_id) else: storage = create_s3_storage(settings.s3) - result = await replay_failed_job(session, storage, JobSettings(), args.job_id, - request_id=args.request_id, actor=args.actor, reason=args.reason) + result = await replay_failed_job( + session, + storage, + JobSettings(), + args.job_id, + request_id=args.request_id, + actor=args.actor, + reason=args.reason, + ) print(json.dumps(result, default=str), flush=True) finally: if storage is not None: @@ -39,10 +52,14 @@ async def execute(args): def main(): - parser = argparse.ArgumentParser(description="Реестр failed и явный replay с аудитом") + parser = argparse.ArgumentParser( + description="Реестр failed и явный replay с аудитом" + ) commands = parser.add_subparsers(dest="command", required=True) listing = commands.add_parser("list-failed") - listing.add_argument("--limit", type=int, choices=range(1, 501), default=50, metavar="1..500") + listing.add_argument( + "--limit", type=int, choices=range(1, 501), default=50, metavar="1..500" + ) for name in ("show", "history", "replay"): command = commands.add_parser(name) command.add_argument("job_id", type=UUID) diff --git a/app/core/job_worker.py b/app/core/job_worker.py index 4c36d0ff..168d186b 100644 --- a/app/core/job_worker.py +++ b/app/core/job_worker.py @@ -12,7 +12,12 @@ from uuid import UUID, uuid4 from PIL import features from pydantic import ValidationError -from app.core.background import TRANSIENT_JOB_ERRORS, log_background_error, shutdown_event, wait_or_stop +from app.core.background import ( + TRANSIENT_JOB_ERRORS, + log_background_error, + shutdown_event, + wait_or_stop, +) from app.core.config import Settings from app.core.database import create_db_engine, create_session_factory from app.core.jobs import JobSettings @@ -34,7 +39,10 @@ class Faults: object_name: str = "optimized" def kill(self, attachment_id: UUID, phase: str) -> None: - print(json.dumps({"fault": phase, "attachment_id": str(attachment_id)}), flush=True) + print( + json.dumps({"fault": phase, "attachment_id": str(attachment_id)}), + flush=True, + ) os.kill(os.getpid(), signal.SIGKILL) def after_object(self, attachment_id: UUID, name: str) -> None: @@ -52,7 +60,14 @@ class Faults: raise ProcessingError("injected_permanent") -async def consume_connected(factory, storage, broker: JobBroker, settings: JobSettings, stop: asyncio.Event, faults: Faults) -> None: +async def consume_connected( + factory, + storage, + broker: JobBroker, + settings: JobSettings, + stop: asyncio.Event, + faults: Faults, +) -> None: consumer = uuid4().hex while not stop.is_set(): try: @@ -68,8 +83,12 @@ async def consume_connected(factory, storage, broker: JobBroker, settings: JobSe try: async with factory() as session: acknowledge = await process_attachment_job( - session, storage, message, settings, - after_object=faults.after_object, before_processing=faults.before_processing, + session, + storage, + message, + settings, + after_object=faults.after_object, + before_processing=faults.before_processing, ) except ProcessingError: await broker.reject_invalid(delivery.id) @@ -107,7 +126,10 @@ async def run_worker(faults: Faults) -> None: storage = create_s3_storage(settings.s3) factory = create_session_factory(engine) async with asyncio.TaskGroup() as group: - tasks = [group.create_task(consume(factory, storage, jobs, stop, faults)) for _ in range(jobs.worker_concurrency)] + tasks = [ + group.create_task(consume(factory, storage, jobs, stop, faults)) + for _ in range(jobs.worker_concurrency) + ] await stop.wait() # Отмена убивает resize-процесс, дожидается SDK и не отправляет ack. for task in tasks: @@ -119,11 +141,22 @@ async def run_worker(faults: Faults) -> None: def main() -> None: - parser = argparse.ArgumentParser(description="Image worker с явными fault-injection hooks") + parser = argparse.ArgumentParser( + description="Image worker с явными fault-injection hooks" + ) modes = parser.add_mutually_exclusive_group() - for flag in ("crash-after-object", "crash-before-ack", "fail-transient", "fail-permanent"): + for flag in ( + "crash-after-object", + "crash-before-ack", + "fail-transient", + "fail-permanent", + ): modes.add_argument("--" + flag, type=UUID, metavar="ATTACHMENT_ID") - parser.add_argument("--object-name", choices=("original", "thumbnail", "crop", "optimized"), default="optimized") + parser.add_argument( + "--object-name", + choices=("original", "thumbnail", "crop", "optimized"), + default="optimized", + ) args = parser.parse_args() faults = Faults(**vars(args)) try: diff --git a/app/core/jobs.py b/app/core/jobs.py index e7badd3e..a67c86da 100644 --- a/app/core/jobs.py +++ b/app/core/jobs.py @@ -29,7 +29,9 @@ class JobSettings(BaseSettings): @model_validator(mode="after") def validate_lease(self) -> Self: if self.job_lease_seconds < self.job_timeout_seconds + 30: - raise ValueError("Lease должен превышать timeout задания минимум на 30 секунд") + raise ValueError( + "Lease должен превышать timeout задания минимум на 30 секунд" + ) if self.job_retry_max_seconds < self.job_retry_seconds: raise ValueError("Retry maximum must be at least the base delay") return self diff --git a/app/core/outbox_publisher.py b/app/core/outbox_publisher.py index 8d23e461..54ef3a2c 100644 --- a/app/core/outbox_publisher.py +++ b/app/core/outbox_publisher.py @@ -8,7 +8,12 @@ import os import signal from uuid import UUID -from app.core.background import TRANSIENT_JOB_ERRORS, log_background_error, shutdown_event, wait_or_stop +from app.core.background import ( + TRANSIENT_JOB_ERRORS, + log_background_error, + shutdown_event, + wait_or_stop, +) from app.core.config import Settings from app.core.database import create_db_engine, create_session_factory from app.core.jobs import JobSettings @@ -18,7 +23,10 @@ from app.storage.broker import JobBroker def crash_before_publish(attachment_id: UUID) -> None: """Реальный SIGKILL после чтения принятой job, до publisher confirm и отметки publish.""" - print(json.dumps({"fault": "before_publish", "attachment_id": str(attachment_id)}), flush=True) + print( + json.dumps({"fault": "before_publish", "attachment_id": str(attachment_id)}), + flush=True, + ) os.kill(os.getpid(), signal.SIGKILL) @@ -39,8 +47,14 @@ async def run_publisher(crash_attachment_id: UUID | None = None) -> None: async with factory() as session: await recover_expired_jobs(session, jobs) published = await publish_next_job( - session, broker, attachment_id=crash_attachment_id, - before_publish=crash_before_publish if crash_attachment_id is not None else None, + session, + broker, + attachment_id=crash_attachment_id, + before_publish=( + crash_before_publish + if crash_attachment_id is not None + else None + ), ) except TRANSIENT_JOB_ERRORS as error: log_background_error("Publisher deferred", error) @@ -53,9 +67,13 @@ async def run_publisher(crash_attachment_id: UUID | None = None) -> None: def main() -> None: - parser = argparse.ArgumentParser(description="Публикация устойчивых jobs в RabbitMQ") + parser = argparse.ArgumentParser( + description="Публикация устойчивых jobs в RabbitMQ" + ) parser.add_argument( - "--crash-before-publish", type=UUID, metavar="ATTACHMENT_ID", + "--crash-before-publish", + type=UUID, + metavar="ATTACHMENT_ID", help="аварийно завершить этот процесс перед публикацией только указанного вложения", ) args = parser.parse_args() diff --git a/app/core/redis.py b/app/core/redis.py index ea525f8b..fb61304d 100644 --- a/app/core/redis.py +++ b/app/core/redis.py @@ -4,6 +4,7 @@ from redis.backoff import NoBackoff from app.core.config import Settings + def create_redis_client(settings: Settings) -> Redis: password = ( settings.redis_password.get_secret_value() if settings.redis_password else None @@ -21,5 +22,3 @@ def create_redis_client(settings: Settings) -> Redis: decode_responses=True, ) return Redis.from_pool(pool) - - diff --git a/app/core/s3.py b/app/core/s3.py index c90137f8..f2de1880 100644 --- a/app/core/s3.py +++ b/app/core/s3.py @@ -6,6 +6,7 @@ from urllib.parse import urlsplit from pydantic import Field, model_validator from pydantic_settings import BaseSettings, SettingsConfigDict + class S3Settings(BaseSettings): endpoint: str = "http://minio:9000" region: str = "us-east-1" @@ -24,12 +25,23 @@ class S3Settings(BaseSettings): cleanup_grace_seconds: int = Field(default=86_400, ge=300) cleanup_interval_seconds: int = Field(default=300, ge=1) - model_config = SettingsConfigDict(env_prefix="S3_", env_file=".env", extra="ignore", frozen=True) + model_config = SettingsConfigDict( + env_prefix="S3_", env_file=".env", extra="ignore", frozen=True + ) @model_validator(mode="after") def validate_endpoint(self): url = urlsplit(self.endpoint) - if (url.scheme not in {"http", "https"} or not url.hostname or url.username - or url.password or url.path not in {"", "/"} or url.query or url.fragment): - raise ValueError("S3_ENDPOINT должен быть http(s) адресом без credentials/пути") + if ( + url.scheme not in {"http", "https"} + or not url.hostname + or url.username + or url.password + or url.path not in {"", "/"} + or url.query + or url.fragment + ): + raise ValueError( + "S3_ENDPOINT должен быть http(s) адресом без credentials/пути" + ) return self diff --git a/app/core/thumbnail_worker.py b/app/core/thumbnail_worker.py index cc3e2be1..2c465107 100644 --- a/app/core/thumbnail_worker.py +++ b/app/core/thumbnail_worker.py @@ -13,11 +13,15 @@ def main() -> int: settings = ImageSettings() memory = settings.memory_mib * 1024 * 1024 resource.setrlimit(resource.RLIMIT_AS, (memory, memory)) - resource.setrlimit(resource.RLIMIT_CPU, (settings.cpu_seconds, settings.cpu_seconds)) + resource.setrlimit( + resource.RLIMIT_CPU, (settings.cpu_seconds, settings.cpu_seconds) + ) source, directory, metadata, content_type, maximum = sys.argv[1:] code = 0 try: - result = create_variants(Path(source), Path(directory), content_type, int(maximum), settings) + result = create_variants( + Path(source), Path(directory), content_type, int(maximum), settings + ) Path(metadata).write_text(result.model_dump_json()) except IMAGE_WORKER_ERRORS as error: code = pipeline_worker_error_code(error) diff --git a/app/domain/attachment_jobs.py b/app/domain/attachment_jobs.py index 5f6d7dcf..ccf99346 100644 --- a/app/domain/attachment_jobs.py +++ b/app/domain/attachment_jobs.py @@ -29,13 +29,17 @@ class Claim: acknowledge: bool = False -async def enqueue_attachment_job(session: AsyncSession, attachment_id: UUID, staging_key: str) -> UUID: +async def enqueue_attachment_job( + session: AsyncSession, attachment_id: UUID, staging_key: str +) -> UUID: """Включить намерение обработки в транзакцию queued metadata; без commit.""" return await jobs.create_job(session, attachment_id, staging_key) async def publish_next_job( - session: AsyncSession, broker: JobBroker, *, + session: AsyncSession, + broker: JobBroker, + *, attachment_id: UUID | None = None, before_publish: Callable[[UUID], None] | None = None, ) -> bool: @@ -45,24 +49,33 @@ async def publish_next_job( if job is not None: if before_publish is not None: before_publish(job.attachment_id) - await broker.publish_job(JobMessage( - job_id=job.id, attachment_id=job.attachment_id, - pipeline_version=job.pipeline_version, attempt=job.attempt + 1, - created_at=job.created_at, - )) + await broker.publish_job( + JobMessage( + job_id=job.id, + attachment_id=job.attachment_id, + pipeline_version=job.pipeline_version, + attempt=job.attempt + 1, + created_at=job.created_at, + ) + ) outbox.mark_published(job) await session.commit() return published -async def claim_job(session: AsyncSession, message: JobMessage, settings: JobSettings) -> Claim: +async def claim_job( + session: AsyncSession, message: JobMessage, settings: JobSettings +) -> Claim: """Дубликат не захватывает живую аренду и не откатывает terminal-состояние.""" row = await jobs.lock_job(session, message.job_id) result = Claim() if row is None: raise ProcessingError("unknown_job") job, attachment = row - if message.attachment_id != attachment.id or message.pipeline_version != job.pipeline_version: + if ( + message.attachment_id != attachment.id + or message.pipeline_version != job.pipeline_version + ): raise ProcessingError("invalid_job") now = await jobs.database_time(session) if job.attempt_limit is None: @@ -77,16 +90,32 @@ async def claim_job(session: AsyncSession, message: JobMessage, settings: JobSet elif message.attempt > job.attempt + 1: raise ProcessingError("invalid_attempt") elif attachment.status == "queued" and job.next_attempt_at <= now: - if job.attempt >= (job.attempt_limit or settings.job_max_attempts) or job.staging_key is None: - jobs.record_failure(job, attachment, now, "retry_exhausted", terminal=True, retry_seconds=0) + if ( + job.attempt >= (job.attempt_limit or settings.job_max_attempts) + or job.staging_key is None + ): + jobs.record_failure( + job, attachment, now, "retry_exhausted", terminal=True, retry_seconds=0 + ) result = Claim(acknowledge=True) else: - result = Claim(work=jobs.start_job(job, attachment, uuid4(), now, settings.job_lease_seconds)) + result = Claim( + work=jobs.start_job( + job, attachment, uuid4(), now, settings.job_lease_seconds + ) + ) await session.commit() return result -async def save_failure(session: AsyncSession, work: JobWork, settings: JobSettings, code: str, *, permanent: bool) -> None: +async def save_failure( + session: AsyncSession, + work: JobWork, + settings: JobSettings, + code: str, + *, + permanent: bool, +) -> None: # Неоднозначный COMMIT проверяется повторным чтением: ready сохраняется. await session.rollback() row = await jobs.lock_job(session, work.job_id) @@ -95,10 +124,14 @@ async def save_failure(session: AsyncSession, work: JobWork, settings: JobSettin if attachment.status == "processing" and job.lease_token == work.lease_token: exhausted = job.attempt >= (job.attempt_limit or settings.job_max_attempts) jobs.record_failure( - job, attachment, await jobs.database_time(session), + job, + attachment, + await jobs.database_time(session), "retry_exhausted" if exhausted and not permanent else code, terminal=permanent or exhausted, - retry_seconds=retry_delay(settings, job.attempt - job.cycle_start_attempt), + retry_seconds=retry_delay( + settings, job.attempt - job.cycle_start_attempt + ), ) job.last_error_code = code job.last_error_permanent = permanent @@ -106,7 +139,9 @@ async def save_failure(session: AsyncSession, work: JobWork, settings: JobSettin async def build_result( - session: AsyncSession, storage: MinioStorage, work: JobWork, + session: AsyncSession, + storage: MinioStorage, + work: JobWork, after_object: Callable[[UUID, str], None] | None = None, ) -> bool: """Фиксированные ключи и блокировка job защищают PUT от устаревшей попытки.""" @@ -129,26 +164,42 @@ async def build_result( # Блокировка остаётся до завершения PUT/COMMIT; reaper использует SKIP LOCKED. # При отмене S3 adapter дожидается SDK-потока до освобождения блокировки. await storage.put_file( - work.original_key, source, size_bytes=work.size_bytes, - content_type=work.content_type, sha256=work.sha256, + work.original_key, + source, + size_bytes=work.size_bytes, + content_type=work.content_type, + sha256=work.sha256, ) if after_object is not None: after_object(work.attachment_id, "original") for variant in result.variants: await storage.put_file( - work.variant_key(variant.name), Path(directory) / VARIANT_FILENAMES[variant.name], - size_bytes=variant.size_bytes, content_type=variant.content_type, sha256=variant.sha256, + work.variant_key(variant.name), + Path(directory) / VARIANT_FILENAMES[variant.name], + size_bytes=variant.size_bytes, + content_type=variant.content_type, + sha256=variant.sha256, ) if after_object is not None: after_object(work.attachment_id, variant.name) - jobs.complete_job(job, attachment, work, [variant.model_dump() for variant in result.variants], await jobs.database_time(session)) + jobs.complete_job( + job, + attachment, + work, + [variant.model_dump() for variant in result.variants], + await jobs.database_time(session), + ) await session.commit() return True async def process_attachment_job( - session: AsyncSession, storage: MinioStorage, message: JobMessage, settings: JobSettings, - *, after_object: Callable[[UUID, str], None] | None = None, + session: AsyncSession, + storage: MinioStorage, + message: JobMessage, + settings: JobSettings, + *, + after_object: Callable[[UUID, str], None] | None = None, before_processing: Callable[[UUID], None] | None = None, ) -> bool: claim = await claim_job(session, message, settings) @@ -160,7 +211,13 @@ async def process_attachment_job( if before_processing is not None: before_processing(work.attachment_id) acknowledge = await build_result(session, storage, work, after_object) - except (ProcessingError, MinioException, StorageHTTPError, OSError, TimeoutError) as error: + except ( + ProcessingError, + MinioException, + StorageHTTPError, + OSError, + TimeoutError, + ) as error: code, permanent = classify_failure(error) await save_failure(session, work, settings, code, permanent=permanent) acknowledge = True @@ -176,8 +233,12 @@ async def recover_expired_jobs(session: AsyncSession, settings: JobSettings) -> job.attempt_limit = settings.job_max_attempts exhausted = job.attempt >= (job.attempt_limit or settings.job_max_attempts) jobs.record_failure( - job, attachment, now, "retry_exhausted" if exhausted else "worker_timeout", - terminal=exhausted, retry_seconds=retry_delay(settings, job.attempt - job.cycle_start_attempt), + job, + attachment, + now, + "retry_exhausted" if exhausted else "worker_timeout", + terminal=exhausted, + retry_seconds=retry_delay(settings, job.attempt - job.cycle_start_attempt), ) job.last_error_code = "worker_timeout" job.last_error_permanent = False @@ -185,8 +246,14 @@ async def recover_expired_jobs(session: AsyncSession, settings: JobSettings) -> return len(rows) -async def cleanup_staging(session: AsyncSession, storage: MinioStorage, settings: JobSettings, *, apply: bool) -> int: - rows = await jobs.terminal_staging(session, settings.staging_retention_seconds, settings.failed_staging_retention_seconds) +async def cleanup_staging( + session: AsyncSession, storage: MinioStorage, settings: JobSettings, *, apply: bool +) -> int: + rows = await jobs.terminal_staging( + session, + settings.staging_retention_seconds, + settings.failed_staging_retention_seconds, + ) if apply: for job, attachment in rows: if job.staging_key != attachment.object_key: diff --git a/app/domain/attachments.py b/app/domain/attachments.py index 80caf109..2a33d010 100644 --- a/app/domain/attachments.py +++ b/app/domain/attachments.py @@ -276,12 +276,20 @@ async def download_attachment( key = record.object_key info = DownloadInfo(result.filename, result.content_type, result.size_bytes) else: - selected = next((item for item in result.variants if item.name == variant), None) - keys = {"thumbnail": record.thumbnail_key, "crop": record.crop_key, "optimized": record.optimized_key} + selected = next( + (item for item in result.variants if item.name == variant), None + ) + keys = { + "thumbnail": record.thumbnail_key, + "crop": record.crop_key, + "optimized": record.optimized_key, + } key = keys.get(variant) if selected is None or key is None: raise not_found() - info = DownloadInfo(VARIANT_FILENAMES[variant], selected.content_type, selected.size_bytes) + info = DownloadInfo( + VARIANT_FILENAMES[variant], selected.content_type, selected.size_bytes + ) await session.rollback() stored = await storage.stat_object(key) if stored.size_bytes != info.size_bytes or stored.content_type != info.content_type: diff --git a/app/domain/health.py b/app/domain/health.py index aabd4d03..c5d6ac31 100644 --- a/app/domain/health.py +++ b/app/domain/health.py @@ -14,12 +14,15 @@ from app.storage.s3 import MinioStorage def liveness() -> Health: - return Health(status='ok') + return Health(status="ok") async def readiness( - engine: AsyncEngine, redis: Redis, storage: MinioStorage | None, - lifecycle: Lifecycle, timeout_seconds: float, + engine: AsyncEngine, + redis: Redis, + storage: MinioStorage | None, + lifecycle: Lifecycle, + timeout_seconds: float, ) -> Health: if lifecycle.draining: raise unavailable() @@ -30,4 +33,4 @@ async def readiness( dependencies = await asyncio.gather(*checks, return_exceptions=True) if not all(result is True for result in dependencies): raise unavailable() - return Health(status='ok') + return Health(status="ok") diff --git a/app/domain/idempotency.py b/app/domain/idempotency.py index 83b13acc..5ad93908 100644 --- a/app/domain/idempotency.py +++ b/app/domain/idempotency.py @@ -57,16 +57,22 @@ async def send_message_idempotent( raise conflict("Idempotency-Key уже использован для другого сообщения") message = await message_storage.get_message(session, previous.message_id) else: - await validate_message_attachments(session, sender, chat_id, new_message.attachment_ids) + await validate_message_attachments( + session, sender, chat_id, new_message.attachment_ids + ) if not await message_storage.reserve_message_slot(session, max_messages): raise unavailable("Достигнут лимит количества сообщений") message = await message_storage.create_message( session, chat_id=chat_id, sender_id=sender.id, text=new_message.text ) - await attachment_storage.bind_attachments(session, message.id, new_message.attachment_ids) + await attachment_storage.bind_attachments( + session, message.id, new_message.attachment_ids + ) await key_storage.save_result(session, scope, fingerprint, message.id) links = await attachment_storage.message_attachment_ids(session, [message.id]) - result = Message.model_validate(message).model_copy(update={"attachment_ids": links[message.id]}) + result = Message.model_validate(message).model_copy( + update={"attachment_ids": links[message.id]} + ) await session.commit() return result diff --git a/app/domain/image_pipeline.py b/app/domain/image_pipeline.py index 823ade9d..c7f63535 100644 --- a/app/domain/image_pipeline.py +++ b/app/domain/image_pipeline.py @@ -15,18 +15,27 @@ def fitted_size(size: tuple[int, int], maximum: int) -> tuple[int, int]: longest = max(size) result = size if longest > maximum: - result = (max(1, size[0] * maximum // longest), max(1, size[1] * maximum // longest)) + result = ( + max(1, size[0] * maximum // longest), + max(1, size[1] * maximum // longest), + ) return result -def describe_variant(name: str, content_type: str, path: Path, size: tuple[int, int]) -> Variant: +def describe_variant( + name: str, content_type: str, path: Path, size: tuple[int, int] +) -> Variant: checksum = hashlib.sha256() with path.open("rb") as source: for chunk in iter(lambda: source.read(65536), b""): checksum.update(chunk) return Variant( - name=name, content_type=content_type, size_bytes=path.stat().st_size, - width=size[0], height=size[1], sha256=checksum.hexdigest(), + name=name, + content_type=content_type, + size_bytes=path.stat().st_size, + width=size[0], + height=size[1], + sha256=checksum.hexdigest(), ) @@ -47,32 +56,49 @@ def create_thumbnail(image: Image.Image, destination: Path, quality: int) -> Var def create_crop(image: Image.Image, destination: Path, quality: int) -> Variant: """Центрированный JPEG 128×128; маленький исходник допускает увеличение.""" - with ImageOps.fit(image, (128, 128), method=Image.Resampling.LANCZOS, centering=(0.5, 0.5)) as crop: + with ImageOps.fit( + image, (128, 128), method=Image.Resampling.LANCZOS, centering=(0.5, 0.5) + ) as crop: save_jpeg(crop, destination, quality) return describe_variant("crop", "image/jpeg", destination, (128, 128)) -def create_optimized(image: Image.Image, destination: Path, settings: ImageSettings) -> Variant: +def create_optimized( + image: Image.Image, destination: Path, settings: ImageSettings +) -> Variant: """WebP ≤1280×1280 без увеличения; alpha сохраняется, метаданные удаляются.""" size = fitted_size(image.size, 1280) with image.resize(size, Image.Resampling.LANCZOS) as resized: with Image.new("RGBA", size) as clean: clean.paste(resized) clean.save( - destination, format="WEBP", lossless=False, - quality=settings.webp_quality, method=settings.webp_method, - exif=b"", icc_profile=b"", xmp=b"", + destination, + format="WEBP", + lossless=False, + quality=settings.webp_quality, + method=settings.webp_method, + exif=b"", + icc_profile=b"", + xmp=b"", ) return describe_variant("optimized", "image/webp", destination, size) -def create_variants(source: Path, directory: Path, content_type: str, max_pixels: int, settings: ImageSettings) -> PipelineResult: +def create_variants( + source: Path, + directory: Path, + content_type: str, + max_pixels: int, + settings: ImageSettings, +) -> PipelineResult: decode_image(source, content_type, max_pixels) with Image.open(source, formats=["PNG", "JPEG"]) as image: ImageOps.exif_transpose(image, in_place=True) with image.convert("RGBA") as pixels: variants = [ - create_thumbnail(pixels, directory / "thumbnail.jpg", settings.jpeg_quality), + create_thumbnail( + pixels, directory / "thumbnail.jpg", settings.jpeg_quality + ), create_crop(pixels, directory / "crop.jpg", settings.jpeg_quality), create_optimized(pixels, directory / "optimized.webp", settings), ] diff --git a/app/domain/job_admin.py b/app/domain/job_admin.py index a1a3494e..6a6be443 100644 --- a/app/domain/job_admin.py +++ b/app/domain/job_admin.py @@ -17,15 +17,28 @@ REPLAY_REASONS = ("dependency_restored", "configuration_fixed", "manual_review") def snapshot(job, attachment): - return dict(job_id=str(job.id), attachment_id=str(attachment.id), status=attachment.status, - attempt=job.attempt, attempt_limit=job.attempt_limit, cycle_start_attempt=job.cycle_start_attempt, - error_code=attachment.error_code, last_error_code=job.last_error_code, - last_error_permanent=job.last_error_permanent, staging_retained=job.staging_key is not None, - next_attempt_at=job.next_attempt_at, lease_until=job.lease_until, finished_at=job.finished_at) + return dict( + job_id=str(job.id), + attachment_id=str(attachment.id), + status=attachment.status, + attempt=job.attempt, + attempt_limit=job.attempt_limit, + cycle_start_attempt=job.cycle_start_attempt, + error_code=attachment.error_code, + last_error_code=job.last_error_code, + last_error_permanent=job.last_error_permanent, + staging_retained=job.staging_key is not None, + next_attempt_at=job.next_attempt_at, + lease_until=job.lease_until, + finished_at=job.finished_at, + ) async def list_failed(session, limit): - return [snapshot(job, attachment) for job, attachment in await job_admin.failed_jobs(session, limit)] + return [ + snapshot(job, attachment) + for job, attachment in await job_admin.failed_jobs(session, limit) + ] async def show_job(session, job_id): @@ -38,20 +51,37 @@ async def show_job(session, job_id): async def history(session, job_id): - return [dict(request_id=str(row.id), actor=row.actor, reason=row.reason, - previous_attempt=row.previous_attempt, attempt_limit=row.attempt_limit, - previous_error_code=row.previous_error_code, created_at=row.created_at) - for row in await job_admin.replay_history(session, job_id)] + return [ + dict( + request_id=str(row.id), + actor=row.actor, + reason=row.reason, + previous_attempt=row.previous_attempt, + attempt_limit=row.attempt_limit, + previous_error_code=row.previous_error_code, + created_at=row.created_at, + ) + for row in await job_admin.replay_history(session, job_id) + ] def same_request(record, job_id, actor, reason): if record.job_id != job_id or record.actor != actor or record.reason != reason: raise JobOperationError("request_conflict") - return {"request_id": str(record.id), "job_id": str(job_id), "already_applied": True} + return { + "request_id": str(record.id), + "job_id": str(job_id), + "already_applied": True, + } -async def replay_failed_job(session, storage, settings, job_id, *, request_id, actor, reason): - if re.fullmatch(r"[A-Za-z0-9._-]{1,64}", actor) is None or reason not in REPLAY_REASONS: +async def replay_failed_job( + session, storage, settings, job_id, *, request_id, actor, reason +): + if ( + re.fullmatch(r"[A-Za-z0-9._-]{1,64}", actor) is None + or reason not in REPLAY_REASONS + ): raise JobOperationError("invalid_audit_fields") row = await jobs.lock_job(session, job_id) if row is None: @@ -67,8 +97,16 @@ async def replay_failed_job(session, storage, settings, job_id, *, request_id, a if job.staging_key is None: raise JobOperationError("source_expired") # Та же блокировка защищает исходник от cleanup до окончания проверки и COMMIT. - work = JobWork(job.id, attachment.id, uuid4(), job.pipeline_version, job.staging_key, - attachment.content_type, attachment.size_bytes, attachment.sha256) + work = JobWork( + job.id, + attachment.id, + uuid4(), + job.pipeline_version, + job.staging_key, + attachment.content_type, + attachment.size_bytes, + attachment.sha256, + ) try: async with asyncio.timeout(settings.job_timeout_seconds): with TemporaryDirectory(prefix="replay-source-") as directory: @@ -81,13 +119,27 @@ async def replay_failed_job(session, storage, settings, job_id, *, request_id, a raise limit = job.attempt + settings.job_max_attempts inserted = await job_admin.insert_replay( - session, id=request_id, job_id=job_id, actor=actor, reason=reason, - previous_attempt=job.attempt, attempt_limit=limit, previous_error_code=job.last_error_code, + session, + id=request_id, + job_id=job_id, + actor=actor, + reason=reason, + previous_attempt=job.attempt, + attempt_limit=limit, + previous_error_code=job.last_error_code, ) if not inserted: - result = same_request(await job_admin.replay_record(session, request_id), job_id, actor, reason) + result = same_request( + await job_admin.replay_record(session, request_id), job_id, actor, reason + ) await session.rollback() return result job_admin.reset_failed(job, attachment, await jobs.database_time(session), limit) await session.commit() - return {"request_id": str(request_id), "job_id": str(job_id), "already_applied": False, "status": "queued", "attempt_limit": limit} + return { + "request_id": str(request_id), + "job_id": str(job_id), + "already_applied": False, + "status": "queued", + "attempt_limit": limit, + } diff --git a/app/domain/job_files.py b/app/domain/job_files.py index b48950d2..b99bda6c 100644 --- a/app/domain/job_files.py +++ b/app/domain/job_files.py @@ -14,7 +14,9 @@ from app.storage.jobs import JobWork from app.storage.s3 import MinioStorage -async def fetch_original(storage: MinioStorage, work: JobWork, destination: Path) -> None: +async def fetch_original( + storage: MinioStorage, work: JobWork, destination: Path +) -> None: if work.size_bytes > storage.settings.max_upload_bytes: raise ProcessingError("image_limits") checksum = hashlib.sha256() @@ -34,12 +36,21 @@ async def fetch_original(storage: MinioStorage, work: JobWork, destination: Path raise ProcessingError("source_mismatch") -async def render_variants(storage: MinioStorage, work: JobWork, source: Path, directory: Path) -> PipelineResult: +async def render_variants( + storage: MinioStorage, work: JobWork, source: Path, directory: Path +) -> PipelineResult: metadata = directory / "variants.json" process = await asyncio.create_subprocess_exec( - sys.executable, "-m", "app.core.thumbnail_worker", str(source), str(directory), - str(metadata), work.content_type, str(storage.settings.max_image_pixels), - stdout=asyncio.subprocess.DEVNULL, stderr=asyncio.subprocess.DEVNULL, + sys.executable, + "-m", + "app.core.thumbnail_worker", + str(source), + str(directory), + str(metadata), + work.content_type, + str(storage.settings.max_image_pixels), + stdout=asyncio.subprocess.DEVNULL, + stderr=asyncio.subprocess.DEVNULL, ) try: code = await process.wait() diff --git a/app/domain/job_retry.py b/app/domain/job_retry.py index 2d9620df..f6a67906 100644 --- a/app/domain/job_retry.py +++ b/app/domain/job_retry.py @@ -10,7 +10,10 @@ from app.errors.exceptions import ProcessingError def retry_delay(settings: JobSettings, cycle_attempt: int) -> float: - ceiling = min(settings.job_retry_max_seconds, settings.job_retry_seconds * 2 ** min(30, max(0, cycle_attempt - 1))) + ceiling = min( + settings.job_retry_max_seconds, + settings.job_retry_seconds * 2 ** min(30, max(0, cycle_attempt - 1)), + ) return random.uniform(ceiling / 2, ceiling) @@ -20,13 +23,29 @@ def classify_failure(error: Exception) -> tuple[str, bool]: if isinstance(error, S3Error): if error.code in {"NoSuchKey", "NoSuchObject"}: return "source_missing", True - if error.code in {"AccessDenied", "InvalidAccessKeyId", "SignatureDoesNotMatch", "NoSuchBucket"}: + if error.code in { + "AccessDenied", + "InvalidAccessKeyId", + "SignatureDoesNotMatch", + "NoSuchBucket", + }: return "storage_configuration", True - if error.code in {"InternalError", "SlowDown", "ServiceUnavailable", "RequestTimeout", "Throttling"}: + if error.code in { + "InternalError", + "SlowDown", + "ServiceUnavailable", + "RequestTimeout", + "Throttling", + }: return "storage_unavailable", False return "storage_rejected", True if isinstance(error, TimeoutError): return "processing_timeout", False - if isinstance(error, OSError) and error.errno in {errno.EACCES, errno.EPERM, errno.EROFS, errno.ENOENT}: + if isinstance(error, OSError) and error.errno in { + errno.EACCES, + errno.EPERM, + errno.EROFS, + errno.ENOENT, + }: return "processing_configuration", True return "processing_unavailable", False diff --git a/app/domain/messages.py b/app/domain/messages.py index f35cbfb3..dae112b5 100644 --- a/app/domain/messages.py +++ b/app/domain/messages.py @@ -32,10 +32,15 @@ async def list_messages( rows = await messages_storage.list_messages(session, chat_id, after, limit + 1) page_rows, has_next = split_page(rows, limit) - links = await attachments_storage.message_attachment_ids(session, [message.id for message in page_rows]) - items = [Message.model_validate(message).model_copy( - update={"attachment_ids": links[message.id]}, - ) for message in page_rows] + links = await attachments_storage.message_attachment_ids( + session, [message.id for message in page_rows] + ) + items = [ + Message.model_validate(message).model_copy( + update={"attachment_ids": links[message.id]}, + ) + for message in page_rows + ] page = MessagePage( items=items, next_cursor=build_next_cursor(KIND_MESSAGES, chat_id, items, has_next), diff --git a/app/endpoints/attachments.py b/app/endpoints/attachments.py index 8387b3e2..ddeddfa9 100644 --- a/app/endpoints/attachments.py +++ b/app/endpoints/attachments.py @@ -12,41 +12,63 @@ from app.domain import attachments from app.endpoints.dependencies import CurrentUser, DbSession, S3Storage, UploadCapacity from app.schemas.attachment import Attachment -router = APIRouter(prefix='/api/v1', tags=['attachments']) +router = APIRouter(prefix="/api/v1", tags=["attachments"]) -@router.post('/chats/{chat_id}/attachments', status_code=202) +@router.post("/chats/{chat_id}/attachments", status_code=202) async def upload( - request: Request, chat_id: UUID, user: CurrentUser, session: DbSession, - storage: S3Storage, capacity: UploadCapacity, + request: Request, + chat_id: UUID, + user: CurrentUser, + session: DbSession, + storage: S3Storage, + capacity: UploadCapacity, filename: Annotated[str, Query(min_length=1, max_length=255)], ) -> Attachment: return await attachments.receive_attachment( - session, storage, user, chat_id, capacity=capacity, filename=filename, - content_type=request.headers.get('content-type', '').lower(), - declared_length=request.headers.get('content-length'), chunks=request.stream(), + session, + storage, + user, + chat_id, + capacity=capacity, + filename=filename, + content_type=request.headers.get("content-type", "").lower(), + declared_length=request.headers.get("content-length"), + chunks=request.stream(), ) -@router.get('/attachments/{attachment_id}') -async def metadata(attachment_id: UUID, user: CurrentUser, session: DbSession) -> Attachment: +@router.get("/attachments/{attachment_id}") +async def metadata( + attachment_id: UUID, user: CurrentUser, session: DbSession +) -> Attachment: return await attachments.get_attachment(session, user, attachment_id) -@router.get('/attachments/{attachment_id}/content') +@router.get("/attachments/{attachment_id}/content") async def content( - attachment_id: UUID, user: CurrentUser, session: DbSession, storage: S3Storage, - variant: Annotated[str, Query()] = 'original', + attachment_id: UUID, + user: CurrentUser, + session: DbSession, + storage: S3Storage, + variant: Annotated[str, Query()] = "original", ) -> StreamingResponse: record, stream = await attachments.download_attachment( - session, storage, user, attachment_id, variant=variant, + session, + storage, + user, + attachment_id, + variant=variant, ) return StreamingResponse( - stream, media_type=record.content_type, + stream, + media_type=record.content_type, headers={ - 'Content-Length': str(record.size_bytes), - 'Content-Disposition': "attachment; filename*=UTF-8''" + quote(record.filename, safe=''), - 'X-Content-Type-Options': 'nosniff', 'Cache-Control': 'private, no-store', + "Content-Length": str(record.size_bytes), + "Content-Disposition": "attachment; filename*=UTF-8''" + + quote(record.filename, safe=""), + "X-Content-Type-Options": "nosniff", + "Cache-Control": "private, no-store", }, background=BackgroundTask(stream.aclose), ) diff --git a/app/endpoints/health.py b/app/endpoints/health.py index 9c28c7d4..2be984b6 100644 --- a/app/endpoints/health.py +++ b/app/endpoints/health.py @@ -3,18 +3,21 @@ from fastapi import APIRouter, Request from app.domain import health from app.schemas.base import Health -router = APIRouter(prefix='/health', tags=['health']) +router = APIRouter(prefix="/health", tags=["health"]) -@router.get('/live') +@router.get("/live") async def liveness() -> Health: return health.liveness() -@router.get('/ready') +@router.get("/ready") async def readiness(request: Request) -> Health: state = request.app.state return await health.readiness( - state.engine, state.redis, state.s3, state.lifecycle, + state.engine, + state.redis, + state.s3, + state.lifecycle, state.settings.readiness_timeout_seconds, ) diff --git a/app/endpoints/messages.py b/app/endpoints/messages.py index 50dc1a18..4bfbd3f3 100644 --- a/app/endpoints/messages.py +++ b/app/endpoints/messages.py @@ -21,9 +21,13 @@ async def send_new_message( idempotency_key: Annotated[str | None, Header(alias="Idempotency-Key")] = None, ) -> Message: return await send_message_idempotent( - session, sender, chat_id, new_message, + session, + sender, + chat_id, + new_message, validate_idempotency_key(idempotency_key), - settings.max_messages, settings.idempotency_ttl_seconds, + settings.max_messages, + settings.idempotency_ttl_seconds, ) diff --git a/app/errors/exceptions.py b/app/errors/exceptions.py index 59d06a96..72cc0782 100644 --- a/app/errors/exceptions.py +++ b/app/errors/exceptions.py @@ -90,14 +90,23 @@ def internal_error() -> ApiError: IMAGE_WORKER_ERRORS = ( - ApiError, Image.DecompressionBombError, Image.DecompressionBombWarning, - MemoryError, UnidentifiedImageError, OSError, ValueError, SyntaxError, + ApiError, + Image.DecompressionBombError, + Image.DecompressionBombWarning, + MemoryError, + UnidentifiedImageError, + OSError, + ValueError, + SyntaxError, ) def image_worker_error_code(error: Exception) -> int: code = 2 - resource_error = isinstance(error, (Image.DecompressionBombError, Image.DecompressionBombWarning, MemoryError)) + resource_error = isinstance( + error, + (Image.DecompressionBombError, Image.DecompressionBombWarning, MemoryError), + ) declared_limit = isinstance(error, ApiError) and error.status_code == 413 if resource_error or declared_limit: code = 3 diff --git a/app/errors/handlers.py b/app/errors/handlers.py index 9dd48190..faa82677 100644 --- a/app/errors/handlers.py +++ b/app/errors/handlers.py @@ -18,14 +18,25 @@ from urllib3.exceptions import HTTPError as StorageHTTPError from app.core.instance import INSTANCE_ID_HEADER from app.core.request_id import REQUEST_ID_HEADER -from app.errors.exceptions import ApiError, internal_error, invalid_request, not_found, unavailable +from app.errors.exceptions import ( + ApiError, + internal_error, + invalid_request, + not_found, + unavailable, +) logger = logging.getLogger(__name__) TRANSIENT_SQLSTATE_CLASSES = ("08", "40", "53", "57") TRANSIENT_SQLSTATES = frozenset({"55P03"}) RECOVERABLE_STORAGE_ERRORS = ( - ApiError, SQLAlchemyError, MinioException, StorageHTTPError, OSError, TimeoutError, + ApiError, + SQLAlchemyError, + MinioException, + StorageHTTPError, + OSError, + TimeoutError, ) @@ -40,12 +51,14 @@ async def operation_timeout(seconds: float, error: ApiError): async def client_disconnect_handler(request: Request, exc: Exception) -> JSONResponse: - return await api_error_handler(request, invalid_request("Загрузка прервана клиентом")) + return await api_error_handler( + request, invalid_request("Загрузка прервана клиентом") + ) def log_recoverable_error(operation: str, error: Exception) -> None: """Согласованный безопасный лог восстановления без содержимого SDK-ошибки.""" - logger.warning('%s deferred: %s', operation, type(error).__name__) + logger.warning("%s deferred: %s", operation, type(error).__name__) def _get_request_id(request: Request) -> str: diff --git a/app/main.py b/app/main.py index de683883..f68247b8 100644 --- a/app/main.py +++ b/app/main.py @@ -52,7 +52,9 @@ def create_app(settings: Settings | None = None) -> FastAPI: application.state.session_factory = create_session_factory(engine) application.state.redis = redis application.state.s3 = s3 - application.state.upload_capacity = create_request_capacity(settings.s3.max_concurrent_uploads) + application.state.upload_capacity = create_request_capacity( + settings.s3.max_concurrent_uploads + ) application.state.request_capacity = create_request_capacity(settings.max_in_flight) application.state.instance_id = resolve_instance_id(settings.instance_id) application.state.lifecycle = Lifecycle() diff --git a/app/models/__init__.py b/app/models/__init__.py index 43097851..d9476c43 100644 --- a/app/models/__init__.py +++ b/app/models/__init__.py @@ -9,4 +9,17 @@ from app.models.idempotency_key import IdempotencyKey from app.models.message import Message from app.models.user import User -__all__ = ["JobReplay", "AttachmentJob", "Base", "Attachment", "MessageAttachment", "UploadIntent", "Chat", "ChatMember", "Counter", "IdempotencyKey", "Message", "User"] +__all__ = [ + "JobReplay", + "AttachmentJob", + "Base", + "Attachment", + "MessageAttachment", + "UploadIntent", + "Chat", + "ChatMember", + "Counter", + "IdempotencyKey", + "Message", + "User", +] diff --git a/app/models/attachment.py b/app/models/attachment.py index 24d075a2..3afebe0d 100644 --- a/app/models/attachment.py +++ b/app/models/attachment.py @@ -18,8 +18,13 @@ class Attachment(Base): "content_type IN ('image/png', 'image/jpeg')", name="content_type" ), CheckConstraint("sha256 ~ '^[a-f0-9]{64}$'", name="sha256"), - CheckConstraint("status IN ('queued', 'processing', 'ready', 'failed')", name="status"), - CheckConstraint("(status = 'failed' AND error_code IS NOT NULL) OR (status <> 'failed' AND error_code IS NULL)", name="error_code"), + CheckConstraint( + "status IN ('queued', 'processing', 'ready', 'failed')", name="status" + ), + CheckConstraint( + "(status = 'failed' AND error_code IS NOT NULL) OR (status <> 'failed' AND error_code IS NULL)", + name="error_code", + ), Index("ix_attachments_chat_id", "chat_id"), ) @@ -36,7 +41,9 @@ class Attachment(Base): thumbnail_key: Mapped[str | None] = mapped_column(String(128)) crop_key: Mapped[str | None] = mapped_column(String(128)) optimized_key: Mapped[str | None] = mapped_column(String(128)) - variants: Mapped[list[dict]] = mapped_column(JSONB, default=list, server_default=text("'[]'::jsonb")) + variants: Mapped[list[dict]] = mapped_column( + JSONB, default=list, server_default=text("'[]'::jsonb") + ) created_at: Mapped[datetime] = mapped_column(server_default=func.clock_timestamp()) updated_at: Mapped[datetime] = mapped_column(server_default=func.clock_timestamp()) diff --git a/app/models/job.py b/app/models/job.py index 00ae6a20..01924ee8 100644 --- a/app/models/job.py +++ b/app/models/job.py @@ -20,7 +20,8 @@ class AttachmentJob(Base): id: Mapped[UUID] = mapped_column(primary_key=True, default=uuid4) attachment_id: Mapped[UUID] = mapped_column( - ForeignKey("attachments.id"), unique=True, + ForeignKey("attachments.id"), + unique=True, ) staging_key: Mapped[str | None] = mapped_column(String(128)) pipeline_version: Mapped[int] = mapped_column(server_default="1") @@ -29,7 +30,9 @@ class AttachmentJob(Base): attempt_limit: Mapped[int | None] last_error_permanent: Mapped[bool | None] published_attempt: Mapped[int] = mapped_column(server_default="0") - next_attempt_at: Mapped[datetime] = mapped_column(server_default=func.clock_timestamp()) + next_attempt_at: Mapped[datetime] = mapped_column( + server_default=func.clock_timestamp() + ) lease_token: Mapped[UUID | None] lease_until: Mapped[datetime | None] last_error_code: Mapped[str | None] = mapped_column(String(64)) diff --git a/app/schemas/attachment.py b/app/schemas/attachment.py index 93911c14..629e26b9 100644 --- a/app/schemas/attachment.py +++ b/app/schemas/attachment.py @@ -35,7 +35,9 @@ class Attachment(Base): VARIANT_FILENAMES = { - "thumbnail": "thumbnail.jpg", "crop": "crop.jpg", "optimized": "optimized.webp", + "thumbnail": "thumbnail.jpg", + "crop": "crop.jpg", + "optimized": "optimized.webp", } diff --git a/app/schemas/job.py b/app/schemas/job.py index 7fb7f341..7b783709 100644 --- a/app/schemas/job.py +++ b/app/schemas/job.py @@ -16,5 +16,7 @@ class JobMessage(BaseModel): pipeline_version: int = Field(default=1, ge=1) attempt: int = Field(ge=1) created_at: AwareDatetime - traceparent: str | None = Field(default=None, pattern=r"^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$") + traceparent: str | None = Field( + default=None, pattern=r"^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$" + ) tracestate: str | None = Field(default=None, max_length=512) diff --git a/app/storage/broker.py b/app/storage/broker.py index 7358a523..1932ffda 100644 --- a/app/storage/broker.py +++ b/app/storage/broker.py @@ -28,26 +28,42 @@ class JobBroker: logging.getLogger("aiormq").setLevel(logging.CRITICAL) async def ensure_ready(self) -> None: - if self.connection is None or self.connection.is_closed or self.channel is None or self.channel.is_closed: + if ( + self.connection is None + or self.connection.is_closed + or self.channel is None + or self.channel.is_closed + ): await self.aclose() self.connection = await aio_pika.connect( - host=self.settings.broker_host, port=self.settings.broker_port, + host=self.settings.broker_host, + port=self.settings.broker_port, login=self.settings.broker_user, password=self.settings.broker_password_file.read_text().strip(), - virtualhost=self.settings.broker_vhost, timeout=5, heartbeat=30, + virtualhost=self.settings.broker_vhost, + timeout=5, + heartbeat=30, + ) + self.channel = await self.connection.channel( + publisher_confirms=True, on_return_raises=True ) - self.channel = await self.connection.channel(publisher_confirms=True, on_return_raises=True) await self.channel.set_qos(prefetch_count=1) - self.queue = await self.channel.declare_queue(self.settings.broker_queue, durable=True) + self.queue = await self.channel.declare_queue( + self.settings.broker_queue, durable=True + ) async def publish_job(self, message: JobMessage) -> None: await self.ensure_ready() await self.channel.default_exchange.publish( aio_pika.Message( body=message.model_dump_json(exclude_none=True).encode(), - content_type="application/json", delivery_mode=aio_pika.DeliveryMode.PERSISTENT, + content_type="application/json", + delivery_mode=aio_pika.DeliveryMode.PERSISTENT, message_id=f"{message.job_id}:{message.attempt}", - ), routing_key=self.settings.broker_queue, mandatory=True, timeout=10, + ), + routing_key=self.settings.broker_queue, + mandatory=True, + timeout=10, ) async def receive(self, consumer: str) -> Delivery: diff --git a/app/storage/health.py b/app/storage/health.py index d6913fe6..2d034b71 100644 --- a/app/storage/health.py +++ b/app/storage/health.py @@ -7,7 +7,7 @@ from sqlalchemy.ext.asyncio import AsyncEngine async def check_database(engine: AsyncEngine) -> bool: async with engine.connect() as connection: - await connection.execute(text('SELECT 1')) + await connection.execute(text("SELECT 1")) return True diff --git a/app/storage/idempotency.py b/app/storage/idempotency.py index 5cfcf709..53e174c3 100644 --- a/app/storage/idempotency.py +++ b/app/storage/idempotency.py @@ -33,7 +33,8 @@ async def get_active_key( IdempotencyKey.method, IdempotencyKey.path, IdempotencyKey.key, - ) == tuple_(*scope), + ) + == tuple_(*scope), IdempotencyKey.created_at > func.clock_timestamp() - timedelta(seconds=ttl_seconds), ) @@ -45,14 +46,20 @@ async def save_result( ) -> None: """Сохраняет новый результат или заменяет истёкший.""" stmt = insert(IdempotencyKey).values( - user_id=scope[0], method=scope[1], path=scope[2], key=scope[3], - request_hash=request_hash, message_id=message_id, + user_id=scope[0], + method=scope[1], + path=scope[2], + key=scope[3], + request_hash=request_hash, + message_id=message_id, ) await session.execute( stmt.on_conflict_do_update( index_elements=[ - IdempotencyKey.user_id, IdempotencyKey.method, - IdempotencyKey.path, IdempotencyKey.key, + IdempotencyKey.user_id, + IdempotencyKey.method, + IdempotencyKey.path, + IdempotencyKey.key, ], set_={ "request_hash": request_hash, diff --git a/app/storage/job_admin.py b/app/storage/job_admin.py index 87782c24..6780924a 100644 --- a/app/storage/job_admin.py +++ b/app/storage/job_admin.py @@ -13,7 +13,8 @@ async def failed_jobs(session, limit: int): select(AttachmentJob, Attachment) .join(Attachment, Attachment.id == AttachmentJob.attachment_id) .where(Attachment.status == "failed") - .order_by(AttachmentJob.finished_at.desc(), AttachmentJob.id).limit(limit) + .order_by(AttachmentJob.finished_at.desc(), AttachmentJob.id) + .limit(limit) ) return result.all() @@ -23,12 +24,24 @@ async def replay_record(session, request_id): async def insert_replay(session, **values) -> bool: - result = await session.scalar(insert(JobReplay).values(**values).on_conflict_do_nothing(index_elements=[JobReplay.id]).returning(JobReplay.id)) + result = await session.scalar( + insert(JobReplay) + .values(**values) + .on_conflict_do_nothing(index_elements=[JobReplay.id]) + .returning(JobReplay.id) + ) return result is not None async def replay_history(session, job_id): - return (await session.scalars(select(JobReplay).where(JobReplay.job_id == job_id).order_by(JobReplay.created_at).limit(100))).all() + return ( + await session.scalars( + select(JobReplay) + .where(JobReplay.job_id == job_id) + .order_by(JobReplay.created_at) + .limit(100) + ) + ).all() def reset_failed(job, attachment, now, attempt_limit): diff --git a/app/storage/jobs.py b/app/storage/jobs.py index d67545b7..e0ca118b 100644 --- a/app/storage/jobs.py +++ b/app/storage/jobs.py @@ -32,7 +32,9 @@ class JobWork: return f"variants/{self.attachment_id.hex}/v{self.pipeline_version}/{filename}" -async def create_job(session: AsyncSession, attachment_id: UUID, staging_key: str) -> UUID: +async def create_job( + session: AsyncSession, attachment_id: UUID, staging_key: str +) -> UUID: job = AttachmentJob(attachment_id=attachment_id, staging_key=staging_key) session.add(job) await session.flush() @@ -58,7 +60,10 @@ async def expired_jobs(session: AsyncSession, limit: int = 20): result = await session.execute( select(AttachmentJob, Attachment) .join(Attachment, Attachment.id == AttachmentJob.attachment_id) - .where(Attachment.status == "processing", AttachmentJob.lease_until < func.clock_timestamp()) + .where( + Attachment.status == "processing", + AttachmentJob.lease_until < func.clock_timestamp(), + ) .order_by(AttachmentJob.lease_until) .limit(limit) .with_for_update(of=AttachmentJob, skip_locked=True) @@ -66,19 +71,35 @@ async def expired_jobs(session: AsyncSession, limit: int = 20): return result.all() -def start_job(job, attachment, token: UUID, now: datetime, lease_seconds: int) -> JobWork: +def start_job( + job, attachment, token: UUID, now: datetime, lease_seconds: int +) -> JobWork: job.attempt += 1 job.lease_token = token job.lease_until = now + timedelta(seconds=lease_seconds) attachment.status = "processing" attachment.updated_at = now return JobWork( - job.id, attachment.id, token, job.pipeline_version, job.staging_key, - attachment.content_type, attachment.size_bytes, attachment.sha256, + job.id, + attachment.id, + token, + job.pipeline_version, + job.staging_key, + attachment.content_type, + attachment.size_bytes, + attachment.sha256, ) -def record_failure(job, attachment, now: datetime, error_code: str, *, terminal: bool, retry_seconds: float) -> None: +def record_failure( + job, + attachment, + now: datetime, + error_code: str, + *, + terminal: bool, + retry_seconds: float, +) -> None: job.lease_token = None job.lease_until = None job.last_error_code = error_code @@ -89,7 +110,9 @@ def record_failure(job, attachment, now: datetime, error_code: str, *, terminal: attachment.updated_at = now -def complete_job(job, attachment, work: JobWork, variants: list[dict], now: datetime) -> None: +def complete_job( + job, attachment, work: JobWork, variants: list[dict], now: datetime +) -> None: attachment.object_key = work.original_key attachment.thumbnail_key = work.variant_key("thumbnail") attachment.crop_key = work.variant_key("crop") @@ -105,15 +128,25 @@ def complete_job(job, attachment, work: JobWork, variants: list[dict], now: date job.finished_at = now -async def terminal_staging(session: AsyncSession, retention: int, failed_retention: int, limit: int = 20): +async def terminal_staging( + session: AsyncSession, retention: int, failed_retention: int, limit: int = 20 +): result = await session.execute( select(AttachmentJob, Attachment) .join(Attachment, Attachment.id == AttachmentJob.attachment_id) .where( AttachmentJob.staging_key.is_not(None), or_( - and_(Attachment.status == "ready", AttachmentJob.finished_at < func.clock_timestamp() - timedelta(seconds=retention)), - and_(Attachment.status == "failed", AttachmentJob.finished_at < func.clock_timestamp() - timedelta(seconds=failed_retention)), + and_( + Attachment.status == "ready", + AttachmentJob.finished_at + < func.clock_timestamp() - timedelta(seconds=retention), + ), + and_( + Attachment.status == "failed", + AttachmentJob.finished_at + < func.clock_timestamp() - timedelta(seconds=failed_retention), + ), ), ) .order_by(AttachmentJob.finished_at) diff --git a/app/storage/outbox.py b/app/storage/outbox.py index e791c841..be50c31b 100644 --- a/app/storage/outbox.py +++ b/app/storage/outbox.py @@ -9,7 +9,9 @@ from app.models.attachment import Attachment from app.models.job import AttachmentJob -async def next_unpublished_job(session: AsyncSession, *, attachment_id: UUID | None = None) -> AttachmentJob | None: +async def next_unpublished_job( + session: AsyncSession, *, attachment_id: UUID | None = None +) -> AttachmentJob | None: statement = ( select(AttachmentJob) .join(Attachment, Attachment.id == AttachmentJob.attachment_id) diff --git a/infra/minio/bootstrap.py b/infra/minio/bootstrap.py index f1c76d70..b8f399f3 100644 --- a/infra/minio/bootstrap.py +++ b/infra/minio/bootstrap.py @@ -81,7 +81,10 @@ def main(): { "Effect": "Allow", "Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"], - "Resource": [f"arn:aws:s3:::{bucket}/{prefix}/*" for prefix in ("originals", "staging", "variants")], + "Resource": [ + f"arn:aws:s3:::{bucket}/{prefix}/*" + for prefix in ("originals", "staging", "variants") + ], }, ], } diff --git a/migrations/versions/49d2c947a3f1_add_idempotency_keys.py b/migrations/versions/49d2c947a3f1_add_idempotency_keys.py index ea3c65fc..153425e2 100644 --- a/migrations/versions/49d2c947a3f1_add_idempotency_keys.py +++ b/migrations/versions/49d2c947a3f1_add_idempotency_keys.py @@ -20,21 +20,26 @@ def upgrade() -> None: sa.Column("request_hash", sa.String(64), nullable=False), sa.Column("message_id", sa.UUID(), nullable=False), sa.Column( - "created_at", sa.DateTime(timezone=True), - server_default=sa.text("clock_timestamp()"), nullable=False, + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("clock_timestamp()"), + nullable=False, ), sa.ForeignKeyConstraint( ["user_id"], ["users.id"], name=op.f("fk_idempotency_keys_user_id_users") ), sa.ForeignKeyConstraint( - ["message_id"], ["messages.id"], + ["message_id"], + ["messages.id"], name=op.f("fk_idempotency_keys_message_id_messages"), ), sa.PrimaryKeyConstraint( "user_id", "method", "path", "key", name=op.f("pk_idempotency_keys") ), ) - op.create_index("ix_idempotency_keys_created_at", "idempotency_keys", ["created_at"]) + op.create_index( + "ix_idempotency_keys_created_at", "idempotency_keys", ["created_at"] + ) def downgrade() -> None: diff --git a/migrations/versions/8f31b94a2d10_add_attachments.py b/migrations/versions/8f31b94a2d10_add_attachments.py index b221dccb..34d3a6d5 100644 --- a/migrations/versions/8f31b94a2d10_add_attachments.py +++ b/migrations/versions/8f31b94a2d10_add_attachments.py @@ -21,14 +21,35 @@ def upgrade() -> None: sa.Column("size_bytes", sa.Integer(), nullable=False), sa.Column("sha256", sa.String(64), nullable=False), sa.Column("status", sa.String(16), server_default="ready", nullable=False), - sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("clock_timestamp()"), nullable=False), - sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("clock_timestamp()"), nullable=False), - sa.CheckConstraint("size_bytes BETWEEN 1 AND 10485760", name=op.f("ck_attachments_size_bytes")), - sa.CheckConstraint("content_type IN ('image/png', 'image/jpeg')", name=op.f("ck_attachments_content_type")), - sa.CheckConstraint("sha256 ~ '^[a-f0-9]{64}$'", name=op.f("ck_attachments_sha256")), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("clock_timestamp()"), + nullable=False, + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + server_default=sa.text("clock_timestamp()"), + nullable=False, + ), + sa.CheckConstraint( + "size_bytes BETWEEN 1 AND 10485760", name=op.f("ck_attachments_size_bytes") + ), + sa.CheckConstraint( + "content_type IN ('image/png', 'image/jpeg')", + name=op.f("ck_attachments_content_type"), + ), + sa.CheckConstraint( + "sha256 ~ '^[a-f0-9]{64}$'", name=op.f("ck_attachments_sha256") + ), sa.CheckConstraint("status = 'ready'", name=op.f("ck_attachments_status")), - sa.ForeignKeyConstraint(["chat_id"], ["chats.id"], name=op.f("fk_attachments_chat_id_chats")), - sa.ForeignKeyConstraint(["owner_id"], ["users.id"], name=op.f("fk_attachments_owner_id_users")), + sa.ForeignKeyConstraint( + ["chat_id"], ["chats.id"], name=op.f("fk_attachments_chat_id_chats") + ), + sa.ForeignKeyConstraint( + ["owner_id"], ["users.id"], name=op.f("fk_attachments_owner_id_users") + ), sa.PrimaryKeyConstraint("id", name=op.f("pk_attachments")), sa.UniqueConstraint("object_key", name=op.f("uq_attachments_object_key")), ) @@ -38,15 +59,31 @@ def upgrade() -> None: sa.Column("message_id", sa.UUID(), nullable=False), sa.Column("attachment_id", sa.UUID(), nullable=False), sa.Column("position", sa.Integer(), nullable=False), - sa.ForeignKeyConstraint(["message_id"], ["messages.id"], ondelete="CASCADE", name=op.f("fk_message_attachments_message_id_messages")), - sa.ForeignKeyConstraint(["attachment_id"], ["attachments.id"], name=op.f("fk_message_attachments_attachment_id_attachments")), - sa.PrimaryKeyConstraint("message_id", "attachment_id", name=op.f("pk_message_attachments")), + sa.ForeignKeyConstraint( + ["message_id"], + ["messages.id"], + ondelete="CASCADE", + name=op.f("fk_message_attachments_message_id_messages"), + ), + sa.ForeignKeyConstraint( + ["attachment_id"], + ["attachments.id"], + name=op.f("fk_message_attachments_attachment_id_attachments"), + ), + sa.PrimaryKeyConstraint( + "message_id", "attachment_id", name=op.f("pk_message_attachments") + ), ) op.create_table( "upload_intents", sa.Column("id", sa.UUID(), nullable=False), sa.Column("object_key", sa.String(128), nullable=False), - sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("clock_timestamp()"), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("clock_timestamp()"), + nullable=False, + ), sa.PrimaryKeyConstraint("id", name=op.f("pk_upload_intents")), sa.UniqueConstraint("object_key", name=op.f("uq_upload_intents_object_key")), ) diff --git a/migrations/versions/abd273076068_add_counters.py b/migrations/versions/abd273076068_add_counters.py index 9c809b06..cb6e08c5 100644 --- a/migrations/versions/abd273076068_add_counters.py +++ b/migrations/versions/abd273076068_add_counters.py @@ -5,29 +5,30 @@ Revises: db06fd3126ff Create Date: 2026-10-26 22:01:01.910396 """ + from typing import Sequence, Union from alembic import op import sqlalchemy as sa - -revision: str = 'abd273076068' -down_revision: Union[str, None] = 'db06fd3126ff' +revision: str = "abd273076068" +down_revision: Union[str, None] = "db06fd3126ff" branch_labels: Union[str, Sequence[str], None] = None depends_on: Union[str, Sequence[str], None] = None def upgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### - op.create_table('counters', - sa.Column('name', sa.String(length=64), nullable=False), - sa.Column('value', sa.BigInteger(), nullable=False), - sa.PrimaryKeyConstraint('name', name=op.f('pk_counters')) + op.create_table( + "counters", + sa.Column("name", sa.String(length=64), nullable=False), + sa.Column("value", sa.BigInteger(), nullable=False), + sa.PrimaryKeyConstraint("name", name=op.f("pk_counters")), ) # ### end Alembic commands ### def downgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### - op.drop_table('counters') + op.drop_table("counters") # ### end Alembic commands ### diff --git a/migrations/versions/c35e91a2f706_async_attachment_jobs.py b/migrations/versions/c35e91a2f706_async_attachment_jobs.py index 302ebc6d..5d42ac5d 100644 --- a/migrations/versions/c35e91a2f706_async_attachment_jobs.py +++ b/migrations/versions/c35e91a2f706_async_attachment_jobs.py @@ -12,12 +12,30 @@ depends_on = None def upgrade() -> None: op.drop_constraint(op.f("ck_attachments_status"), "attachments", type_="check") - op.create_check_constraint(op.f("ck_attachments_status"), "attachments", "status IN ('queued', 'processing', 'ready', 'failed')") + op.create_check_constraint( + op.f("ck_attachments_status"), + "attachments", + "status IN ('queued', 'processing', 'ready', 'failed')", + ) op.alter_column("attachments", "status", server_default="queued") op.add_column("attachments", sa.Column("error_code", sa.String(64), nullable=True)) - op.add_column("attachments", sa.Column("thumbnail_key", sa.String(128), nullable=True)) - op.add_column("attachments", sa.Column("variants", postgresql.JSONB(), nullable=False, server_default=sa.text("'[]'::jsonb"))) - op.create_check_constraint(op.f("ck_attachments_error_code"), "attachments", "(status = 'failed' AND error_code IS NOT NULL) OR (status <> 'failed' AND error_code IS NULL)") + op.add_column( + "attachments", sa.Column("thumbnail_key", sa.String(128), nullable=True) + ) + op.add_column( + "attachments", + sa.Column( + "variants", + postgresql.JSONB(), + nullable=False, + server_default=sa.text("'[]'::jsonb"), + ), + ) + op.create_check_constraint( + op.f("ck_attachments_error_code"), + "attachments", + "(status = 'failed' AND error_code IS NOT NULL) OR (status <> 'failed' AND error_code IS NULL)", + ) op.create_table( "attachment_jobs", sa.Column("id", sa.UUID(), nullable=False), @@ -25,31 +43,62 @@ def upgrade() -> None: sa.Column("staging_key", sa.String(128), nullable=True), sa.Column("pipeline_version", sa.Integer(), server_default="1", nullable=False), sa.Column("attempt", sa.Integer(), server_default="0", nullable=False), - sa.Column("published_attempt", sa.Integer(), server_default="0", nullable=False), - sa.Column("next_attempt_at", sa.DateTime(timezone=True), server_default=sa.text("clock_timestamp()"), nullable=False), + sa.Column( + "published_attempt", sa.Integer(), server_default="0", nullable=False + ), + sa.Column( + "next_attempt_at", + sa.DateTime(timezone=True), + server_default=sa.text("clock_timestamp()"), + nullable=False, + ), sa.Column("lease_token", sa.UUID(), nullable=True), sa.Column("lease_until", sa.DateTime(timezone=True), nullable=True), sa.Column("last_error_code", sa.String(64), nullable=True), sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True), - sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("clock_timestamp()"), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("clock_timestamp()"), + nullable=False, + ), sa.PrimaryKeyConstraint("id", name=op.f("pk_attachment_jobs")), - sa.ForeignKeyConstraint(["attachment_id"], ["attachments.id"], name=op.f("fk_attachment_jobs_attachment_id_attachments")), - sa.UniqueConstraint("attachment_id", name=op.f("uq_attachment_jobs_attachment_id")), - sa.CheckConstraint("pipeline_version = 1", name=op.f("ck_attachment_jobs_pipeline_version")), - sa.CheckConstraint("attempt >= 0 AND published_attempt >= 0", name=op.f("ck_attachment_jobs_attempt")), + sa.ForeignKeyConstraint( + ["attachment_id"], + ["attachments.id"], + name=op.f("fk_attachment_jobs_attachment_id_attachments"), + ), + sa.UniqueConstraint( + "attachment_id", name=op.f("uq_attachment_jobs_attachment_id") + ), + sa.CheckConstraint( + "pipeline_version = 1", name=op.f("ck_attachment_jobs_pipeline_version") + ), + sa.CheckConstraint( + "attempt >= 0 AND published_attempt >= 0", + name=op.f("ck_attachment_jobs_attempt"), + ), ) op.create_index("ix_attachment_jobs_due", "attachment_jobs", ["next_attempt_at"]) op.create_index("ix_attachment_jobs_lease", "attachment_jobs", ["lease_until"]) # Прежние оригиналы остаются источником; worker создаст отсутствующий thumbnail. - op.execute("INSERT INTO attachment_jobs (id, attachment_id, staging_key) SELECT id, id, object_key FROM attachments") - op.execute("UPDATE attachments SET status = 'queued', updated_at = clock_timestamp()") + op.execute( + "INSERT INTO attachment_jobs (id, attachment_id, staging_key) SELECT id, id, object_key FROM attachments" + ) + op.execute( + "UPDATE attachments SET status = 'queued', updated_at = clock_timestamp()" + ) def downgrade() -> None: connection = op.get_bind() - unfinished = connection.scalar(sa.text("SELECT count(*) FROM attachments WHERE status <> 'ready'")) + unfinished = connection.scalar( + sa.text("SELECT count(*) FROM attachments WHERE status <> 'ready'") + ) if unfinished: - raise RuntimeError("Перед downgrade нужно завершить или вручную разобрать все queued/processing/failed вложения") + raise RuntimeError( + "Перед downgrade нужно завершить или вручную разобрать все queued/processing/failed вложения" + ) op.drop_index("ix_attachment_jobs_lease", table_name="attachment_jobs") op.drop_index("ix_attachment_jobs_due", table_name="attachment_jobs") op.drop_table("attachment_jobs") @@ -58,5 +107,7 @@ def downgrade() -> None: op.drop_column("attachments", "thumbnail_key") op.drop_column("attachments", "error_code") op.drop_constraint(op.f("ck_attachments_status"), "attachments", type_="check") - op.create_check_constraint(op.f("ck_attachments_status"), "attachments", "status = 'ready'") + op.create_check_constraint( + op.f("ck_attachments_status"), "attachments", "status = 'ready'" + ) op.alter_column("attachments", "status", server_default="ready") diff --git a/migrations/versions/e7048b6da913_attachment_crop_webp.py b/migrations/versions/e7048b6da913_attachment_crop_webp.py index 2576cad3..b02f44f4 100644 --- a/migrations/versions/e7048b6da913_attachment_crop_webp.py +++ b/migrations/versions/e7048b6da913_attachment_crop_webp.py @@ -11,7 +11,9 @@ depends_on = None def upgrade() -> None: op.add_column("attachments", sa.Column("crop_key", sa.String(128), nullable=True)) - op.add_column("attachments", sa.Column("optimized_key", sa.String(128), nullable=True)) + op.add_column( + "attachments", sa.Column("optimized_key", sa.String(128), nullable=True) + ) # Старый staging сменяется надёжным ready-оригиналом, но не теряется из журнала cleanup. op.execute(""" INSERT INTO upload_intents (id, object_key) @@ -37,7 +39,11 @@ def upgrade() -> None: def downgrade() -> None: - active = op.get_bind().scalar(sa.text("SELECT count(*) FROM attachments WHERE status IN ('queued', 'processing')")) + active = op.get_bind().scalar( + sa.text( + "SELECT count(*) FROM attachments WHERE status IN ('queued', 'processing')" + ) + ) if active: raise RuntimeError("Перед downgrade необходимо завершить активные задания") op.execute(""" diff --git a/migrations/versions/f9183d27a640_job_replay.py b/migrations/versions/f9183d27a640_job_replay.py index 7e6ed24c..0c495fbf 100644 --- a/migrations/versions/f9183d27a640_job_replay.py +++ b/migrations/versions/f9183d27a640_job_replay.py @@ -10,19 +10,36 @@ depends_on = None def upgrade() -> None: - op.add_column("attachment_jobs", sa.Column("cycle_start_attempt", sa.Integer(), nullable=False, server_default="0")) - op.add_column("attachment_jobs", sa.Column("attempt_limit", sa.Integer(), nullable=True)) - op.add_column("attachment_jobs", sa.Column("last_error_permanent", sa.Boolean(), nullable=True)) + op.add_column( + "attachment_jobs", + sa.Column( + "cycle_start_attempt", sa.Integer(), nullable=False, server_default="0" + ), + ) + op.add_column( + "attachment_jobs", sa.Column("attempt_limit", sa.Integer(), nullable=True) + ) + op.add_column( + "attachment_jobs", + sa.Column("last_error_permanent", sa.Boolean(), nullable=True), + ) op.create_table( "job_replays", sa.Column("id", sa.Uuid(), primary_key=True), - sa.Column("job_id", sa.Uuid(), sa.ForeignKey("attachment_jobs.id"), nullable=False), + sa.Column( + "job_id", sa.Uuid(), sa.ForeignKey("attachment_jobs.id"), nullable=False + ), sa.Column("actor", sa.String(64), nullable=False), sa.Column("reason", sa.String(32), nullable=False), sa.Column("previous_attempt", sa.Integer(), nullable=False), sa.Column("attempt_limit", sa.Integer(), nullable=False), sa.Column("previous_error_code", sa.String(64), nullable=True), - sa.Column("created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.clock_timestamp()), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.func.clock_timestamp(), + ), ) op.create_index("ix_job_replays_job_id", "job_replays", ["job_id"]) diff --git a/scripts/check_contract.py b/scripts/check_contract.py index dcb34dfa..d31b8a3f 100644 --- a/scripts/check_contract.py +++ b/scripts/check_contract.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 """Offline structural checks; full schema validation is a separate target.""" + import ast import json from pathlib import Path @@ -7,48 +8,71 @@ import re ROOT = Path(__file__).resolve().parents[1] + def main(): - lab = json.loads((ROOT/'lab.json').read_text())['number'] - spec = json.loads((ROOT/'contracts/openapi.json').read_text()) - assert spec['openapi']=='3.1.0' - assert spec['info']['version']=='1.%d.0'%lab + lab = json.loads((ROOT / "lab.json").read_text())["number"] + spec = json.loads((ROOT / "contracts/openapi.json").read_text()) + assert spec["openapi"] == "3.1.0" + assert spec["info"]["version"] == "1.%d.0" % lab operation_ids = set() + def walk(value): - if isinstance(value,dict): - if '$ref' in value: - pointer = value['$ref'] - assert pointer.startswith('#/'), 'unexpected external reference' + if isinstance(value, dict): + if "$ref" in value: + pointer = value["$ref"] + assert pointer.startswith("#/"), "unexpected external reference" target = spec - for part in pointer[2:].split('/'): - target = target[part.replace('~1','/').replace('~0','~')] + for part in pointer[2:].split("/"): + target = target[part.replace("~1", "/").replace("~0", "~")] for v in value.values(): walk(v) - elif isinstance(value,list): + elif isinstance(value, list): for v in value: walk(v) + walk(spec) - for route,methods in spec['paths'].items(): - placeholders = set(re.findall(r'\{([^}]+)\}',route)) - for method,op in methods.items(): - assert op['operationId'] not in operation_ids, 'duplicate operationId' - operation_ids.add(op['operationId']) - params = {p['name'] for p in op.get('parameters',[]) if p['in']=='path' and p.get('required')} - assert placeholders==params, 'path parameter mismatch: '+route - for response in op['responses'].values(): - assert 'X-Request-Id' in response['headers'] - if lab>=3: - assert 'X-Instance-Id' in response['headers'] - assert ('/api/v1/auth/sessions' in spec['paths']) == (lab>=2) - assert ('/api/v1/attachments/{attachment_id}' in spec['paths']) == (lab>=4) - for file in list((ROOT/'tests').glob('*.py'))+list((ROOT/'scripts').glob('*.py')): - ast.parse(file.read_text(),filename=str(file)) - for file in ROOT.rglob('*.json'): - if any(part in {'.git','.venv','node_modules','vendor','output'} for part in file.parts): + for route, methods in spec["paths"].items(): + placeholders = set(re.findall(r"\{([^}]+)\}", route)) + for method, op in methods.items(): + assert op["operationId"] not in operation_ids, "duplicate operationId" + operation_ids.add(op["operationId"]) + params = { + p["name"] + for p in op.get("parameters", []) + if p["in"] == "path" and p.get("required") + } + assert placeholders == params, "path parameter mismatch: " + route + for response in op["responses"].values(): + assert "X-Request-Id" in response["headers"] + if lab >= 3: + assert "X-Instance-Id" in response["headers"] + assert ("/api/v1/auth/sessions" in spec["paths"]) == (lab >= 2) + assert ("/api/v1/attachments/{attachment_id}" in spec["paths"]) == (lab >= 4) + for file in list((ROOT / "tests").glob("*.py")) + list( + (ROOT / "scripts").glob("*.py") + ): + ast.parse(file.read_text(), filename=str(file)) + for file in ROOT.rglob("*.json"): + if any( + part in {".git", ".venv", "node_modules", "vendor", "output"} + for part in file.parts + ): continue json.loads(file.read_text()) - for required in ('README.md','COURSE.md','CONTRIBUTING.md','REPORT.md','contracts/README.md','tests/smoke.py','tests/README.md'): - assert (ROOT/required).is_file(), 'missing '+required - print('PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.') + for required in ( + "README.md", + "COURSE.md", + "CONTRIBUTING.md", + "REPORT.md", + "contracts/README.md", + "tests/smoke.py", + "tests/README.md", + ): + assert (ROOT / required).is_file(), "missing " + required + print( + "PASS: offline skeleton structure, local refs and Python syntax; application NOT tested." + ) -if __name__=='__main__': + +if __name__ == "__main__": main() diff --git a/scripts/check_jobs.py b/scripts/check_jobs.py index dc724b66..76519c51 100644 --- a/scripts/check_jobs.py +++ b/scripts/check_jobs.py @@ -16,7 +16,7 @@ from scripts.check_outbox_recovery import COMPOSE, ROOT, compose, require_compos from scripts.deployment import ComposeProject from tests.client import Client, check -PROBE = ''' +PROBE = """ import asyncio, hashlib, json from uuid import UUID from sqlalchemy import select @@ -43,169 +43,387 @@ async def main(): result['objects']=sorted(objects,key=lambda x:x['name']);print(json.dumps(result,default=str)) finally: await storage.aclose();await engine.dispose() asyncio.run(main()) -''' +""" def probe(attachment_id): - code = 'ATTACHMENT_ID = ' + repr(str(UUID(attachment_id))) + '\n' + PROBE - result = subprocess.run(COMPOSE + ['exec', '-T', 'api', 'python', '-'], input=code, text=True, capture_output=True, cwd=ROOT, timeout=30) - check(result.returncode == 0, 'Probe failed; diagnostic output hidden') + code = "ATTACHMENT_ID = " + repr(str(UUID(attachment_id))) + "\n" + PROBE + result = subprocess.run( + COMPOSE + ["exec", "-T", "api", "python", "-"], + input=code, + text=True, + capture_output=True, + cwd=ROOT, + timeout=30, + ) + check(result.returncode == 0, "Probe failed; diagnostic output hidden") return json.loads(result.stdout) def admin(*arguments, expected=0): - result = compose('run', '--rm', '--no-deps', 'worker', 'python', '-m', 'app.core.job_admin', *arguments) - check(result.returncode == expected, 'Unexpected operator command exit status') + result = compose( + "run", + "--rm", + "--no-deps", + "worker", + "python", + "-m", + "app.core.job_admin", + *arguments + ) + check(result.returncode == expected, "Unexpected operator command exit status") return json.loads(result.stdout) def verify_result(client, actor, attachment, original): ready = client.wait_ready(actor, attachment, wait_seconds=180) - check(ready['id'] == attachment['id'], 'Attachment identity changed') - path = '/api/v1/attachments/' + ready['id'] + '/content' - check(client.call('GET', path, actor=actor)['bytes'] == original, 'Original changed') - check({v['name'] for v in ready['variants']} == {'thumbnail','crop','optimized'}, 'Incomplete pipeline') - for variant in ready['variants']: - data = client.call('GET', path+'?variant='+variant['name'], actor=actor)['bytes'] - check(hashlib.sha256(data).hexdigest() == variant['sha256'], 'Checksum mismatch') + check(ready["id"] == attachment["id"], "Attachment identity changed") + path = "/api/v1/attachments/" + ready["id"] + "/content" + check( + client.call("GET", path, actor=actor)["bytes"] == original, "Original changed" + ) + check( + {v["name"] for v in ready["variants"]} == {"thumbnail", "crop", "optimized"}, + "Incomplete pipeline", + ) + for variant in ready["variants"]: + data = client.call("GET", path + "?variant=" + variant["name"], actor=actor)[ + "bytes" + ] + check( + hashlib.sha256(data).hexdigest() == variant["sha256"], "Checksum mismatch" + ) with Image.open(BytesIO(data)) as image: - check(image.size == (variant['width'],variant['height']), 'Metadata dimensions mismatch') - check(image.format == ('WEBP' if variant['name']=='optimized' else 'JPEG'), 'Wrong format') - check(not image.getexif() and not image.info.get('icc_profile') and not image.info.get('xmp'), 'Metadata leaked') - if variant['name']=='crop':check(image.size==(128,128),'Wrong crop size') - snapshot=probe(ready['id']) - check(len(snapshot['objects']) == 4, 'Expected exactly one original and three variants') + check( + image.size == (variant["width"], variant["height"]), + "Metadata dimensions mismatch", + ) + check( + image.format == ("WEBP" if variant["name"] == "optimized" else "JPEG"), + "Wrong format", + ) + check( + not image.getexif() + and not image.info.get("icc_profile") + and not image.info.get("xmp"), + "Metadata leaked", + ) + if variant["name"] == "crop": + check(image.size == (128, 128), "Wrong crop size") + snapshot = probe(ready["id"]) + check( + len(snapshot["objects"]) == 4, + "Expected exactly one original and three variants", + ) return snapshot def crash(client, actor, chat, mode): - name='messenger-worker-fault-'+uuid4().hex + name = "messenger-worker-fault-" + uuid4().hex try: - require_compose('stop','worker') - attachment,original=client.attachment(actor,chat) - result=compose('run','--rm','--no-deps','--name',name,'worker','python','-m','app.core.job_worker','--'+mode,attachment['id']) - phase='after_object:optimized' if mode=='crash-after-object' else 'before_ack' - marker=json.dumps({'fault':phase,'attachment_id':attachment['id']}) - check(result.returncode==137 and marker in result.stdout,'Targeted crash not observed') - before=probe(attachment['id']) - check(before['status']==('processing' if mode=='crash-after-object' else 'ready'),'Unexpected crash state') - check(len(before['objects'])==4,'Crash must occur after all four PUTs') + require_compose("stop", "worker") + attachment, original = client.attachment(actor, chat) + result = compose( + "run", + "--rm", + "--no-deps", + "--name", + name, + "worker", + "python", + "-m", + "app.core.job_worker", + "--" + mode, + attachment["id"], + ) + phase = ( + "after_object:optimized" if mode == "crash-after-object" else "before_ack" + ) + marker = json.dumps({"fault": phase, "attachment_id": attachment["id"]}) + check( + result.returncode == 137 and marker in result.stdout, + "Targeted crash not observed", + ) + before = probe(attachment["id"]) + check( + before["status"] + == ("processing" if mode == "crash-after-object" else "ready"), + "Unexpected crash state", + ) + check(len(before["objects"]) == 4, "Crash must occur after all four PUTs") finally: - subprocess.run(['docker','rm','-f',name],capture_output=True,timeout=30) - require_compose('start','worker') - after=verify_result(client,actor,attachment,original) - if mode=='crash-before-ack': + subprocess.run(["docker", "rm", "-f", name], capture_output=True, timeout=30) + require_compose("start", "worker") + after = verify_result(client, actor, attachment, original) + if mode == "crash-before-ack": # Wait for RabbitMQ to deliver and acknowledge the ready duplicate. wait_queue_empty() - after=probe(attachment['id']) - check(after==before,'Ready duplicate rewrote result') + after = probe(attachment["id"]) + check(after == before, "Ready duplicate rewrote result") else: - check(after['attempt']==before['attempt']+1,'Lease did not recover the interrupted attempt') - check([(x['name'],x['sha256']) for x in before['objects']]==[(x['name'],x['sha256']) for x in after['objects']],'Replay changed object content') - return dict(scenario=mode,attachment_id=attachment['id'],exit_code=137,before=before,after=after) + check( + after["attempt"] == before["attempt"] + 1, + "Lease did not recover the interrupted attempt", + ) + check( + [(x["name"], x["sha256"]) for x in before["objects"]] + == [(x["name"], x["sha256"]) for x in after["objects"]], + "Replay changed object content", + ) + return dict( + scenario=mode, + attachment_id=attachment["id"], + exit_code=137, + before=before, + after=after, + ) -def retry_replay(client,actor,chat,permanent): - name='messenger-retry-fault-'+uuid4().hex - samples=[] - configuration=ComposeProject().config['services']['worker']['environment'] - retry_base=float(configuration['JOB_RETRY_SECONDS']) - retry_max=float(configuration['JOB_RETRY_MAX_SECONDS']) +def retry_replay(client, actor, chat, permanent): + name = "messenger-retry-fault-" + uuid4().hex + samples = [] + configuration = ComposeProject().config["services"]["worker"]["environment"] + retry_base = float(configuration["JOB_RETRY_SECONDS"]) + retry_max = float(configuration["JOB_RETRY_MAX_SECONDS"]) try: - require_compose('stop','worker') - attachment,original=client.attachment(actor,chat) - mode='fail-permanent' if permanent else 'fail-transient' - require_compose('run','-d','--no-deps','--name',name,'worker','python','-m','app.core.job_worker','--'+mode,attachment['id']) - deadline=time.monotonic()+180 - while time.monotonic()=1 and queued_before['durable'], 'No durable queued message before restart') - result=subprocess.run([str(ROOT/'scripts/restart.sh')],cwd=ROOT,capture_output=True,timeout=180) - check(result.returncode==0,'Restart hook failed') - check('worker' not in require_compose('ps','--status','running','--services').splitlines(),'Restart started deliberately stopped worker') - queued_after=queue_state() - check(queued_after['messages_ready']>=1 and queued_after['durable'], 'Broker lost queued delivery across restart') - client.message(actor,chat,'session and chat survived broker/API restart') - finally:require_compose('start','worker') - after=verify_result(client,actor,attachment,original) - return dict(scenario='worker_stopped_restart_all',attachment_id=attachment['id'],queue_before=queued_before,queue_after=queued_after,after=after) + queued_before = queue_state() + check( + queued_before["messages_ready"] >= 1 and queued_before["durable"], + "No durable queued message before restart", + ) + result = subprocess.run( + [str(ROOT / "scripts/restart.sh")], + cwd=ROOT, + capture_output=True, + timeout=180, + ) + check(result.returncode == 0, "Restart hook failed") + check( + "worker" + not in require_compose( + "ps", "--status", "running", "--services" + ).splitlines(), + "Restart started deliberately stopped worker", + ) + queued_after = queue_state() + check( + queued_after["messages_ready"] >= 1 and queued_after["durable"], + "Broker lost queued delivery across restart", + ) + client.message(actor, chat, "session and chat survived broker/API restart") + finally: + require_compose("start", "worker") + after = verify_result(client, actor, attachment, original) + return dict( + scenario="worker_stopped_restart_all", + attachment_id=attachment["id"], + queue_before=queued_before, + queue_after=queued_after, + after=after, + ) def main(): - parser=argparse.ArgumentParser() - parser.add_argument('--base-url',required=True) - parser.add_argument('--ca-file') - parser.add_argument('--scenario',choices=['all','persistence','crash-after-object','crash-before-ack','fail-transient','fail-permanent'],default='all') - parser.add_argument('--output',default='evidence/jobs.json') - args=parser.parse_args() - client=Client(args.base_url,args.ca_file,timeout=40) - aliases={r.instance_id for r in ComposeProject().replicas() if r.running} - check(client.call('GET','/health/live',schema='Health')['headers'].get('X-Instance-Id') in aliases,'Wrong Compose project for BASE_URL') - actor,_,_,chat=client.bootstrap() - results=[] - operations={'persistence':persistence,'crash-after-object':lambda *x:crash(*x,'crash-after-object'),'crash-before-ack':lambda *x:crash(*x,'crash-before-ack'),'fail-transient':lambda *x:retry_replay(*x,False),'fail-permanent':lambda *x:retry_replay(*x,True)} - selected=operations.values() if args.scenario=='all' else [operations[args.scenario]] + parser = argparse.ArgumentParser() + parser.add_argument("--base-url", required=True) + parser.add_argument("--ca-file") + parser.add_argument( + "--scenario", + choices=[ + "all", + "persistence", + "crash-after-object", + "crash-before-ack", + "fail-transient", + "fail-permanent", + ], + default="all", + ) + parser.add_argument("--output", default="evidence/jobs.json") + args = parser.parse_args() + client = Client(args.base_url, args.ca_file, timeout=40) + aliases = {r.instance_id for r in ComposeProject().replicas() if r.running} + check( + client.call("GET", "/health/live", schema="Health")["headers"].get( + "X-Instance-Id" + ) + in aliases, + "Wrong Compose project for BASE_URL", + ) + actor, _, _, chat = client.bootstrap() + results = [] + operations = { + "persistence": persistence, + "crash-after-object": lambda *x: crash(*x, "crash-after-object"), + "crash-before-ack": lambda *x: crash(*x, "crash-before-ack"), + "fail-transient": lambda *x: retry_replay(*x, False), + "fail-permanent": lambda *x: retry_replay(*x, True), + } + selected = ( + operations.values() if args.scenario == "all" else [operations[args.scenario]] + ) for operation in selected: - result=operation(client,actor,chat);results.append(result) - Path(args.output).parent.mkdir(parents=True,exist_ok=True) - Path(args.output).write_text(json.dumps(dict(status='in_progress',results=results),indent=2)+'\n') - print('PASS: '+result['scenario'],flush=True) - Path(args.output).write_text(json.dumps(dict(status='passed',results=results),indent=2)+'\n') + result = operation(client, actor, chat) + results.append(result) + Path(args.output).parent.mkdir(parents=True, exist_ok=True) + Path(args.output).write_text( + json.dumps(dict(status="in_progress", results=results), indent=2) + "\n" + ) + print("PASS: " + result["scenario"], flush=True) + Path(args.output).write_text( + json.dumps(dict(status="passed", results=results), indent=2) + "\n" + ) -if __name__=='__main__':main() +if __name__ == "__main__": + main() diff --git a/scripts/check_outbox_recovery.py b/scripts/check_outbox_recovery.py index 288257e9..c52471a4 100644 --- a/scripts/check_outbox_recovery.py +++ b/scripts/check_outbox_recovery.py @@ -19,7 +19,13 @@ COMPOSE = ["docker", "compose", "--env-file", ".env"] def compose(*arguments: str, timeout: int = 90) -> subprocess.CompletedProcess: - return subprocess.run(COMPOSE + list(arguments), cwd=ROOT, text=True, capture_output=True, timeout=timeout) + return subprocess.run( + COMPOSE + list(arguments), + cwd=ROOT, + text=True, + capture_output=True, + timeout=timeout, + ) def require_compose(*arguments: str) -> str: @@ -29,54 +35,112 @@ def require_compose(*arguments: str) -> str: def main() -> None: - parser = argparse.ArgumentParser(description="Отказ publisher после COMMIT перед RabbitMQ publish") + parser = argparse.ArgumentParser( + description="Отказ publisher после COMMIT перед RabbitMQ publish" + ) parser.add_argument("--base-url", required=True) parser.add_argument("--ca-file") args = parser.parse_args() - running = set(require_compose("ps", "--status", "running", "--services").splitlines()) - check({"api", "postgres", "minio", "broker", "publisher", "worker"} <= running, "Start the application stack before this scenario") + running = set( + require_compose("ps", "--status", "running", "--services").splitlines() + ) + check( + {"api", "postgres", "minio", "broker", "publisher", "worker"} <= running, + "Start the application stack before this scenario", + ) client = Client(args.base_url, args.ca_file, timeout=40) project = ComposeProject() aliases = {replica.instance_id for replica in project.replicas() if replica.running} live = client.call("GET", "/health/live", schema="Health") - check(live["headers"].get("X-Instance-Id") in aliases, "BASE_URL does not belong to the selected Compose project") + check( + live["headers"].get("X-Instance-Id") in aliases, + "BASE_URL does not belong to the selected Compose project", + ) alice, _, _, chat = client.bootstrap() attachment = None fault_container = "messenger-outbox-fault-" + uuid4().hex try: require_compose("stop", "publisher") attachment, original = client.attachment(alice, chat) - message = client.message(alice, chat, "", attachments=[attachment["id"]])["json"] - client.call("GET", f"/api/v1/attachments/{attachment['id']}/content", expected=409, actor=alice) - killed = compose( - "run", "--rm", "--no-deps", "--name", fault_container, "publisher", "python", "-m", - "app.core.outbox_publisher", "--crash-before-publish", attachment["id"], + message = client.message(alice, chat, "", attachments=[attachment["id"]])[ + "json" + ] + client.call( + "GET", + f"/api/v1/attachments/{attachment['id']}/content", + expected=409, + actor=alice, ) - marker = json.dumps({"fault": "before_publish", "attachment_id": attachment["id"]}) - check(killed.returncode == 137 and marker in killed.stdout, "Expected targeted SIGKILL before publish was not observed") - pending = client.call("GET", f"/api/v1/attachments/{attachment['id']}", actor=alice, schema="Attachment")["json"] + killed = compose( + "run", + "--rm", + "--no-deps", + "--name", + fault_container, + "publisher", + "python", + "-m", + "app.core.outbox_publisher", + "--crash-before-publish", + attachment["id"], + ) + marker = json.dumps( + {"fault": "before_publish", "attachment_id": attachment["id"]} + ) + check( + killed.returncode == 137 and marker in killed.stdout, + "Expected targeted SIGKILL before publish was not observed", + ) + pending = client.call( + "GET", + f"/api/v1/attachments/{attachment['id']}", + actor=alice, + schema="Attachment", + )["json"] check(pending["status"] == "queued", "Crash must leave accepted job queued") finally: # Если docker compose run оборвался по timeout, одноразовый контейнер тоже удаляется. try: - subprocess.run(["docker", "rm", "-f", fault_container], capture_output=True, timeout=30) + subprocess.run( + ["docker", "rm", "-f", fault_container], capture_output=True, timeout=30 + ) finally: require_compose("start", "publisher") ready = client.wait_ready(alice, attachment, wait_seconds=60) check(ready["id"] == attachment["id"], "Recovered a different attachment") - check({variant["name"] for variant in ready["variants"]} == {"thumbnail", "crop", "optimized"}, "Missing required variants") + check( + {variant["name"] for variant in ready["variants"]} + == {"thumbnail", "crop", "optimized"}, + "Missing required variants", + ) path = f"/api/v1/attachments/{attachment['id']}/content" - check(client.call("GET", path, actor=alice)["bytes"] == original, "Original changed after recovery") + check( + client.call("GET", path, actor=alice)["bytes"] == original, + "Original changed after recovery", + ) for variant in ready["variants"]: - downloaded = client.call("GET", path + "?variant=" + variant["name"], actor=alice)["bytes"] - check(len(downloaded) == variant["size_bytes"] and hashlib.sha256(downloaded).hexdigest() == variant["sha256"], "Recovered variant checksum mismatch") - print(json.dumps({ - "scenario": "commit_before_publish", "attachment_id": ready["id"], - "message_id": message["id"], "publisher_exit_code": killed.returncode, - "status_after_crash": pending["status"], "status_after_restart": ready["status"], - "variants": [variant["name"] for variant in ready["variants"]], - "reuploaded": False, - })) + downloaded = client.call( + "GET", path + "?variant=" + variant["name"], actor=alice + )["bytes"] + check( + len(downloaded) == variant["size_bytes"] + and hashlib.sha256(downloaded).hexdigest() == variant["sha256"], + "Recovered variant checksum mismatch", + ) + print( + json.dumps( + { + "scenario": "commit_before_publish", + "attachment_id": ready["id"], + "message_id": message["id"], + "publisher_exit_code": killed.returncode, + "status_after_crash": pending["status"], + "status_after_restart": ready["status"], + "variants": [variant["name"] for variant in ready["variants"]], + "reuploaded": False, + } + ) + ) if __name__ == "__main__": diff --git a/scripts/check_s3.py b/scripts/check_s3.py index f94f6a9e..eddadaf0 100644 --- a/scripts/check_s3.py +++ b/scripts/check_s3.py @@ -7,7 +7,7 @@ import time from scripts.deployment import ComposeProject from tests.client import Client, arguments, check, run -PROBE = r''' +PROBE = r""" import asyncio, json, sys from pathlib import Path from uuid import UUID @@ -62,60 +62,91 @@ async def main(): await storage.aclose() await engine.dispose() asyncio.run(main()) -''' +""" def main(): def configure(parser): - parser.add_argument('--with-outage', action='store_true', help='временно остановить MinIO выбранного проекта') - args = arguments('MinIO private bucket and failure checks', configure) + parser.add_argument( + "--with-outage", + action="store_true", + help="временно остановить MinIO выбранного проекта", + ) + + args = arguments("MinIO private bucket and failure checks", configure) project = ComposeProject() client = Client(args.base_url, args.ca_file, timeout=40) alice, _, _, chat = client.bootstrap() aliases = {replica.instance_id for replica in project.replicas() if replica.running} - live = client.call('GET', '/health/live', schema='Health') - check(live['headers'].get('X-Instance-Id') in aliases, 'BASE_URL does not belong to the selected Compose project') + live = client.call("GET", "/health/live", schema="Health") + check( + live["headers"].get("X-Instance-Id") in aliases, + "BASE_URL does not belong to the selected Compose project", + ) attachment, original = client.attachment(alice, chat) attachment = client.wait_ready(alice, attachment) # Повторный bootstrap не должен очистить bucket или сменить используемые credentials. for _ in range(2): - project.run('run', '--rm', '--no-deps', 'minio-bootstrap', timeout=100) + project.run("run", "--rm", "--no-deps", "minio-bootstrap", timeout=100) replicas = project.replicas() - check(bool(replicas), 'API not running') - result = subprocess.run(['docker', 'exec', '-i', replicas[0].id, 'python', '-c', PROBE], - input=json.dumps({'attachment_id': attachment['id']}), - text=True, capture_output=True, timeout=20) - check(result.returncode == 0, 'private bucket probe failed (details hidden)') + check(bool(replicas), "API not running") + result = subprocess.run( + ["docker", "exec", "-i", replicas[0].id, "python", "-c", PROBE], + input=json.dumps({"attachment_id": attachment["id"]}), + text=True, + capture_output=True, + timeout=20, + ) + check(result.returncode == 0, "private bucket probe failed (details hidden)") statuses = json.loads(result.stdout) - check(statuses['GET'] == statuses['PUT'] == 403, 'anonymous S3 access allowed') - check(statuses['admin_denied'] and statuses['other_buckets_denied'] and statuses['root_files_absent'], 'API has excessive access') - check(statuses['console_unreachable'], 'console reachable from API network') - check(statuses['temp_originals_absent'], 'API retained temporary originals') - check(not project.config['services']['minio'].get('ports'), 'MinIO publishes host ports') - env = project.config['services']['minio']['environment'] - check(env['MINIO_BROWSER'] == 'off', 'browser enabled') - check('127.0.0.1:9001' in project.config['services']['minio']['command'], 'console is not bound to loopback') - path = '/api/v1/attachments/' + attachment['id'] + '/content' - check(client.call('GET', path, actor=alice)['bytes'] == original, 'bootstrap changed original') + check(statuses["GET"] == statuses["PUT"] == 403, "anonymous S3 access allowed") + check( + statuses["admin_denied"] + and statuses["other_buckets_denied"] + and statuses["root_files_absent"], + "API has excessive access", + ) + check(statuses["console_unreachable"], "console reachable from API network") + check(statuses["temp_originals_absent"], "API retained temporary originals") + check( + not project.config["services"]["minio"].get("ports"), + "MinIO publishes host ports", + ) + env = project.config["services"]["minio"]["environment"] + check(env["MINIO_BROWSER"] == "off", "browser enabled") + check( + "127.0.0.1:9001" in project.config["services"]["minio"]["command"], + "console is not bound to loopback", + ) + path = "/api/v1/attachments/" + attachment["id"] + "/content" + check( + client.call("GET", path, actor=alice)["bytes"] == original, + "bootstrap changed original", + ) if args.with_outage: try: - project.run('stop', 'minio') - client.call('GET', path, expected=503, actor=alice) + project.run("stop", "minio") + client.call("GET", path, expected=503, actor=alice) finally: - project.run('start', 'minio') + project.run("start", "minio") deadline = time.monotonic() + 60 while True: try: - client.call('GET', '/health/ready', schema='Health') + client.call("GET", "/health/ready", schema="Health") break except Exception: if time.monotonic() >= deadline: raise time.sleep(0.5) - check(client.call('GET', path, actor=alice)['bytes'] == original, 'original lost after MinIO restart') - print('PASS: idempotent bootstrap, anonymous GET/PUT denied, restricted API credentials, private console' - + (', MinIO outage/recovery' if args.with_outage else '')) + check( + client.call("GET", path, actor=alice)["bytes"] == original, + "original lost after MinIO restart", + ) + print( + "PASS: idempotent bootstrap, anonymous GET/PUT denied, restricted API credentials, private console" + + (", MinIO outage/recovery" if args.with_outage else "") + ) -if __name__ == '__main__': +if __name__ == "__main__": run(main) diff --git a/scripts/deployment.py b/scripts/deployment.py index fba70f5e..c629e6e8 100644 --- a/scripts/deployment.py +++ b/scripts/deployment.py @@ -31,8 +31,12 @@ class ComposeProject: def docker(self, *args: str, timeout: int = 90, env=None) -> str: """Выполняет Docker-команду с ограничением времени.""" result = subprocess.run( - ["docker", *args], cwd=ROOT, text=True, capture_output=True, - timeout=timeout, env={**os.environ, **(env or {})}, + ["docker", *args], + cwd=ROOT, + text=True, + capture_output=True, + timeout=timeout, + env={**os.environ, **(env or {})}, ) if result.returncode: raise RuntimeError(result.stderr.strip() or result.stdout.strip()) @@ -53,8 +57,17 @@ class ComposeProject: '"health":{{if .State.Health}}{{json .State.Health.Status}}{{else}}null{{end}}}' ) rows = self.docker("inspect", "--format", template, *ids).splitlines() - configured = str(self.config["services"]["api"].get("environment", {}).get("INSTANCE_ID") or "").strip() + configured = str( + self.config["services"]["api"].get("environment", {}).get("INSTANCE_ID") + or "" + ).strip() return [ - Replica(row["id"], row["image"], configured or row["hostname"], row["running"], row["health"]) + Replica( + row["id"], + row["image"], + configured or row["hostname"], + row["running"], + row["health"], + ) for row in map(json.loads, rows) ] diff --git a/scripts/init_local.py b/scripts/init_local.py index a3fb687a..7477cd09 100644 --- a/scripts/init_local.py +++ b/scripts/init_local.py @@ -14,7 +14,15 @@ def main() -> None: destination.chmod(0o600) directory = root / "secrets" directory.mkdir(mode=0o700, exist_ok=True) - for name in ("db_password", "redis_password", "broker_password", "minio_root_user", "minio_root_password", "s3_access_key_id", "s3_secret_access_key"): + for name in ( + "db_password", + "redis_password", + "broker_password", + "minio_root_user", + "minio_root_password", + "s3_access_key_id", + "s3_secret_access_key", + ): path = directory / name try: descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o444) diff --git a/scripts/validate_openapi.py b/scripts/validate_openapi.py index 7e1d1afc..8351b1c4 100644 --- a/scripts/validate_openapi.py +++ b/scripts/validate_openapi.py @@ -1,23 +1,27 @@ #!/usr/bin/env python3 """Full OpenAPI and JSON Schema checks; install requirements-dev.txt first.""" + import json from pathlib import Path from openapi_spec_validator import validate from jsonschema import Draft202012Validator root = Path(__file__).resolve().parents[1] -spec = json.loads((root/'contracts/openapi.json').read_text()) +spec = json.loads((root / "contracts/openapi.json").read_text()) validate(spec) from referencing import Registry, Resource + # The base URI resolves local #/components references inside examples. -resource = dict(spec, **{'$schema': 'https://json-schema.org/draft/2020-12/schema'}) -registry = Registry().with_resource('urn:messenger:openapi', Resource.from_contents(resource)) -for name, example in json.loads((root/'contracts/examples.json').read_text()).items(): - schema = {'$ref': 'urn:messenger:openapi#/components/schemas/' + name} +resource = dict(spec, **{"$schema": "https://json-schema.org/draft/2020-12/schema"}) +registry = Registry().with_resource( + "urn:messenger:openapi", Resource.from_contents(resource) +) +for name, example in json.loads((root / "contracts/examples.json").read_text()).items(): + schema = {"$ref": "urn:messenger:openapi#/components/schemas/" + name} Draft202012Validator(schema, registry=registry).validate(example) -for path in (root/'contracts').glob('*.schema.json'): +for path in (root / "contracts").glob("*.schema.json"): schema = json.loads(path.read_text()) Draft202012Validator.check_schema(schema) - for example in schema.get('examples',[]): + for example in schema.get("examples", []): Draft202012Validator(schema).validate(example) -print('PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.') +print("PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.") diff --git a/student-tests/test_attachments.py b/student-tests/test_attachments.py index 20237f1f..dba63f94 100644 --- a/student-tests/test_attachments.py +++ b/student-tests/test_attachments.py @@ -28,7 +28,7 @@ from app.storage import attachments as repository from app.storage.s3 import ObjectInfo from helpers import bearer, create_chat, create_user, idempotency_header -PNG = Path('tests/fixtures/sample.png').read_bytes() +PNG = Path("tests/fixtures/sample.png").read_bytes() class MemoryStorage: @@ -55,12 +55,15 @@ class MemoryStorage: info = None if key in self.objects: data, metadata = self.objects[key] - info = ObjectInfo(key, len(data), metadata['content_type'], datetime.now(timezone.utc)) + info = ObjectInfo( + key, len(data), metadata["content_type"], datetime.now(timezone.utc) + ) return info async def open_object(self, key): async def chunks(): yield self.objects[key][0] + return chunks() async def check_available(self): @@ -76,23 +79,40 @@ async def context(client, app): return storage, alice, bob, eve, chat -async def upload(client, actor, chat, data=PNG, filename='фото.png', content_type='image/png'): +async def upload( + client, actor, chat, data=PNG, filename="фото.png", content_type="image/png" +): return await client.post( - f"/api/v1/chats/{chat['id']}/attachments", params={'filename': filename}, - headers={**bearer(actor), 'Content-Type': content_type}, content=data, + f"/api/v1/chats/{chat['id']}/attachments", + params={"filename": filename}, + headers={**bearer(actor), "Content-Type": content_type}, + content=data, ) async def finish_attachment(client, storage, actor, attachment_id): engine = client._transport.app.state.engine async with AsyncSession(engine) as session: - job = await session.scalar(select(AttachmentJob).where(AttachmentJob.attachment_id == UUID(attachment_id))) - message = JobMessage(job_id=job.id, attachment_id=job.attachment_id, - pipeline_version=job.pipeline_version, attempt=job.attempt + 1, created_at=job.created_at) + job = await session.scalar( + select(AttachmentJob).where( + AttachmentJob.attachment_id == UUID(attachment_id) + ) + ) + message = JobMessage( + job_id=job.id, + attachment_id=job.attachment_id, + pipeline_version=job.pipeline_version, + attempt=job.attempt + 1, + created_at=job.created_at, + ) await session.rollback() - assert await process_attachment_job(session, storage, message, JobSettings(_env_file=None)) - result = await client.get(f'/api/v1/attachments/{attachment_id}', headers=bearer(actor)) - assert result.json()['status'] == 'ready' + assert await process_attachment_job( + session, storage, message, JobSettings(_env_file=None) + ) + result = await client.get( + f"/api/v1/attachments/{attachment_id}", headers=bearer(actor) + ) + assert result.json()["status"] == "ready" return result.json() @@ -101,37 +121,52 @@ async def test_upload_download_and_attachment_only_message(client, context): response = await upload(client, alice, chat) assert response.status_code == 202, response.text attachment = response.json() - assert attachment['status'] == 'queued' - before = await client.get(f"/api/v1/attachments/{attachment['id']}/content", headers=bearer(alice)) + assert attachment["status"] == "queued" + before = await client.get( + f"/api/v1/attachments/{attachment['id']}/content", headers=bearer(alice) + ) assert before.status_code == 409 - attachment = await finish_attachment(client, storage, alice, attachment['id']) - assert attachment['sha256'] == hashlib.sha256(PNG).hexdigest() - assert {v['name'] for v in attachment['variants']} == {'thumbnail', 'crop', 'optimized'} - assert attachment['error_code'] is None - assert 'object_key' not in attachment - id = attachment['id'] + attachment = await finish_attachment(client, storage, alice, attachment["id"]) + assert attachment["sha256"] == hashlib.sha256(PNG).hexdigest() + assert {v["name"] for v in attachment["variants"]} == { + "thumbnail", + "crop", + "optimized", + } + assert attachment["error_code"] is None + assert "object_key" not in attachment + id = attachment["id"] for actor in (alice, bob): - content = await client.get(f'/api/v1/attachments/{id}/content', headers=bearer(actor)) + content = await client.get( + f"/api/v1/attachments/{id}/content", headers=bearer(actor) + ) assert content.status_code == 200 and content.content == PNG - assert content.headers['content-type'] == 'image/png' + assert content.headers["content-type"] == "image/png" key = idempotency_header() - body = {'text': '', 'attachment_ids': [id]} + body = {"text": "", "attachment_ids": [id]} path = f"/api/v1/chats/{chat['id']}/messages" message = await client.post(path, json=body, headers={**bearer(bob), **key}) assert message.status_code == 201, message.text repeat = await client.post(path, json=body, headers={**bearer(bob), **key}) assert repeat.json() == message.json() history = await client.get(path, headers=bearer(alice)) - assert history.json()['items'][0]['attachment_ids'] == [id] - for suffix in ('', '/content'): - denied = await client.get(f'/api/v1/attachments/{id}{suffix}', headers=bearer(eve)) + assert history.json()["items"][0]["attachment_ids"] == [id] + for suffix in ("", "/content"): + denied = await client.get( + f"/api/v1/attachments/{id}{suffix}", headers=bearer(eve) + ) assert denied.status_code == 404 @pytest.mark.parametrize( - 'data,content_type', - [(b'not an image', 'image/png'), (PNG[:30], 'image/png'), (PNG, 'image/jpeg'), (PNG, 'application/octet-stream')], - ids=['not-image', 'truncated-png', 'mime-spoofing', 'unsupported-type'], + "data,content_type", + [ + (b"not an image", "image/png"), + (PNG[:30], "image/png"), + (PNG, "image/jpeg"), + (PNG, "application/octet-stream"), + ], + ids=["not-image", "truncated-png", "mime-spoofing", "unsupported-type"], ) async def test_invalid_image_and_mime_are_415(client, context, data, content_type): storage, alice, _, _, chat = context @@ -143,17 +178,21 @@ async def test_invalid_image_and_mime_are_415(client, context, data, content_typ async def test_jpeg_is_stored_without_reencoding(client, context): storage, alice, _, _, chat = context output = BytesIO() - Image.new('RGB', (32, 32), 'blue').save(output, format='JPEG') + Image.new("RGB", (32, 32), "blue").save(output, format="JPEG") original = output.getvalue() - response = await upload(client, alice, chat, original, content_type='image/jpeg') + response = await upload(client, alice, chat, original, content_type="image/jpeg") assert response.status_code == 202, response.text - id = response.json()['id'] + id = response.json()["id"] await finish_attachment(client, storage, alice, id) - downloaded = await client.get(f'/api/v1/attachments/{id}/content', headers=bearer(alice)) + downloaded = await client.get( + f"/api/v1/attachments/{id}/content", headers=bearer(alice) + ) assert downloaded.content == original -@pytest.mark.parametrize('filename', ['../a.png', 'a\\b.png', 'a\x00.png', 'a\n.png', 'x' * 256]) +@pytest.mark.parametrize( + "filename", ["../a.png", "a\\b.png", "a\x00.png", "a\n.png", "x" * 256] +) async def test_unsafe_filename_is_400(client, context, filename): storage, alice, _, _, chat = context response = await upload(client, alice, chat, filename=filename) @@ -163,9 +202,11 @@ async def test_unsafe_filename_is_400(client, context, filename): async def test_oversize_chunked_upload_is_413(client, context): storage, alice, _, _, chat = context + async def chunks(): for _ in range(11): - yield b'x' * 1048576 + yield b"x" * 1048576 + response = await upload(client, alice, chat, chunks()) assert response.status_code == 413 assert not storage.objects @@ -175,7 +216,12 @@ async def test_oversize_content_length_is_413(client, context): _, alice, _, _, chat = context response = await client.post( f"/api/v1/chats/{chat['id']}/attachments?filename=a.png", - headers={**bearer(alice), 'Content-Type': 'image/png', 'Content-Length': '10485761'}, content=PNG, + headers={ + **bearer(alice), + "Content-Type": "image/png", + "Content-Length": "10485761", + }, + content=PNG, ) assert response.status_code == 413 @@ -185,30 +231,47 @@ async def test_exact_byte_limit_accepts_valid_png(client, context): maximum = 10_485_760 # Валидный ancillary chunk вместо данных после IEND. payload = bytes(maximum - len(PNG) - 12) - kind = b'npAD' - padding = struct.pack('!I', len(payload)) + kind + payload + struct.pack('!I', zlib.crc32(kind + payload)) + kind = b"npAD" + padding = ( + struct.pack("!I", len(payload)) + + kind + + payload + + struct.pack("!I", zlib.crc32(kind + payload)) + ) image = PNG[:-12] + padding + PNG[-12:] response = await upload(client, alice, chat, image) assert response.status_code == 202, response.text - assert response.json()['size_bytes'] == maximum - assert response.json()['sha256'] == hashlib.sha256(image).hexdigest() + assert response.json()["size_bytes"] == maximum + assert response.json()["sha256"] == hashlib.sha256(image).hexdigest() async def test_exact_pixel_limit_accepts_valid_png(client, context): _, alice, _, _, chat = context output = BytesIO() - with Image.new('RGB', (8000, 5000), 'blue') as image: - image.save(output, format='PNG') + with Image.new("RGB", (8000, 5000), "blue") as image: + image.save(output, format="PNG") response = await upload(client, alice, chat, output.getvalue()) assert response.status_code == 202, response.text async def test_pixel_limit_before_decoding(client, context): _, alice, _, _, chat = context + def chunk(kind, data): - return struct.pack('!I', len(data)) + kind + data + struct.pack('!I', zlib.crc32(kind + data)) - header = struct.pack('!IIBBBBB', 8000, 5001, 8, 2, 0, 0, 0) - image = b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', header) + chunk(b'IDAT', zlib.compress(b'\0')) + chunk(b'IEND', b'') + return ( + struct.pack("!I", len(data)) + + kind + + data + + struct.pack("!I", zlib.crc32(kind + data)) + ) + + header = struct.pack("!IIBBBBB", 8000, 5001, 8, 2, 0, 0, 0) + image = ( + b"\x89PNG\r\n\x1a\n" + + chunk(b"IHDR", header) + + chunk(b"IDAT", zlib.compress(b"\0")) + + chunk(b"IEND", b"") + ) response = await upload(client, alice, chat, image) assert response.status_code == 413, response.text @@ -217,10 +280,10 @@ async def test_unknown_and_other_chat_attachment_are_404(client, context): _, alice, bob, _, chat = context attachment = (await upload(client, alice, chat)).json() other_chat = await create_chat(client, alice, bob) - for id in (attachment['id'], str(uuid4())): + for id in (attachment["id"], str(uuid4())): response = await client.post( f"/api/v1/chats/{other_chat['id']}/messages", - json={'text': '', 'attachment_ids': [id]}, + json={"text": "", "attachment_ids": [id]}, headers={**bearer(alice), **idempotency_header()}, ) assert response.status_code == 404 @@ -228,10 +291,12 @@ async def test_unknown_and_other_chat_attachment_are_404(client, context): async def test_duplicate_ids_are_400(client, context): _, alice, _, _, chat = context - id = (await upload(client, alice, chat)).json()['id'] - response = await client.post(f"/api/v1/chats/{chat['id']}/messages", - json={'text': '', 'attachment_ids': [id, id]}, - headers={**bearer(alice), **idempotency_header()}) + id = (await upload(client, alice, chat)).json()["id"] + response = await client.post( + f"/api/v1/chats/{chat['id']}/messages", + json={"text": "", "attachment_ids": [id, id]}, + headers={**bearer(alice), **idempotency_header()}, + ) assert response.status_code == 400 @@ -239,20 +304,24 @@ async def test_empty_ids_and_absent_ids_have_same_idempotency(client, context): _, alice, _, _, chat = context headers = {**bearer(alice), **idempotency_header()} path = f"/api/v1/chats/{chat['id']}/messages" - first = await client.post(path, json={'text': 'same'}, headers=headers) - second = await client.post(path, json={'text': 'same', 'attachment_ids': []}, headers=headers) + first = await client.post(path, json={"text": "same"}, headers=headers) + second = await client.post( + path, json={"text": "same", "attachment_ids": []}, headers=headers + ) assert first.status_code == second.status_code == 201 assert first.json() == second.json() async def test_s3_outage_is_503(client, context): storage, alice, _, _, chat = context - id = (await upload(client, alice, chat)).json()['id'] + id = (await upload(client, alice, chat)).json()["id"] await finish_attachment(client, storage, alice, id) storage.down = True - response = await client.get(f'/api/v1/attachments/{id}/content', headers=bearer(alice)) + response = await client.get( + f"/api/v1/attachments/{id}/content", headers=bearer(alice) + ) assert response.status_code == 503 - assert (await client.get('/health/ready')).status_code == 503 + assert (await client.get("/health/ready")).status_code == 503 async def test_uncertain_put_is_compensated(client, context, db_engine): @@ -267,25 +336,31 @@ async def test_uncertain_put_is_compensated(client, context, db_engine): async def test_failure_after_s3_write_compensates(client, context, monkeypatch): storage, alice, _, _, chat = context + async def fail(*args, **kwargs): - raise OperationalError('injected', {}, None) - monkeypatch.setattr(repository, 'create_attachment', fail) + raise OperationalError("injected", {}, None) + + monkeypatch.setattr(repository, "create_attachment", fail) response = await upload(client, alice, chat) assert response.status_code == 503 assert not storage.objects -async def test_ambiguous_successful_commit_keeps_original(client, context, db_engine, monkeypatch): +async def test_ambiguous_successful_commit_keeps_original( + client, context, db_engine, monkeypatch +): storage, alice, _, _, chat = context original = AsyncSession.commit count = 0 + async def uncertain(session): nonlocal count count += 1 await original(session) if count == 2: - raise OperationalError('lost commit acknowledgement', {}, None) - monkeypatch.setattr(AsyncSession, 'commit', uncertain) + raise OperationalError("lost commit acknowledgement", {}, None) + + monkeypatch.setattr(AsyncSession, "commit", uncertain) response = await upload(client, alice, chat) assert response.status_code == 503 assert len(storage.objects) == 1 @@ -318,10 +393,16 @@ async def test_cleanup_keeps_fresh_and_skips_locked_uploads(client, context, db_ storage, _, _, _, _ = context old_id, fresh_id = uuid4(), uuid4() async with AsyncSession(db_engine) as session: - session.add_all([ - UploadIntent(id=old_id, object_key='originals/old', created_at=datetime.now(timezone.utc) - timedelta(days=2)), - UploadIntent(id=fresh_id, object_key='originals/fresh'), - ]) + session.add_all( + [ + UploadIntent( + id=old_id, + object_key="originals/old", + created_at=datetime.now(timezone.utc) - timedelta(days=2), + ), + UploadIntent(id=fresh_id, object_key="originals/fresh"), + ] + ) await session.commit() async with AsyncSession(db_engine) as uploader, AsyncSession(db_engine) as cleaner: await repository.lock_upload(uploader, old_id) @@ -335,9 +416,11 @@ async def test_cleanup_keeps_fresh_and_skips_locked_uploads(client, context, db_ async def test_client_disconnect_before_put_has_no_side_effects(client, context): storage, alice, _, _, chat = context + async def interrupted(): yield PNG[:16] raise ClientDisconnect() + response = await upload(client, alice, chat, interrupted()) assert response.status_code == 400 assert not storage.objects @@ -345,10 +428,14 @@ async def test_client_disconnect_before_put_has_no_side_effects(client, context) async def test_upload_deadline_cancels_validation(client, context, monkeypatch): storage, alice, _, _, chat = context - storage.settings = storage.settings.model_copy(update={'upload_timeout_seconds': 0.02}) + storage.settings = storage.settings.model_copy( + update={"upload_timeout_seconds": 0.02} + ) + async def stalled(*args): await asyncio.sleep(10) - monkeypatch.setattr(domain, 'validate_image', stalled) + + monkeypatch.setattr(domain, "validate_image", stalled) response = await upload(client, alice, chat) assert response.status_code == 400 assert not storage.objects @@ -358,11 +445,13 @@ async def test_cancelled_put_is_compensated(client, context, db_engine, monkeypa storage, alice, _, _, chat = context entered = asyncio.Event() original = storage.put_file + async def interrupted(*args, **kwargs): await original(*args, **kwargs) entered.set() await asyncio.sleep(10) - monkeypatch.setattr(storage, 'put_file', interrupted) + + monkeypatch.setattr(storage, "put_file", interrupted) task = asyncio.create_task(upload(client, alice, chat)) await asyncio.wait_for(entered.wait(), timeout=10) task.cancel() @@ -375,14 +464,19 @@ async def test_cancelled_put_is_compensated(client, context, db_engine, monkeypa async def test_cleanup_never_deletes_ready_attachment(client, context, db_engine): storage, alice, _, _, chat = context - id = (await upload(client, alice, chat)).json()['id'] + id = (await upload(client, alice, chat)).json()["id"] await finish_attachment(client, storage, alice, id) async with AsyncSession(db_engine) as session: attachment = await session.get(Attachment, UUID(id)) key = attachment.object_key # Искусственно оставляем старый журнал у уже подтверждённого объекта. - session.add(UploadIntent(id=uuid4(), object_key=key, - created_at=datetime.now(timezone.utc) - timedelta(days=2))) + session.add( + UploadIntent( + id=uuid4(), + object_key=key, + created_at=datetime.now(timezone.utc) - timedelta(days=2), + ) + ) await session.commit() result = await domain.reconcile_attachments(session, storage, dry_run=False) assert result.examined == 1 and result.deleted == 0 diff --git a/student-tests/test_concurrency.py b/student-tests/test_concurrency.py index 66860669..bf07e6f8 100644 --- a/student-tests/test_concurrency.py +++ b/student-tests/test_concurrency.py @@ -8,7 +8,14 @@ from sqlalchemy import text from app.main import create_app from conftest import close_app, make_client -from helpers import PASSWORD, basic_header, bearer, create_user, idempotency_header, register +from helpers import ( + PASSWORD, + basic_header, + bearer, + create_user, + idempotency_header, + register, +) async def test_parallel_registration_of_same_username_creates_exactly_one_user( diff --git a/student-tests/test_history_plan.py b/student-tests/test_history_plan.py index 704c9ccd..b19afcac 100644 --- a/student-tests/test_history_plan.py +++ b/student-tests/test_history_plan.py @@ -29,14 +29,12 @@ async def seed_messages( ) -> None: async with engine.begin() as connection: await connection.execute( - text( - """ + text(""" INSERT INTO messages (id, chat_id, sender_id, text, created_at) SELECT gen_random_uuid(), :chat_id, :sender_id, 'message ' || g, now() - make_interval(secs => :count - g) FROM generate_series(1, :count) AS g - """ - ), + """), {"chat_id": chat_id, "sender_id": sender_id, "count": count}, ) await connection.execute(text("ANALYZE messages")) diff --git a/student-tests/test_idempotency.py b/student-tests/test_idempotency.py index 9455e3d4..7c3aa49d 100644 --- a/student-tests/test_idempotency.py +++ b/student-tests/test_idempotency.py @@ -16,10 +16,15 @@ from helpers import bearer, create_chat, create_user, idempotency_header, send_m @pytest.fixture async def replicas(request, make_settings): overrides = getattr(request, "param", {}) - apps = [create_app(make_settings(instance_id=name, **overrides)) for name in ("api-a", "api-b")] + apps = [ + create_app(make_settings(instance_id=name, **overrides)) + for name in ("api-a", "api-b") + ] try: async with AsyncExitStack() as stack: - clients = [await stack.enter_async_context(make_client(app)) for app in apps] + clients = [ + await stack.enter_async_context(make_client(app)) for app in apps + ] yield clients finally: for app in apps: @@ -54,7 +59,11 @@ async def test_replay_and_conflict_across_replicas(replicas, db_engine): original = await send_message(first, user, chat, "Привет", "same-key") repeated = await second.post( f"/api/v1/chats/{chat['id']}/messages", - headers={**bearer(user), **idempotency_header("same-key"), "Content-Type": "application/json"}, + headers={ + **bearer(user), + **idempotency_header("same-key"), + "Content-Type": "application/json", + }, content='{ "text" : "\\u041f\\u0440\\u0438\\u0432\\u0435\\u0442" }', ) conflict = await send_message(second, user, chat, "другой текст", "same-key") @@ -63,23 +72,38 @@ async def test_replay_and_conflict_across_replicas(replicas, db_engine): assert original.headers["X-Instance-Id"] != repeated.headers["X-Instance-Id"] assert conflict.status_code == 409 async with db_engine.connect() as connection: - assert (await connection.execute(text("SELECT count(*) FROM messages"))).scalar_one() == 1 - assert (await connection.execute(text("SELECT value FROM counters WHERE name='messages'"))).scalar_one() == 1 + assert ( + await connection.execute(text("SELECT count(*) FROM messages")) + ).scalar_one() == 1 + assert ( + await connection.execute( + text("SELECT value FROM counters WHERE name='messages'") + ) + ).scalar_one() == 1 async def test_parallel_replays_create_one_message(replicas, db_engine): user = await create_user(replicas[0]) chat = await create_chat(replicas[0], user) - responses = await asyncio.gather(*( - send_message(replicas[index % 2], user, chat, "parallel", "one-key") - for index in range(24) - )) + responses = await asyncio.gather( + *( + send_message(replicas[index % 2], user, chat, "parallel", "one-key") + for index in range(24) + ) + ) assert all(response.status_code == 201 for response in responses) assert len({response.json()["id"] for response in responses}) == 1 - assert {response.headers["X-Instance-Id"] for response in responses} == {"api-a", "api-b"} + assert {response.headers["X-Instance-Id"] for response in responses} == { + "api-a", + "api-b", + } async with db_engine.connect() as connection: - assert (await connection.execute(text("SELECT count(*) FROM messages"))).scalar_one() == 1 - assert (await connection.execute(text("SELECT count(*) FROM idempotency_keys"))).scalar_one() == 1 + assert ( + await connection.execute(text("SELECT count(*) FROM messages")) + ).scalar_one() == 1 + assert ( + await connection.execute(text("SELECT count(*) FROM idempotency_keys")) + ).scalar_one() == 1 async def test_key_scope_separates_users_and_chats(client): @@ -101,9 +125,11 @@ async def test_expired_key_can_be_reused_without_removing_message(client, db_eng chat = await create_chat(client, user) first = await send_message(client, user, chat, "old", "expiring-key") async with db_engine.begin() as connection: - await connection.execute(text( - "UPDATE idempotency_keys SET created_at = clock_timestamp() - interval '25 hours'" - )) + await connection.execute( + text( + "UPDATE idempotency_keys SET created_at = clock_timestamp() - interval '25 hours'" + ) + ) second = await send_message(client, user, chat, "new", "expiring-key") assert first.status_code == second.status_code == 201 assert first.json()["id"] != second.json()["id"] @@ -127,7 +153,9 @@ async def test_failed_transaction_does_not_poison_key(client, db_engine, monkeyp assert failed.status_code == 503 async with db_engine.connect() as connection: for table in ("messages", "idempotency_keys", "counters"): - assert (await connection.execute(text(f"SELECT count(*) FROM {table}"))).scalar_one() == 0 + assert ( + await connection.execute(text(f"SELECT count(*) FROM {table}")) + ).scalar_one() == 0 monkeypatch.setattr(key_storage, "save_result", original) recovered = await send_message(client, user, chat, "retry", "retry-key") assert recovered.status_code == 201 @@ -148,9 +176,15 @@ async def test_replay_works_when_message_limit_is_full(replicas): async def test_replay_checks_session_and_membership(client, db_engine): user = await create_user(client) chat = await create_chat(client, user) - assert (await send_message(client, user, chat, "private", "private-key")).status_code == 201 + assert ( + await send_message(client, user, chat, "private", "private-key") + ).status_code == 201 async with db_engine.begin() as connection: await connection.execute(text("DELETE FROM chat_members")) - assert (await send_message(client, user, chat, "private", "private-key")).status_code == 404 + assert ( + await send_message(client, user, chat, "private", "private-key") + ).status_code == 404 await client.delete("/api/v1/auth/sessions/current", headers=bearer(user)) - assert (await send_message(client, user, chat, "private", "private-key")).status_code == 401 + assert ( + await send_message(client, user, chat, "private", "private-key") + ).status_code == 401 diff --git a/student-tests/test_image_pipeline.py b/student-tests/test_image_pipeline.py index bbed62f2..f317fa59 100644 --- a/student-tests/test_image_pipeline.py +++ b/student-tests/test_image_pipeline.py @@ -18,54 +18,96 @@ from app.storage.jobs import JobWork from test_attachments import MemoryStorage -@pytest.mark.parametrize('size,transparent', [((1600,800),False),((24,12),True)]) -async def test_variant_geometry_metadata_and_alpha(tmp_path,size,transparent): - source=tmp_path/'source.png' - with Image.new('RGBA',size,(255,0,0,0 if transparent else 255)) as image: - exif=Image.Exif();exif[270]='private description' - image.save(source,format='PNG',exif=exif) - result=create_variants(source,tmp_path,'image/png',40_000_000,ImageSettings(_env_file=None)) - variants={v.name:v for v in result.variants} - assert (variants['crop'].width,variants['crop'].height)==(128,128) - assert (variants['thumbnail'].width,variants['thumbnail'].height)==((256,128) if size[0]>256 else size) - assert (variants['optimized'].width,variants['optimized'].height)==((1280,640) if size[0]>1280 else size) - for name in ('thumbnail.jpg','crop.jpg','optimized.webp'): - with Image.open(tmp_path/name) as image: - assert not image.getexif() and not image.info.get('icc_profile') and not image.info.get('xmp') +@pytest.mark.parametrize("size,transparent", [((1600, 800), False), ((24, 12), True)]) +async def test_variant_geometry_metadata_and_alpha(tmp_path, size, transparent): + source = tmp_path / "source.png" + with Image.new("RGBA", size, (255, 0, 0, 0 if transparent else 255)) as image: + exif = Image.Exif() + exif[270] = "private description" + image.save(source, format="PNG", exif=exif) + result = create_variants( + source, tmp_path, "image/png", 40_000_000, ImageSettings(_env_file=None) + ) + variants = {v.name: v for v in result.variants} + assert (variants["crop"].width, variants["crop"].height) == (128, 128) + assert (variants["thumbnail"].width, variants["thumbnail"].height) == ( + (256, 128) if size[0] > 256 else size + ) + assert (variants["optimized"].width, variants["optimized"].height) == ( + (1280, 640) if size[0] > 1280 else size + ) + for name in ("thumbnail.jpg", "crop.jpg", "optimized.webp"): + with Image.open(tmp_path / name) as image: + assert ( + not image.getexif() + and not image.info.get("icc_profile") + and not image.info.get("xmp") + ) if transparent: - if name.endswith('webp'):assert image.getpixel((0,0))[3]==0 - else:assert min(image.getpixel((0,0)))>=250 + if name.endswith("webp"): + assert image.getpixel((0, 0))[3] == 0 + else: + assert min(image.getpixel((0, 0))) >= 250 async def test_exif_orientation_applied(tmp_path): - source=tmp_path/'source.jpg' - with Image.new('RGB',(400,200),'blue') as image: - exif=Image.Exif();exif[274]=6 - image.save(source,exif=exif) - result=create_variants(source,tmp_path,'image/jpeg',40_000_000,ImageSettings(_env_file=None)) - assert (result.variants[0].width,result.variants[0].height)==(128,256) + source = tmp_path / "source.jpg" + with Image.new("RGB", (400, 200), "blue") as image: + exif = Image.Exif() + exif[274] = 6 + image.save(source, exif=exif) + result = create_variants( + source, tmp_path, "image/jpeg", 40_000_000, ImageSettings(_env_file=None) + ) + assert (result.variants[0].width, result.variants[0].height) == (128, 256) -@pytest.mark.parametrize('failure', ['memory','cpu']) -async def test_worker_os_limits(failure,tmp_path): - body='return bytearray(2 * 1024 ** 3)' if failure=='memory' else 'while True: pass' - code="from app.core import thumbnail_worker as w\ndef fail(*args):\n "+body+"\nw.create_variants=fail\nraise SystemExit(w.main())" - result=await asyncio.to_thread(subprocess.run,[sys.executable,'-c',code,'source',str(tmp_path),str(tmp_path/'metadata'),'image/png','40000000'], - env={**os.environ,'IMAGE_MEMORY_MIB':'256','IMAGE_CPU_SECONDS':'1'},capture_output=True,timeout=15) - assert result.returncode==(3 if failure=='memory' else -9) - assert not (tmp_path/'metadata').exists() +@pytest.mark.parametrize("failure", ["memory", "cpu"]) +async def test_worker_os_limits(failure, tmp_path): + body = ( + "return bytearray(2 * 1024 ** 3)" if failure == "memory" else "while True: pass" + ) + code = ( + "from app.core import thumbnail_worker as w\ndef fail(*args):\n " + + body + + "\nw.create_variants=fail\nraise SystemExit(w.main())" + ) + result = await asyncio.to_thread( + subprocess.run, + [ + sys.executable, + "-c", + code, + "source", + str(tmp_path), + str(tmp_path / "metadata"), + "image/png", + "40000000", + ], + env={**os.environ, "IMAGE_MEMORY_MIB": "256", "IMAGE_CPU_SECONDS": "1"}, + capture_output=True, + timeout=15, + ) + assert result.returncode == (3 if failure == "memory" else -9) + assert not (tmp_path / "metadata").exists() -async def test_render_timeout_kills_child(monkeypatch,tmp_path): - create=asyncio.create_subprocess_exec - children=[] - async def slow_child(*args,**kwargs): - child=await create(sys.executable,'-c','import time;time.sleep(60)',**kwargs) +async def test_render_timeout_kills_child(monkeypatch, tmp_path): + create = asyncio.create_subprocess_exec + children = [] + + async def slow_child(*args, **kwargs): + child = await create( + sys.executable, "-c", "import time;time.sleep(60)", **kwargs + ) children.append(child) return child - monkeypatch.setattr(asyncio,'create_subprocess_exec',slow_child) - work=JobWork(uuid4(),uuid4(),uuid4(),1,'staging/test','image/png',1,'0'*64) + + monkeypatch.setattr(asyncio, "create_subprocess_exec", slow_child) + work = JobWork( + uuid4(), uuid4(), uuid4(), 1, "staging/test", "image/png", 1, "0" * 64 + ) with pytest.raises(TimeoutError): - async with asyncio.timeout(.2): - await render_variants(MemoryStorage(),work,tmp_path/'source',tmp_path) - assert len(children)==1 and children[0].returncode is not None + async with asyncio.timeout(0.2): + await render_variants(MemoryStorage(), work, tmp_path / "source", tmp_path) + assert len(children) == 1 and children[0].returncode is not None diff --git a/student-tests/test_instance.py b/student-tests/test_instance.py index 8b0d932b..1f262474 100644 --- a/student-tests/test_instance.py +++ b/student-tests/test_instance.py @@ -92,7 +92,9 @@ async def test_instance_header_is_present_on_success_and_errors( assert response.headers["X-Instance-Id"] == "api-test" assert response.headers["X-Request-Id"] if status >= 400: - assert response.json()["error"]["request_id"] == response.headers["X-Request-Id"] + assert ( + response.json()["error"]["request_id"] == response.headers["X-Request-Id"] + ) async def test_instance_id_is_stable_while_request_ids_are_distinct(instance_client): @@ -100,7 +102,9 @@ async def test_instance_id_is_stable_while_request_ids_are_distinct(instance_cli second = await instance_client.get("/__test__/request-id") assert first.status_code == second.status_code == 200 - assert first.headers["X-Instance-Id"] == second.headers["X-Instance-Id"] == "api-test" + assert ( + first.headers["X-Instance-Id"] == second.headers["X-Instance-Id"] == "api-test" + ) assert first.headers["X-Request-Id"] != second.headers["X-Request-Id"] for response in (first, second): assert response.json()["request_id"] == response.headers["X-Request-Id"] diff --git a/student-tests/test_jobs.py b/student-tests/test_jobs.py index 4a08bc26..3582a09a 100644 --- a/student-tests/test_jobs.py +++ b/student-tests/test_jobs.py @@ -13,7 +13,12 @@ from sqlalchemy.ext.asyncio import AsyncSession from app.core.jobs import JobSettings from app.domain import attachment_jobs -from app.domain.attachment_jobs import claim_job, process_attachment_job, cleanup_staging, build_result +from app.domain.attachment_jobs import ( + claim_job, + process_attachment_job, + cleanup_staging, + build_result, +) from app.domain.job_admin import replay_failed_job from app.domain.job_retry import classify_failure, retry_delay from app.errors.exceptions import JobOperationError, ProcessingError @@ -26,50 +31,85 @@ from test_attachments import context, upload, finish_attachment async def envelope(session, attachment_id): - job = await session.scalar(select(AttachmentJob).where(AttachmentJob.attachment_id == UUID(attachment_id))) - return JobMessage(job_id=job.id, attachment_id=job.attachment_id, pipeline_version=1, - attempt=job.attempt + 1, created_at=job.created_at) + job = await session.scalar( + select(AttachmentJob).where(AttachmentJob.attachment_id == UUID(attachment_id)) + ) + return JobMessage( + job_id=job.id, + attachment_id=job.attachment_id, + pipeline_version=1, + attempt=job.attempt + 1, + created_at=job.created_at, + ) async def test_retry_budget_backoff_replay_and_duplicate(client, context, db_engine): storage, alice, _, _, chat = context attachment = (await upload(client, alice, chat)).json() settings = JobSettings(_env_file=None, job_max_attempts=3) + def fail(_): raise TimeoutError + async with AsyncSession(db_engine, expire_on_commit=False) as session: - first = await envelope(session, attachment['id']) + first = await envelope(session, attachment["id"]) for attempt in range(1, 4): - message = await envelope(session, attachment['id']) - assert await process_attachment_job(session, storage, message, settings, before_processing=fail) + message = await envelope(session, attachment["id"]) + assert await process_attachment_job( + session, storage, message, settings, before_processing=fail + ) job, item = await jobs.lock_job(session, first.job_id) assert job.attempt == attempt delay = (job.next_attempt_at - item.updated_at).total_seconds() assert 5 * 2 ** (attempt - 1) / 2 <= delay <= 5 * 2 ** (attempt - 1) - assert item.status == ('failed' if attempt == 3 else 'queued') + assert item.status == ("failed" if attempt == 3 else "queued") assert job.last_error_permanent is False - job.next_attempt_at = await jobs.database_time(session) - timedelta(seconds=1) + job.next_attempt_at = await jobs.database_time(session) - timedelta( + seconds=1 + ) await session.commit() - assert item.error_code == 'retry_exhausted' + assert item.error_code == "retry_exhausted" request_id = uuid4() - result = await replay_failed_job(session, storage, settings, first.job_id, request_id=request_id, - actor='test', reason='dependency_restored') - assert result['attempt_limit'] == 6 + result = await replay_failed_job( + session, + storage, + settings, + first.job_id, + request_id=request_id, + actor="test", + reason="dependency_restored", + ) + assert result["attempt_limit"] == 6 # Old envelope cannot consume the fresh replay budget. assert await process_attachment_job(session, storage, first, settings) job, item = await jobs.lock_job(session, first.job_id) - assert job.attempt == 3 and item.status == 'queued' + assert job.attempt == 3 and item.status == "queued" await session.commit() - message = await envelope(session, attachment['id']) + message = await envelope(session, attachment["id"]) assert await process_attachment_job(session, storage, message, settings) before = dict(storage.objects) assert await process_attachment_job(session, storage, message, settings) assert storage.objects == before - repeat = await replay_failed_job(session, storage, settings, first.job_id, request_id=request_id, - actor='test', reason='dependency_restored') - assert repeat['already_applied'] - with pytest.raises(JobOperationError, match='job_not_failed'): - await replay_failed_job(session, storage, settings, first.job_id, request_id=uuid4(), actor='test', reason='manual_review') + repeat = await replay_failed_job( + session, + storage, + settings, + first.job_id, + request_id=request_id, + actor="test", + reason="dependency_restored", + ) + assert repeat["already_applied"] + with pytest.raises(JobOperationError, match="job_not_failed"): + await replay_failed_job( + session, + storage, + settings, + first.job_id, + request_id=uuid4(), + actor="test", + reason="manual_review", + ) await session.rollback() assert len((await session.scalars(select(JobReplay))).all()) == 1 @@ -77,26 +117,40 @@ async def test_retry_budget_backoff_replay_and_duplicate(client, context, db_eng async def test_permanent_failure_and_missing_source_replay(client, context, db_engine): storage, alice, _, _, chat = context attachment = (await upload(client, alice, chat)).json() + def fail(_): - raise ProcessingError('image_limits') + raise ProcessingError("image_limits") + settings = JobSettings(_env_file=None) async with AsyncSession(db_engine, expire_on_commit=False) as session: - message = await envelope(session, attachment['id']) - await process_attachment_job(session, storage, message, settings, before_processing=fail) + message = await envelope(session, attachment["id"]) + await process_attachment_job( + session, storage, message, settings, before_processing=fail + ) job, item = await jobs.lock_job(session, message.job_id) - assert item.status == 'failed' and job.attempt == 1 and job.last_error_permanent + assert item.status == "failed" and job.attempt == 1 and job.last_error_permanent job.staging_key = None await session.commit() - with pytest.raises(JobOperationError, match='source_expired'): - await replay_failed_job(session, storage, settings, job.id, request_id=uuid4(), actor='test', reason='manual_review') + with pytest.raises(JobOperationError, match="source_expired"): + await replay_failed_job( + session, + storage, + settings, + job.id, + request_id=uuid4(), + actor="test", + reason="manual_review", + ) -async def test_stale_worker_cannot_write_and_cleanup_keeps_active(client, context, db_engine): +async def test_stale_worker_cannot_write_and_cleanup_keeps_active( + client, context, db_engine +): storage, alice, _, _, chat = context attachment = (await upload(client, alice, chat)).json() settings = JobSettings(_env_file=None) async with AsyncSession(db_engine, expire_on_commit=False) as session: - message = await envelope(session, attachment['id']) + message = await envelope(session, attachment["id"]) claim = await claim_job(session, message, settings) job, item = await jobs.lock_job(session, message.job_id) job.lease_token = uuid4() @@ -112,18 +166,33 @@ async def test_stale_worker_cannot_write_and_cleanup_keeps_active(client, contex async def test_cleanup_ready_preserves_final_objects(client, context, db_engine): storage, alice, _, _, chat = context attachment = (await upload(client, alice, chat)).json() - await finish_attachment(client, storage, alice, attachment['id']) + await finish_attachment(client, storage, alice, attachment["id"]) async with AsyncSession(db_engine, expire_on_commit=False) as session: - message = await envelope(session, attachment['id']) + message = await envelope(session, attachment["id"]) job, item = await jobs.lock_job(session, message.job_id) staging = job.staging_key job.finished_at = await jobs.database_time(session) - timedelta(days=2) await session.commit() - assert await cleanup_staging(session, storage, JobSettings(_env_file=None), apply=False) == 1 + assert ( + await cleanup_staging( + session, storage, JobSettings(_env_file=None), apply=False + ) + == 1 + ) assert staging in storage.objects - assert await cleanup_staging(session, storage, JobSettings(_env_file=None), apply=True) == 1 + assert ( + await cleanup_staging( + session, storage, JobSettings(_env_file=None), apply=True + ) + == 1 + ) assert staging not in storage.objects and len(storage.objects) == 4 - assert await cleanup_staging(session, storage, JobSettings(_env_file=None), apply=True) == 0 + assert ( + await cleanup_staging( + session, storage, JobSettings(_env_file=None), apply=True + ) + == 0 + ) async def test_retry_classification_and_cap(): @@ -131,44 +200,64 @@ async def test_retry_classification_and_cap(): for attempt in (1, 2, 6, 100): cap = min(120, 5 * 2 ** min(30, attempt - 1)) assert all(cap / 2 <= retry_delay(settings, attempt) <= cap for _ in range(100)) - assert classify_failure(TimeoutError()) == ('processing_timeout', False) - assert classify_failure(OSError(errno.EACCES, 'denied'))[1] - assert not classify_failure(OSError(errno.ENOSPC, 'full'))[1] + assert classify_failure(TimeoutError()) == ("processing_timeout", False) + assert classify_failure(OSError(errno.EACCES, "denied"))[1] + assert not classify_failure(OSError(errno.ENOSPC, "full"))[1] -async def test_timeout_records_retry_and_clears_lease(client, context, db_engine, monkeypatch): +async def test_timeout_records_retry_and_clears_lease( + client, context, db_engine, monkeypatch +): storage, alice, _, _, chat = context attachment = (await upload(client, alice, chat)).json() + async def stalled(*args, **kwargs): await asyncio.sleep(60) - monkeypatch.setattr(attachment_jobs, 'build_result', stalled) - settings = JobSettings(_env_file=None).model_copy(update={'job_timeout_seconds': .05}) + + monkeypatch.setattr(attachment_jobs, "build_result", stalled) + settings = JobSettings(_env_file=None).model_copy( + update={"job_timeout_seconds": 0.05} + ) async with AsyncSession(db_engine, expire_on_commit=False) as session: - message = await envelope(session, attachment['id']) + message = await envelope(session, attachment["id"]) assert await process_attachment_job(session, storage, message, settings) job, item = await jobs.lock_job(session, message.job_id) - assert item.status == 'queued' and job.last_error_code == 'processing_timeout' + assert item.status == "queued" and job.last_error_code == "processing_timeout" assert job.lease_token is None and job.lease_until is None -async def test_failed_retention_and_source_checksum_protect_replay(client, context, db_engine): +async def test_failed_retention_and_source_checksum_protect_replay( + client, context, db_engine +): storage, alice, _, _, chat = context attachment = (await upload(client, alice, chat)).json() + def fail(_): - raise ProcessingError('invalid_image') + raise ProcessingError("invalid_image") + settings = JobSettings(_env_file=None) async with AsyncSession(db_engine, expire_on_commit=False) as session: - message = await envelope(session, attachment['id']) - await process_attachment_job(session, storage, message, settings, before_processing=fail) + message = await envelope(session, attachment["id"]) + await process_attachment_job( + session, storage, message, settings, before_processing=fail + ) job, item = await jobs.lock_job(session, message.job_id) key = job.staging_key job.finished_at = await jobs.database_time(session) - timedelta(days=2) await session.commit() assert await cleanup_staging(session, storage, settings, apply=True) == 0 data, metadata = storage.objects[key] - storage.objects[key] = (b'x' * len(data), metadata) - with pytest.raises(JobOperationError, match='source_mismatch'): - await replay_failed_job(session, storage, settings, message.job_id, request_id=uuid4(), actor='test', reason='manual_review') + storage.objects[key] = (b"x" * len(data), metadata) + with pytest.raises(JobOperationError, match="source_mismatch"): + await replay_failed_job( + session, + storage, + settings, + message.job_id, + request_id=uuid4(), + actor="test", + reason="manual_review", + ) await session.rollback() assert not (await session.scalars(select(JobReplay))).all() job, item = await jobs.lock_job(session, message.job_id) diff --git a/student-tests/test_lifecycle.py b/student-tests/test_lifecycle.py index 1a659b74..11e4dcb1 100644 --- a/student-tests/test_lifecycle.py +++ b/student-tests/test_lifecycle.py @@ -20,9 +20,9 @@ async def test_draining_keeps_liveness_but_rejects_new_work(make_settings): app.state.lifecycle.start_draining() try: async with make_client(app) as client: - assert (await client.get('/health/live')).status_code == 200 - assert (await client.get('/health/ready')).status_code == 503 - assert (await client.get('/api/v1/users/me')).status_code == 503 + assert (await client.get("/health/live")).status_code == 200 + assert (await client.get("/health/ready")).status_code == 503 + assert (await client.get("/api/v1/users/me")).status_code == 503 finally: await close_app(app) @@ -36,13 +36,13 @@ async def test_readiness_timeout_cancels_dependency_checks(make_settings, monkey finally: cancelled.append(True) - monkeypatch.setattr(health, 'check_database', stalled) - monkeypatch.setattr(health, 'check_redis', stalled) + monkeypatch.setattr(health, "check_database", stalled) + monkeypatch.setattr(health, "check_redis", stalled) app = create_app(make_settings(readiness_timeout_seconds=0.03)) try: async with make_client(app) as client: started = time.monotonic() - response = await client.get('/health/ready') + response = await client.get("/health/ready") assert response.status_code == 503 assert time.monotonic() - started < 1 assert len(cancelled) == 2 diff --git a/student-tests/test_limits.py b/student-tests/test_limits.py index 3bd68c0d..9cfca1d3 100644 --- a/student-tests/test_limits.py +++ b/student-tests/test_limits.py @@ -6,7 +6,13 @@ import httpx from app.main import create_app from conftest import close_app, make_client -from helpers import assert_request_id, bearer, create_chat, create_user, idempotency_header +from helpers import ( + assert_request_id, + bearer, + create_chat, + create_user, + idempotency_header, +) def _assert_error(response: httpx.Response, status: int, code: str) -> None: @@ -25,12 +31,16 @@ async def test_message_limit_rejects_without_deleting_confirmed_messages(make_se for index in range(2): ok = await client.post( - url, headers={**bearer(user), **idempotency_header()}, json={"text": f"confirmed-{index}"} + url, + headers={**bearer(user), **idempotency_header()}, + json={"text": f"confirmed-{index}"}, ) assert ok.status_code == 201 rejected = await client.post( - url, headers={**bearer(user), **idempotency_header()}, json={"text": "rejected"} + url, + headers={**bearer(user), **idempotency_header()}, + json={"text": "rejected"}, ) _assert_error(rejected, 503, "unavailable") @@ -65,12 +75,18 @@ async def test_in_flight_limit_returns_429_and_recovers(make_settings, monkeypat user = await create_user(client) chat = await create_chat(client, user) - monkeypatch.setattr(message_endpoint, "send_message_idempotent", blocking_send_message) + monkeypatch.setattr( + message_endpoint, "send_message_idempotent", blocking_send_message + ) url = f"/api/v1/chats/{chat['id']}/messages" held = [ asyncio.create_task( - client.post(url, headers={**bearer(user), **idempotency_header()}, json={"text": f"held-{i}"}) + client.post( + url, + headers={**bearer(user), **idempotency_header()}, + json={"text": f"held-{i}"}, + ) ) for i in range(2) ] @@ -79,7 +95,11 @@ async def test_in_flight_limit_returns_429_and_recovers(make_settings, monkeypat rejected = await asyncio.gather( *( - client.post(url, headers={**bearer(user), **idempotency_header()}, json={"text": f"rejected-{i}"}) + client.post( + url, + headers={**bearer(user), **idempotency_header()}, + json={"text": f"rejected-{i}"}, + ) for i in range(4) ) ) @@ -92,7 +112,11 @@ async def test_in_flight_limit_returns_429_and_recovers(make_settings, monkeypat release.set() assert all(r.status_code == 201 for r in await asyncio.gather(*held)) - recovered = await client.post(url, headers={**bearer(user), **idempotency_header()}, json={"text": "after"}) + recovered = await client.post( + url, + headers={**bearer(user), **idempotency_header()}, + json={"text": "after"}, + ) assert recovered.status_code == 201 assert_request_id(recovered) await close_app(app) diff --git a/student-tests/test_migrations.py b/student-tests/test_migrations.py index f8efed1b..c515ade1 100644 --- a/student-tests/test_migrations.py +++ b/student-tests/test_migrations.py @@ -8,7 +8,19 @@ from sqlalchemy import inspect, text from app.models import Base from conftest import reset_schema, upgrade_head -EXPECTED_TABLES = {"users", "chats", "chat_members", "messages", "counters", "idempotency_keys", "attachments", "message_attachments", "upload_intents", "attachment_jobs", "job_replays"} +EXPECTED_TABLES = { + "users", + "chats", + "chat_members", + "messages", + "counters", + "idempotency_keys", + "attachments", + "message_attachments", + "upload_intents", + "attachment_jobs", + "job_replays", +} @pytest.fixture @@ -62,6 +74,6 @@ async def test_models_and_migrations_are_in_sync(migrated_engine): async with migrated_engine.connect() as connection: diff = await connection.run_sync(compare) - assert diff == [], ( - f"Создайте миграцию: alembic revision --autogenerate. Diff: {diff}" - ) + assert ( + diff == [] + ), f"Создайте миграцию: alembic revision --autogenerate. Diff: {diff}" diff --git a/student-tests/test_s3_stream.py b/student-tests/test_s3_stream.py index b168a626..67fc364b 100644 --- a/student-tests/test_s3_stream.py +++ b/student-tests/test_s3_stream.py @@ -10,10 +10,12 @@ from app.storage.s3 import ObjectStream, run_sync async def test_sdk_thread_finishes_before_cancellation_returns(): entered, release, finished = Event(), Event(), Event() + def operation(): entered.set() release.wait(timeout=3) finished.set() + task = asyncio.create_task(run_sync(operation)) await asyncio.to_thread(entered.wait, 2) task.cancel() @@ -31,10 +33,12 @@ async def test_response_discarded_when_get_is_cancelled(): entered, release = Event(), Event() discarded = [] response = object() + def operation(): entered.set() release.wait(timeout=3) return response + task = asyncio.create_task(run_sync(operation, discard_result=discarded.append)) await asyncio.to_thread(entered.wait, 2) task.cancel() @@ -48,9 +52,12 @@ async def test_response_discarded_when_get_is_cancelled(): async def test_stream_closes_even_before_first_iteration(): calls = [] - response = SimpleNamespace(close=lambda: calls.append('close'), release_conn=lambda: calls.append('release')) + response = SimpleNamespace( + close=lambda: calls.append("close"), + release_conn=lambda: calls.append("release"), + ) storage = SimpleNamespace(settings=S3Settings(_env_file=None)) stream = ObjectStream(storage, response) await stream.aclose() await stream.aclose() - assert calls == ['close', 'release'] + assert calls == ["close", "release"] diff --git a/tests/client.py b/tests/client.py index f0499a29..c9ca1ae9 100644 --- a/tests/client.py +++ b/tests/client.py @@ -1,4 +1,5 @@ """HTTP helpers for public black-box tests; Python 3.9+, standard library only.""" + import base64 import datetime as dt import hashlib @@ -13,205 +14,336 @@ import urllib.request import uuid ROOT = Path(__file__).resolve().parents[1] -LAB = json.loads((ROOT / 'lab.json').read_text())['number'] -SPEC = json.loads((ROOT / 'contracts/openapi.json').read_text()) +LAB = json.loads((ROOT / "lab.json").read_text())["number"] +SPEC = json.loads((ROOT / "contracts/openapi.json").read_text()) + class Failure(AssertionError): pass + def check(condition, message): if not condition: raise Failure(message) -def timestamp(value): - check(isinstance(value, str) and value.endswith('Z'), 'timestamp must be UTC ending in Z') - try: - return dt.datetime.fromisoformat(value[:-1] + '+00:00') - except ValueError as exc: - raise Failure('invalid RFC3339 timestamp') from exc -def validate(value, schema, where='response'): +def timestamp(value): + check( + isinstance(value, str) and value.endswith("Z"), + "timestamp must be UTC ending in Z", + ) + try: + return dt.datetime.fromisoformat(value[:-1] + "+00:00") + except ValueError as exc: + raise Failure("invalid RFC3339 timestamp") from exc + + +def validate(value, schema, where="response"): """Validate the JSON Schema subset used by the supplied contract, not arbitrary OpenAPI.""" - if '$ref' in schema: + if "$ref" in schema: target = SPEC - for part in schema['$ref'].split('/')[1:]: + for part in schema["$ref"].split("/")[1:]: target = target[part] return validate(value, target, where) - kinds = schema.get('type', []) + kinds = schema.get("type", []) if isinstance(kinds, str): kinds = [kinds] - matches = {'null':value is None, 'boolean':isinstance(value,bool), 'integer':isinstance(value,int) and not isinstance(value,bool), 'number':isinstance(value,(int,float)) and not isinstance(value,bool), 'string':isinstance(value,str), 'array':isinstance(value,list), 'object':isinstance(value,dict)} + matches = { + "null": value is None, + "boolean": isinstance(value, bool), + "integer": isinstance(value, int) and not isinstance(value, bool), + "number": isinstance(value, (int, float)) and not isinstance(value, bool), + "string": isinstance(value, str), + "array": isinstance(value, list), + "object": isinstance(value, dict), + } if kinds: - check(any(matches.get(k,False) for k in kinds), where + ': incorrect type') - if 'enum' in schema: - check(value in schema['enum'], where + ': invalid enum') - if 'const' in schema: - check(value == schema['const'], where + ': invalid const') - if isinstance(value,dict): - props = schema.get('properties',{}) - check(all(k in value for k in schema.get('required',[])), where + ': missing required fields') - if schema.get('additionalProperties') is False: - check(not set(value).difference(props), where + ': unexpected fields') + check(any(matches.get(k, False) for k in kinds), where + ": incorrect type") + if "enum" in schema: + check(value in schema["enum"], where + ": invalid enum") + if "const" in schema: + check(value == schema["const"], where + ": invalid const") + if isinstance(value, dict): + props = schema.get("properties", {}) + check( + all(k in value for k in schema.get("required", [])), + where + ": missing required fields", + ) + if schema.get("additionalProperties") is False: + check(not set(value).difference(props), where + ": unexpected fields") for key, sub in props.items(): if key in value: - validate(value[key], sub, where + '.' + key) - if isinstance(value,list): - check(len(value) >= schema.get('minItems',0), where + ': too few items') - check(len(value) <= schema.get('maxItems',float('inf')), where + ': too many items') - if schema.get('uniqueItems'): - encoded = [json.dumps(x,sort_keys=True) for x in value] - check(len(encoded)==len(set(encoded)), where + ': duplicate items') - if 'items' in schema: + validate(value[key], sub, where + "." + key) + if isinstance(value, list): + check(len(value) >= schema.get("minItems", 0), where + ": too few items") + check( + len(value) <= schema.get("maxItems", float("inf")), + where + ": too many items", + ) + if schema.get("uniqueItems"): + encoded = [json.dumps(x, sort_keys=True) for x in value] + check(len(encoded) == len(set(encoded)), where + ": duplicate items") + if "items" in schema: for item in value: - validate(item, schema['items'], where + '[]') - if isinstance(value,str): - check(len(value) >= schema.get('minLength',0), where + ': too short') - check(len(value) <= schema.get('maxLength',float('inf')), where + ': too long') - if 'pattern' in schema: - check(re.search(schema['pattern'], value) is not None, where + ': pattern mismatch') - if schema.get('format') == 'uuid': + validate(item, schema["items"], where + "[]") + if isinstance(value, str): + check(len(value) >= schema.get("minLength", 0), where + ": too short") + check(len(value) <= schema.get("maxLength", float("inf")), where + ": too long") + if "pattern" in schema: + check( + re.search(schema["pattern"], value) is not None, + where + ": pattern mismatch", + ) + if schema.get("format") == "uuid": try: uuid.UUID(value) except ValueError as exc: - raise Failure(where + ': invalid UUID') from exc - if schema.get('format') == 'date-time': + raise Failure(where + ": invalid UUID") from exc + if schema.get("format") == "date-time": timestamp(value) - if isinstance(value,(int,float)) and not isinstance(value,bool): - check(value >= schema.get('minimum',-float('inf')), where + ': below minimum') - check(value <= schema.get('maximum',float('inf')), where + ': above maximum') - if 'anyOf' in schema: - for alternative in schema['anyOf']: + if isinstance(value, (int, float)) and not isinstance(value, bool): + check(value >= schema.get("minimum", -float("inf")), where + ": below minimum") + check(value <= schema.get("maximum", float("inf")), where + ": above maximum") + if "anyOf" in schema: + for alternative in schema["anyOf"]: try: validate(value, alternative, where) break except Failure: pass else: - raise Failure(where + ': no anyOf branch matches') + raise Failure(where + ": no anyOf branch matches") + def shape(value, name): - validate(value, SPEC['components']['schemas'][name]) + validate(value, SPEC["components"]["schemas"][name]) return value + class NoRedirect(urllib.request.HTTPRedirectHandler): def redirect_request(self, req, fp, code, msg, headers, newurl): return None + class Client: def __init__(self, base_url=None, ca_file=None, timeout=10): - self.base = (base_url or os.environ.get('BASE_URL') or ('https://localhost:8443' if LAB==7 else 'http://localhost:8080')).rstrip('/') - check(self.base.startswith(('http://','https://')), 'BASE_URL must be HTTP(S)') - check(LAB != 7 or self.base.startswith('https://'), 'lab7 requires HTTPS') + self.base = ( + base_url + or os.environ.get("BASE_URL") + or ("https://localhost:8443" if LAB == 7 else "http://localhost:8080") + ).rstrip("/") + check(self.base.startswith(("http://", "https://")), "BASE_URL must be HTTP(S)") + check(LAB != 7 or self.base.startswith("https://"), "lab7 requires HTTPS") self.timeout = timeout - context = ssl.create_default_context(cafile=ca_file or os.environ.get('CA_FILE') or None) - self.opener = urllib.request.build_opener(NoRedirect(), urllib.request.HTTPSHandler(context=context)) + context = ssl.create_default_context( + cafile=ca_file or os.environ.get("CA_FILE") or None + ) + self.opener = urllib.request.build_opener( + NoRedirect(), urllib.request.HTTPSHandler(context=context) + ) - def call(self, method, path, expected=200, body=None, actor=None, headers=None, raw=None, schema=None): - h = {'Accept':'application/json'} + def call( + self, + method, + path, + expected=200, + body=None, + actor=None, + headers=None, + raw=None, + schema=None, + ): + h = {"Accept": "application/json"} if actor: - h['X-User-Id' if LAB==1 else 'Authorization'] = actor['id'] if LAB==1 else 'Bearer ' + actor['token'] + h["X-User-Id" if LAB == 1 else "Authorization"] = ( + actor["id"] if LAB == 1 else "Bearer " + actor["token"] + ) if body is not None: - raw = json.dumps(body,ensure_ascii=False).encode('utf-8') - h['Content-Type'] = 'application/json' + raw = json.dumps(body, ensure_ascii=False).encode("utf-8") + h["Content-Type"] = "application/json" h.update(headers or {}) - req = urllib.request.Request(self.base + path, data=raw, headers=h, method=method) + req = urllib.request.Request( + self.base + path, data=raw, headers=h, method=method + ) start = time.monotonic() try: response = self.opener.open(req, timeout=self.timeout) except urllib.error.HTTPError as error: response = error - except (urllib.error.URLError,TimeoutError,OSError) as exc: + except (urllib.error.URLError, TimeoutError, OSError) as exc: # Do not print URL query, headers, request body, or credentials. - raise Failure(method + ' ' + path.split('?')[0] + ': connection/TLS failure (' + type(exc).__name__ + ')') from None + raise Failure( + method + + " " + + path.split("?")[0] + + ": connection/TLS failure (" + + type(exc).__name__ + + ")" + ) from None with response: status = response.code rh = response.headers - data = response.read(16*1024*1024 + 1) - check(len(data) <= 16*1024*1024, 'response exceeds test safety limit') - allowed = [expected] if isinstance(expected,int) else expected - check(status in allowed, method + ' ' + path.split('?')[0] + ': expected ' + str(allowed) + ', received ' + str(status)) - check(bool(rh.get('X-Request-Id')), 'missing X-Request-Id') - if LAB>=3: - check(bool(rh.get('X-Instance-Id')), 'missing X-Instance-Id') + data = response.read(16 * 1024 * 1024 + 1) + check(len(data) <= 16 * 1024 * 1024, "response exceeds test safety limit") + allowed = [expected] if isinstance(expected, int) else expected + check( + status in allowed, + method + + " " + + path.split("?")[0] + + ": expected " + + str(allowed) + + ", received " + + str(status), + ) + check(bool(rh.get("X-Request-Id")), "missing X-Request-Id") + if LAB >= 3: + check(bool(rh.get("X-Instance-Id")), "missing X-Instance-Id") payload = None - if schema or status>=400: - check(rh.get_content_type()=='application/json', 'expected application/json') + if schema or status >= 400: + check( + rh.get_content_type() == "application/json", "expected application/json" + ) try: payload = json.loads(data) - except (ValueError,UnicodeError): - raise Failure('invalid response JSON') from None - shape(payload, 'Error' if status>=400 else schema) - if status>=400: - check(payload['error']['request_id']==rh.get('X-Request-Id'),'error request_id differs from header') - if status==401 and LAB>=2: - scheme = 'Basic' if path=='/api/v1/auth/sessions' else 'Bearer' - check(rh.get('WWW-Authenticate','').lower().startswith(scheme.lower()), 'incorrect WWW-Authenticate challenge') - if status==204: - check(not data, '204 must not contain body') - return {'status':status,'headers':rh,'json':payload,'bytes':data,'elapsed':time.monotonic()-start} + except (ValueError, UnicodeError): + raise Failure("invalid response JSON") from None + shape(payload, "Error" if status >= 400 else schema) + if status >= 400: + check( + payload["error"]["request_id"] == rh.get("X-Request-Id"), + "error request_id differs from header", + ) + if status == 401 and LAB >= 2: + scheme = "Basic" if path == "/api/v1/auth/sessions" else "Bearer" + check( + rh.get("WWW-Authenticate", "").lower().startswith(scheme.lower()), + "incorrect WWW-Authenticate challenge", + ) + if status == 204: + check(not data, "204 must not contain body") + return { + "status": status, + "headers": rh, + "json": payload, + "bytes": data, + "elapsed": time.monotonic() - start, + } def user(self, label): - username = 'u_' + label + '_' + uuid.uuid4().hex[:12] - password = 'T3st-' + uuid.uuid4().hex - body = {'username':username,'display_name':'Студент ' + label} - if LAB>=2: - body['password'] = password - user = self.call('POST','/api/v1/users',201,body=body,schema='User')['json'] - check(user['username']==username,'username changed') - actor = {'id':user['id'],'user':user,'registration':body} - if LAB>=2: - credential = base64.b64encode((username+':'+password).encode()).decode() - session = self.call('POST','/api/v1/auth/sessions',201,headers={'Authorization':'Basic '+credential},schema='Session')['json'] - check(session['user']==user,'session user mismatch') - check(timestamp(session['expires_at']) > dt.datetime.now(dt.timezone.utc),'session already expired') - actor['token'] = session['token'] + username = "u_" + label + "_" + uuid.uuid4().hex[:12] + password = "T3st-" + uuid.uuid4().hex + body = {"username": username, "display_name": "Студент " + label} + if LAB >= 2: + body["password"] = password + user = self.call("POST", "/api/v1/users", 201, body=body, schema="User")["json"] + check(user["username"] == username, "username changed") + actor = {"id": user["id"], "user": user, "registration": body} + if LAB >= 2: + credential = base64.b64encode((username + ":" + password).encode()).decode() + session = self.call( + "POST", + "/api/v1/auth/sessions", + 201, + headers={"Authorization": "Basic " + credential}, + schema="Session", + )["json"] + check(session["user"] == user, "session user mismatch") + check( + timestamp(session["expires_at"]) > dt.datetime.now(dt.timezone.utc), + "session already expired", + ) + actor["token"] = session["token"] return actor def bootstrap(self): - self.call('GET','/health/live',schema='Health') - self.call('GET','/health/ready',schema='Health') - alice, bob, eve = (self.user(label) for label in ('alice','bob','eve')) - chat = self.call('POST','/api/v1/chats',201,actor=alice,body={'title':'Контрактный тест','member_ids':[alice['id'],bob['id']]},schema='Chat')['json'] - check(set(chat['member_ids'])=={alice['id'],bob['id']},'chat member mismatch') - return alice,bob,eve,chat + self.call("GET", "/health/live", schema="Health") + self.call("GET", "/health/ready", schema="Health") + alice, bob, eve = (self.user(label) for label in ("alice", "bob", "eve")) + chat = self.call( + "POST", + "/api/v1/chats", + 201, + actor=alice, + body={"title": "Контрактный тест", "member_ids": [alice["id"], bob["id"]]}, + schema="Chat", + )["json"] + check( + set(chat["member_ids"]) == {alice["id"], bob["id"]}, "chat member mismatch" + ) + return alice, bob, eve, chat def message(self, actor, chat, text, key=None, attachments=None): - body = {'text':text} + body = {"text": text} if attachments is not None: - body['attachment_ids'] = attachments - headers = {'Idempotency-Key':key or uuid.uuid4().hex} if LAB>=3 else {} - return self.call('POST','/api/v1/chats/'+chat['id']+'/messages',201,actor=actor,body=body,headers=headers,schema='Message') + body["attachment_ids"] = attachments + headers = {"Idempotency-Key": key or uuid.uuid4().hex} if LAB >= 3 else {} + return self.call( + "POST", + "/api/v1/chats/" + chat["id"] + "/messages", + 201, + actor=actor, + body=body, + headers=headers, + schema="Message", + ) def attachment(self, actor, chat): - content = (ROOT / 'tests/fixtures/sample.png').read_bytes() - att = self.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=sample.png',201 if LAB==4 else 202,actor=actor,raw=content,headers={'Content-Type':'image/png'},schema='Attachment')['json'] - check(att['status']==('ready' if LAB==4 else 'queued'),'incorrect upload status') - check(att['chat_id']==chat['id'] and att['owner_id']==actor['id'],'attachment ownership mismatch') - check(att['sha256']==hashlib.sha256(content).hexdigest() and att['size_bytes']==len(content),'original checksum/size mismatch') - return att,content + content = (ROOT / "tests/fixtures/sample.png").read_bytes() + att = self.call( + "POST", + "/api/v1/chats/" + chat["id"] + "/attachments?filename=sample.png", + 201 if LAB == 4 else 202, + actor=actor, + raw=content, + headers={"Content-Type": "image/png"}, + schema="Attachment", + )["json"] + check( + att["status"] == ("ready" if LAB == 4 else "queued"), + "incorrect upload status", + ) + check( + att["chat_id"] == chat["id"] and att["owner_id"] == actor["id"], + "attachment ownership mismatch", + ) + check( + att["sha256"] == hashlib.sha256(content).hexdigest() + and att["size_bytes"] == len(content), + "original checksum/size mismatch", + ) + return att, content def wait_ready(self, actor, att, wait_seconds=60): - deadline = time.monotonic()+wait_seconds - while time.monotonic()=2 and i+length<=len(data),'invalid JPEG segment length') + check(i + 2 <= len(data), "truncated JPEG segment") + length = int.from_bytes(data[i : i + 2], "big") + check(length >= 2 and i + length <= len(data), "invalid JPEG segment length") if marker in sof: - check(length>=8,'invalid JPEG SOF') - return (int.from_bytes(data[i+5:i+7],'big'),int.from_bytes(data[i+3:i+5],'big')) + check(length >= 8, "invalid JPEG SOF") + return ( + int.from_bytes(data[i + 5 : i + 7], "big"), + int.from_bytes(data[i + 3 : i + 5], "big"), + ) i += length - raise Failure('JPEG dimensions not found') + raise Failure("JPEG dimensions not found") diff --git a/tests/load.py b/tests/load.py index ad81d596..8d022bad 100644 --- a/tests/load.py +++ b/tests/load.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 """Small closed-loop HTTP write benchmark; not an open-loop capacity proof.""" + from concurrent.futures import ThreadPoolExecutor import json import math @@ -11,48 +12,78 @@ from client import Client, Failure, LAB, arguments, check, run def main(): def configure(p): - p.add_argument('--duration',type=float,default=30) - p.add_argument('--concurrency',type=int,default=4) - p.add_argument('--output',default=None,help='Optional JSON metrics, no session tokens') - args = arguments('Synthetic POST message load, closed loop; creates users/messages',configure) - check(1<=args.duration<=3600,'duration must be 1..3600 seconds') - check(1<=args.concurrency<=128,'concurrency must be 1..128') - c = Client(args.base_url,args.ca_file) - alice,bob,eve,chat = c.bootstrap() + p.add_argument("--duration", type=float, default=30) + p.add_argument("--concurrency", type=int, default=4) + p.add_argument( + "--output", default=None, help="Optional JSON metrics, no session tokens" + ) + + args = arguments( + "Synthetic POST message load, closed loop; creates users/messages", configure + ) + check(1 <= args.duration <= 3600, "duration must be 1..3600 seconds") + check(1 <= args.concurrency <= 128, "concurrency must be 1..128") + c = Client(args.base_url, args.ca_file) + alice, bob, eve, chat = c.bootstrap() barrier = threading.Barrier(args.concurrency) + def worker(worker_id): - local = Client(args.base_url,args.ca_file,timeout=5) + local = Client(args.base_url, args.ca_file, timeout=5) barrier.wait() - deadline = time.monotonic()+args.duration - latencies,errors = [],0 + deadline = time.monotonic() + args.duration + latencies, errors = [], 0 index = 0 - while time.monotonic()=2,'persistence requires lab2+') - check(args.mode!='failover' or LAB>=3,'failover requires lab3+') + p.add_argument("--mode", choices=["persistence", "failover"], required=True) + p.add_argument( + "--action-script", + required=True, + help="Executable script authored by student; restarts stack or stops ONE API replica", + ) + + args = arguments( + "Explicit restart/failover scenario; modifies only synthetic API data", + configure, + ) + check(LAB >= 2, "persistence requires lab2+") + check(args.mode != "failover" or LAB >= 3, "failover requires lab3+") script = Path(args.action_script).expanduser().resolve() - check(script.is_file() and os.access(script,os.X_OK),'action script must exist and be executable (chmod +x)') - check('.example.' not in script.name,'copy and implement the example hook first') - c = Client(args.base_url,args.ca_file,timeout=2) - alice,bob,eve,chat = c.bootstrap() - sent = c.message(alice,chat,'Сохранить при отказе') - original = sent['json'] - target = sent['headers'].get('X-Instance-Id','') + check( + script.is_file() and os.access(script, os.X_OK), + "action script must exist and be executable (chmod +x)", + ) + check(".example." not in script.name, "copy and implement the example hook first") + c = Client(args.base_url, args.ca_file, timeout=2) + alice, bob, eve, chat = c.bootstrap() + sent = c.message(alice, chat, "Сохранить при отказе") + original = sent["json"] + target = sent["headers"].get("X-Instance-Id", "") attachment = content = None - if LAB>=4: - attachment,content = c.attachment(alice,chat) - if LAB>=5: - attachment = c.wait_ready(alice,attachment) + if LAB >= 4: + attachment, content = c.attachment(alice, chat) + if LAB >= 5: + attachment = c.wait_ready(alice, attachment) env = dict(os.environ) - env['TARGET_INSTANCE_ID'] = target + env["TARGET_INSTANCE_ID"] = target observed = set() - if args.mode=='failover': + if args.mode == "failover": for _ in range(40): - observed.add(c.call('GET','/api/v1/users/me',actor=alice,schema='User')['headers']['X-Instance-Id']) - check(len(observed)>=2,'less than two API instances observed before failure; verify LB/session affinity') - print('Executing explicit '+args.mode+' hook; hook output suppressed to avoid leaking secrets.') + observed.add( + c.call("GET", "/api/v1/users/me", actor=alice, schema="User")[ + "headers" + ]["X-Instance-Id"] + ) + check( + len(observed) >= 2, + "less than two API instances observed before failure; verify LB/session affinity", + ) + print( + "Executing explicit " + + args.mode + + " hook; hook output suppressed to avoid leaking secrets." + ) # No shell interpolation and no Docker commands in this test. - proc = subprocess.Popen([str(script)],env=env,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL) + proc = subprocess.Popen( + [str(script)], env=env, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL + ) started = time.monotonic() - successes,failures,after_instances = [],0,[] + successes, failures, after_instances = [], 0, [] try: - if args.mode=='persistence': + if args.mode == "persistence": try: code = proc.wait(timeout=90) except subprocess.TimeoutExpired: - raise Failure('restart hook exceeded 90 seconds') from None - check(code==0,'restart hook failed (inspect your script locally)') - deadline = time.monotonic()+90 + raise Failure("restart hook exceeded 90 seconds") from None + check(code == 0, "restart hook failed (inspect your script locally)") + deadline = time.monotonic() + 90 while True: try: - c.call('GET','/health/ready',schema='Health') - c.call('GET','/api/v1/users/me',actor=alice,schema='User') + c.call("GET", "/health/ready", schema="Health") + c.call("GET", "/api/v1/users/me", actor=alice, schema="User") break except Failure: - check(time.monotonic()=5,'too few successful writes after stop') - check(target not in after_instances[-5:],'target replica still serves requests; hook must leave it stopped') - print(json.dumps({'observed_instances_before':len(observed),'successful_writes':len(successes),'failed_attempts':failures,'first_recovery_seconds':round(first_recovery,3)},ensure_ascii=False)) + check(proc.poll() == 0, "stop-one hook did not finish successfully") + check( + first_recovery is not None and first_recovery <= 10, + "writes did not recover within 10 seconds from hook start", + ) + check(len(after_instances) >= 5, "too few successful writes after stop") + check( + target not in after_instances[-5:], + "target replica still serves requests; hook must leave it stopped", + ) + print( + json.dumps( + { + "observed_instances_before": len(observed), + "successful_writes": len(successes), + "failed_attempts": failures, + "first_recovery_seconds": round(first_recovery, 3), + }, + ensure_ascii=False, + ) + ) # Existing token, no re-login. Read all confirmations and reject duplicates. - messages = history(c,bob,chat) - ids = [m['id'] for m in messages] - check(len(ids)==len(set(ids)),'duplicated message IDs in history') - check(any(m==original for m in messages),'confirmed pre-failure message lost or altered') + messages = history(c, bob, chat) + ids = [m["id"] for m in messages] + check(len(ids) == len(set(ids)), "duplicated message IDs in history") + check( + any(m == original for m in messages), + "confirmed pre-failure message lost or altered", + ) for msg in successes: - check(any(m==msg for m in messages),'confirmed in-flight message lost or altered') + check( + any(m == msg for m in messages), + "confirmed in-flight message lost or altered", + ) # Bodies are unique in this scenario: an uncertain request may exist once, never twice. - texts = [m['text'] for m in messages] - check(len(texts)==len(set(texts)),'uncertain retry created duplicate message') + texts = [m["text"] for m in messages] + check( + len(texts) == len(set(texts)), "uncertain retry created duplicate message" + ) if attachment: - c.call('GET','/api/v1/attachments/'+attachment['id'],actor=bob,schema='Attachment') - downloaded = c.call('GET','/api/v1/attachments/'+attachment['id']+'/content',actor=bob) - check(downloaded['bytes']==content,'attachment lost or changed across failure') - print('PASS: '+args.mode+' preserved confirmed data and existing session. Hook scope needs operator evidence.') + c.call( + "GET", + "/api/v1/attachments/" + attachment["id"], + actor=bob, + schema="Attachment", + ) + downloaded = c.call( + "GET", "/api/v1/attachments/" + attachment["id"] + "/content", actor=bob + ) + check( + downloaded["bytes"] == content, + "attachment lost or changed across failure", + ) + print( + "PASS: " + + args.mode + + " preserved confirmed data and existing session. Hook scope needs operator evidence." + ) finally: if proc.poll() is None: proc.terminate() @@ -125,5 +202,6 @@ def main(): proc.kill() proc.wait() -if __name__=='__main__': + +if __name__ == "__main__": run(main) diff --git a/tests/smoke.py b/tests/smoke.py index 4cbfa558..19db4747 100644 --- a/tests/smoke.py +++ b/tests/smoke.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 """One complete public acceptance scenario, cumulative by lab number.""" + import base64 import hashlib import uuid @@ -7,76 +8,165 @@ from client import Client, LAB, arguments, check, run, timestamp def main(): - args = arguments('Messenger public smoke; creates synthetic users and messages') - c = Client(args.base_url,args.ca_file) - alice,bob,eve,chat = c.bootstrap() - path = '/api/v1/chats/'+chat['id'] - c.call('POST','/api/v1/users',409,body=alice['registration']) - c.call('GET','/api/v1/users/me',401) - c.call('GET',path,404,actor=eve) - me = c.call('GET','/api/v1/users/me',actor=alice,schema='User')['json'] - check(me==alice['user'],'current user mismatch') - listed = c.call('GET','/api/v1/chats?limit=100',actor=bob,schema='ChatPage')['json'] - check(any(x['id']==chat['id'] for x in listed['items']),'member cannot list chat') - hidden = c.call('GET','/api/v1/chats?limit=100',actor=eve,schema='ChatPage')['json'] - check(all(x['id']!=chat['id'] for x in hidden['items']),'chat leaks to outsider') + args = arguments("Messenger public smoke; creates synthetic users and messages") + c = Client(args.base_url, args.ca_file) + alice, bob, eve, chat = c.bootstrap() + path = "/api/v1/chats/" + chat["id"] + c.call("POST", "/api/v1/users", 409, body=alice["registration"]) + c.call("GET", "/api/v1/users/me", 401) + c.call("GET", path, 404, actor=eve) + me = c.call("GET", "/api/v1/users/me", actor=alice, schema="User")["json"] + check(me == alice["user"], "current user mismatch") + listed = c.call("GET", "/api/v1/chats?limit=100", actor=bob, schema="ChatPage")[ + "json" + ] + check( + any(x["id"] == chat["id"] for x in listed["items"]), "member cannot list chat" + ) + hidden = c.call("GET", "/api/v1/chats?limit=100", actor=eve, schema="ChatPage")[ + "json" + ] + check(all(x["id"] != chat["id"] for x in hidden["items"]), "chat leaks to outsider") key = uuid.uuid4().hex - first = c.message(alice,chat,'Привет, мир 👋',key=key)['json'] - second = c.message(bob,chat,'Сообщение Боба')['json'] - check(first['sender_id']==alice['id'] and second['sender_id']==bob['id'],'sender was not derived from actor') - check(first['chat_id']==second['chat_id']==chat['id'],'wrong chat in message') - check(first['text']=='Привет, мир 👋' and second['text']=='Сообщение Боба','message text changed') - page1 = c.call('GET',path+'/messages?limit=1',actor=bob,schema='MessagePage')['json'] - check(len(page1['items'])==1 and page1['next_cursor'],'first page must have one item and cursor') + first = c.message(alice, chat, "Привет, мир 👋", key=key)["json"] + second = c.message(bob, chat, "Сообщение Боба")["json"] + check( + first["sender_id"] == alice["id"] and second["sender_id"] == bob["id"], + "sender was not derived from actor", + ) + check(first["chat_id"] == second["chat_id"] == chat["id"], "wrong chat in message") + check( + first["text"] == "Привет, мир 👋" and second["text"] == "Сообщение Боба", + "message text changed", + ) + page1 = c.call("GET", path + "/messages?limit=1", actor=bob, schema="MessagePage")[ + "json" + ] + check( + len(page1["items"]) == 1 and page1["next_cursor"], + "first page must have one item and cursor", + ) from urllib.parse import quote - page2 = c.call('GET',path+'/messages?limit=1&cursor='+quote(page1['next_cursor'],safe=''),actor=alice,schema='MessagePage')['json'] - check(len(page2['items'])==1 and page2['next_cursor'] is None,'second page must finish history') - ordered = sorted([first,second],key=lambda x:(timestamp(x['created_at']),uuid.UUID(x['id']).int)) - check(page1['items']+page2['items']==ordered,'pagination lost, duplicated or reordered a message') - h = {'Idempotency-Key':uuid.uuid4().hex} if LAB>=3 else {} - c.call('POST',path+'/messages',400,actor=alice,body={'text':''},headers=h) - c.call('POST',path+'/messages',404,actor=eve,body={'text':'чужое'},headers=h) - c.call('GET',path+'/messages?limit=0',400,actor=alice) - if LAB>=2: - bad = base64.b64encode((alice['registration']['username']+':wrong-password').encode()).decode() - c.call('POST','/api/v1/auth/sessions',401,headers={'Authorization':'Basic '+bad}) - good = base64.b64encode((alice['registration']['username']+':'+alice['registration']['password']).encode()).decode() - c.call('GET','/api/v1/users/me',401,headers={'Authorization':'Basic '+good}) - c.call('GET','/api/v1/users/me',401,headers={'X-User-Id':alice['id']}) - me = c.call('GET','/api/v1/users/me',actor=alice,headers={'X-User-Id':eve['id']},schema='User')['json'] - check(me['id']==alice['id'],'X-User-Id overrode bearer identity') - if LAB>=3: - replay = c.message(alice,chat,first['text'],key=key)['json'] - check(replay==first,'idempotency replay differs') - c.call('POST',path+'/messages',409,actor=alice,body={'text':'другой body'},headers={'Idempotency-Key':key}) - c.call('POST',path+'/messages',400,actor=alice,body={'text':'без ключа'}) - if LAB>=4: - att,content = c.attachment(alice,chat) - # A queued attachment must already be linkable to its own chat. - attached = c.message(alice,chat,'',attachments=[att['id']])['json'] - check(attached['attachment_ids']==[att['id']],'message lost attachment') - ready = c.wait_ready(bob,att) if LAB>=5 else att - check(ready['error_code'] is None,'ready attachment has an error') - download = c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=bob) - check(download['bytes']==content,'download differs from original') - check(download['headers'].get_content_type()=='image/png','original content type mismatch') - for suffix in ('','/content'): - c.call('GET','/api/v1/attachments/'+att['id']+suffix,404,actor=eve) - if LAB>=5: - variants = [x for x in ready['variants'] if x['name']=='thumbnail'] - check(len(variants)==1,'expected exactly one thumbnail') - v = variants[0] - check((v['width'],v['height'])==(256,160),'thumbnail metadata has wrong dimensions') - thumb = c.call('GET','/api/v1/attachments/'+att['id']+'/content?variant=thumbnail',actor=bob) - check(thumb['headers'].get_content_type()=='image/jpeg','thumbnail content type mismatch') - check(len(thumb['bytes'])==v['size_bytes'] and hashlib.sha256(thumb['bytes']).hexdigest()==v['sha256'],'thumbnail checksum mismatch') - from image_probe import jpeg_size - check(jpeg_size(thumb['bytes'])==(256,160),'actual JPEG dimensions mismatch') - if LAB>=2: - c.call('DELETE','/api/v1/auth/sessions/current',204,actor=alice) - c.call('GET','/api/v1/users/me',401,actor=alice) - c.call('DELETE','/api/v1/auth/sessions/current',401,actor=alice) - print('PASS: lab%d public API scenario; infrastructure and grading evidence remain separate.' % LAB) -if __name__=='__main__': + page2 = c.call( + "GET", + path + "/messages?limit=1&cursor=" + quote(page1["next_cursor"], safe=""), + actor=alice, + schema="MessagePage", + )["json"] + check( + len(page2["items"]) == 1 and page2["next_cursor"] is None, + "second page must finish history", + ) + ordered = sorted( + [first, second], + key=lambda x: (timestamp(x["created_at"]), uuid.UUID(x["id"]).int), + ) + check( + page1["items"] + page2["items"] == ordered, + "pagination lost, duplicated or reordered a message", + ) + h = {"Idempotency-Key": uuid.uuid4().hex} if LAB >= 3 else {} + c.call("POST", path + "/messages", 400, actor=alice, body={"text": ""}, headers=h) + c.call( + "POST", path + "/messages", 404, actor=eve, body={"text": "чужое"}, headers=h + ) + c.call("GET", path + "/messages?limit=0", 400, actor=alice) + if LAB >= 2: + bad = base64.b64encode( + (alice["registration"]["username"] + ":wrong-password").encode() + ).decode() + c.call( + "POST", + "/api/v1/auth/sessions", + 401, + headers={"Authorization": "Basic " + bad}, + ) + good = base64.b64encode( + ( + alice["registration"]["username"] + + ":" + + alice["registration"]["password"] + ).encode() + ).decode() + c.call( + "GET", "/api/v1/users/me", 401, headers={"Authorization": "Basic " + good} + ) + c.call("GET", "/api/v1/users/me", 401, headers={"X-User-Id": alice["id"]}) + me = c.call( + "GET", + "/api/v1/users/me", + actor=alice, + headers={"X-User-Id": eve["id"]}, + schema="User", + )["json"] + check(me["id"] == alice["id"], "X-User-Id overrode bearer identity") + if LAB >= 3: + replay = c.message(alice, chat, first["text"], key=key)["json"] + check(replay == first, "idempotency replay differs") + c.call( + "POST", + path + "/messages", + 409, + actor=alice, + body={"text": "другой body"}, + headers={"Idempotency-Key": key}, + ) + c.call("POST", path + "/messages", 400, actor=alice, body={"text": "без ключа"}) + if LAB >= 4: + att, content = c.attachment(alice, chat) + # A queued attachment must already be linkable to its own chat. + attached = c.message(alice, chat, "", attachments=[att["id"]])["json"] + check(attached["attachment_ids"] == [att["id"]], "message lost attachment") + ready = c.wait_ready(bob, att) if LAB >= 5 else att + check(ready["error_code"] is None, "ready attachment has an error") + download = c.call( + "GET", "/api/v1/attachments/" + att["id"] + "/content", actor=bob + ) + check(download["bytes"] == content, "download differs from original") + check( + download["headers"].get_content_type() == "image/png", + "original content type mismatch", + ) + for suffix in ("", "/content"): + c.call("GET", "/api/v1/attachments/" + att["id"] + suffix, 404, actor=eve) + if LAB >= 5: + variants = [x for x in ready["variants"] if x["name"] == "thumbnail"] + check(len(variants) == 1, "expected exactly one thumbnail") + v = variants[0] + check( + (v["width"], v["height"]) == (256, 160), + "thumbnail metadata has wrong dimensions", + ) + thumb = c.call( + "GET", + "/api/v1/attachments/" + att["id"] + "/content?variant=thumbnail", + actor=bob, + ) + check( + thumb["headers"].get_content_type() == "image/jpeg", + "thumbnail content type mismatch", + ) + check( + len(thumb["bytes"]) == v["size_bytes"] + and hashlib.sha256(thumb["bytes"]).hexdigest() == v["sha256"], + "thumbnail checksum mismatch", + ) + from image_probe import jpeg_size + + check( + jpeg_size(thumb["bytes"]) == (256, 160), + "actual JPEG dimensions mismatch", + ) + if LAB >= 2: + c.call("DELETE", "/api/v1/auth/sessions/current", 204, actor=alice) + c.call("GET", "/api/v1/users/me", 401, actor=alice) + c.call("DELETE", "/api/v1/auth/sessions/current", 401, actor=alice) + print( + "PASS: lab%d public API scenario; infrastructure and grading evidence remain separate." + % LAB + ) + + +if __name__ == "__main__": run(main)