From 5ae3d771520e34a1b08e58e30d7320e9ff7043a1 Mon Sep 17 00:00:00 2001 From: Artyom Date: Thu, 29 Oct 2026 03:51:58 +0300 Subject: [PATCH] Carry over the lab 5 solution Bring the application, own tests, Dockerfile, toolchain, compose stack, HAProxy, k6, RustFS and RabbitMQ configuration, migrations, hooks, scenario scripts and the CI job from the lab 5 solution commit 64d5c67, keep the lab 5 Makefile targets on top of the unchanged skeleton targets, and append the lab 5 solution variables to the skeleton .env.example. The package and the application follow the contract version 1.6.0, and the image, the throwaway test containers and the broker vhost description are tagged lab6 so they do not collide with a lab 5 stand on the same host. This is the starting point of the lab 6 cycle: lint, own tests and the offline skeleton check pass before any lab 6 change. Co-Authored-By: Claude Opus 5.5 (1M context) --- .dockerignore | 24 + .env.example | 48 + .github/workflows/application.yml | 109 ++ Dockerfile | 48 + Makefile | 200 ++- compose.yaml | 400 ++++++ infra/compose.faults.yaml | 18 + infra/haproxy/haproxy.cfg | 71 + infra/k6/messages.js | 106 ++ infra/rabbitmq/definitions.json | 44 + infra/rabbitmq/enabled_plugins | 1 + infra/rabbitmq/rabbitmq.conf | 21 + infra/s3/api-policy.json | 23 + infra/s3/cli.sh | 12 + infra/s3/init.sh | 26 + infra/s3/lifecycle.json | 10 + infra/s3/worker-policy.json | 24 + pyproject.toml | 124 ++ scripts/broker-secrets.sh | 60 + scripts/broker.sh | 21 + scripts/detection.sh | 20 + scripts/evidence-attachments.sh | 40 + scripts/evidence-cache.sh | 24 + scripts/evidence-stack.sh | 51 + scripts/evidence-storage.sh | 44 + scripts/failover-each.sh | 19 + scripts/failover-evidence.sh | 28 + scripts/k6.sh | 10 + scripts/lab4_scenarios.py | 231 +++ scripts/lab4_seed.py | 352 +++++ scripts/lb-wait.sh | 11 + scripts/limits.py | 185 +++ scripts/make_images.py | 354 +++++ scripts/persistence-evidence.sh | 14 + scripts/pipeline_evidence.py | 105 ++ scripts/pipeline_suite.py | 1033 ++++++++++++++ scripts/probe.py | 436 ++++++ scripts/restart.sh | 15 + scripts/restore.sh | 5 + scripts/rolling-update.sh | 20 + scripts/s3-check.sh | 66 + scripts/secrets.sh | 24 + scripts/sql/explain-history.sql | 59 + scripts/sql/seed-history.sql | 40 + scripts/stop-one.sh | 9 + scripts/upload_memory.py | 117 ++ scripts/worker.sh | 12 + src/messenger/__init__.py | 0 src/messenger/admin.py | 37 + src/messenger/attachments/__init__.py | 0 .../attachments/application/__init__.py | 0 src/messenger/attachments/application/bus.py | 5 + .../attachments/application/commands.py | 49 + .../attachments/application/errors.py | 2 + .../attachments/application/handlers.py | 158 ++ .../attachments/application/pipeline.py | 301 ++++ .../attachments/application/ports.py | 155 ++ .../attachments/application/queries.py | 57 + src/messenger/attachments/config.py | 49 + src/messenger/attachments/domain/__init__.py | 0 src/messenger/attachments/domain/errors.py | 45 + src/messenger/attachments/domain/events.py | 41 + src/messenger/attachments/domain/jobs.py | 95 ++ src/messenger/attachments/domain/models.py | 103 ++ src/messenger/attachments/domain/values.py | 46 + src/messenger/attachments/domain/variants.py | 35 + .../attachments/entrypoints/__init__.py | 0 .../attachments/entrypoints/admin.py | 50 + .../attachments/entrypoints/errors.py | 20 + .../attachments/entrypoints/heartbeat.py | 44 + src/messenger/attachments/entrypoints/http.py | 77 + src/messenger/attachments/entrypoints/ioc.py | 268 ++++ .../attachments/entrypoints/journal.py | 78 + .../attachments/entrypoints/relay.py | 73 + .../attachments/entrypoints/schemas.py | 84 ++ .../attachments/entrypoints/sweeper.py | 53 + .../attachments/entrypoints/worker.py | 50 + .../attachments/infrastructure/__init__.py | 0 .../attachments/infrastructure/faults.py | 32 + .../attachments/infrastructure/images.py | 206 +++ .../attachments/infrastructure/jitter.py | 10 + .../attachments/infrastructure/postgres.py | 461 ++++++ .../attachments/infrastructure/rabbitmq.py | 37 + .../attachments/infrastructure/s3.py | 104 ++ src/messenger/gunicorn_conf.py | 66 + src/messenger/identity/__init__.py | 0 .../identity/application/__init__.py | 0 src/messenger/identity/application/bus.py | 5 + .../identity/application/commands.py | 22 + .../identity/application/handlers.py | 76 + src/messenger/identity/application/ports.py | 48 + src/messenger/identity/application/queries.py | 43 + src/messenger/identity/config.py | 12 + src/messenger/identity/domain/__init__.py | 0 src/messenger/identity/domain/errors.py | 17 + src/messenger/identity/domain/events.py | 25 + src/messenger/identity/domain/models.py | 62 + src/messenger/identity/domain/values.py | 54 + .../identity/entrypoints/__init__.py | 0 .../identity/entrypoints/authentication.py | 98 ++ src/messenger/identity/entrypoints/errors.py | 12 + src/messenger/identity/entrypoints/http.py | 57 + src/messenger/identity/entrypoints/ioc.py | 112 ++ src/messenger/identity/entrypoints/journal.py | 20 + src/messenger/identity/entrypoints/schemas.py | 42 + .../identity/infrastructure/__init__.py | 0 .../identity/infrastructure/argon2.py | 35 + .../identity/infrastructure/postgres.py | 124 ++ .../identity/infrastructure/valkey.py | 102 ++ src/messenger/main.py | 162 +++ src/messenger/messaging/__init__.py | 0 .../messaging/application/__init__.py | 0 src/messenger/messaging/application/bus.py | 5 + .../messaging/application/commands.py | 22 + .../messaging/application/handlers.py | 73 + src/messenger/messaging/application/ports.py | 52 + .../messaging/application/queries.py | 73 + src/messenger/messaging/config.py | 8 + src/messenger/messaging/domain/__init__.py | 0 src/messenger/messaging/domain/errors.py | 37 + src/messenger/messaging/domain/events.py | 24 + src/messenger/messaging/domain/keys.py | 3 + src/messenger/messaging/domain/models.py | 177 +++ src/messenger/messaging/domain/values.py | 40 + .../messaging/entrypoints/__init__.py | 0 src/messenger/messaging/entrypoints/errors.py | 22 + src/messenger/messaging/entrypoints/http.py | 90 ++ src/messenger/messaging/entrypoints/ioc.py | 75 + .../messaging/entrypoints/journal.py | 27 + .../messaging/entrypoints/schemas.py | 85 ++ .../messaging/infrastructure/__init__.py | 0 .../messaging/infrastructure/cursor.py | 59 + .../messaging/infrastructure/postgres.py | 219 +++ src/messenger/migrate.py | 70 + src/messenger/migrations/0001_init.sql | 46 + .../migrations/0002_idempotency_keys.sql | 10 + src/messenger/migrations/0003_attachments.sql | 37 + src/messenger/migrations/0004_jobs.sql | 70 + .../migrations/0005_failed_registry.sql | 1 + src/messenger/relay.py | 66 + src/messenger/shared/__init__.py | 0 src/messenger/shared/application/__init__.py | 0 src/messenger/shared/application/commands.py | 2 + src/messenger/shared/application/errors.py | 2 + .../shared/application/messagebus.py | 124 ++ src/messenger/shared/application/ports.py | 42 + src/messenger/shared/config.py | 49 + src/messenger/shared/domain/__init__.py | 0 src/messenger/shared/domain/errors.py | 4 + src/messenger/shared/domain/events.py | 6 + src/messenger/shared/entrypoints/__init__.py | 0 src/messenger/shared/entrypoints/actor.py | 16 + src/messenger/shared/entrypoints/config.py | 10 + src/messenger/shared/entrypoints/errors.py | 136 ++ src/messenger/shared/entrypoints/health.py | 25 + src/messenger/shared/entrypoints/ioc.py | 54 + src/messenger/shared/entrypoints/pipeline.py | 175 +++ src/messenger/shared/entrypoints/schemas.py | 21 + .../shared/infrastructure/__init__.py | 0 src/messenger/shared/infrastructure/clock.py | 17 + src/messenger/shared/infrastructure/ids.py | 8 + .../shared/infrastructure/logging.py | 113 ++ .../shared/infrastructure/postgres.py | 119 ++ src/messenger/worker.py | 78 + student-tests/adapters/test_rabbitmq.py | 259 ++++ student-tests/adapters/test_s3_storage.py | 95 ++ student-tests/api/test_access.py | 84 ++ student-tests/api/test_attachments.py | 326 +++++ student-tests/api/test_bounded_contexts.py | 289 ++++ student-tests/api/test_cache.py | 118 ++ student-tests/api/test_concurrency.py | 88 ++ student-tests/api/test_consistency.py | 188 +++ student-tests/api/test_drain.py | 110 ++ student-tests/api/test_events.py | 124 ++ student-tests/api/test_idempotency.py | 145 ++ student-tests/api/test_instance.py | 64 + student-tests/api/test_message_attachments.py | 149 ++ student-tests/api/test_migrations.py | 240 ++++ student-tests/api/test_negative.py | 196 +++ student-tests/api/test_operability.py | 144 ++ student-tests/api/test_pagination.py | 155 ++ student-tests/api/test_pipeline.py | 303 ++++ student-tests/api/test_pool.py | 27 + student-tests/api/test_recovery.py | 307 ++++ student-tests/api/test_sessions.py | 172 +++ student-tests/api/test_timeout.py | 97 ++ student-tests/api/test_upload_consistency.py | 254 ++++ student-tests/api/test_wire_format.py | 69 + student-tests/conftest.py | 213 +++ student-tests/fakes.py | 81 ++ student-tests/pictures.py | 35 + student-tests/pipeline.py | 164 +++ student-tests/stack/test_attachment_faults.py | 472 ++++++ student-tests/stack/test_faults.py | 229 +++ student-tests/stand.py | 132 ++ student-tests/unit/test_attachments_domain.py | 96 ++ student-tests/unit/test_authentication.py | 64 + student-tests/unit/test_cursor.py | 56 + student-tests/unit/test_domain.py | 239 ++++ student-tests/unit/test_pipeline_domain.py | 93 ++ student-tests/unit/test_render_pool.py | 91 ++ student-tests/unit/test_renderer.py | 126 ++ student-tests/unit/test_timeouts.py | 60 + uv.lock | 1269 +++++++++++++++++ 204 files changed, 18547 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/application.yml create mode 100644 Dockerfile create mode 100644 compose.yaml create mode 100644 infra/compose.faults.yaml create mode 100644 infra/haproxy/haproxy.cfg create mode 100644 infra/k6/messages.js create mode 100644 infra/rabbitmq/definitions.json create mode 100644 infra/rabbitmq/enabled_plugins create mode 100644 infra/rabbitmq/rabbitmq.conf create mode 100644 infra/s3/api-policy.json create mode 100755 infra/s3/cli.sh create mode 100755 infra/s3/init.sh create mode 100644 infra/s3/lifecycle.json create mode 100644 infra/s3/worker-policy.json create mode 100644 pyproject.toml create mode 100755 scripts/broker-secrets.sh create mode 100755 scripts/broker.sh create mode 100755 scripts/detection.sh create mode 100755 scripts/evidence-attachments.sh create mode 100755 scripts/evidence-cache.sh create mode 100755 scripts/evidence-stack.sh create mode 100755 scripts/evidence-storage.sh create mode 100755 scripts/failover-each.sh create mode 100755 scripts/failover-evidence.sh create mode 100755 scripts/k6.sh create mode 100755 scripts/lab4_scenarios.py create mode 100755 scripts/lab4_seed.py create mode 100755 scripts/lb-wait.sh create mode 100644 scripts/limits.py create mode 100644 scripts/make_images.py create mode 100755 scripts/persistence-evidence.sh create mode 100644 scripts/pipeline_evidence.py create mode 100644 scripts/pipeline_suite.py create mode 100755 scripts/probe.py create mode 100755 scripts/restart.sh create mode 100755 scripts/restore.sh create mode 100755 scripts/rolling-update.sh create mode 100755 scripts/s3-check.sh create mode 100755 scripts/secrets.sh create mode 100644 scripts/sql/explain-history.sql create mode 100644 scripts/sql/seed-history.sql create mode 100755 scripts/stop-one.sh create mode 100755 scripts/upload_memory.py create mode 100755 scripts/worker.sh create mode 100644 src/messenger/__init__.py create mode 100644 src/messenger/admin.py create mode 100644 src/messenger/attachments/__init__.py create mode 100644 src/messenger/attachments/application/__init__.py create mode 100644 src/messenger/attachments/application/bus.py create mode 100644 src/messenger/attachments/application/commands.py create mode 100644 src/messenger/attachments/application/errors.py create mode 100644 src/messenger/attachments/application/handlers.py create mode 100644 src/messenger/attachments/application/pipeline.py create mode 100644 src/messenger/attachments/application/ports.py create mode 100644 src/messenger/attachments/application/queries.py create mode 100644 src/messenger/attachments/config.py create mode 100644 src/messenger/attachments/domain/__init__.py create mode 100644 src/messenger/attachments/domain/errors.py create mode 100644 src/messenger/attachments/domain/events.py create mode 100644 src/messenger/attachments/domain/jobs.py create mode 100644 src/messenger/attachments/domain/models.py create mode 100644 src/messenger/attachments/domain/values.py create mode 100644 src/messenger/attachments/domain/variants.py create mode 100644 src/messenger/attachments/entrypoints/__init__.py create mode 100644 src/messenger/attachments/entrypoints/admin.py create mode 100644 src/messenger/attachments/entrypoints/errors.py create mode 100644 src/messenger/attachments/entrypoints/heartbeat.py create mode 100644 src/messenger/attachments/entrypoints/http.py create mode 100644 src/messenger/attachments/entrypoints/ioc.py create mode 100644 src/messenger/attachments/entrypoints/journal.py create mode 100644 src/messenger/attachments/entrypoints/relay.py create mode 100644 src/messenger/attachments/entrypoints/schemas.py create mode 100644 src/messenger/attachments/entrypoints/sweeper.py create mode 100644 src/messenger/attachments/entrypoints/worker.py create mode 100644 src/messenger/attachments/infrastructure/__init__.py create mode 100644 src/messenger/attachments/infrastructure/faults.py create mode 100644 src/messenger/attachments/infrastructure/images.py create mode 100644 src/messenger/attachments/infrastructure/jitter.py create mode 100644 src/messenger/attachments/infrastructure/postgres.py create mode 100644 src/messenger/attachments/infrastructure/rabbitmq.py create mode 100644 src/messenger/attachments/infrastructure/s3.py create mode 100644 src/messenger/gunicorn_conf.py create mode 100644 src/messenger/identity/__init__.py create mode 100644 src/messenger/identity/application/__init__.py create mode 100644 src/messenger/identity/application/bus.py create mode 100644 src/messenger/identity/application/commands.py create mode 100644 src/messenger/identity/application/handlers.py create mode 100644 src/messenger/identity/application/ports.py create mode 100644 src/messenger/identity/application/queries.py create mode 100644 src/messenger/identity/config.py create mode 100644 src/messenger/identity/domain/__init__.py create mode 100644 src/messenger/identity/domain/errors.py create mode 100644 src/messenger/identity/domain/events.py create mode 100644 src/messenger/identity/domain/models.py create mode 100644 src/messenger/identity/domain/values.py create mode 100644 src/messenger/identity/entrypoints/__init__.py create mode 100644 src/messenger/identity/entrypoints/authentication.py create mode 100644 src/messenger/identity/entrypoints/errors.py create mode 100644 src/messenger/identity/entrypoints/http.py create mode 100644 src/messenger/identity/entrypoints/ioc.py create mode 100644 src/messenger/identity/entrypoints/journal.py create mode 100644 src/messenger/identity/entrypoints/schemas.py create mode 100644 src/messenger/identity/infrastructure/__init__.py create mode 100644 src/messenger/identity/infrastructure/argon2.py create mode 100644 src/messenger/identity/infrastructure/postgres.py create mode 100644 src/messenger/identity/infrastructure/valkey.py create mode 100644 src/messenger/main.py create mode 100644 src/messenger/messaging/__init__.py create mode 100644 src/messenger/messaging/application/__init__.py create mode 100644 src/messenger/messaging/application/bus.py create mode 100644 src/messenger/messaging/application/commands.py create mode 100644 src/messenger/messaging/application/handlers.py create mode 100644 src/messenger/messaging/application/ports.py create mode 100644 src/messenger/messaging/application/queries.py create mode 100644 src/messenger/messaging/config.py create mode 100644 src/messenger/messaging/domain/__init__.py create mode 100644 src/messenger/messaging/domain/errors.py create mode 100644 src/messenger/messaging/domain/events.py create mode 100644 src/messenger/messaging/domain/keys.py create mode 100644 src/messenger/messaging/domain/models.py create mode 100644 src/messenger/messaging/domain/values.py create mode 100644 src/messenger/messaging/entrypoints/__init__.py create mode 100644 src/messenger/messaging/entrypoints/errors.py create mode 100644 src/messenger/messaging/entrypoints/http.py create mode 100644 src/messenger/messaging/entrypoints/ioc.py create mode 100644 src/messenger/messaging/entrypoints/journal.py create mode 100644 src/messenger/messaging/entrypoints/schemas.py create mode 100644 src/messenger/messaging/infrastructure/__init__.py create mode 100644 src/messenger/messaging/infrastructure/cursor.py create mode 100644 src/messenger/messaging/infrastructure/postgres.py create mode 100644 src/messenger/migrate.py create mode 100644 src/messenger/migrations/0001_init.sql create mode 100644 src/messenger/migrations/0002_idempotency_keys.sql create mode 100644 src/messenger/migrations/0003_attachments.sql create mode 100644 src/messenger/migrations/0004_jobs.sql create mode 100644 src/messenger/migrations/0005_failed_registry.sql create mode 100644 src/messenger/relay.py create mode 100644 src/messenger/shared/__init__.py create mode 100644 src/messenger/shared/application/__init__.py create mode 100644 src/messenger/shared/application/commands.py create mode 100644 src/messenger/shared/application/errors.py create mode 100644 src/messenger/shared/application/messagebus.py create mode 100644 src/messenger/shared/application/ports.py create mode 100644 src/messenger/shared/config.py create mode 100644 src/messenger/shared/domain/__init__.py create mode 100644 src/messenger/shared/domain/errors.py create mode 100644 src/messenger/shared/domain/events.py create mode 100644 src/messenger/shared/entrypoints/__init__.py create mode 100644 src/messenger/shared/entrypoints/actor.py create mode 100644 src/messenger/shared/entrypoints/config.py create mode 100644 src/messenger/shared/entrypoints/errors.py create mode 100644 src/messenger/shared/entrypoints/health.py create mode 100644 src/messenger/shared/entrypoints/ioc.py create mode 100644 src/messenger/shared/entrypoints/pipeline.py create mode 100644 src/messenger/shared/entrypoints/schemas.py create mode 100644 src/messenger/shared/infrastructure/__init__.py create mode 100644 src/messenger/shared/infrastructure/clock.py create mode 100644 src/messenger/shared/infrastructure/ids.py create mode 100644 src/messenger/shared/infrastructure/logging.py create mode 100644 src/messenger/shared/infrastructure/postgres.py create mode 100644 src/messenger/worker.py create mode 100644 student-tests/adapters/test_rabbitmq.py create mode 100644 student-tests/adapters/test_s3_storage.py create mode 100644 student-tests/api/test_access.py create mode 100644 student-tests/api/test_attachments.py create mode 100644 student-tests/api/test_bounded_contexts.py create mode 100644 student-tests/api/test_cache.py create mode 100644 student-tests/api/test_concurrency.py create mode 100644 student-tests/api/test_consistency.py create mode 100644 student-tests/api/test_drain.py create mode 100644 student-tests/api/test_events.py create mode 100644 student-tests/api/test_idempotency.py create mode 100644 student-tests/api/test_instance.py create mode 100644 student-tests/api/test_message_attachments.py create mode 100644 student-tests/api/test_migrations.py create mode 100644 student-tests/api/test_negative.py create mode 100644 student-tests/api/test_operability.py create mode 100644 student-tests/api/test_pagination.py create mode 100644 student-tests/api/test_pipeline.py create mode 100644 student-tests/api/test_pool.py create mode 100644 student-tests/api/test_recovery.py create mode 100644 student-tests/api/test_sessions.py create mode 100644 student-tests/api/test_timeout.py create mode 100644 student-tests/api/test_upload_consistency.py create mode 100644 student-tests/api/test_wire_format.py create mode 100644 student-tests/conftest.py create mode 100644 student-tests/fakes.py create mode 100644 student-tests/pictures.py create mode 100644 student-tests/pipeline.py create mode 100644 student-tests/stack/test_attachment_faults.py create mode 100644 student-tests/stack/test_faults.py create mode 100644 student-tests/stand.py create mode 100644 student-tests/unit/test_attachments_domain.py create mode 100644 student-tests/unit/test_authentication.py create mode 100644 student-tests/unit/test_cursor.py create mode 100644 student-tests/unit/test_domain.py create mode 100644 student-tests/unit/test_pipeline_domain.py create mode 100644 student-tests/unit/test_render_pool.py create mode 100644 student-tests/unit/test_renderer.py create mode 100644 student-tests/unit/test_timeouts.py create mode 100644 uv.lock diff --git a/.dockerignore b/.dockerignore index bc16e556..bee74195 100644 --- a/.dockerignore +++ b/.dockerignore @@ -14,7 +14,31 @@ backups *.pfx *.jks .venv +venv __pycache__ +*.pyc +*.pyo +.pytest_cache +.ruff_cache +.mypy_cache +.ty_cache node_modules evidence +openspec +student-tests +tests +scripts +examples +contracts +infra +*.md +!README.md +Makefile +Dockerfile* +compose*.yaml +.dockerignore +.gitignore .DS_Store +lab.json +requirements-dev.txt +.claude diff --git a/.env.example b/.env.example index 2b326558..72acc0ad 100644 --- a/.env.example +++ b/.env.example @@ -28,3 +28,51 @@ OTEL_SERVICE_NAME=messenger-api OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318 OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab +# Solution-specific: published LB ports, replica limits, workers and pool per replica, +# LB queue per replica, drain window before closing the listener on SIGTERM, PostgreSQL +# limits, load generator limits (profile bench), cursor key file. API_PORT/SHORTTTL_PORT of lab2 are gone: +# only the load balancer is published. +LB_HTTP_PORT=8080 +LB_STATS_PORT=8404 +API_CPUS=1 +API_MEMORY=512m +WEB_CONCURRENCY=2 +DB_POOL_MAX_SIZE=8 +API_SERVER_MAXCONN=64 +DRAIN_SECONDS=2 +PG_MAX_CONNECTIONS=200 +PG_CPUS=2 +PG_MEMORY=1g +K6_CPUS=2 +K6_MEMORY=512m +CURSOR_SECRET_FILE=/run/secrets/cursor_secret +# Solution-specific for lab4: S3 client timeouts (connect + read, two attempts stay below +# REQUEST_TIMEOUT_SECONDS=8), parallel image checks per worker, RustFS memory limit. +S3_CONNECT_TIMEOUT_SECONDS=1 +S3_READ_TIMEOUT_SECONDS=2 +IMAGE_CHECK_CONCURRENCY=2 +S3_MEMORY=512m +# Solution-specific for lab4 grade 4: upload bodies buffered at once per worker, grace period +# before an abandoned pending upload is swept, sweep interval and rows per sweep pass. +UPLOAD_CONCURRENCY=3 +UPLOAD_GRACE_SECONDS=900 +UPLOAD_SWEEP_INTERVAL_SECONDS=30 +UPLOAD_SWEEP_BATCH=100 +# Fixed glibc mmap threshold for the API: buffers above 128 KiB are returned to the system on free. +MALLOC_MMAP_THRESHOLD_=131072 +# Solution-specific for lab5: RabbitMQ management UI on 127.0.0.1, backoff base of a transient +# retry (doubles per attempt up to 60 s), relay poll and sweep intervals, worker limits and the +# PostgreSQL pool of relay and worker. FAULT_POINT is set only by infra/compose.faults.yaml. +BROKER_MANAGEMENT_PORT=15672 +JOB_BACKOFF_BASE_SECONDS=2 +OUTBOX_POLL_SECONDS=1 +SWEEP_INTERVAL_SECONDS=5 +WORKER_CPUS=2 +WORKER_MEMORY=1g +PIPELINE_DB_POOL_MAX_SIZE=4 +# Solution-specific for lab5 grade 4: render time per job (the render process is killed after it), +# address space of one render process (WORKER_MEMORY >= WORKER_CONCURRENCY * RENDER_MEMORY_MB plus +# ~120 MiB of the worker itself), jobs per render process before it is replaced. +RENDER_TIMEOUT_SECONDS=30 +RENDER_MEMORY_MB=448 +RENDER_TASKS_PER_CHILD=100 diff --git a/.github/workflows/application.yml b/.github/workflows/application.yml new file mode 100644 index 00000000..2c612823 --- /dev/null +++ b/.github/workflows/application.yml @@ -0,0 +1,109 @@ +name: Application + +on: [push, pull_request] + +permissions: + contents: read + +jobs: + quality: + name: Линт, типы и собственные тесты + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v5 + with: + enable-cache: true + - run: uv sync --locked --group dev + - run: make lint VENV=.venv + - run: make student-test VENV=.venv + + acceptance: + name: Стенд Compose с RabbitMQ, relay и worker за HAProxy, smoke, pipeline, отказ реплики, рестарт и отказы зависимостей + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v5 + with: + enable-cache: true + - run: uv sync --locked --group dev + + - name: Поднять стенд из чистого clone + run: make up + + - name: Публичный smoke через балансировщик + run: make test + + - name: Топология брокера, consumer с prefetch и права клиентов + run: make topology + + - name: Уровни «3» и «4» pipeline — worker остановлен, рестарт брокера, дубль, S3, попытки, варианты и пределы, падения в окнах 2–5, аренда, replay, DLQ, jitter, retention + run: | + make suite-preflight + make level4 + + - name: Сообщение задания в очереди, 409 до ready и SIGTERM relay и worker + run: make pipeline-evidence + + - name: Вложение после остановки принявшей его реплики и при недоступном S3 + run: | + make scenario SCENARIO=failover-upload + make scenario SCENARIO=s3-outage + + - name: Приватный bucket, ограниченный ключ API и повторный bootstrap + run: make s3-check + + - name: Негативный suite вложений, компенсация, уборка после grace, обрыв и зависание загрузки + run: make upload-faults + + - name: Несколько реплик на одном токене и гонка одного ключа + run: make lb-check + + - name: Остановка реплики под запросами, затем возврат + run: | + make test-failover ACTION_SCRIPT=scripts/stop-one.sh + docker compose ps -a api + make restore + + - name: Рестарт LB, API и хранилищ с сохранением volumes + run: make test-persistence ACTION_SCRIPT=scripts/restart.sh + + - name: Отказы Valkey и PostgreSQL, гонка и rollback + run: make test-faults + + - name: Лимиты, порты, секреты, volumes и сети по политике + run: make limits + + - name: Время обнаружения отказа балансировщиком (SIGTERM, kill, pause, PostgreSQL) + run: make detection + + - name: Наружу опубликован только балансировщик + run: | + docker compose ps --format '{{.Service}} {{.Ports}}' | awk '$2 != "" && $1 != "lb" && $1 != "broker" {print; bad=1} END {exit bad}' + docker compose ps --format '{{.Service}} {{.Ports}}' | awk '$1 == "broker" && $2 !~ /^127\.0\.0\.1:/ {print; bad=1} END {exit bad}' + + - name: Процессы работают не от root + run: | + docker compose top | awk 'NR>1 && $3=="0" {print; bad=1} END {exit bad}' + + - name: Корректная остановка relay и worker по SIGTERM + run: | + start=$(date +%s) + docker compose stop -t 10 relay worker + elapsed=$(( $(date +%s) - start )) + codes=$(docker inspect -f '{{.State.ExitCode}}' $(docker compose ps -aq relay worker) | sort -u) + echo "остановка за ${elapsed} с, коды завершения ${codes}" + test "$elapsed" -le 10 && test "$codes" = "0" + + - name: Корректная остановка реплик API по SIGTERM + run: | + start=$(date +%s) + docker compose kill --signal SIGTERM api + codes=$(docker wait $(docker compose ps -aq api) | sort -u) + elapsed=$(( $(date +%s) - start )) + echo "остановка за ${elapsed} с, коды завершения ${codes}" + test "$elapsed" -le 10 && test "$codes" = "0" + + - name: Журналы стенда + if: always() + run: docker compose logs --tail 80 diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..1f4e6fed --- /dev/null +++ b/Dockerfile @@ -0,0 +1,48 @@ +FROM python:3.14-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d AS builder + +COPY --from=ghcr.io/astral-sh/uv:0.9.7@sha256:ba4857bf2a068e9bc0e64eed8563b065908a4cd6bfb66b531a9c424c8e25e142 /uv /usr/local/bin/uv + +ENV UV_COMPILE_BYTECODE=1 \ + UV_LINK_MODE=copy \ + UV_PYTHON_DOWNLOADS=never \ + UV_PROJECT_ENVIRONMENT=/opt/venv + +WORKDIR /build + +COPY pyproject.toml uv.lock README.md ./ +RUN uv sync --locked --no-dev --no-install-project --no-editable + +COPY src ./src +RUN uv sync --locked --no-dev --no-editable + + +FROM python:3.14-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d AS runtime + +ENV PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + PATH="/opt/venv/bin:$PATH" \ + HOST=0.0.0.0 \ + PORT=8080 \ + LOG_LEVEL=INFO \ + WEB_CONCURRENCY=2 \ + REQUEST_TIMEOUT_SECONDS=8 \ + DRAIN_SECONDS=2 \ + SHUTDOWN_TIMEOUT_SECONDS=5 + +RUN groupadd --system --gid 10001 messenger \ + && useradd --system --uid 10001 --gid messenger --no-create-home --shell /usr/sbin/nologin messenger + +COPY --from=builder --chown=root:root /opt/venv /opt/venv + +WORKDIR /app + +USER 10001:10001 + +EXPOSE 8080 + +HEALTHCHECK --interval=10s --timeout=3s --start-period=5s --retries=3 \ + CMD ["python", "-c", "import urllib.request,sys;sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8080/health/live',timeout=2).status==200 else 1)"] + +CMD ["python", "-m", "gunicorn", \ + "--config", "python:messenger.gunicorn_conf", \ + "messenger.main:create_app()"] diff --git a/Makefile b/Makefile index 5d86d171..ccde1de0 100644 --- a/Makefile +++ b/Makefile @@ -4,10 +4,38 @@ CA_FILE ?= ACTION_SCRIPT ?= TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) -.PHONY: help check validate test load test-persistence test-failover +VENV ?= .venv +UV ?= uv +COMPOSE ?= docker compose +RATE ?= 100 +API_IMAGE ?= mtusi-messenger:lab6 +FROM_REV ?= +SCRIPTS_PY = scripts/probe.py scripts/limits.py scripts/lab4_scenarios.py scripts/lab4_seed.py \ + scripts/upload_memory.py scripts/make_images.py scripts/pipeline_suite.py \ + scripts/pipeline_evidence.py +SCENARIO ?= failover-upload +PSQL = $(COMPOSE) exec -T postgres psql -U messenger -d messenger -X -q + +.PHONY: help check validate test load test-persistence test-failover \ + install lint format typecheck student-test \ + secrets up down reset-data migrate ps logs restore lb-check failover-evidence stack-evidence \ + persistence-evidence storage-evidence cache-evidence test-faults seed explain \ + limits detection failover-each rolling-update lab4-level4 \ + scenario s3-check attachments-evidence upload-faults upload-memory \ + images queues topology worker-stop worker-start broker-restart suite-preflight level3 \ + pipeline-evidence level4 replay failed dlq help: @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" + @echo "install/lint/typecheck/student-test: студенческий тулчейн" + @echo "secrets/up/down/migrate/ps/logs: стенд Docker Compose; reset-data: намеренное удаление volumes" + @echo "restore: вернуть остановленные реплики; lb-check: реплики на одном токене и 20 гонок одного ключа" + @echo "lab4-level4 = limits, lb-check, detection, failover-each, rolling-update [FROM_REV=<коммит>]; RATE=$(RATE) rps" + @echo "scenario SCENARIO=failover-upload|s3-outage → evidence/<сценарий>.json; s3-check; attachments-evidence" + @echo "upload-faults: негативный suite, компенсация, уборка, обрыв и зависание загрузки против стенда" + @echo "upload-memory: пик памяти реплик при 10/20/40 одновременных загрузках 1 и 10 MiB → evidence/upload-memory/" + @echo "lab5: queues, topology, worker-stop/worker-start, broker-restart, images, suite-preflight, level3, level4" + @echo "lab5 оператор: failed (реестр), replay ID=, dlq (просмотр DLQ без извлечения)" check: $(PYTHON) scripts/check_contract.py @@ -28,3 +56,173 @@ test-persistence: test-failover: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2) $(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) + +install: + $(UV) sync --group dev + +lint: + $(VENV)/bin/ruff check src student-tests $(SCRIPTS_PY) + $(VENV)/bin/ruff format --check src student-tests $(SCRIPTS_PY) + $(VENV)/bin/ty check src student-tests + +format: + $(VENV)/bin/ruff check --fix src student-tests $(SCRIPTS_PY) + $(VENV)/bin/ruff format src student-tests $(SCRIPTS_PY) + +typecheck: + $(VENV)/bin/ty check src student-tests + +student-test: + $(VENV)/bin/pytest + +secrets: + sh scripts/secrets.sh + sh scripts/broker-secrets.sh + +up: secrets + $(COMPOSE) build + $(COMPOSE) up -d --wait + +down: + $(COMPOSE) down + +reset-data: + @echo "Удаляются volumes с учебными данными стенда" + $(COMPOSE) down -v --remove-orphans + +migrate: + $(COMPOSE) run --rm migrate + +ps: + $(COMPOSE) ps + +logs: + $(COMPOSE) logs --tail 200 + +restore: + scripts/restore.sh + +lb-check: + $(PYTHON) scripts/probe.py check --rounds 20 --out evidence/lb-check.json + +limits: + $(PYTHON) scripts/limits.py --out evidence/limits.txt + +detection: + sh scripts/detection.sh evidence/detection + +failover-each: + RATE=$(RATE) sh scripts/failover-each.sh evidence/failover-each + +rolling-update: +ifneq ($(FROM_REV),) + git archive "$(FROM_REV)" | docker build -q -t "$(API_IMAGE)" - + $(COMPOSE) up -d --no-deps --force-recreate --wait api + scripts/lb-wait.sh +endif + $(COMPOSE) build -q api + RATE=$(RATE) $(PYTHON) scripts/probe.py load --expect replace --duration 90 \ + --out evidence/rolling$(if $(FROM_REV),-from-$(FROM_REV)) \ + -- scripts/rolling-update.sh + +lab4-level4: limits lb-check detection failover-each rolling-update + +scenario: + $(PYTHON) scripts/lab4_scenarios.py $(SCENARIO) --base-url "$(BASE_URL)" --restore --output evidence/$(SCENARIO).json + scripts/lb-wait.sh + +s3-check: + sh scripts/s3-check.sh > evidence/s3-security.txt; status=$$?; cat evidence/s3-security.txt; exit $$status + +attachments-evidence: + sh scripts/evidence-attachments.sh + +upload-faults: + { echo "date: $$(date -u +%FT%TZ) commit: $$(git rev-parse --short HEAD) BASE_URL=$(BASE_URL)"; \ + BASE_URL="$(BASE_URL)" $(VENV)/bin/pytest -m stack -v -s --no-header --tb=short \ + -p no:cacheprovider student-tests/stack/test_attachment_faults.py; } > evidence/upload-faults.txt; \ + status=$$?; cat evidence/upload-faults.txt; exit $$status + +upload-memory: + $(PYTHON) scripts/upload_memory.py --base-url "$(BASE_URL)" + +failover-evidence: + sh scripts/failover-evidence.sh "$(BASE_URL)" + +stack-evidence: + sh scripts/evidence-stack.sh + +persistence-evidence: + sh scripts/persistence-evidence.sh "$(BASE_URL)" > evidence/persistence.txt + cat evidence/persistence.txt + +storage-evidence: + sh scripts/evidence-storage.sh + +cache-evidence: + sh scripts/evidence-cache.sh > evidence/cache.txt + cat evidence/cache.txt + +test-faults: + BASE_URL="$(BASE_URL)" $(VENV)/bin/pytest -m stack -v -s student-tests/stack + +seed: + $(PSQL) -f - < scripts/sql/seed-history.sql + +explain: seed + { echo "date: $$(date -u +%FT%TZ) commit: $$(git rev-parse --short HEAD)"; \ + echo "\$$ make explain # scripts/sql/explain-history.sql"; \ + $(PSQL) -f - < scripts/sql/explain-history.sql; } > evidence/explain-history.txt + cat evidence/explain-history.txt + +images: + $(PYTHON) scripts/make_images.py + +queues: + sh scripts/broker.sh queues + +topology: + { echo "date: $$(date -u +%FT%TZ) commit: $$(git rev-parse --short HEAD)"; \ + echo "$$ rabbitmqctl list_queues -p messenger name type durable policy messages"; \ + $(COMPOSE) exec -T broker rabbitmqctl -q list_queues -p messenger name type durable policy messages; \ + echo "$$ rabbitmqctl list_consumers -p messenger"; \ + $(COMPOSE) exec -T broker rabbitmqctl -q list_consumers -p messenger queue_name ack_required prefetch_count; \ + echo "$$ rabbitmqctl list_permissions -p messenger"; \ + $(COMPOSE) exec -T broker rabbitmqctl -q list_permissions -p messenger; \ + echo "$$ rabbitmqctl list_users"; \ + $(COMPOSE) exec -T broker rabbitmqctl -q list_users; \ + echo "$$ docker compose ps"; \ + $(COMPOSE) ps --format '{{.Service}}\t{{.Status}}\t{{.Ports}}'; } > evidence/topology.txt + cat evidence/topology.txt + +worker-stop: + sh scripts/worker.sh stop + +worker-start: + sh scripts/worker.sh start + +broker-restart: + sh scripts/broker.sh restart + +suite-preflight: + BASE_URL="$(BASE_URL)" $(PYTHON) scripts/pipeline_suite.py --preflight + +level3: + BASE_URL="$(BASE_URL)" $(PYTHON) scripts/pipeline_suite.py --level 3 --output evidence/level3.json + +level4: + BASE_URL="$(BASE_URL)" $(PYTHON) scripts/pipeline_suite.py --level 4 --output evidence/level4.json + +failed: + $(COMPOSE) run --rm --no-deps -T worker python -m messenger.admin failed + +replay: + @test -n "$(ID)" || (echo "Set ID="; exit 2) + $(COMPOSE) run --rm --no-deps -T worker python -m messenger.admin replay --attachment-id "$(ID)" + +dlq: + sh scripts/broker.sh dlq + +pipeline-evidence: + $(PYTHON) scripts/pipeline_evidence.py --base-url "$(BASE_URL)" > evidence/pipeline-stand.txt; \ + status=$$?; cat evidence/pipeline-stand.txt; exit $$status diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..f464e1ef --- /dev/null +++ b/compose.yaml @@ -0,0 +1,400 @@ +x-app: &app + image: ${API_IMAGE:-mtusi-messenger:lab6} + networks: [backend] + +x-s3-env: &s3-env + S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + S3_CONNECT_TIMEOUT_SECONDS: ${S3_CONNECT_TIMEOUT_SECONDS:-1} + S3_READ_TIMEOUT_SECONDS: ${S3_READ_TIMEOUT_SECONDS:-2} + MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} + MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} + IMAGE_CHECK_CONCURRENCY: ${IMAGE_CHECK_CONCURRENCY:-2} + UPLOAD_CONCURRENCY: ${UPLOAD_CONCURRENCY:-3} + UPLOAD_GRACE_SECONDS: ${UPLOAD_GRACE_SECONDS:-900} + UPLOAD_SWEEP_INTERVAL_SECONDS: ${UPLOAD_SWEEP_INTERVAL_SECONDS:-30} + UPLOAD_SWEEP_BATCH: ${UPLOAD_SWEEP_BATCH:-100} + +x-pipeline: &pipeline + image: ${API_IMAGE:-mtusi-messenger:lab6} + pull_policy: never + restart: unless-stopped + init: true + read_only: true + tmpfs: [/tmp] + security_opt: ["no-new-privileges:true"] + cap_drop: [ALL] + stop_grace_period: 10s + networks: [backend] + healthcheck: + test: ["CMD", "python", "-c", "import os,sys,time;sys.exit(time.time()-os.stat('/tmp/heartbeat').st_mtime>30)"] + interval: 10s + timeout: 5s + retries: 3 + start_period: 30s + start_interval: 2s + depends_on: + migrate: + condition: service_completed_successfully + s3-init: + condition: service_completed_successfully + broker: + condition: service_healthy + +x-pipeline-env: &pipeline-env + <<: *s3-env + DATABASE_URL_FILE: /run/secrets/database_url + BROKER_URL_FILE: /run/secrets/broker_url + DB_POOL_MAX_SIZE: ${PIPELINE_DB_POOL_MAX_SIZE:-4} + LOG_LEVEL: ${LOG_LEVEL:-INFO} + HEARTBEAT_FILE: /tmp/heartbeat + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60} + JOB_BACKOFF_BASE_SECONDS: ${JOB_BACKOFF_BASE_SECONDS:-2} + STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400} + +x-s3-client: &s3-client + image: rustfs/rc:v0.1.36@sha256:ab024bfebee49a750ce886b4c70963ccd9ddaa03f491704a90710641d7a26699 + read_only: true + tmpfs: ["/tmp:size=16m"] + environment: + S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + RC_CONFIG_DIR: /tmp/rc + volumes: ["./infra/s3:/s3:ro"] + secrets: + [s3_root_user, s3_root_password, s3_access_key_id, s3_secret_access_key, + s3_worker_access_key_id, s3_worker_secret_access_key] + cap_drop: [ALL] + security_opt: ["no-new-privileges:true"] + restart: "no" + networks: [backend] + deploy: + resources: + limits: {cpus: "0.5", memory: 64m} + +services: + lb: + image: haproxy:3.2.26-alpine@sha256:5893840576071e6217cd52ebdcc14f454c6f7a631056fa6f4e0f40d3cac4215b + environment: + API_SERVER_MAXCONN: ${API_SERVER_MAXCONN:-64} + volumes: [./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro] + ports: + - "127.0.0.1:${LB_HTTP_PORT:-8080}:8080" + - "127.0.0.1:${LB_STATS_PORT:-8404}:8404" + networks: [edge, backend] + healthcheck: + test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8404/healthz"] + interval: 2s + timeout: 2s + retries: 3 + start_period: 10s + sysctls: + net.ipv4.ip_local_port_range: "10240 65535" + net.ipv4.tcp_tw_reuse: "1" + stop_signal: SIGUSR1 + stop_grace_period: 8s + restart: unless-stopped + read_only: true + cap_drop: [ALL] + deploy: + resources: + limits: {cpus: "${LB_CPUS:-1}", memory: 128m} + + api: + <<: *app + build: . + environment: + <<: *s3-env + DATABASE_URL_FILE: /run/secrets/database_url + CACHE_URL_FILE: /run/secrets/cache_url + CURSOR_SECRET_FILE: /run/secrets/cursor_secret + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400} + WEB_CONCURRENCY: ${WEB_CONCURRENCY:-2} + DB_POOL_MAX_SIZE: ${DB_POOL_MAX_SIZE:-8} + REQUEST_TIMEOUT_SECONDS: ${REQUEST_TIMEOUT_SECONDS:-8} + DB_ACQUIRE_TIMEOUT_SECONDS: ${DB_ACQUIRE_TIMEOUT_SECONDS:-1} + DB_COMMAND_TIMEOUT_SECONDS: ${DB_COMMAND_TIMEOUT_SECONDS:-2} + DRAIN_SECONDS: ${DRAIN_SECONDS:-2} + SHUTDOWN_TIMEOUT_SECONDS: ${SHUTDOWN_TIMEOUT_SECONDS:-5} + LOG_LEVEL: ${LOG_LEVEL:-INFO} + MALLOC_MMAP_THRESHOLD_: ${MALLOC_MMAP_THRESHOLD_:-131072} + secrets: [database_url, cache_url, cursor_secret, s3_access_key_id, s3_secret_access_key] + healthcheck: + test: ["CMD", "python", "-c", "import sys,urllib.request;sys.exit(urllib.request.urlopen('http://127.0.0.1:8080/health/ready',timeout=2).status!=200)"] + interval: 5s + timeout: 3s + retries: 3 + start_period: 30s + start_interval: 1s + stop_grace_period: 8s + restart: unless-stopped + read_only: true + tmpfs: [/tmp] + sysctls: + net.ipv4.tcp_rmem: "4096 65536 262144" + net.ipv4.tcp_wmem: "4096 65536 262144" + depends_on: + migrate: + condition: service_completed_successfully + valkey: + condition: service_started + s3-init: + condition: service_completed_successfully + deploy: + replicas: ${API_REPLICAS:-2} + resources: + limits: {cpus: "${API_CPUS:-1}", memory: "${API_MEMORY:-512m}"} + + migrate: + <<: *app + pull_policy: never + command: ["python", "-m", "messenger.migrate"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + LOG_LEVEL: ${LOG_LEVEL:-INFO} + secrets: [database_url] + healthcheck: + disable: true + restart: "no" + depends_on: + postgres: + condition: service_healthy + deploy: + resources: + limits: {cpus: "0.5", memory: 256m} + + postgres: + image: postgres:17.11-alpine@sha256:266d3c257594e195b309ae9d1805e54c788f1d7b375f56bf6c367aac6c917e7b + user: "70:70" + command: + - postgres + - -c + - max_connections=${PG_MAX_CONNECTIONS:-200} + - -c + - shared_buffers=${PG_SHARED_BUFFERS:-256MB} + - -c + - shared_preload_libraries=pg_stat_statements + - -c + - pg_stat_statements.track=top + - -c + - track_io_timing=on + - -c + - idle_in_transaction_session_timeout=10s + environment: + POSTGRES_DB: messenger + POSTGRES_USER: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + secrets: [postgres_password] + volumes: [pgdata:/var/lib/postgresql/data] + shm_size: 256m + healthcheck: + test: ["CMD", "pg_isready", "-h", "127.0.0.1", "-U", "messenger", "-d", "messenger"] + interval: 2s + timeout: 3s + retries: 30 + start_period: 10s + stop_grace_period: 10s + restart: unless-stopped + networks: [backend] + deploy: + resources: + limits: {cpus: "${PG_CPUS:-2}", memory: "${PG_MEMORY:-1g}"} + + valkey: + image: valkey/valkey:8.1.11-alpine@sha256:2602ace2be6569bc2bbb06f873776516ab9089821525be34f4305c67eb9db02a + user: "999:1000" + command: + - valkey-server + - /run/secrets/valkey.conf + - --appendonly + - "yes" + - --appendfsync + - always + - --maxmemory + - 64mb + - --maxmemory-policy + - volatile-lru + secrets: [valkey.conf] + volumes: [valkeydata:/data] + healthcheck: + test: ["CMD-SHELL", "REDISCLI_AUTH=\"$$(sed -n 's/^requirepass //p' /run/secrets/valkey.conf)\" valkey-cli ping | grep -q PONG"] + interval: 2s + timeout: 3s + retries: 30 + stop_grace_period: 10s + restart: unless-stopped + networks: [backend] + deploy: + resources: + limits: {cpus: "0.5", memory: 128m} + + s3: + image: rustfs/rustfs:1.0.4@sha256:ed9210d6411d77ffceb70d94b964a3cb7d225983368bc25bebe01e995293dcc2 + environment: + RUSTFS_ACCESS_KEY_FILE: /run/secrets/s3_root_user + RUSTFS_SECRET_KEY_FILE: /run/secrets/s3_root_password + RUSTFS_ADDRESS: ":9000" + RUSTFS_CONSOLE_ENABLE: "false" + RUSTFS_REGION: ${S3_REGION:-us-east-1} + RUSTFS_OBS_LOG_DIRECTORY: "" + RUSTFS_OBS_LOGGER_LEVEL: ${S3_LOG_LEVEL:-warn} + secrets: [s3_root_user, s3_root_password] + volumes: [s3data:/data] + read_only: true + tmpfs: ["/tmp:size=64m"] + cap_drop: [ALL] + security_opt: ["no-new-privileges:true"] + healthcheck: + test: ["CMD", "curl", "-fsS", "-o", "/dev/null", "http://127.0.0.1:9000/health/ready"] + interval: 2s + timeout: 3s + retries: 30 + start_period: 20s + stop_grace_period: 10s + restart: unless-stopped + networks: [backend] + deploy: + resources: + limits: {cpus: "1", memory: "${S3_MEMORY:-512m}"} + + s3-init: + <<: *s3-client + entrypoint: ["/bin/sh", "/s3/init.sh"] + depends_on: + s3: + condition: service_healthy + + s3-cli: + <<: *s3-client + profiles: [tools] + entrypoint: ["/bin/sh", "/s3/cli.sh"] + + broker: + image: rabbitmq:4.1.8-management-alpine@sha256:54ef5ff505352c1beef05a97ee9bd6ab95b3559a76629ba0119b1d63d8d6e4b2 + hostname: broker + user: "100:101" + restart: unless-stopped + read_only: true + tmpfs: [/tmp] + security_opt: ["no-new-privileges:true"] + cap_drop: [ALL] + ports: + - "127.0.0.1:${BROKER_MANAGEMENT_PORT:-15672}:15672" + networks: [backend, edge] + volumes: + - broker-data:/var/lib/rabbitmq + - ./infra/rabbitmq/rabbitmq.conf:/etc/rabbitmq/conf.d/20-messenger.conf:ro + - ./infra/rabbitmq/enabled_plugins:/etc/rabbitmq/enabled_plugins:ro + - ./infra/rabbitmq/definitions.json:/etc/rabbitmq/definitions.d/10-topology.json:ro + secrets: + - source: rabbitmq_users + target: /etc/rabbitmq/definitions.d/20-users.json + - rabbitmq_ops_password + healthcheck: + test: ["CMD", "rabbitmq-diagnostics", "-q", "check_port_connectivity"] + interval: 10s + timeout: 10s + retries: 5 + start_period: 60s + start_interval: 2s + stop_grace_period: 10s + deploy: + resources: + limits: {cpus: "1", memory: 512m} + + relay: + <<: *pipeline + command: ["python", "-m", "messenger.relay"] + environment: + <<: *pipeline-env + OUTBOX_POLL_SECONDS: ${OUTBOX_POLL_SECONDS:-1} + SWEEP_INTERVAL_SECONDS: ${SWEEP_INTERVAL_SECONDS:-5} + secrets: + - database_url + - source: broker_url_api + target: broker_url + - s3_access_key_id + - s3_secret_access_key + deploy: + resources: + limits: {cpus: "0.5", memory: 192m} + + worker: + <<: *pipeline + command: ["python", "-m", "messenger.worker"] + environment: + <<: *pipeline-env + WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2} + RENDER_TIMEOUT_SECONDS: ${RENDER_TIMEOUT_SECONDS:-30} + RENDER_MEMORY_MB: ${RENDER_MEMORY_MB:-448} + RENDER_TASKS_PER_CHILD: ${RENDER_TASKS_PER_CHILD:-100} + secrets: + - database_url + - source: broker_url_worker + target: broker_url + - source: s3_worker_access_key_id + target: s3_access_key_id + - source: s3_worker_secret_access_key + target: s3_secret_access_key + deploy: + resources: + limits: {cpus: "${WORKER_CPUS:-2}", memory: "${WORKER_MEMORY:-1g}"} + + k6: + image: grafana/k6:2.3.0@sha256:2dd4756ec6e14e62778916e1eb850108e4b499304c02df56760fa0bdb0fba0b8 + profiles: [bench] + environment: + BASE_URL: http://lb:8080 + volumes: [./infra/k6:/scripts:ro] + networks: [edge] + deploy: + resources: + limits: {cpus: "${K6_CPUS:-2}", memory: "${K6_MEMORY:-512m}"} + +secrets: + postgres_password: + file: ./secrets/postgres_password + database_url: + file: ./secrets/database_url + valkey.conf: + file: ./secrets/valkey.conf + cache_url: + file: ./secrets/cache_url + cursor_secret: + file: ./secrets/cursor_secret + s3_root_user: + file: ./secrets/s3_root_user + s3_root_password: + file: ./secrets/s3_root_password + s3_access_key_id: + file: ./secrets/s3_access_key_id + s3_secret_access_key: + file: ./secrets/s3_secret_access_key + s3_worker_access_key_id: + file: ./secrets/s3_worker_access_key_id + s3_worker_secret_access_key: + file: ./secrets/s3_worker_secret_access_key + rabbitmq_users: + file: ./secrets/rabbitmq_users.json + rabbitmq_ops_password: + file: ./secrets/rabbitmq_ops_password + broker_url_api: + file: ./secrets/broker_url_api + broker_url_worker: + file: ./secrets/broker_url_worker + +volumes: + pgdata: + valkeydata: + s3data: + broker-data: + +networks: + edge: + backend: + internal: true diff --git a/infra/compose.faults.yaml b/infra/compose.faults.yaml new file mode 100644 index 00000000..6f00dcfa --- /dev/null +++ b/infra/compose.faults.yaml @@ -0,0 +1,18 @@ +services: + relay: + restart: "no" + environment: + FAULT_POINT: ${RELAY_FAULT_POINT:-} + OUTBOX_POLL_SECONDS: "0.5" + SWEEP_INTERVAL_SECONDS: "1" + STAGING_RETENTION_SECONDS: ${FAULT_STAGING_RETENTION_SECONDS:-${STAGING_RETENTION_SECONDS:-86400}} + + worker: + restart: "no" + environment: + FAULT_POINT: ${WORKER_FAULT_POINT:-} + FAULT_STALL_SECONDS: ${FAULT_STALL_SECONDS:-15} + JOB_TIMEOUT_SECONDS: ${FAULT_JOB_TIMEOUT_SECONDS:-5} + JOB_BACKOFF_BASE_SECONDS: ${FAULT_BACKOFF_BASE_SECONDS:-1} + RENDER_TIMEOUT_SECONDS: ${FAULT_RENDER_TIMEOUT_SECONDS:-${RENDER_TIMEOUT_SECONDS:-30}} + RENDER_MEMORY_MB: ${FAULT_RENDER_MEMORY_MB:-${RENDER_MEMORY_MB:-448}} diff --git a/infra/haproxy/haproxy.cfg b/infra/haproxy/haproxy.cfg new file mode 100644 index 00000000..91d9a2e1 --- /dev/null +++ b/infra/haproxy/haproxy.cfg @@ -0,0 +1,71 @@ +global + log stdout format raw local0 info + maxconn 4096 + hard-stop-after 5s + tune.bufsize 32768 + tune.rcvbuf.client 65536 + tune.rcvbuf.server 65536 + tune.sndbuf.client 65536 + tune.sndbuf.server 65536 + +resolvers docker + nameserver dns 127.0.0.11:53 + accepted_payload_size 8192 + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + hold nx 2s + hold other 2s + hold refused 2s + hold timeout 2s + +defaults + mode http + log global + option dontlognull + option dontlog-normal + log-format "%ci [%tr] %ft %b/%s %TR/%Tw/%Tc/%Tr/%Ta %ST %B %tsc %ac/%fc/%bc/%sc/%rc %sq/%bq rid=%[var(txn.request_id)] %HM %HPO" + option http-server-close + option redispatch 1 + retries 2 + retry-on conn-failure empty-response + timeout connect 500ms + timeout client 30s + timeout server 10s + timeout http-request 5s + timeout http-keep-alive 5s + timeout queue 3s + timeout check 1s + http-error status 502 content-type application/json hdr X-Request-Id "%[var(txn.request_id)]" lf-string '{"error":{"code":"unavailable","message":"Upstream replica failed","request_id":"%[var(txn.request_id)]"}}' + http-error status 503 content-type application/json hdr X-Request-Id "%[var(txn.request_id)]" lf-string '{"error":{"code":"unavailable","message":"No ready API replica","request_id":"%[var(txn.request_id)]"}}' + http-error status 504 content-type application/json hdr X-Request-Id "%[var(txn.request_id)]" lf-string '{"error":{"code":"unavailable","message":"Upstream timeout","request_id":"%[var(txn.request_id)]"}}' + +frontend public + bind :8080 + option forwardfor + http-request set-var(txn.request_id) req.fhdr(X-Request-Id) if { req.fhdr(X-Request-Id) -m len 1:200 } + http-request set-var(txn.request_id) uuid(4) unless { var(txn.request_id) -m found } + http-request set-header X-Request-Id %[var(txn.request_id)] + default_backend api + +backend api + balance roundrobin + option httpchk + http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host lb + http-check expect status 200 + acl message_path path_reg ^/api/v1/chats/[^/]+/messages$ + acl has_key req.hdr(Idempotency-Key) -m found + http-request set-var(txn.replay_safe) bool(1) if METH_GET + http-request set-var(txn.replay_safe) bool(1) if METH_POST message_path has_key + http-request disable-l7-retry unless { var(txn.replay_safe) -m bool } + default-server check init-state fully-down inter 1s fastinter 500ms downinter 2s fall 2 rise 2 resolvers docker resolve-prefer ipv4 init-addr none maxconn "${API_SERVER_MAXCONN-64}" + server-template api 8 api:8080 + +frontend stats + bind :8404 + monitor-uri /healthz + monitor fail if { nbsrv(api) lt 1 } + http-request use-service prometheus-exporter if { path /metrics } + stats enable + stats uri /stats + stats refresh 5s diff --git a/infra/k6/messages.js b/infra/k6/messages.js new file mode 100644 index 00000000..722cf980 --- /dev/null +++ b/infra/k6/messages.js @@ -0,0 +1,106 @@ +import http from 'k6/http'; +import { check } from 'k6'; +import encoding from 'k6/encoding'; +import exec from 'k6/execution'; + +const BASE = __ENV.BASE_URL || 'http://lb:8080'; +const RATE = Number(__ENV.RATE || 50); +const WARMUP = Number(__ENV.WARMUP || 10); +const DURATION = Number(__ENV.DURATION || 60); +const USERS = Math.max(2, Number(__ENV.USERS || 20)); +const READ_RATIO = Number(__ENV.READ_RATIO || 0.2); +const VUS = Number(__ENV.VUS || Math.max(10, Math.ceil(RATE / 10))); +const MAX_VUS = Number(__ENV.MAX_VUS || Math.max(VUS, Math.min(VUS * 4, 1000))); +const RUN = __ENV.RUN_ID || Date.now().toString(36); +const SUMMARY = __ENV.SUMMARY || '/out/summary.json'; +const JSON_HEADERS = { 'Content-Type': 'application/json' }; + +const phase = (startTime, duration) => ({ + executor: 'constant-arrival-rate', + rate: RATE, + timeUnit: '1s', + startTime: `${startTime}s`, + duration: `${duration}s`, + preAllocatedVUs: VUS, + maxVUs: MAX_VUS, + gracefulStop: '5s', +}); + +const measured = ['http_reqs', 'http_req_failed', 'http_req_duration', 'dropped_iterations', 'iterations']; + +export const options = { + setupTimeout: '180s', + summaryTrendStats: ['avg', 'min', 'med', 'p(90)', 'p(95)', 'p(99)', 'max'], + scenarios: Object.assign(WARMUP > 0 ? { warmup: phase(0, WARMUP) } : {}, { measure: phase(WARMUP, DURATION) }), + thresholds: Object.fromEntries([ + ...measured.map((metric) => [`${metric}{scenario:measure}`, []]), + ['http_req_duration{scenario:measure,name:post_message}', []], + ['http_req_duration{scenario:measure,name:get_history}', []], + ]), +}; + +function created(response, what) { + if (response.status !== 201) { + throw new Error(`${what}: HTTP ${response.status}`); + } + return response.json(); +} + +const bearer = (token, extra) => Object.assign({ Authorization: `Bearer ${token}` }, extra || {}); + +export function setup() { + const actors = []; + for (let i = 0; i < USERS; i++) { + const username = `k6_${RUN}_${i}`.slice(0, 32); + const password = `K6-${RUN}-${i}-synthetic`; + const user = created( + http.post(`${BASE}/api/v1/users`, JSON.stringify({ username, display_name: `k6 ${i}`, password }), { headers: JSON_HEADERS }), + 'register', + ); + const basic = encoding.b64encode(`${username}:${password}`); + const session = created( + http.post(`${BASE}/api/v1/auth/sessions`, null, { headers: { Authorization: `Basic ${basic}` } }), + 'login', + ); + actors.push({ id: user.id, token: session.token }); + } + const chats = []; + for (let i = 0; i < USERS; i += 2) { + const [a, b] = [actors[i], actors[(i + 1) % USERS]]; + const body = JSON.stringify({ title: `k6 ${RUN} ${i}`, member_ids: [a.id, b.id] }); + const chat = created(http.post(`${BASE}/api/v1/chats`, body, { headers: bearer(a.token, JSON_HEADERS) }), 'chat'); + chats.push({ id: chat.id, tokens: [a.token, b.token] }); + } + console.log('LOAD-START'); + return { chats }; +} + +export default function (data) { + const n = exec.scenario.iterationInTest; + const chat = data.chats[n % data.chats.length]; + const token = chat.tokens[Math.floor(n / data.chats.length) % 2]; + if (Math.random() < READ_RATIO) { + const response = http.get(`${BASE}/api/v1/chats/${chat.id}/messages?limit=50`, { + headers: bearer(token), + tags: { name: 'get_history' }, + timeout: '10s', + }); + check(response, { 'history 200': (r) => r.status === 200 }); + return; + } + const key = `${RUN}-${exec.scenario.name}-${n}`; + const response = http.post(`${BASE}/api/v1/chats/${chat.id}/messages`, JSON.stringify({ text: `k6 ${key}` }), { + headers: bearer(token, Object.assign({ 'Idempotency-Key': key }, JSON_HEADERS)), + tags: { name: 'post_message' }, + timeout: '10s', + }); + check(response, { 'message 201': (r) => r.status === 201 }); +} + +export function handleSummary(data) { + delete data.setup_data; + const reqs = data.metrics['http_reqs{scenario:measure}']; + const failed = data.metrics['http_req_failed{scenario:measure}']; + const line = `measure: ${reqs ? reqs.values.count : 0} requests, failed ratio ${failed ? failed.values.rate : 'n/a'}\n`; + return { [SUMMARY]: JSON.stringify(data, null, 1), stdout: line }; +} diff --git a/infra/rabbitmq/definitions.json b/infra/rabbitmq/definitions.json new file mode 100644 index 00000000..c5f466e9 --- /dev/null +++ b/infra/rabbitmq/definitions.json @@ -0,0 +1,44 @@ +{ + "rabbit_version": "4.1.8", + "vhosts": [ + {"name": "messenger", "description": "lab6 attachment pipeline", "default_queue_type": "quorum"} + ], + "exchanges": [ + {"name": "attachments", "vhost": "messenger", "type": "direct", "durable": true, "auto_delete": false, "internal": false, "arguments": {}}, + {"name": "attachments.dlx", "vhost": "messenger", "type": "fanout", "durable": true, "auto_delete": false, "internal": false, "arguments": {}} + ], + "queues": [ + {"name": "attachments.process", "vhost": "messenger", "durable": true, "auto_delete": false, "arguments": {"x-queue-type": "quorum"}}, + {"name": "attachments.process.dlq", "vhost": "messenger", "durable": true, "auto_delete": false, "arguments": {"x-queue-type": "quorum"}} + ], + "bindings": [ + {"source": "attachments", "vhost": "messenger", "destination": "attachments.process", "destination_type": "queue", "routing_key": "attachment.process.v1", "arguments": {}}, + {"source": "attachments.dlx", "vhost": "messenger", "destination": "attachments.process.dlq", "destination_type": "queue", "routing_key": "", "arguments": {}} + ], + "policies": [ + { + "vhost": "messenger", + "name": "attachments-process", + "pattern": "^attachments\\.process$", + "apply-to": "queues", + "priority": 10, + "definition": { + "dead-letter-exchange": "attachments.dlx", + "dead-letter-strategy": "at-least-once", + "overflow": "reject-publish", + "delivery-limit": 10 + } + }, + { + "vhost": "messenger", + "name": "attachments-dlq", + "pattern": "^attachments\\.process\\.dlq$", + "apply-to": "queues", + "priority": 10, + "definition": { + "max-length": 100000, + "overflow": "reject-publish" + } + } + ] +} diff --git a/infra/rabbitmq/enabled_plugins b/infra/rabbitmq/enabled_plugins new file mode 100644 index 00000000..cc972a47 --- /dev/null +++ b/infra/rabbitmq/enabled_plugins @@ -0,0 +1 @@ +[rabbitmq_management,rabbitmq_prometheus]. diff --git a/infra/rabbitmq/rabbitmq.conf b/infra/rabbitmq/rabbitmq.conf new file mode 100644 index 00000000..06b35dcf --- /dev/null +++ b/infra/rabbitmq/rabbitmq.conf @@ -0,0 +1,21 @@ +listeners.tcp.default = 5672 +management.tcp.port = 15672 +prometheus.tcp.port = 15692 + +loopback_users.guest = true +log.console = true +log.console.level = info +log.file = false + +definitions.import_backend = local_filesystem +definitions.local.path = /etc/rabbitmq/definitions.d + +vm_memory_high_watermark.absolute = 256MiB +disk_free_limit.absolute = 256MB +raft.wal_max_size_bytes = 64000000 + +consumer_timeout = 300000 +heartbeat = 30 + +deprecated_features.permit.transient_nonexcl_queues = false +deprecated_features.permit.global_qos = false diff --git a/infra/s3/api-policy.json b/infra/s3/api-policy.json new file mode 100644 index 00000000..1e46aa4f --- /dev/null +++ b/infra/s3/api-policy.json @@ -0,0 +1,23 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "MessengerApiObjects", + "Effect": "Allow", + "Action": [ + "s3:GetObject", + "s3:PutObject", + "s3:DeleteObject", + "s3:AbortMultipartUpload", + "s3:ListMultipartUploadParts" + ], + "Resource": ["arn:aws:s3:::messenger-attachments/*"] + }, + { + "Sid": "MessengerApiBucket", + "Effect": "Allow", + "Action": ["s3:ListBucket", "s3:ListBucketMultipartUploads", "s3:GetBucketLocation"], + "Resource": ["arn:aws:s3:::messenger-attachments"] + } + ] +} diff --git a/infra/s3/cli.sh b/infra/s3/cli.sh new file mode 100755 index 00000000..9e0937f9 --- /dev/null +++ b/infra/s3/cli.sh @@ -0,0 +1,12 @@ +#!/bin/sh +set -eu +secret() { tr -d '\r\n' < "/run/secrets/$1"; } +endpoint="${S3_ENDPOINT:-http://s3:9000}" +RC_HOST_root="${endpoint%%://*}://$(secret s3_root_user):$(secret s3_root_password)@${endpoint#*://}" +RC_HOST_api="${endpoint%%://*}://$(secret s3_access_key_id):$(secret s3_secret_access_key)@${endpoint#*://}" +export RC_HOST_root RC_HOST_api +rc alias set anon "$endpoint" --anonymous >/dev/null +case "${1:-}" in + sh|wget) exec "$@" ;; +esac +exec rc "$@" diff --git a/infra/s3/init.sh b/infra/s3/init.sh new file mode 100755 index 00000000..c9bd299f --- /dev/null +++ b/infra/s3/init.sh @@ -0,0 +1,26 @@ +#!/bin/sh +set -eu +secret() { tr -d '\r\n' < "/run/secrets/$1"; } +bucket="${S3_BUCKET:-messenger-attachments}" +endpoint="${S3_ENDPOINT:-http://s3:9000}" +policy="${S3_API_POLICY:-messenger-api}" +RC_HOST_root="${endpoint%%://*}://$(secret s3_root_user):$(secret s3_root_password)@${endpoint#*://}" +export RC_HOST_root +api_key=$(secret s3_access_key_id) +rc ready root --timeout 10 >/dev/null +rc bucket create --ignore-existing "root/$bucket" >/dev/null +rc bucket anonymous set private "root/$bucket" >/dev/null +sed "s/messenger-attachments/$bucket/g" /s3/api-policy.json > /tmp/api-policy.json +rc admin policy create root "$policy" /tmp/api-policy.json >/dev/null +rc admin user add root "$api_key" "$(secret s3_secret_access_key)" >/dev/null +rc admin policy attach root "$policy" --user "$api_key" >/dev/null +worker_key=$(secret s3_worker_access_key_id) +sed "s/messenger-attachments/$bucket/g" /s3/worker-policy.json > /tmp/worker-policy.json +rc admin policy create root messenger-worker /tmp/worker-policy.json >/dev/null +rc admin user add root "$worker_key" "$(secret s3_worker_secret_access_key)" >/dev/null +rc admin policy attach root messenger-worker --user "$worker_key" >/dev/null +rc bucket lifecycle rule import "root/$bucket" /s3/lifecycle.json >/dev/null +rc bucket anonymous get "root/$bucket" --json | grep -q '"permission": *"private"' +rc admin user info root "$api_key" --json | grep -q "\"$policy\"" +rc admin user info root "$worker_key" --json | grep -q '"messenger-worker"' +echo "s3-init: bucket=$bucket anonymous=private policy=$policy worker=messenger-worker lifecycle=abort-incomplete-uploads" diff --git a/infra/s3/lifecycle.json b/infra/s3/lifecycle.json new file mode 100644 index 00000000..ab206a07 --- /dev/null +++ b/infra/s3/lifecycle.json @@ -0,0 +1,10 @@ +{ + "rules": [ + { + "id": "abort-incomplete-uploads", + "status": "Enabled", + "prefix": "", + "abortIncompleteMultipartUploadDays": 1 + } + ] +} diff --git a/infra/s3/worker-policy.json b/infra/s3/worker-policy.json new file mode 100644 index 00000000..b7d4f805 --- /dev/null +++ b/infra/s3/worker-policy.json @@ -0,0 +1,24 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "MessengerWorkerRead", + "Effect": "Allow", + "Action": ["s3:GetObject"], + "Resource": [ + "arn:aws:s3:::messenger-attachments/staging/*", + "arn:aws:s3:::messenger-attachments/originals/*", + "arn:aws:s3:::messenger-attachments/variants/*" + ] + }, + { + "Sid": "MessengerWorkerWrite", + "Effect": "Allow", + "Action": ["s3:PutObject", "s3:AbortMultipartUpload", "s3:ListMultipartUploadParts"], + "Resource": [ + "arn:aws:s3:::messenger-attachments/originals/*", + "arn:aws:s3:::messenger-attachments/variants/*" + ] + } + ] +} diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..f3371162 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,124 @@ +[project] +name = "mtusi-messenger" +version = "1.6.0" +description = "Лабораторная 6 «Наблюдаемость: OpenTelemetry и Grafana», МТУСИ" +requires-python = ">=3.14,<3.15" +dependencies = [ + "fastapi>=0.118.0", + "uvicorn[standard]>=0.37.0", + "pydantic>=2.11.0", + "pydantic-settings>=2.11.0", + "dishka>=1.7.0", + "structlog>=26.1.0", + "gunicorn>=26.2.0", + "orjson>=3.12.0", + "asyncpg>=0.32.0", + "argon2-cffi>=25.1.0", + "uvicorn-worker>=0.4.0", + "redis>=8.1.0", + "aiobotocore>=3.10.0", + "pillow>=12.0.0", + "faststream[rabbit]>=0.7.2,<0.8", + "dishka-faststream>=0.1", +] + +[dependency-groups] +dev = [ + "pytest>=8.4.0", + "pytest-asyncio>=1.2.0", + "httpx>=0.28.0", + "ruff>=0.14.0", + "ty>=0.0.1a21", +] + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["src/messenger"] + +[tool.ruff] +line-length = 100 +target-version = "py314" +src = ["src", "student-tests"] +extend-exclude = ["tests", "scripts/check_contract.py", "scripts/validate_openapi.py"] +per-file-target-version = { "scripts/*.py" = "py39" } + +[tool.ruff.format] +indent-style = "space" +quote-style = "double" + +[tool.ruff.lint] +select = [ + "E", + "W", + "F", + "I", + "N", + "UP", + "B", + "A", + "C4", + "DTZ", + "T20", + "SIM", + "TID", + "PTH", + "RUF", + "ASYNC", + "S", + "ARG", + "PL", +] +ignore = [ + "PLR0913", + "RUF001", + "RUF002", + "RUF003", + "S104", + "BLE001", +] + +[tool.ruff.lint.per-file-ignores] +"student-tests/**" = ["S101", "S105", "S106", "S603", "S607", "PLR2004", "ARG001", "ARG002", "T20", "TID251"] +"scripts/*.py" = ["T20", "S603", "S607", "S310", "PLR2004", "TID251"] +"scripts/pipeline_suite.py" = ["T20", "S603", "S607", "S310", "PLR2004", "TID251", "E402", "S608"] +"scripts/pipeline_evidence.py" = ["T20", "S603", "S607", "S310", "PLR2004", "TID251", "E402"] +"src/messenger/*/infrastructure/**" = ["TID251", "ARG002"] +"src/messenger/*/domain/errors.py" = ["N818"] +"src/messenger/*/application/errors.py" = ["N818"] +"src/messenger/*/entrypoints/**" = ["TID251", "N818", "PLR0917", "S105"] +"src/messenger/shared/config.py" = ["ARG002", "ARG003"] +"src/messenger/main.py" = ["TID251"] +"src/messenger/migrate.py" = ["TID251"] +"src/messenger/relay.py" = ["TID251"] +"src/messenger/worker.py" = ["TID251"] +"src/messenger/admin.py" = ["TID251"] +"src/messenger/gunicorn_conf.py" = ["TID251", "ARG002", "RUF012"] + +[tool.ruff.lint.flake8-tidy-imports.banned-api] +"messenger.shared.infrastructure".msg = "Домен и слой приложения не импортируют инфраструктуру: зависимость идёт через порт." +"messenger.identity.infrastructure".msg = "Домен и слой приложения не импортируют инфраструктуру: зависимость идёт через порт." +"messenger.messaging.infrastructure".msg = "Домен и слой приложения не импортируют инфраструктуру: зависимость идёт через порт." +"messenger.attachments.infrastructure".msg = "Домен и слой приложения не импортируют инфраструктуру: зависимость идёт через порт." + +[tool.ruff.lint.isort] +known-first-party = ["messenger"] + +[tool.ty.src] +include = ["src", "student-tests"] + +[tool.ty.rules] +unresolved-import = "error" +invalid-argument-type = "error" +possibly-missing-attribute = "warn" + +[tool.pytest.ini_options] +testpaths = ["student-tests"] +asyncio_mode = "auto" +asyncio_default_fixture_loop_scope = "function" +addopts = "-q --strict-markers -m 'not stack'" +markers = [ + "stack: black-box проверки против поднятого compose-стенда (make test-faults)", +] diff --git a/scripts/broker-secrets.sh b/scripts/broker-secrets.sh new file mode 100755 index 00000000..4d0ee49c --- /dev/null +++ b/scripts/broker-secrets.sh @@ -0,0 +1,60 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +umask 077 +mkdir -p secrets + +for role in api worker ops; do + file="secrets/rabbitmq_${role}_password" + [ -s "$file" ] || od -An -tx1 -N24 /dev/urandom | tr -d ' \n' >"$file" +done + +[ -s secrets/s3_worker_access_key_id ] || printf 'worker%s' "$(od -An -tx1 -N8 /dev/urandom | tr -d ' \n')" >secrets/s3_worker_access_key_id +[ -s secrets/s3_worker_secret_access_key ] || od -An -tx1 -N32 /dev/urandom | tr -d ' \n' >secrets/s3_worker_secret_access_key + +for role in api worker; do + file="secrets/broker_url_${role}" + [ -s "$file" ] || printf 'amqp://messenger-%s:%s@%s:5672/messenger' \ + "$role" "$(cat "secrets/rabbitmq_${role}_password")" "${BROKER_HOST:-broker}" >"$file" +done + +python3 - <<'PY' +import base64 +import hashlib +import json +import os + + +def user(role, tags): + salt = os.urandom(4) + with open("secrets/rabbitmq_%s_password" % role, "rb") as source: + secret = source.read().strip() + digest = hashlib.sha256(salt + secret).digest() + return { + "name": "messenger-" + role, + "password_hash": base64.b64encode(salt + digest).decode(), + "hashing_algorithm": "rabbit_password_hashing_sha256", + "tags": tags, + } + + +def grant(role, configure, write, read): + return {"user": "messenger-" + role, "vhost": "messenger", "configure": configure, "write": write, "read": read} + + +definitions = { + "users": [user("api", []), user("worker", []), user("ops", ["administrator"])], + "permissions": [ + grant("api", "^$", "^attachments$", "^$"), + grant("worker", "^$", "^$", "^attachments\\.process$"), + grant("ops", ".*", ".*", ".*"), + ], +} +with open("secrets/rabbitmq_users.json", "w") as target: + json.dump(definitions, target, indent=2) + target.write("\n") +PY +chmod 700 secrets +chmod 644 secrets/rabbitmq_users.json secrets/rabbitmq_ops_password secrets/broker_url_api secrets/broker_url_worker \ + secrets/s3_worker_access_key_id secrets/s3_worker_secret_access_key +echo "broker and S3 worker secrets ready in secrets/ (values not printed)" diff --git a/scripts/broker.sh b/scripts/broker.sh new file mode 100755 index 00000000..1828288f --- /dev/null +++ b/scripts/broker.sh @@ -0,0 +1,21 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +COMPOSE=${COMPOSE:-docker compose} +service=${BROKER_SERVICE:-broker} +dlq() { + $COMPOSE exec -T "$service" sh -c 'RABBITMQADMIN_USERNAME=messenger-ops \ + RABBITMQADMIN_PASSWORD="$(cat /run/secrets/rabbitmq_ops_password)" \ + exec rabbitmqadmin --non-interactive -V messenger get messages \ + -q attachments.process.dlq -c "$1" -a ack_requeue_true' sh "${1:-10}" +} +queues() { + $COMPOSE exec -T "$service" rabbitmqctl -q list_queues -p messenger \ + name type messages_ready messages_unacknowledged consumers +} +case "${1:-}" in + restart) $COMPOSE restart -t 20 "$service" && $COMPOSE up -d --no-deps --no-recreate --wait --wait-timeout 90 "$service" ;; + queues) queues ;; + dlq) dlq "${2:-10}" ;; + *) echo "usage: $0 restart|queues|dlq [count]" >&2; exit 2 ;; +esac diff --git a/scripts/detection.sh b/scripts/detection.sh new file mode 100755 index 00000000..5ef0d35a --- /dev/null +++ b/scripts/detection.sh @@ -0,0 +1,20 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +out=${1:-evidence/detection} +mkdir -p "$out" +scripts/restore.sh +replica() { docker compose ps -q "${API_SERVICE:-api}" | head -1; } +detect() { python3 scripts/probe.py detect "$@"; } +failed=0 +detect --max 1.5 --within-action --restore scripts/restore.sh --out "$out/sigterm.json" \ + -- docker stop -t "${STOP_TIMEOUT:-6}" "$(replica)" || failed=1 +detect --max 1.5 --restore scripts/restore.sh --out "$out/kill.json" \ + -- docker kill "$(replica)" || failed=1 +hung=$(replica) +detect --max 3.5 --restore "docker unpause $hung && scripts/lb-wait.sh" --out "$out/pause.json" \ + -- docker pause "$hung" || failed=1 +db=$(docker compose ps -q postgres) +detect --max 1.5 --until 0 --lb-status 503 --restore "docker start $db && scripts/lb-wait.sh" \ + --out "$out/postgres.json" -- docker stop "$db" || failed=1 +exit "$failed" diff --git a/scripts/evidence-attachments.sh b/scripts/evidence-attachments.sh new file mode 100755 index 00000000..87c241a2 --- /dev/null +++ b/scripts/evidence-attachments.sh @@ -0,0 +1,40 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +psql() { docker compose exec -T postgres psql -U messenger -d messenger -X -c "$1"; } +value() { docker compose exec -T postgres psql -U messenger -d messenger -X -tA -c "$1"; } +cli() { docker compose run --rm --no-deps -T s3-cli "$@"; } +bucket=${S3_BUCKET:-messenger-attachments} +{ + echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse --short HEAD)" + echo "\$ docker compose ps --format '{{.Service}} {{.Status}} {{.Ports}}'" + docker compose ps -a --format '{{.Service}} {{.Status}} {{.Ports}}' + for table in attachments message_attachments; do + echo "\$ psql -c '\\d $table'" + psql "\\d $table" + done + echo "\$ psql: столбцы bytea во всей схеме (хеши токенов и отпечатки, не файлы)" + psql "SELECT table_name, column_name FROM information_schema.columns WHERE table_schema = 'public' AND data_type = 'bytea' ORDER BY 1, 2" + echo "\$ psql: вложения по статусу и максимальный размер строки attachments" + psql "SELECT status, count(*), max(pg_column_size(a.*)) AS max_row_bytes FROM attachments a GROUP BY status" + echo "\$ rc object list --recursive root/$bucket/attachments/ | count" + cli object list --recursive "root/$bucket/attachments/" --json | grep -c '"key"' || true + echo "\$ rc object list root/$bucket/ (первые 3 объекта, ключи назначены сервером)" + cli object list --recursive "root/$bucket/attachments/" | head -3 + row=$(value "SELECT object_key || ' ' || sha256 || ' ' || size_bytes FROM attachments WHERE status = 'ready' ORDER BY created_at DESC LIMIT 1") + key=${row%% *} + echo "\$ сверка вне API: sha256 и размер последнего ready-вложения в PostgreSQL против байтов объекта в bucket" + echo "postgres: ${row#* }" + echo "s3: $(cli sh -c "rc object show root/$bucket/$key | sha256sum | cut -d' ' -f1") $(cli sh -c "rc object show root/$bucket/$key | wc -c" | tr -d ' ')" + echo "\$ docker inspect api: ReadonlyRootfs, tmpfs, volumes/bind mounts" + for id in $(docker compose ps -q api); do + docker inspect -f '{{.Config.Hostname}} ro={{.HostConfig.ReadonlyRootfs}} tmpfs={{json .HostConfig.Tmpfs}} mounts={{range .Mounts}}{{.Type}}:{{.Destination}} {{end}}' "$id" + done + echo "\$ docker inspect s3: volume данных" + docker inspect -f '{{range .Mounts}}{{.Type}}:{{.Name}}:{{.Destination}} {{end}}' "$(docker compose ps -q s3)" + echo "\$ секреты сервисов: root-ключи S3 только у s3/s3-init/s3-cli" + docker compose config --format json | python3 -c 'import json,sys +for name, service in sorted(json.load(sys.stdin)["services"].items()): + print(name, sorted(s["source"] for s in service.get("secrets", [])))' +} > evidence/storage-layout.txt +cat evidence/storage-layout.txt diff --git a/scripts/evidence-cache.sh b/scripts/evidence-cache.sh new file mode 100755 index 00000000..ad017628 --- /dev/null +++ b/scripts/evidence-cache.sh @@ -0,0 +1,24 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +cli() { docker compose exec -T valkey sh -c "REDISCLI_AUTH=\"\$(sed -n 's/^requirepass //p' /run/secrets/valkey.conf)\" valkey-cli $1"; } +psql() { docker compose exec -T postgres psql -U messenger -d messenger -tA -c "$1"; } +echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse --short HEAD)" +echo "\$ valkey-cli config get appendonly appendfsync maxmemory maxmemory-policy" +cli "config get appendonly" ; cli "config get appendfsync"; cli "config get maxmemory"; cli "config get maxmemory-policy" +echo "\$ valkey-cli info persistence | grep aof_enabled,aof_last_write_status" +cli "info persistence" | grep -E '^aof_enabled|^aof_last_write_status' +echo "\$ valkey-cli dbsize; info stats | grep keyspace" +cli dbsize +cli "info stats" | grep -E '^keyspace_(hits|misses)' +key=$(cli "--scan --pattern session:* --count 1000" | head -1) +hash=${key#session:} +echo "\$ сравнение срока ключа кэша и срока сессии для одного ключа (значение и токен не печатаются)" +echo "valkey pexpiretime, ms: $(cli "pexpiretime $key")" +echo "postgres expires_at, ms: $(psql "SELECT (extract(epoch FROM expires_at) * 1000)::bigint FROM sessions WHERE token_hash = decode('$hash', 'hex')")" +echo "\$ количество отметок отзыва и закэшированных сессий" +revoked=0; cached=0 +for k in $(cli "--scan --pattern session:* --count 1000"); do + if [ "$(cli "get $k")" = "-" ]; then revoked=$((revoked + 1)); else cached=$((cached + 1)); fi +done +echo "tombstones: $revoked, cached sessions: $cached" diff --git a/scripts/evidence-stack.sh b/scripts/evidence-stack.sh new file mode 100755 index 00000000..7a4bc96d --- /dev/null +++ b/scripts/evidence-stack.sh @@ -0,0 +1,51 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +base=${1:-http://$(docker compose port lb 8080)} +stats=http://$(docker compose port lb 8404)/stats\;csv +header() { echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse --short HEAD)"; } +{ + header + echo "\$ docker compose --env-file .env config --quiet && echo valid" + docker compose --env-file .env config --quiet && echo valid + echo "\$ docker compose ps --format '{{.Service}} {{.Name}} {{.Status}} {{.Ports}}'" + docker compose ps --format '{{.Service}} {{.Name}} {{.Status}} {{.Ports}}' + echo "\$ docker compose config | grep -c container_name" + docker compose config | grep -c container_name || true + echo "\$ docker port <каждая реплика api> # реплики не публикуют порт" + for id in $(docker compose ps -q api); do echo "$(docker inspect -f '{{.Config.Hostname}}' "$id"): [$(docker port "$id")]"; done + echo "\$ docker inspect: hostname = X-Instance-Id, сети, лимиты CPU (nano) и памяти" + for id in $(docker compose ps -q); do + docker inspect -f '{{index .Config.Labels "com.docker.compose.service"}} {{.Config.Hostname}} {{.Name}} nets={{range $k, $v := .NetworkSettings.Networks}}{{$k}},{{end}} cpus={{.HostConfig.NanoCpus}} mem={{.HostConfig.Memory}} user={{.Config.User}} ro={{.HostConfig.ReadonlyRootfs}}' "$id" + done | sort + echo "\$ curl LB stats: backend api, server status check" + curl -fsS "$stats" | python3 -c 'import csv,sys +for r in csv.DictReader(sys.stdin.read().lstrip("# ").splitlines()): + if r["pxname"] == "api" and r["status"] != "MAINT (resolution)": + print(r["svname"], r["status"], r["check_status"], r["check_code"])' + echo "\$ for i in 1..6: curl -D - $base/health/live | X-Instance-Id" + for _ in 1 2 3 4 5 6; do curl -fsS -D - -o /dev/null "$base/health/live" | tr -d '\r' | grep -i '^x-instance-id'; done +} > evidence/stack.txt +{ + header + echo "host: $(sysctl -n machdep.cpu.brand_string 2>/dev/null || uname -m), $(sysctl -n hw.ncpu 2>/dev/null) CPU, $(($(sysctl -n hw.memsize 2>/dev/null || echo 0) / 1073741824)) GiB RAM, $(sw_vers -productName 2>/dev/null) $(sw_vers -productVersion 2>/dev/null)" + echo "docker: $(docker version -f '{{.Server.Version}}'), VM: $(docker info -f '{{.NCPU}}') CPU, $(docker info -f '{{.MemTotal}}') bytes" + echo "compose: $(docker compose version --short)" + echo "images:" + grep -h -E 'image:|@sha256' compose.yaml Dockerfile | sed 's/^ */ /' + echo "python in image: $(docker exec "$(docker compose ps -q api | head -1)" python -V)" + echo "haproxy: $(docker compose exec -T lb haproxy -v | head -1)" + echo "rustfs: $(docker compose exec -T s3 rustfs --version 2>/dev/null | head -1)" + echo "rabbitmq: $(docker compose exec -T broker rabbitmqctl -q version 2>/dev/null | head -1)" + echo "packages (uv.lock):" + for name in fastapi uvicorn uvicorn-worker gunicorn asyncpg argon2-cffi pydantic pydantic-settings dishka structlog redis aiobotocore botocore aiohttp pillow \ + faststream aio-pika aiormq dishka-faststream; do + printf ' %s %s\n' "$name" "$(sed -n "/^name = \"$name\"$/{n;s/version = \"\(.*\)\"/\1/p;}" uv.lock)" + done + echo "process users (service uid command):" + for id in $(docker compose ps -q); do + service=$(docker inspect -f '{{index .Config.Labels "com.docker.compose.service"}}' "$id") + docker top "$id" -eo pid,uid,comm | awk -v s="$service" 'NR > 1 {print " " s, $2, $3}' + done | sort -u +} > evidence/environment.txt +cat evidence/stack.txt evidence/environment.txt diff --git a/scripts/evidence-storage.sh b/scripts/evidence-storage.sh new file mode 100755 index 00000000..b913aff7 --- /dev/null +++ b/scripts/evidence-storage.sh @@ -0,0 +1,44 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +psql() { docker compose exec -T postgres psql -U messenger -d messenger -c "$1"; } +header() { echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse --short HEAD)"; } +counts="SELECT (SELECT count(*) FROM users) users, (SELECT count(*) FROM sessions) sessions, (SELECT count(*) FROM chats) chats, (SELECT count(*) FROM chat_members) members, (SELECT count(*) FROM messages) messages, (SELECT count(*) FROM idempotency_keys) keys, (SELECT count(*) FROM attachments) attachments, (SELECT count(*) FROM message_attachments) links, (SELECT count(*) FROM schema_migrations) versions" +kdf="SELECT substring(password_hash from '^[$]argon2id[$]v=[0-9]+[$]m=[0-9]+,t=[0-9]+,p=[0-9]+[$]') AS kdf_prefix, count(*) AS users, min(length(password_hash)) AS hash_length FROM users GROUP BY 1" +tokens="SELECT octet_length(token_hash) AS stored_bytes, count(*) AS sessions FROM sessions GROUP BY 1" +{ + header + for table in users sessions chats chat_members messages idempotency_keys attachments message_attachments schema_migrations; do + echo "\$ psql -c '\\d+ $table'" + psql "\\d+ $table" + done + echo "\$ psql -c 'SELECT version, left(checksum, 12), applied_at FROM schema_migrations'" + psql "SELECT version, left(checksum, 12) AS checksum, applied_at FROM schema_migrations" +} > evidence/schema.txt +{ + header + echo "\$ psql -c \"$counts\"" + psql "$counts" + echo "\$ make migrate" + docker compose run --rm migrate 2>/dev/null + echo "\$ psql -c \"$counts\"" + psql "$counts" +} > evidence/migrations.txt +{ + header + echo "\$ psql -c \"$kdf\"" + psql "$kdf" + echo "\$ psql -c \"$tokens\"" + psql "$tokens" +} > evidence/kdf.txt +{ + header + echo "\$ git ls-files secrets .env backups volumes" + git ls-files secrets .env backups volumes + echo "(пусто: секретов, .env, дампов и volumes в Git нет)" + echo "\$ git check-ignore -v secrets/database_url secrets/postgres_password secrets/s3_root_password secrets/s3_secret_access_key .env backups/" + git check-ignore -v secrets/database_url secrets/postgres_password secrets/s3_root_password secrets/s3_secret_access_key .env backups/ + echo "\$ docker volume ls --filter name=mtusi2026lab6" + docker volume ls --filter name=mtusi2026lab6 +} > evidence/secrets.txt +echo "evidence/{schema,migrations,kdf,secrets}.txt обновлены" diff --git a/scripts/failover-each.sh b/scripts/failover-each.sh new file mode 100755 index 00000000..64050ef7 --- /dev/null +++ b/scripts/failover-each.sh @@ -0,0 +1,19 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +out=${1:-evidence/failover-each} +rm -rf "$out" +mkdir -p "$out" +scripts/restore.sh +hosts=$(docker inspect -f '{{.Config.Hostname}}' $(docker compose ps -q "${API_SERVICE:-api}")) +failed=0 +set -- +for host in $hosts; do + TARGET_INSTANCE_ID=$host python3 scripts/probe.py load --expect dip --out "$out/$host" \ + -- sh -c "scripts/stop-one.sh && sleep ${HOLD:-10} && scripts/restore.sh" || failed=1 + scripts/restore.sh + set -- "$@" "$out/$host/result.json" +done +python3 -c 'import json,sys; print(json.dumps([json.load(open(p)) for p in sys.argv[1:]], ensure_ascii=False, indent=2))' "$@" > "$out/summary.tmp" +mv "$out/summary.tmp" "$out/summary.json" +exit "$failed" diff --git a/scripts/failover-evidence.sh b/scripts/failover-evidence.sh new file mode 100755 index 00000000..fb30bd36 --- /dev/null +++ b/scripts/failover-evidence.sh @@ -0,0 +1,28 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +replicas() { + docker inspect -f '{{.Config.Hostname}} {{.State.Status}} started={{.State.StartedAt}} finished={{.State.FinishedAt}}' $(docker compose ps -aq "${API_SERVICE:-api}") +} +tmp=evidence/failover.tmp +since=$(date -u +%FT%TZ) +{ + echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse --short HEAD)" + echo "== before: api replicas (hostname = X-Instance-Id), state, StartedAt, FinishedAt" + replicas + echo "\$ make test-failover ACTION_SCRIPT=scripts/stop-one.sh" + make --no-print-directory test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL="${1:-http://localhost:8080}" + echo "== after: api replicas" + replicas +} > "$tmp" 2>&1 +stopped=$(replicas | awk '$2 == "exited" {print $1}') +{ + echo "== stopped replica: $stopped; shutdown: signal, drain window with readiness 503, close, exit" + docker logs --since "$since" "$stopped" 2>&1 | grep -E 'Handling signal|Draining|"status":503|Shutting down|Waiting for conn|Finished server|service.stopped|SIGTERM' | tail -16 + echo "\$ scripts/restore.sh" + scripts/restore.sh + echo "== after restore" + replicas +} >> "$tmp" 2>&1 +mv "$tmp" "evidence/failover-$stopped.txt" +cat "evidence/failover-$stopped.txt" diff --git a/scripts/k6.sh b/scripts/k6.sh new file mode 100755 index 00000000..01d95f29 --- /dev/null +++ b/scripts/k6.sh @@ -0,0 +1,10 @@ +#!/bin/sh +set -eu +out=${1:?usage: k6.sh [k6 run args]} +shift +mkdir -p "$out" +out=$(cd "$out" && pwd) +cd "$(dirname "$0")/.." +exec docker compose --profile bench run --rm --no-deps -T -u "$(id -u):$(id -g)" -v "$out:/out" \ + -e RATE -e WARMUP -e DURATION -e USERS -e READ_RATIO -e VUS -e MAX_VUS -e RUN_ID \ + k6 run --quiet "$@" "/scripts/${K6_SCRIPT:-messages.js}" diff --git a/scripts/lab4_scenarios.py b/scripts/lab4_scenarios.py new file mode 100755 index 00000000..9ac01c73 --- /dev/null +++ b/scripts/lab4_scenarios.py @@ -0,0 +1,231 @@ +#!/usr/bin/env python3 +import argparse +import hashlib +import http.client +import json +import os +import subprocess +import sys +import time +import uuid +from pathlib import Path +from urllib.parse import urlsplit + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +client = __import__("client") +Client, Failure, check = client.Client, client.Failure, client.check +FIXTURE = (ROOT / "tests/fixtures/sample.png").read_bytes() + + +def compose(*args, stdin=None): + cmd = ["docker", "compose", *args] + return subprocess.run(cmd, cwd=ROOT, input=stdin, capture_output=True, text=True, check=False) + + +def upload(c, actor, chat, expected=202): + path = f"/api/v1/chats/{chat['id']}/attachments?filename=sample.png" + schema = "Attachment" if expected == 202 else None + return c.call( + "POST", + path, + expected, + actor=actor, + raw=FIXTURE, + headers={"Content-Type": "image/png"}, + schema=schema, + ) + + +def content(c, actor, attachment, expected=200): + return c.call("GET", f"/api/v1/attachments/{attachment['id']}/content", expected, actor=actor) + + +def timed(fn): + started = time.monotonic() + try: + response = fn() + except Failure as exc: + return { + "status": "failure", + "seconds": round(time.monotonic() - started, 3), + "error": str(exc), + } + seconds = round(time.monotonic() - started, 3) + return { + "status": response["status"], + "seconds": seconds, + "instance": response["headers"].get("X-Instance-Id"), + } + + +def wait(fn, seconds): + deadline = time.monotonic() + seconds + while True: + try: + return fn() + except Failure: + if time.monotonic() > deadline: + raise + time.sleep(0.2) + + +def failover_upload(c, args): + alice, bob, _, chat = c.bootstrap() + sent = upload(c, alice, chat) + att, target = sent["json"], sent["headers"]["X-Instance-Id"] + c.wait_ready(bob, att) + started = time.monotonic() + hook = subprocess.run( + [str(ROOT / "scripts/stop-one.sh")], + cwd=ROOT, + env=dict(os.environ, TARGET_INSTANCE_ID=target), + capture_output=True, + text=True, + check=False, + ) + check(hook.returncode == 0, f"stop-one.sh failed: {hook.stderr.strip()}") + hook_seconds = time.monotonic() - started + got = wait(lambda: content(c, bob, att), 15) + recovered = time.monotonic() - started + served = got["headers"].get("X-Instance-Id") + check(served != target, "download served by the stopped replica") + digest = hashlib.sha256(got["bytes"]).hexdigest() + check( + digest == att["sha256"] == hashlib.sha256(FIXTURE).hexdigest(), + "sha256 mismatch after failover", + ) + if args.restore: + compose("up", "-d", "--wait", "--no-recreate", "--no-deps", args.service) + return { + "scenario": "failover-upload", + "upload_instance": target, + "download_instance": served, + "sha256_equal": True, + "hook_seconds": round(hook_seconds, 3), + "download_ok_after_seconds": round(recovered, 3), + "restored": args.restore, + } + + +def s3_outage(c, args): + alice, bob, _, chat = c.bootstrap() + att = c.wait_ready(bob, upload(c, alice, chat)["json"]) + check( + compose("stop", "-t", "10", args.s3_service).returncode == 0, "docker compose stop failed" + ) + try: + during = { + "upload": timed(lambda: upload(c, alice, chat, [503])), + "metadata": timed( + lambda: c.call("GET", f"/api/v1/attachments/{att['id']}", [200, 503], actor=bob) + ), + "download": timed(lambda: content(c, bob, att, 503)), + "text_message": timed(lambda: c.message(alice, chat, "S3 недоступен, текст работает")), + "health_ready": timed(lambda: c.call("GET", "/health/ready", [200, 503])), + } + finally: + started = time.monotonic() + compose("up", "-d", "--wait", "--no-deps", "--no-recreate", args.s3_service) + recovered = round(time.monotonic() - started, 3) + got = wait(lambda: content(c, bob, att), 30) + check(got["bytes"] == FIXTURE, "attachment changed across S3 outage") + expected = {"upload": 503, "download": 503, "text_message": 201} + return { + "scenario": "s3-outage", + "during_outage": during, + "expected": expected, + "as_expected": all(during[k]["status"] == v for k, v in expected.items()), + "s3_back_healthy_seconds": recovered, + "download_after_recovery": "sha256 equal", + } + + +def abort_upload(c, args): + alice, _, _, chat = c.bootstrap() + name = f"abort-{uuid.uuid4().hex}.png" + parts = urlsplit(c.base) + factory = http.client.HTTPSConnection if parts.scheme == "https" else http.client.HTTPConnection + conn = factory(parts.netloc, timeout=10) + conn.putrequest("POST", f"/api/v1/chats/{chat['id']}/attachments?filename={name}") + headers = { + "Authorization": f"Bearer {alice['token']}", + "Content-Type": "image/png", + "Content-Length": str(args.declared), + "Accept": "application/json", + } + for key, value in headers.items(): + conn.putheader(key, value) + conn.endheaders() + conn.send(FIXTURE[:8] + os.urandom(args.sent - 8)) + conn.close() + time.sleep(args.settle) + user, db = os.environ.get("PGUSER", "messenger"), os.environ.get("PGDATABASE", "messenger") + sql = "SELECT status, count(*) FROM attachments WHERE filename = :'name' GROUP BY status;\n" + out = compose( + "exec", + "-T", + "postgres", + "psql", + "-qAt", + "-F", + " ", + "-U", + user, + "-d", + db, + "-v", + f"name={name}", + stdin=sql, + ) + rows = out.stdout.split() + counts = dict(zip(rows[::2], (int(x) for x in rows[1::2]))) + alive = c.call("GET", "/api/v1/users/me", actor=alice, schema="User")["status"] + return { + "scenario": "abort-upload", + "declared_bytes": args.declared, + "sent_bytes": args.sent, + "settle_seconds": args.settle, + "rows_by_status": counts, + "ready_rows": counts.get("ready", 0), + "api_still_serves": alive == 200, + } + + +def main(): + p = argparse.ArgumentParser( + description="Lab4 failure scenarios over the public HTTP contract plus docker compose" + ) + p.add_argument("scenario", choices=["failover-upload", "s3-outage", "abort-upload"]) + p.add_argument("--base-url", default=None) + p.add_argument("--ca-file", default=None) + p.add_argument("--timeout", type=float, default=15) + p.add_argument("--service", default="api") + p.add_argument("--s3-service", default="s3") + p.add_argument( + "--restore", action="store_true", help="failover-upload: start the stopped replica again" + ) + p.add_argument("--declared", type=int, default=5 * 1024 * 1024) + p.add_argument("--sent", type=int, default=256 * 1024) + p.add_argument("--settle", type=float, default=3) + p.add_argument("--output", metavar="FILE") + args = p.parse_args() + c = Client(args.base_url, args.ca_file, timeout=args.timeout) + run = { + "failover-upload": failover_upload, + "s3-outage": s3_outage, + "abort-upload": abort_upload, + }[args.scenario] + try: + report = {"ok": True, **run(c, args)} + except Failure as exc: + report = {"scenario": args.scenario, "ok": False, "error": str(exc)} + text = json.dumps(report, ensure_ascii=False, indent=2) + if args.output: + Path(args.output).write_text(text + "\n") + print(text) + sys.exit(0 if report["ok"] else 1) + + +if __name__ == "__main__": + main() diff --git a/scripts/lab4_seed.py b/scripts/lab4_seed.py new file mode 100755 index 00000000..d37d23a3 --- /dev/null +++ b/scripts/lab4_seed.py @@ -0,0 +1,352 @@ +#!/usr/bin/env python3 +import argparse +import base64 +import hashlib +import http.client +import json +import os +import re +import struct +import subprocess +import tempfile +import threading +import time +import uuid +import zlib +from pathlib import Path +from urllib.parse import urlsplit + +ROOT = Path(__file__).resolve().parents[1] +MAX_UPLOAD = 10 * 1024 * 1024 +COLUMNS = ( + "id", + "chat_id", + "owner_id", + "filename", + "content_type", + "size_bytes", + "sha256", + "width", + "height", + "object_key", + "status", +) + + +def chunk(tag, data): + return ( + struct.pack(">I", len(data)) + + tag + + data + + struct.pack(">I", zlib.crc32(tag + data) & 0xFFFFFFFF) + ) + + +def png(width, height): + raw = b"".join(b"\x00" + os.urandom(width * 3) for _ in range(height)) + header = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0) + return ( + b"\x89PNG\r\n\x1a\n" + + chunk(b"IHDR", header) + + chunk(b"IDAT", zlib.compress(raw, 1)) + + chunk(b"IEND", b"") + ) + + +def sized_png(target): + height = max(1, target // (1024 * 3 + 1)) + body = png(1024, height) + while len(body) > min(target, MAX_UPLOAD) and height > 1: + height -= max(1, (len(body) - target) // 3073 + 1) + body = png(1024, height) + return body + + +def run(cmd, stdin=None): + out = subprocess.run(cmd, cwd=ROOT, input=stdin, capture_output=True, check=False) + if out.returncode: + raise SystemExit( + f"command failed: {' '.join(cmd[:6])}\n{out.stderr.decode(errors='replace')[-500:]}" + ) + return out.stdout.decode() + + +def psql(sql, stdin=None): + user, db = os.environ.get("PGUSER", "messenger"), os.environ.get("PGDATABASE", "messenger") + psql = [ + "docker", + "compose", + "exec", + "-T", + "postgres", + "psql", + "-v", + "ON_ERROR_STOP=1", + "-qAt", + "-F", + " ", + ] + return run([*psql, "-U", user, "-d", db, "-c", sql], stdin) + + +def direct(args): + bucket = os.environ.get("S3_BUCKET", "messenger-attachments") + prefix = os.environ.get("S3_PREFIX", "attachments/") + if args.chat_id and args.owner_id: + chat_id, owner_id = args.chat_id, args.owner_id + else: + found = psql( + "SELECT chat_id, owner_id FROM attachments ORDER BY created_at LIMIT 1" + ).split() + if len(found) != 2: + raise SystemExit( + "no attachment rows yet: upload one via API or pass --chat-id and --owner-id" + ) + chat_id, owner_id = found + rows = [] + started = time.monotonic() + with tempfile.TemporaryDirectory(prefix="lab4-seed-") as tmp: + Path(tmp).chmod(0o755) + for i in range(args.count): + aid = str(uuid.uuid4()) + body = png(8, 8) + path = Path(tmp) / aid + path.write_bytes(body) + path.chmod(0o644) + digest = hashlib.sha256(body).hexdigest() + rows.append( + f"{aid},{chat_id},{owner_id},seed-{i}.png,image/png,{len(body)},{digest},8,8,{prefix}{aid},ready" + ) + generated = time.monotonic() + cli = [ + "docker", + "compose", + "run", + "--rm", + "--no-deps", + "-T", + "-v", + f"{tmp}:/seed:ro", + "s3-cli", + ] + run([*cli, "put", "--recursive", "--no-progress", "/seed/", f"root/{bucket}/{prefix}"]) + uploaded = time.monotonic() + columns = ", ".join(COLUMNS) + psql( + f"COPY attachments ({columns}) FROM STDIN WITH (FORMAT csv)", + "".join(f"{r}\n" for r in rows).encode(), + ) + seconds = { + "generate": round(generated - started, 3), + "s3_put": round(uploaded - generated, 3), + "pg_copy": round(time.monotonic() - uploaded, 3), + } + return {"mode": "direct", "count": args.count, "seconds": seconds} + + +class Http: + def __init__(self, base, timeout): + parts = urlsplit(base) + self.cls = ( + http.client.HTTPSConnection if parts.scheme == "https" else http.client.HTTPConnection + ) + self.host, self.timeout, self.local = parts.netloc, timeout, threading.local() + + def call(self, method, path, body=None, headers=None, token=None): + conn = getattr(self.local, "conn", None) or self.cls(self.host, timeout=self.timeout) + self.local.conn = conn + h = {"Accept": "application/json"} + if token: + h["Authorization"] = "Bearer " + token + if isinstance(body, dict): + body, h["Content-Type"] = json.dumps(body).encode(), "application/json" + h.update(headers or {}) + try: + conn.request(method, path, body=body, headers=h) + response = conn.getresponse() + data = response.read() + except (OSError, http.client.HTTPException): + conn.close() + self.local.conn = None + return 0, {}, b"" + return response.status, {k.lower(): v for k, v in response.getheaders()}, data + + +def login(api, label): + name = f"seed_{label}_{uuid.uuid4().hex[:12]}" + password = "Seed-" + uuid.uuid4().hex + body = {"username": name, "display_name": label, "password": password} + status, _, data = api.call("POST", "/api/v1/users", body) + if status != 201: + raise SystemExit(f"user create failed: {status}") + user = json.loads(data) + basic = base64.b64encode(f"{name}:{password}".encode()).decode() + status, _, data = api.call( + "POST", "/api/v1/auth/sessions", headers={"Authorization": f"Basic {basic}"} + ) + if status != 201: + raise SystemExit(f"login failed: {status}") + return user["id"], json.loads(data)["token"] + + +def mib(text): + m = re.match(r"([\d.]+)\s*([KMG]i?B)", text) + if not m: + return 0.0 + scale = {"KiB": 1 / 1024, "KB": 1 / 1024, "GiB": 1024, "GB": 1024} + return float(m.group(1)) * scale.get(m.group(2), 1) + + +def sampler(ids, peaks, stop): + while not stop.is_set(): + cmd = ["docker", "stats", "--no-stream", "--format", "{{.ID}} {{.MemUsage}}", *ids] + out = subprocess.run(cmd, capture_output=True, text=True, check=False).stdout + for line in out.splitlines(): + cid, usage = line.split(" ", 1) + peaks[cid] = max(peaks.get(cid, 0.0), mib(usage)) + + +def cgroup(cid, name): + cmd = ["docker", "exec", cid, "cat", f"/sys/fs/cgroup/{name}"] + return subprocess.run(cmd, capture_output=True, text=True, check=False).stdout.strip() + + +def percentile(values, q): + return ( + round(sorted(values)[min(len(values) - 1, int(q * len(values)))] * 1000, 1) + if values + else None + ) + + +def over_http(args): + api = Http(args.base_url, args.timeout) + alice, token = login(api, "alice") + bob, _ = login(api, "bob") + status, _, data = api.call( + "POST", "/api/v1/chats", {"title": "seed", "member_ids": [alice, bob]}, token=token + ) + if status != 201: + raise SystemExit(f"chat create failed: {status}") + chat = json.loads(data)["id"] + fixed = sized_png(args.size) if args.size else None + counter, lock, results = iter(range(args.count)), threading.Lock(), [] + ids = run(["docker", "compose", "ps", "-q", args.service]).split() if args.sample else [] + idle = {cid: round(int(cgroup(cid, "memory.current") or 0) / 1048576, 1) for cid in ids} + peaks, stop = {}, threading.Event() + watcher = threading.Thread(target=sampler, args=(ids, peaks, stop), daemon=True) + if ids: + watcher.start() + + def worker(): + while True: + with lock: + i = next(counter, None) + if i is None: + return + body = fixed or png(8, 8) + began = time.monotonic() + path = f"/api/v1/chats/{chat}/attachments?filename=seed-{i}.png" + status, headers, data = api.call( + "POST", path, body, {"Content-Type": "image/png"}, token + ) + created = json.loads(data)["id"] if status == 201 else None + results.append( + (status, time.monotonic() - began, headers.get("x-instance-id", ""), created) + ) + + started = time.monotonic() + threads = [threading.Thread(target=worker) for _ in range(args.concurrency)] + for t in threads: + t.start() + for t in threads: + t.join() + elapsed = time.monotonic() - started + stop.set() + if ids: + watcher.join() + created = [r[3] for r in results if r[3]] + for start in range(0, min(args.link, len(created)), 10): + body = {"text": "", "attachment_ids": created[start : min(start + 10, args.link)]} + api.call( + "POST", + f"/api/v1/chats/{chat}/messages", + body, + {"Idempotency-Key": uuid.uuid4().hex}, + token, + ) + if args.ids_out: + Path(args.ids_out).write_text("".join(f"{i}\n" for i in created)) + codes, instances = {}, {} + for status, _, instance, _ in results: + codes[str(status)] = codes.get(str(status), 0) + 1 + instances[instance] = instances.get(instance, 0) + 1 + ok = [r[1] for r in results if r[0] == 201] + memory = {} + for cid in ids: + peak = cgroup(cid, "memory.peak") + events = dict(line.split() for line in cgroup(cid, "memory.events").splitlines()) + memory[cid[:12]] = { + "idle_mib": idle[cid], + "docker_stats_peak_mib": round(peaks.get(cid[:12], 0.0), 1), + "cgroup_memory_peak_mib": round(int(peak) / 1048576, 1) if peak.isdigit() else None, + "oom_kill": int(events.get("oom_kill", 0)), + } + return { + "mode": "http", + "count": args.count, + "concurrency": args.concurrency, + "body_bytes": len(fixed) if fixed else "~200 (unique 8x8 PNG)", + "status_codes": codes, + "instances": len([i for i in instances if i]), + "linked_in_messages": min(args.link, len(created)), + "elapsed_seconds": round(elapsed, 3), + "successful_rps": round(len(ok) / elapsed, 2) if elapsed else None, + "latency_ms": { + "p50": percentile(ok, 0.5), + "p95": percentile(ok, 0.95), + "p99": percentile(ok, 0.99), + "max": percentile(ok, 1.0), + }, + "memory": memory, + } + + +def main(): + p = argparse.ArgumentParser( + description="Lab4 attachment volume generator: HTTP uploads or direct S3+PG seeding" + ) + p.add_argument("--mode", choices=["http", "direct"], default="http") + p.add_argument("--count", type=int, default=1000) + p.add_argument("--concurrency", type=int, default=16) + p.add_argument( + "--size", + type=int, + default=0, + help="http: bytes per upload (one PNG reused); 0 = unique 8x8 PNGs", + ) + p.add_argument( + "--link", type=int, default=0, help="http: reference the first N attachments from messages" + ) + p.add_argument( + "--sample", + action="store_true", + help="http: sample docker stats and memory.peak of API containers", + ) + p.add_argument("--service", default="api") + p.add_argument("--base-url", default=os.environ.get("BASE_URL", "http://localhost:8080")) + p.add_argument("--timeout", type=float, default=60) + p.add_argument("--ids-out", metavar="FILE") + p.add_argument("--chat-id") + p.add_argument("--owner-id") + p.add_argument("--output", metavar="FILE") + args = p.parse_args() + report = direct(args) if args.mode == "direct" else over_http(args) + text = json.dumps(report, ensure_ascii=False, indent=2) + if args.output: + Path(args.output).write_text(text + "\n") + print(text) + + +if __name__ == "__main__": + main() diff --git a/scripts/lb-wait.sh b/scripts/lb-wait.sh new file mode 100755 index 00000000..f1ead888 --- /dev/null +++ b/scripts/lb-wait.sh @@ -0,0 +1,11 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +service=${API_SERVICE:-api} +want=${1:-$(docker compose ps -q "$service" | wc -l | tr -d ' ')} +url=${LB_STATS_URL:-http://$(docker compose port lb 8404)/stats;csv} +deadline=$(($(date +%s) + ${WAIT_TIMEOUT:-60})) +until [ "$(curl -fsS "$url" | awk -F, -v b="$service" '$1 == b && $2 != "BACKEND" && $18 == "UP"' | wc -l)" -ge "$want" ]; do + [ "$(date +%s)" -lt "$deadline" ] || { echo "LB does not see $want UP $service servers" >&2; exit 4; } + sleep 0.5 +done diff --git a/scripts/limits.py b/scripts/limits.py new file mode 100644 index 00000000..c1da642a --- /dev/null +++ b/scripts/limits.py @@ -0,0 +1,185 @@ +#!/usr/bin/env python3 +import argparse +import json +import subprocess +import sys +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +S3_CLIENT_SECRETS = [ + "s3_access_key_id", + "s3_root_password", + "s3_root_user", + "s3_secret_access_key", + "s3_worker_access_key_id", + "s3_worker_secret_access_key", +] +POLICY = { + "lb": {"secrets": [], "volumes": [], "networks": ["backend", "edge"], "ports": True}, + "api": { + "secrets": [ + "cache_url", + "cursor_secret", + "database_url", + "s3_access_key_id", + "s3_secret_access_key", + ], + "volumes": [], + "networks": ["backend"], + }, + "migrate": {"secrets": ["database_url"], "volumes": [], "networks": ["backend"]}, + "postgres": {"secrets": ["postgres_password"], "volumes": ["pgdata"], "networks": ["backend"]}, + "valkey": {"secrets": ["valkey.conf"], "volumes": ["valkeydata"], "networks": ["backend"]}, + "k6": {"secrets": [], "volumes": [], "networks": ["edge"]}, + "s3": { + "secrets": ["s3_root_password", "s3_root_user"], + "volumes": ["s3data"], + "networks": ["backend"], + }, + "s3-init": {"secrets": S3_CLIENT_SECRETS, "volumes": [], "networks": ["backend"]}, + "s3-cli": {"secrets": S3_CLIENT_SECRETS, "volumes": [], "networks": ["backend"]}, + "broker": { + "secrets": ["rabbitmq_ops_password", "rabbitmq_users"], + "volumes": ["broker-data"], + "networks": ["backend", "edge"], + "ports": True, + }, + "relay": { + "secrets": ["broker_url_api", "database_url", "s3_access_key_id", "s3_secret_access_key"], + "volumes": [], + "networks": ["backend"], + }, + "worker": { + "secrets": [ + "broker_url_worker", + "database_url", + "s3_worker_access_key_id", + "s3_worker_secret_access_key", + ], + "volumes": [], + "networks": ["backend"], + }, +} +PUBLISHING = frozenset(name for name, rule in POLICY.items() if rule.get("ports")) + + +def run(*command): + done = subprocess.run(command, cwd=str(ROOT), capture_output=True, text=True, check=False) + return done.returncode, done.stdout.strip() + + +def declared(name, service, networks, problems): + limits = service.get("deploy", {}).get("resources", {}).get("limits", {}) + secrets = sorted(s["source"] for s in service.get("secrets", [])) + volumes = sorted(v["source"] for v in service.get("volumes", []) if v["type"] == "volume") + nets = sorted(service.get("networks", {})) + ports = [ + f"{p.get('host_ip', '0.0.0.0')}:{p.get('published')}" for p in service.get("ports", []) + ] + want = POLICY.get(name) + if want is None: + problems.append(f"{name}: service is not covered by the policy") + want = {"secrets": [], "volumes": [], "networks": []} + if not limits.get("cpus") or not limits.get("memory"): + problems.append(f"{name}: no CPU or memory limit") + if "container_name" in service: + problems.append(f"{name}: container_name prevents scaling") + for kind, actual in (("secrets", secrets), ("volumes", volumes), ("networks", nets)): + if actual != want[kind]: + problems.append(f"{name}: {kind} {actual}, policy {want[kind]}") + if ports and not want.get("ports"): + problems.append(f"{name}: publishes {ports} in bypass of the balancer") + if any(not port.startswith("127.0.0.1:") for port in ports): + problems.append(f"{name}: publishes {ports} beyond loopback") + if name == "api" and not service.get("read_only"): + problems.append("api: root filesystem is writable") + internal = [n for n in nets if networks.get(n, {}).get("internal")] + return [ + name, + f"{limits.get('cpus', '-')}/{int(limits.get('memory') or 0) >> 20}MiB", + ",".join(ports) or "-", + ",".join(secrets) or "-", + ",".join(volumes) or "-", + ",".join(f"{n}{'(internal)' if n in internal else ''}" for n in nets), + str(service.get("read_only", False)).lower(), + ] + + +def running(problems): + _, ids = run("docker", "compose", "ps", "-q") + template = ( + '{{index .Config.Labels "com.docker.compose.service"}} {{.Config.Hostname}} ' + "{{.HostConfig.NanoCpus}} {{.HostConfig.Memory}} {{json .HostConfig.PortBindings}}" + ) + _, found = run("docker", "inspect", "-f", template, *ids.split()) + rows = [] + for line in found.splitlines(): + service, host, cpus, memory, bindings = line.split(" ", 4) + published = [ + f"{b['HostIp']}:{b['HostPort']}->{port}" + for port, items in (json.loads(bindings) or {}).items() + for b in items or [] + ] + if int(cpus) <= 0 or int(memory) <= 0: + problems.append(f"container {service}/{host}: no CPU or memory limit applied") + if published and service not in PUBLISHING: + problems.append(f"container {service}/{host}: publishes {published}") + rows.append( + [service, host, f"{int(cpus) / 1e9:g}", f"{int(memory) >> 20}MiB", ",".join(published)] + ) + return sorted(rows) + + +def table(header, rows): + widths = [max(len(str(r[i])) for r in [header, *rows]) for i in range(len(header))] + lines = [header, ["-" * w for w in widths], *rows] + return [" ".join(str(c).ljust(w) for c, w in zip(r, widths)).rstrip() for r in lines] + + +def main(): + parser = argparse.ArgumentParser(description="limits, ports, secrets, volumes and networks") + parser.add_argument("--out", help="text report, written only after a complete check") + args = parser.parse_args() + problems = [] + _, raw = run( + "docker", + "compose", + "--profile", + "bench", + "--profile", + "tools", + "config", + "--format", + "json", + ) + config = json.loads(raw) + services = [ + declared(name, service, config.get("networks", {}), problems) + for name, service in sorted(config["services"].items()) + ] + containers = running(problems) + if not any(row[0] == "api" for row in containers): + problems.append("no running api replicas to inspect") + lines = [ + f"date: {time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())}", + "$ docker compose --profile bench --profile tools config --format json (declared)", + *table(["service", "cpus/mem", "ports", "secrets", "volumes", "networks", "ro"], services), + "", + "$ docker inspect (running containers: hostname = X-Instance-Id, applied limits)", + *table(["service", "hostname", "cpus", "memory", "published"], containers), + "", + *(f"FAIL: {p}" for p in problems), + "PASS: limits, ports, secrets, volumes and networks match the policy" + if not problems + else f"FAIL: {len(problems)} violation(s)", + ] + text = "\n".join(lines) + "\n" + if args.out: + Path(args.out).write_text(text) + print(text, end="") + return 1 if problems else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/make_images.py b/scripts/make_images.py new file mode 100644 index 00000000..f4b34596 --- /dev/null +++ b/scripts/make_images.py @@ -0,0 +1,354 @@ +#!/usr/bin/env python3 +import argparse +import json +import struct +import zlib +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +MAX_BYTES = 10485760 +MAX_PIXELS = 40000000 +REJECTED_OR_FAILED = {"upload": [202, 415], "final": "failed", "error_code": "invalid_image"} +GIF = ( + b"GIF89a\x01\x00\x01\x00\x80\x00\x00\x00\x00\x00\xff\xff\xff!\xf9\x04\x01\x00\x00\x00\x00" + b",\x00\x00\x00\x00\x01\x00\x01\x00\x00\x02\x02D\x01\x00;" +) + + +def chunk(kind, data): + return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data)) + + +def png(width, height, rows, color_type=2, depth=8, *, extra=(), level=6): + packer = zlib.compressobj(level) + body = b"".join(packer.compress(b"\x00" + row) for row in rows) + packer.flush() + header = struct.pack(">IIBBBBB", width, height, depth, color_type, 0, 0, 0) + return ( + b"\x89PNG\r\n\x1a\n" + + chunk(b"IHDR", header) + + b"".join(extra) + + chunk(b"IDAT", body) + + chunk(b"IEND", b"") + ) + + +def ramp(value, size): + return value * 255 // max(1, size - 1) + + +def gradient(width, height, channels): + for y in range(height): + yield bytes( + channel + for x in range(width) + for channel in (ramp(x, width), ramp(y, height), 160, ramp(x + y, width + height))[ + :channels + ] + ) + + +def blank(width, height, channels, depth=8): + row = bytes(width * channels * depth // 8) + for _ in range(height): + yield row + + +def padded(data, total): + pad = total - len(data) - 12 + return data[:33] + chunk(b"paDd", bytes(pad)) + data[33:] + + +class Bits: + def __init__(self): + self.out = bytearray() + self.acc = 0 + self.size = 0 + + def put(self, value, length): + self.acc = (self.acc << length) | value + self.size += length + while self.size >= 8: + self.size -= 8 + byte = (self.acc >> self.size) & 0xFF + self.out += b"\xff\x00" if byte == 0xFF else bytes((byte,)) + self.acc &= (1 << self.size) - 1 + + def zeros(self, count): + head = min(count, (8 - self.size) % 8) + self.put(0, head) + count -= head + self.out += bytes(count // 8) + self.put(0, count % 8) + + def flush(self): + if self.size: + self.put((1 << (8 - self.size)) - 1, 8 - self.size) + return bytes(self.out) + + +def segment(marker, payload): + return b"\xff" + bytes((marker,)) + struct.pack(">H", len(payload) + 2) + payload + + +def ycbcr(red, green, blue): + return ( + 0.299 * red + 0.587 * green + 0.114 * blue, + 128 - 0.168736 * red - 0.331264 * green + 0.5 * blue, + 128 + 0.5 * red - 0.418688 * green - 0.081312 * blue, + ) + + +def exif(orientation): + entry = struct.pack(">HHIHH", 0x0112, 3, 1, orientation, 0) + tiff = b"MM\x00\x2a\x00\x00\x00\x08" + struct.pack(">H", 1) + entry + struct.pack(">I", 0) + return segment(0xE1, b"Exif\x00\x00" + tiff) + + +def jpeg(width, height, mode="rgb", orientation=None, uniform=False): + components = {"gray": 1, "rgb": 3, "cmyk": 4}[mode] + head = b"\xff\xd8" + ( + exif(orientation) + if orientation + else segment(0xE0, b"JFIF\x00\x01\x01\x00\x00\x01\x00\x01\x00\x00") + ) + if mode == "cmyk": + head += segment(0xEE, b"Adobe\x00\x64\x00\x00\x00\x00\x00") + head += segment(0xDB, b"\x00" + bytes([1] * 64)) + head += segment( + 0xC0, + struct.pack(">BHHB", 8, height, width, components) + + b"".join(bytes((i + 1, 0x11, 0)) for i in range(components)), + ) + head += segment(0xC4, b"\x00" + bytes([0, 0, 0, 12] + [0] * 12) + bytes(range(12))) + head += segment(0xC4, b"\x10" + bytes([1] + [0] * 15) + b"\x00") + head += segment( + 0xDA, + bytes((components,)) + + b"".join(bytes((i + 1, 0)) for i in range(components)) + + b"\x00\x3f\x00", + ) + columns, rows = (width + 7) // 8, (height + 7) // 8 + bits, previous = Bits(), [0] * components + for block in range(1 if uniform else columns * rows): + bx, by = block % columns, block // columns + red, green, blue = ramp(bx, columns), ramp(by, rows), 160 + values = {"gray": (red,), "rgb": ycbcr(red, green, blue), "cmyk": (red, green, blue, 64)}[ + mode + ] + for index, value in enumerate(values): + dc = round(8 * (value - 128)) + diff, previous[index] = dc - previous[index], dc + category = abs(diff).bit_length() + bits.put(category, 4) + if category: + bits.put(diff if diff > 0 else diff + (1 << category) - 1, category) + bits.put(0, 1) + if uniform: + bits.zeros((columns * rows - 1) * components * 5) + return head + bits.flush() + b"\xff\xd9" + + +def fit(width, height, box, upscale=False): + if not upscale and max(width, height) <= box: + return [width, height] + if width >= height: + return [box, max(1, height * box // width)] + return [max(1, width * box // height), box] + + +def ready(width, height, orientation=1): + if orientation in (5, 6, 7, 8): + width, height = height, width + return { + "upload": [202], + "final": "ready", + "variants": { + "thumbnail": fit(width, height, 256), + "crop": [128, 128], + "optimized": fit(width, height, 1280), + }, + } + + +def catalog(heavy): + small = png(64, 64, gradient(64, 64, 3)) + truncated_png = png(320, 200, gradient(320, 200, 3)) + idat = truncated_png.index(b"IDAT") + corrupt = bytearray(truncated_png) + corrupt[idat + 40 : idat + 48] = b"\x00" * 8 + palette = bytes(channel for index in range(256) for channel in (index, 255 - index, 128)) + alpha = bytes(index for index in range(256)) + items = [ + ( + "png-rgb-320x200.png", + "image/png", + lambda: png(320, 200, gradient(320, 200, 3)), + ready(320, 200), + ), + ( + "png-rgba-300x300.png", + "image/png", + lambda: png(300, 300, gradient(300, 300, 4), 6), + ready(300, 300), + ), + ( + "png-palette-trns-200x100.png", + "image/png", + lambda: png( + 200, + 100, + (bytes((x + y) % 256 for x in range(200)) for y in range(100)), + 3, + extra=(chunk(b"PLTE", palette), chunk(b"tRNS", alpha)), + ), + ready(200, 100), + ), + ( + "png-gray-17x1000.png", + "image/png", + lambda: png(17, 1000, gradient(17, 1000, 1), 0), + ready(17, 1000), + ), + ( + "png-gray-alpha-640x480.png", + "image/png", + lambda: png(640, 480, gradient(640, 480, 2), 4), + ready(640, 480), + ), + ( + "png-gray16-300x200.png", + "image/png", + lambda: png(300, 200, gradient(600, 200, 1), 0, 16), + ready(300, 200), + ), + ( + "png-rgba16-64x64.png", + "image/png", + lambda: png(64, 64, gradient(512, 64, 1), 6, 16), + ready(64, 64), + ), + ("png-1x1.png", "image/png", lambda: png(1, 1, gradient(1, 1, 3)), ready(1, 1)), + ( + "png-wide-3000x2.png", + "image/png", + lambda: png(3000, 2, gradient(3000, 2, 3)), + ready(3000, 2), + ), + ( + "png-small-50x30.png", + "image/png", + lambda: png(50, 30, gradient(50, 30, 3)), + ready(50, 30), + ), + ("jpeg-rgb-1000x750.jpg", "image/jpeg", lambda: jpeg(1000, 750), ready(1000, 750)), + ("jpeg-gray-801x601.jpg", "image/jpeg", lambda: jpeg(801, 601, "gray"), ready(801, 601)), + ("jpeg-cmyk-400x300.jpg", "image/jpeg", lambda: jpeg(400, 300, "cmyk"), ready(400, 300)), + ( + "jpeg-exif6-400x200.jpg", + "image/jpeg", + lambda: jpeg(400, 200, orientation=6), + ready(400, 200, 6), + ), + ( + "jpeg-exif3-300x200.jpg", + "image/jpeg", + lambda: jpeg(300, 200, orientation=3), + ready(300, 200, 3), + ), + ("jpeg-rgb-4000x3000.jpg", "image/jpeg", lambda: jpeg(4000, 3000), ready(4000, 3000)), + ("png-exactly-10mib.png", "image/png", lambda: padded(small, MAX_BYTES), ready(64, 64)), + ( + "png-over-10mib.png", + "image/png", + lambda: padded(small, MAX_BYTES + 1), + {"upload": [413]}, + ), + ( + "png-over-pixels-8001x5000.png", + "image/png", + lambda: png(8001, 5000, blank(8001, 5000, 3), level=9), + {"upload": [413]}, + ), + ( + "jpeg-over-pixels-10000x5000.jpg", + "image/jpeg", + lambda: jpeg(10000, 5000, uniform=True), + {"upload": [413]}, + ), + ("png-truncated.png", "image/png", lambda: truncated_png[: idat + 600], REJECTED_OR_FAILED), + ("png-corrupt-idat.png", "image/png", lambda: bytes(corrupt), REJECTED_OR_FAILED), + ("jpeg-truncated.jpg", "image/jpeg", lambda: jpeg(1000, 750)[:3000], REJECTED_OR_FAILED), + ( + "png-zero-width.png", + "image/png", + zero_width, + {"upload": [400, 415]}, + ), + ( + "not-an-image.png", + "image/png", + lambda: b"plain text, not an image\n" * 8, + {"upload": [415]}, + ), + ("gif-declared-as-png.png", "image/png", lambda: GIF, {"upload": [415]}), + ] + if heavy: + items.append( + ( + "png-at-pixel-limit-8000x5000.png", + "image/png", + lambda: png(8000, 5000, blank(8000, 5000, 3), level=9), + ready(8000, 5000), + ) + ) + return items + + +def zero_width(): + header = struct.pack(">IIBBBBB", 0, 10, 8, 2, 0, 0, 0) + return ( + b"\x89PNG\r\n\x1a\n" + + chunk(b"IHDR", header) + + chunk(b"IDAT", zlib.compress(b"")) + + chunk(b"IEND", b"") + ) + + +def build(out, heavy=False, only=None): + out.mkdir(parents=True, exist_ok=True) + manifest = [] + for name, content_type, make, expect in catalog(heavy): + if only and name not in only: + continue + data = make() + (out / name).write_bytes(data) + manifest.append( + {"file": name, "content_type": content_type, "size_bytes": len(data), "expect": expect} + ) + (out / "manifest.json").write_text( + json.dumps( + {"rules": {"max_bytes": MAX_BYTES, "max_pixels": MAX_PIXELS}, "images": manifest}, + indent=2, + ) + + "\n" + ) + return manifest + + +def main(): + parser = argparse.ArgumentParser( + description="Generate lab5 test images and manifest.json with expected outcomes" + ) + parser.add_argument("--out", default=str(ROOT / "build/lab5-images")) + parser.add_argument( + "--heavy", action="store_true", help="also write a 40 Mpx image at the pixel limit" + ) + parser.add_argument("--only", default="", help="comma separated file names") + args = parser.parse_args() + manifest = build(Path(args.out), args.heavy, {x for x in args.only.split(",") if x}) + for item in manifest: + print(f"{item['file']:36} {item['size_bytes']:>10} {item['expect']['upload']}") + + +if __name__ == "__main__": + main() diff --git a/scripts/persistence-evidence.sh b/scripts/persistence-evidence.sh new file mode 100755 index 00000000..f10ce0c8 --- /dev/null +++ b/scripts/persistence-evidence.sh @@ -0,0 +1,14 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +started() { + for id in $(docker compose ps -aq); do + docker inspect --format '{{index .Config.Labels "com.docker.compose.service"}} {{.Config.Hostname}} {{.State.StartedAt}} {{.State.Status}}' "$id" + done | sort +} +echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse --short HEAD)" +echo "== before: service hostname StartedAt status" +started +make --no-print-directory test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL="${1:-http://localhost:8080}" +echo "== after: service hostname StartedAt status" +started diff --git a/scripts/pipeline_evidence.py b/scripts/pipeline_evidence.py new file mode 100644 index 00000000..5ed7b9c3 --- /dev/null +++ b/scripts/pipeline_evidence.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python3 +import argparse +import json +import os +import subprocess +import sys +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) + +from client import Client, check + +COMPOSE = os.environ.get("COMPOSE", "docker compose").split() +PEEK = ( + 'RABBITMQADMIN_USERNAME=messenger-ops RABBITMQADMIN_PASSWORD="$(cat /run/secrets/' + 'rabbitmq_ops_password)" exec rabbitmqadmin -V messenger ' + "--table-style ascii get messages -q attachments.process -a ack_requeue_true" +) +QUEUES = "rabbitmqctl -q list_queues -p messenger name messages_ready messages_unacknowledged" + + +def compose(*argv): + done = subprocess.run( + [*COMPOSE, *argv], cwd=ROOT, capture_output=True, text=True, timeout=180, check=False + ) + check(done.returncode == 0, f"{' '.join(argv[:3])}: {done.stderr.strip()[-300:]}") + return done.stdout + + +def ready_messages(): + for line in compose("exec", "-T", "broker", *QUEUES.split()).splitlines(): + name, ready, _ = [*line.split("\t"), "", ""][:3] + if name == "attachments.process": + return int(ready) + return 0 + + +def say(line): + print(line, flush=True) + + +def stop(service): + began = time.monotonic() + compose("stop", "-t", "10", service) + seconds = time.monotonic() - began + container = compose("ps", "-aq", service).split()[0] + code = subprocess.run( + ["docker", "inspect", "-f", "{{.State.ExitCode}}", container], + capture_output=True, + text=True, + check=True, + ).stdout.strip() + say(f"$ docker compose stop -t 10 {service} -> {seconds:.1f} s, exit code {code}") + check(seconds <= 10 and code == "0", f"{service} did not stop cleanly on SIGTERM") + + +def start(service): + compose("up", "-d", "--no-deps", "--wait", "--wait-timeout", "90", service) + say(f"$ docker compose up -d --no-deps --wait {service} -> healthy") + + +def main(): + parser = argparse.ArgumentParser(description="Lab5 stand evidence: queue, message, SIGTERM") + parser.add_argument("--base-url", default=os.environ.get("BASE_URL", "http://localhost:8080")) + args = parser.parse_args() + client = Client(args.base_url) + alice, bob, _, chat = client.bootstrap() + say(f"date: {time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())}") + stop("worker") + attachment, _ = client.attachment(alice, chat) + say(f"upload -> 202 status={attachment['status']} id={attachment['id']}") + path = f"/api/v1/attachments/{attachment['id']}" + for variant in ("original", "thumbnail"): + client.call("GET", f"{path}/content?variant={variant}", 409, actor=bob) + say(f"GET content?variant={variant} while worker is stopped -> 409") + deadline = time.monotonic() + 30 + while ready_messages() < 1: + check(time.monotonic() < deadline, "the job did not reach the broker") + time.sleep(0.5) + say("$ rabbitmqctl list_queues -p messenger name messages_ready messages_unacknowledged") + say(compose("exec", "-T", "broker", *QUEUES.split()).rstrip()) + say("$ rabbitmqadmin get messages -q attachments.process -a ack_requeue_true") + peeked = compose("exec", "-T", "broker", "sh", "-c", PEEK) + say(peeked.rstrip()) + body = json.loads(peeked[peeked.index('{"') : peeked.index("}", peeked.index('{"')) + 1]) + say(f"payload keys: {sorted(body)}") + check(body["attachment_id"] == attachment["id"], "the queued job is for another attachment") + stop("relay") + start("relay") + start("worker") + ready = client.wait_ready(bob, attachment) + names = [(v["name"], v["width"], v["height"]) for v in ready["variants"]] + say(f"after worker start -> status={ready['status']} variants={names}") + deadline = time.monotonic() + 30 + while ready_messages(): + check(time.monotonic() < deadline, "the queue did not drain") + time.sleep(0.5) + say(compose("exec", "-T", "broker", *QUEUES.split()).rstrip()) + say("PASS: pipeline stand evidence") + + +if __name__ == "__main__": + main() diff --git a/scripts/pipeline_suite.py b/scripts/pipeline_suite.py new file mode 100644 index 00000000..1388f7fe --- /dev/null +++ b/scripts/pipeline_suite.py @@ -0,0 +1,1033 @@ +#!/usr/bin/env python3 +import argparse +import hashlib +import json +import os +import shlex +import struct +import subprocess +import sys +import time +import urllib.request +import uuid +from datetime import datetime, timezone +from pathlib import Path +from urllib.parse import quote + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) + +import make_images +from client import Client, Failure, check +from image_probe import jpeg_size + +FIXTURE = ROOT / "tests/fixtures/sample.png" +HEAVY = "png-at-pixel-limit-8000x5000.png" +EXCHANGE, ROUTING_KEY = "attachments", "attachment.process.v1" +QUEUE, DLQ = "attachments.process", "attachments.process.dlq" +RABBITMQADMIN = ( + 'RABBITMQADMIN_USERNAME=messenger-ops RABBITMQADMIN_PASSWORD="$(cat /run/secrets/' + 'rabbitmq_ops_password)" exec rabbitmqadmin --non-interactive -V messenger "$@"' +) +PSQL = 'exec psql -X -q -A -t -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB" "$@"' +ERRORS = (Failure, OSError, ValueError, KeyError, subprocess.TimeoutExpired) +SCENARIOS = ( + ("smoke_pipeline", 3, "upload, ready, variants match metadata"), + ("worker_down_broker_restart", 3, "worker down, API and broker restart"), + ("duplicate_after_ready", 3, "redelivery of a finished job"), + ("poison_image", 3, "permanent error without retries"), + ("s3_outage_recovers", 3, "transient error with retry"), + ("retry_exhausted", 3, "transient error until JOB_MAX_ATTEMPTS"), + ("image_matrix", 4, "limits and variants"), + ("crash_commit_publish", 4, "after COMMIT, before publish"), + ("crash_publish_confirm", 4, "after publish, before confirm"), + ("crash_after_s3_before_commit", 4, "after S3 write, before COMMIT"), + ("crash_after_commit_before_ack", 4, "after COMMIT, before ack"), + ("lease_expiry_fencing", 4, "stalled worker, lease expiry"), + ("replay_failed", 4, "operator replay"), + ("malformed_message_dead_lettered", 4, "message that is not a job goes to the DLQ"), + ("worker_limits", 4, "render time and memory limits, 40 Mpx in the worker limit"), + ("retry_backoff_jitter", 4, "backoff with equal jitter between attempts"), + ("failed_retention", 4, "failed staging kept until retention, then replay expires"), + ("crash_before_side_effect", 5, "after claim, before side effects"), + ("worker_kill_mid_task", 5, "SIGKILL during processing"), + ("duplicate_in_flight", 5, "two copies of one job in flight"), + ("reorder_attempts", 5, "attempt 2 delivered before attempt 1"), + ("reorder_versions", 5, "v2 delivered before v1"), + ("stale_after_new_version", 5, "v1 job after v2 is published"), +) +QUERIES = """ +attachment: + SELECT status, published_version, error_code, staging_key FROM attachments WHERE id = :'id' + +jobs: + SELECT job_id, pipeline_version, status, attempt FROM jobs + WHERE attachment_id = :'id' ORDER BY pipeline_version + +outbox: + SELECT o.payload, o.published_at IS NOT NULL AS published FROM outbox o JOIN jobs j USING (job_id) + WHERE j.attachment_id = :'id' ORDER BY o.id + +variants: + SELECT pipeline_version, name, sha256, size_bytes, object_key FROM attachment_variants + WHERE attachment_id = :'id' ORDER BY pipeline_version, name + +events: + SELECT kind, attempt FROM job_events WHERE attachment_id = :'id' ORDER BY id + +backoff: + SELECT attempt, (detail->>'delay_seconds')::float8 AS delay FROM job_events + WHERE attachment_id = :'id' AND kind = 'retry_scheduled' ORDER BY id + +nonterminal: + SELECT id, status FROM attachments + WHERE id = ANY(:'ids'::uuid[]) AND status NOT IN ('ready', 'failed') + +known: + SELECT id FROM attachments WHERE id = ANY(:'ids'::uuid[]) + +duplicate_effects: + SELECT attachment_id, pipeline_version, name, count(*) AS copies FROM attachment_variants + WHERE attachment_id = ANY(:'ids'::uuid[]) GROUP BY 1, 2, 3 HAVING count(*) > 1 + +attempts: + SELECT job_id, attempt FROM jobs WHERE attachment_id = ANY(:'ids'::uuid[]) AND attempt > :max + +pending_outbox: + SELECT o.job_id FROM outbox o JOIN jobs j USING (job_id) + WHERE j.attachment_id = ANY(:'ids'::uuid[]) AND o.published_at IS NULL + +unpublished_version: + SELECT a.id, a.published_version, max(j.pipeline_version) AS done FROM attachments a + JOIN jobs j ON j.attachment_id = a.id AND j.status = 'done' WHERE a.id = ANY(:'ids'::uuid[]) + GROUP BY a.id, a.published_version HAVING a.published_version <> max(j.pipeline_version) + +staging_left: + SELECT id FROM attachments + WHERE id = ANY(:'ids'::uuid[]) AND status = 'ready' AND staging_key IS NOT NULL + +object_keys: + SELECT object_key AS key FROM attachment_variants + UNION ALL SELECT original_key FROM attachments WHERE original_key IS NOT NULL + UNION ALL SELECT staging_key FROM attachments WHERE staging_key IS NOT NULL + +columns: + SELECT table_name, column_name FROM information_schema.columns + WHERE table_schema = current_schema() +""" +SQL = { + name.strip(): " ".join(body.split()) + for name, body in (block.split(":\n", 1) for block in QUERIES.strip().split("\n\n")) +} +REQUIRED_COLUMNS = { + "attachments": "id status published_version error_code staging_key original_key", + "attachment_variants": "attachment_id pipeline_version name sha256 size_bytes object_key", + "jobs": "job_id attachment_id pipeline_version status attempt", + "outbox": "job_id payload published_at", + "job_events": "job_id attachment_id kind attempt", +} + + +def now(): + return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%fZ") + + +def run(cmd, stdin=None, env=None, timeout=180): + done = subprocess.run( + cmd, input=stdin, capture_output=True, text=True, env=env, timeout=timeout, check=False + ) + if done.returncode: + raise Failure(f"{' '.join(cmd[:7])}: exit {done.returncode}: {done.stderr.strip()[-400:]}") + return done.stdout + + +def poll(probe, timeout, what, interval=0.5): + deadline = time.monotonic() + timeout + while True: + value = probe() + if value: + return value + if time.monotonic() > deadline: + raise Failure(f"timeout {timeout}s waiting for {what}") + time.sleep(interval) + + +def http_ready(base_url): + try: + with urllib.request.urlopen(base_url.rstrip("/") + "/health/ready", timeout=3) as response: + return response.status == 200 + except OSError: + return False + + +def image_size(data, content_type): + if content_type == "image/jpeg": + return jpeg_size(data) + if content_type == "image/png": + return struct.unpack(">II", data[16:24]) + check(data[:4] == b"RIFF" and data[8:12] == b"WEBP", "optimized is not WebP") + if data[12:16] == b"VP8 ": + width, height = struct.unpack("> 14) & 0x3FFF) + 1 + return int.from_bytes(data[24:27], "little") + 1, int.from_bytes(data[27:30], "little") + 1 + + +class Stack: + def __init__(self, args): + self.args = args + self.base = shlex.split(args.compose) + self.faults = shlex.split(args.compose_faults) + + def dc(self, *argv, faults=False, stdin=None, env=None, timeout=180): + command = [*(self.faults if faults else self.base), *argv] + return run(command, stdin, dict(os.environ, **(env or {})), timeout) + + def up(self, *services, faults=False, env=None): + wait = ["up", "-d", "--no-deps", "--no-recreate", "--wait", "--wait-timeout", "90"] + if faults or env: + wait.remove("--no-recreate") + self.dc(*wait, *services, faults=faults, env=env) + + def launch(self, service, point="", **overrides): + variable = "WORKER_FAULT_POINT" if service == self.args.worker else "RELAY_FAULT_POINT" + self.up(service, faults=True, env={variable: point, **overrides}) + + def cgroup(self, service, name): + return self.dc("exec", "-T", service, "cat", "/sys/fs/cgroup/" + name).strip() + + def restore(self): + self.dc("up", "-d", "--no-deps", "--wait", self.args.relay, self.args.worker) + + def stop(self, *services): + self.dc("stop", "-t", "10", *services) + + def kill(self, service): + self.dc("kill", "-s", "KILL", service) + + def restart(self, *services): + self.dc("restart", "-t", "20", *services) + self.up(*services) + + def state(self, service): + text = self.dc("ps", "-a", "--format", "json", service).strip() + rows = ( + json.loads(text) if text.startswith("[") else [json.loads(x) for x in text.split("\n")] + ) + check(len(rows) == 1, "expected one container for " + service) + return rows[0]["State"], rows[0].get("ExitCode") + + def wait_exit(self, service, timeout=60): + def probe(): + state, code = self.state(service) + return (code,) if state in ("exited", "dead") else None + + return poll(probe, timeout, service + " exit")[0] + + def sql(self, name, **params): + query = f"SELECT coalesce(json_agg(q), '[]'::json) FROM ({SQL[name]}) q;\n" + variables = [x for key, value in params.items() for x in ("-v", f"{key}={value}")] + out = self.dc( + "exec", "-T", self.args.postgres, "sh", "-c", PSQL, "sh", *variables, stdin=query + ) + return json.loads(out) + + def queues(self): + columns = "name messages_ready messages_unacknowledged --formatter json" + listing = f"exec -T {self.args.broker} rabbitmqctl -q list_queues -p messenger {columns}" + rows = json.loads(self.dc(*listing.split())) + return {q["name"]: (q["messages_ready"], q["messages_unacknowledged"]) for q in rows} + + def rabbitmqadmin(self, *argv, stdin=None): + return self.dc( + "exec", "-T", self.args.broker, "sh", "-c", RABBITMQADMIN, "sh", *argv, stdin=stdin + ) + + def publish(self, payload): + properties = {"delivery_mode": 2, "content_type": "application/json", "type": ROUTING_KEY} + properties["message_id"] = payload["job_id"] + command = f"publish message -e {EXCHANGE} -k {ROUTING_KEY} -f - -p".split() + out = self.rabbitmqadmin(*command, json.dumps(properties), stdin=json.dumps(payload)) + check("routed successfully" in out, "broker did not route the injected job") + + def admin(self, *argv): + command = shlex.split(self.args.admin) + list(argv) + out = self.dc("run", "--rm", "--no-deps", "-T", self.args.worker, *command) + return json.loads(out.strip().splitlines()[-1]) + + +class Api: + def __init__(self, args): + self.base_url = args.base_url + self.client = Client(args.base_url, args.ca_file) + self.alice, self.bob, self.eve, self.chat = self.client.bootstrap() + self.accepted, self.traces, self.final = [], {}, {} + + def upload(self, data, kind="image/png", name="sample.png", expected=202, final=("ready",)): + path = f"/api/v1/chats/{self.chat['id']}/attachments?filename={quote(name)}" + allowed = [expected] if isinstance(expected, int) else expected + headers = {"Content-Type": kind} + response = self.client.call( + "POST", path, allowed, actor=self.alice, raw=data, headers=headers, schema="Attachment" + ) + if response["status"] != 202: + return None + body = response["json"] + snapshot = ( + body["status"] == "queued" and body["variants"] == [] and body["error_code"] is None + ) + check(snapshot, "202 is not a queued snapshot") + same = body["sha256"] == hashlib.sha256(data).hexdigest() and body["size_bytes"] == len( + data + ) + check(same, "202 checksum or size differs from the uploaded bytes") + self.accepted.append(body["id"]) + self.final[body["id"]] = set(final) + self.traces[body["id"]] = ["queued"] + return body + + def fixture(self, **kwargs): + return self.upload(FIXTURE.read_bytes(), **kwargs) + + def get(self, ident): + path = "/api/v1/attachments/" + ident + body = self.client.call("GET", path, actor=self.bob, schema="Attachment")["json"] + self.traces.setdefault(ident, []).append(body["status"]) + return body + + def wait(self, ident, final=("ready",), timeout=90): + def probe(): + body = self.get(ident) + failed = body["status"] == "failed" and "failed" not in final + check(not failed, f"attachment reached failed: {body['error_code']}") + return body if body["status"] in final else None + + return poll(probe, timeout, f"{ident} {'/'.join(final)}", 0.3) + + def content(self, ident, variant=None, expected=200): + path = f"/api/v1/attachments/{ident}/content" + (f"?variant={variant}" if variant else "") + return self.client.call("GET", path, expected, actor=self.bob) + + def message(self, text, attachments=None, expected=201): + if expected == 201: + return self.client.message(self.alice, self.chat, text, attachments=attachments)["json"] + body = {"text": text, "attachment_ids": attachments or []} + headers = {"Idempotency-Key": os.urandom(8).hex()} + path = f"/api/v1/chats/{self.chat['id']}/messages" + return self.client.call( + "POST", path, expected, actor=self.alice, body=body, headers=headers + ) + + +class Suite: + def __init__(self, args): + self.args = args + self.stack = Stack(args) + self.api = None + self.samples = {} + self.dead_letters = 0 + + def scenarios(self): + return [(name, level, window, getattr(self, name)) for name, level, window in SCENARIOS] + + def drained(self, timeout=60): + poll(lambda: self.stack.queues()[QUEUE] == (0, 0), timeout, "main queue drained") + + def queued(self, count, timeout=30): + poll(lambda: self.stack.queues()[QUEUE][0] >= count, timeout, f"{count} ready messages") + + def record(self, ident): + row = self.stack.sql("attachment", id=ident)[0] + self.samples.setdefault(ident, []).append(row["published_version"]) + self.api.traces.setdefault(ident, []).append(row["status"]) + return row + + def snapshot(self, ident): + body = self.api.get(ident) + public = sorted(body["variants"], key=lambda v: v["name"]) + published = self.record(ident)["published_version"] + rows = self.stack.sql("variants", id=ident) + return {"status": body["status"], "public": public, "rows": rows, "published": published} + + def payload(self, ident, version=1): + rows = self.stack.sql("outbox", id=ident) + found = [r["payload"] for r in rows if r["payload"]["pipeline_version"] == version] + check(bool(found), f"no outbox payload for v{version}") + return found[-1] + + def verify(self, ready, expect=None): + names = sorted(v["name"] for v in ready["variants"]) + check(len(names) == len(set(names)), "duplicate variant names in API") + missing = set(self.args.expect_variants.split(",")) - set(names) + check(not missing, f"missing variants {sorted(missing)}") + original = self.api.content(ready["id"]) + data = original["bytes"] + same = ( + hashlib.sha256(data).hexdigest() == ready["sha256"] and len(data) == ready["size_bytes"] + ) + check(same, "original bytes differ from metadata") + check( + original["headers"].get_content_type() == ready["content_type"], "original type differs" + ) + for variant in ready["variants"]: + name = variant["name"] + got = self.api.content(ready["id"], name) + data, kind = got["bytes"], got["headers"].get_content_type() + check(kind == variant["content_type"], name + " content type differs") + same = len(data) == variant["size_bytes"] + check( + same and hashlib.sha256(data).hexdigest() == variant["sha256"], + name + " bytes differ", + ) + size = (variant["width"], variant["height"]) + check(tuple(image_size(data, kind)) == size, name + " real size differs from metadata") + wanted = (expect or {}).get(name) + slack = 1 if name == "optimized" else 0 + fits = not wanted or all(abs(a - b) <= slack for a, b in zip(wanted, size)) + check(fits, f"{name} is {size}, expected {wanted}") + return names + + def crash(self, service, point, observe): + self.drained() + self.stack.launch(service, point) + ident = self.api.fixture()["id"] + code = self.stack.wait_exit(service) + check(code == 137, f"{service} exited with {code}, expected 137 at {point}") + seen = observe(ident) + self.stack.launch(service) + ready = self.api.wait(ident, timeout=self.args.recovery_timeout) + poll(lambda: all(r["published"] for r in self.stack.sql("outbox", id=ident)), 30, "outbox") + self.drained() + self.verify(ready) + variants = self.stack.sql("variants", id=ident) + return { + "attachment_id": ident, + "exit_code": code, + "after_crash": seen, + "variants": variants, + } + + def smoke_pipeline(self): + att = self.api.fixture() + ready = self.api.wait(att["id"]) + return { + "attachment_id": att["id"], + "variants": self.verify(ready, {"thumbnail": [256, 160]}), + } + + def worker_down_broker_restart(self): + steps = [] + + def mark(step, **extra): + steps.append({"at": now(), "step": step, **extra}) + + self.stack.stop(self.args.worker) + mark("worker stopped") + att = self.api.fixture() + mark("upload 202 queued", attachment_id=att["id"]) + self.api.content(att["id"], expected=409) + mark("content 409 while queued") + message = self.api.message("", [att["id"]]) + self.api.message("text messages still work") + path = f"/api/v1/chats/{self.api.chat['id']}/messages?limit=5" + self.api.client.call("GET", path, actor=self.api.bob, schema="MessagePage") + mark("message with attachment 201, other messages work", message_id=message["id"]) + self.queued(1) + mark("job is in the broker queue", queues=self.stack.queues()) + self.stack.restart(*self.args.api, self.args.relay) + poll(lambda: http_ready(self.api.base_url), 90, "API ready") + mark("API replicas and relay restarted") + self.stack.restart(self.args.broker) + self.queued(1) + mark("broker restarted with its volume, job survived", queues=self.stack.queues()) + self.stack.launch(self.args.worker) + mark("worker started") + ready = self.api.wait(att["id"]) + mark("ready", variants=self.verify(ready)) + return {"attachment_id": att["id"], "timeline": steps} + + def duplicate_after_ready(self): + ident = self.api.fixture()["id"] + self.api.wait(ident) + self.drained() + before = self.snapshot(ident) + self.stack.publish(self.payload(ident)) + self.drained() + time.sleep(1) + check(self.snapshot(ident) == before, "duplicate delivery changed the published result") + return {"attachment_id": ident, "events": self.stack.sql("events", id=ident)} + + def poison_image(self): + data = make_images.png(320, 200, make_images.gradient(320, 200, 3)) + broken = data[: data.index(b"IDAT") + 600] + att = self.api.upload(broken, expected=[202, 415], final=("failed",)) + if att is None: + return {"rejected_by_api": 415} + failed = self.api.wait(att["id"], ("failed",)) + check( + failed["error_code"] == "invalid_image", f"unexpected error_code {failed['error_code']}" + ) + jobs = self.stack.sql("jobs", id=att["id"]) + check(all(j["attempt"] == 1 for j in jobs), "permanent error was retried") + self.api.content(att["id"], expected=409) + self.api.message("", [att["id"]], expected=409) + return {"attachment_id": att["id"], "error_code": failed["error_code"], "jobs": jobs} + + def s3_outage_recovers(self): + self.stack.stop(self.args.worker) + ident = self.api.fixture()["id"] + self.queued(1) + self.stack.stop(self.args.s3) + try: + self.stack.launch(self.args.worker) + poll( + lambda: any(j["attempt"] > 1 for j in self.stack.sql("jobs", id=ident)), 60, "retry" + ) + status = self.record(ident)["status"] + check(status in ("queued", "processing"), "transient error produced " + status) + finally: + self.stack.up(self.args.s3) + poll(lambda: http_ready(self.api.base_url), 90, "API ready") + self.verify(self.api.wait(ident, timeout=self.args.recovery_timeout)) + return {"attachment_id": ident, "jobs": self.stack.sql("jobs", id=ident)} + + def exhaust(self): + self.drained() + self.stack.launch(self.args.worker, "transient_error") + ident = self.api.fixture(final=("failed", "ready"))["id"] + failed = self.api.wait(ident, ("failed",), timeout=self.args.recovery_timeout) + self.stack.launch(self.args.worker) + jobs = self.stack.sql("jobs", id=ident) + check(failed["error_code"] == "retry_exhausted", f"unexpected {failed['error_code']}") + check(max(j["attempt"] for j in jobs) == self.args.max_attempts, "attempts != max") + return ident, jobs + + def retry_exhausted(self): + ident, jobs = self.exhaust() + self.api.final[ident] = {"failed"} + self.api.content(ident, expected=409) + self.api.message("", [ident], expected=409) + return {"attachment_id": ident, "jobs": jobs} + + def replay_failed(self): + ident, _ = self.exhaust() + self.api.final[ident] = {"ready"} + first = self.stack.admin("replay", "--attachment-id", ident) + self.verify(self.api.wait(ident, timeout=self.args.recovery_timeout)) + self.drained() + before = self.snapshot(ident) + second = self.stack.admin("replay", "--attachment-id", ident) + self.drained() + check(self.snapshot(ident) == before, "replay of a ready attachment changed the result") + events = self.stack.sql("events", id=ident) + check(any("replay" in e["kind"] for e in events), "replay left no audit event") + return { + "attachment_id": ident, + "replay": first, + "replay_of_ready": second, + "events": events, + } + + def malformed_message_dead_lettered(self): + before = self.stack.queues()[DLQ][0] + self.stack.publish({"schema_version": 2, "job_id": str(uuid.uuid4())}) + poll(lambda: self.stack.queues()[DLQ][0] > before, 30, "dead letter") + self.dead_letters += 1 + return {"dead_letters": self.stack.queues()[DLQ][0] - before} + + def limited(self, data, **overrides): + self.stack.launch(self.args.worker, **overrides) + att = self.api.upload(data, name=HEAVY, final=("failed",)) + failed = self.api.wait(att["id"], ("failed",), self.args.recovery_timeout) + check(failed["error_code"] == "processing_failed", f"limit gave {failed['error_code']}") + jobs = self.stack.sql("jobs", id=att["id"]) + check([j["attempt"] for j in jobs] == [1], "a limit breach was retried") + state, _ = self.stack.state(self.args.worker) + check(state == "running", "worker died on a limit breach: " + state) + return {"attachment_id": att["id"], "error_code": failed["error_code"], "worker": state} + + def worker_limits(self): + folder = Path(self.args.images) / "limits" + make_images.build(folder, True, {HEAVY}) + data = (folder / HEAVY).read_bytes() + self.drained() + timed = self.limited(data, FAULT_RENDER_TIMEOUT_SECONDS=str(self.args.tight_timeout)) + memory = self.limited(data, FAULT_RENDER_MEMORY_MB=str(self.args.tight_memory_mb)) + self.stack.launch(self.args.worker) + att = self.api.upload(data, name=HEAVY) + ready = self.api.wait(att["id"], timeout=self.args.recovery_timeout) + sizes = {"thumbnail": [256, 160], "crop": [128, 128], "optimized": [1280, 800]} + names = self.verify(ready, sizes) + peak = int(self.stack.cgroup(self.args.worker, "memory.peak")) + limit = int(self.stack.cgroup(self.args.worker, "memory.max")) + check(peak < limit, f"worker peak {peak} reached its limit {limit}") + return { + "image": {"file": HEAVY, "bytes": len(data), "pixels": 8000 * 5000}, + "render_timeout": {"seconds": self.args.tight_timeout, **timed}, + "render_memory": {"mib": self.args.tight_memory_mb, **memory}, + "default_limits": { + "attachment_id": att["id"], + "variants": names, + "memory_peak_mib": round(peak / 2**20, 1), + "memory_max_mib": round(limit / 2**20, 1), + }, + } + + def retry_backoff_jitter(self): + ident, jobs = self.exhaust() + self.api.final[ident] = {"failed"} + rows = self.stack.sql("backoff", id=ident) + check([r["attempt"] for r in rows] == [2, 3, 4, 5], f"unexpected retries {rows}") + shares = [] + for row in rows: + ceiling = min(60.0, self.args.backoff_base * 2 ** (row["attempt"] - 2)) + share = row["delay"] / ceiling + check(0.5 <= share <= 1.0, f"delay {row['delay']} outside [{ceiling / 2}, {ceiling}]") + shares.append(round(share, 3)) + check(len(set(shares)) > 1, f"no jitter: shares {shares}") + return {"attachment_id": ident, "delays": rows, "share_of_ceiling": shares, "jobs": jobs} + + def failed_retention(self): + ident, _ = self.exhaust() + self.api.final[ident] = {"failed"} + kept = self.record(ident)["staging_key"] + check(kept is not None, "failed staging was removed before the retention period") + try: + self.stack.launch(self.args.relay, FAULT_STAGING_RETENTION_SECONDS="1") + poll(lambda: self.record(ident)["staging_key"] is None, 30, "failed staging cleanup") + finally: + self.stack.launch(self.args.relay) + replay = self.stack.admin("replay", "--attachment-id", ident) + check(replay["result"] == "expired", f"replay after retention gave {replay}") + check(self.api.get(ident)["status"] == "failed", "expired replay changed the status") + return {"attachment_id": ident, "staging_before": "kept", "replay": replay} + + def image_matrix(self): + folder = Path(self.args.images) + results = [] + for item in make_images.build(folder, self.args.heavy): + expect, name = item["expect"], item["file"] + ready = expect.get("final") == "ready" + data = (folder / name).read_bytes() + final = ("ready",) if ready else ("failed", "ready") + att = self.api.upload(data, item["content_type"], name, expect["upload"], final) + row = {"file": name, "http": 202 if att else "rejected"} + if att: + done = self.api.wait(att["id"], ("ready", "failed"), self.args.recovery_timeout) + row.update(status=done["status"], error_code=done["error_code"]) + if ready: + check(done["status"] == "ready", name + " did not become ready") + row["variants"] = self.verify(done, expect["variants"]) + else: + self.api.final[att["id"]] = {"failed"} + code = expect.get("error_code") + check(done["error_code"] == code, f"{name} should fail with {code}") + results.append(row) + return {"images": results} + + def crash_commit_publish(self): + def observe(ident): + outbox = self.stack.sql("outbox", id=ident) + check(outbox and not any(r["published"] for r in outbox), "outbox already published") + check(self.stack.queues()[QUEUE] == (0, 0), "job reached the broker before publish") + return {"outbox_published": False, "status": self.record(ident)["status"]} + + return self.crash(self.args.relay, "after_commit_before_publish", observe) + + def crash_publish_confirm(self): + def observe(ident): + outbox = [r["published"] for r in self.stack.sql("outbox", id=ident)] + check(outbox == [False], "outbox row was marked before the crash point") + return {"outbox_published": outbox, "queues": self.stack.queues()} + + result = self.crash(self.args.relay, "after_publish_before_confirm", observe) + kinds = [e["kind"] for e in self.stack.sql("events", id=result["attachment_id"])] + check("duplicate_noop" in kinds, "the republished copy was not a no-op: " + str(kinds)) + return {**result, "events": kinds} + + def crash_after_s3_before_commit(self): + def observe(ident): + check(not self.stack.sql("variants", id=ident), "variants committed before the crash") + return {"status": self.record(ident)["status"], "variant_rows": 0} + + return self.crash(self.args.worker, "after_s3_write_before_commit", observe) + + def crash_after_commit_before_ack(self): + def observe(ident): + row = self.record(ident) + check(row["status"] == "ready", "result was not committed before the crash point") + variants = self.stack.sql("variants", id=ident) + return {"status": row["status"], "variants": variants, "queues": self.stack.queues()} + + result = self.crash(self.args.worker, "after_commit_before_ack", observe) + same = result["variants"] == result["after_crash"]["variants"] + check(same, "redelivery after COMMIT changed variants") + return result + + def crash_before_side_effect(self): + def observe(ident): + return { + "status": self.record(ident)["status"], + "jobs": self.stack.sql("jobs", id=ident), + } + + return self.crash(self.args.worker, "after_claim_before_side_effect", observe) + + def lease_expiry_fencing(self): + self.drained() + self.stack.launch(self.args.worker, "stall_after_claim") + ident = self.api.fixture()["id"] + self.verify(self.api.wait(ident, timeout=self.args.recovery_timeout)) + before = self.snapshot(ident) + time.sleep(self.args.stall_seconds + 3) + after = self.snapshot(ident) + self.stack.launch(self.args.worker) + check(before == after, "stalled worker overwrote the result after its lease expired") + jobs, events = self.stack.sql("jobs", id=ident), self.stack.sql("events", id=ident) + check(any(e["kind"] == "fenced" for e in events), "stalled holder was not fenced") + return {"attachment_id": ident, "jobs": jobs, "events": events} + + def worker_kill_mid_task(self): + self.drained() + self.stack.launch(self.args.worker, "stall_after_claim") + ident = self.api.fixture()["id"] + poll(lambda: self.record(ident)["status"] == "processing", 30, "processing") + self.stack.kill(self.args.worker) + code = self.stack.wait_exit(self.args.worker) + self.stack.launch(self.args.worker) + ready = self.api.wait(ident, timeout=self.args.recovery_timeout) + self.drained() + self.verify(ready) + return {"attachment_id": ident, "exit_code": code, "jobs": self.stack.sql("jobs", id=ident)} + + def duplicate_in_flight(self): + self.stack.stop(self.args.worker) + ident = self.api.fixture()["id"] + self.queued(1) + self.stack.publish(self.payload(ident)) + self.queued(2) + self.stack.launch(self.args.worker) + ready = self.api.wait(ident) + self.drained() + self.verify(ready) + variants, events = self.stack.sql("variants", id=ident), self.stack.sql("events", id=ident) + return {"attachment_id": ident, "variants": variants, "events": events} + + def reorder_attempts(self): + self.stack.stop(self.args.worker, self.args.relay) + ident = self.api.fixture()["id"] + original = self.payload(ident) + self.stack.publish({**original, "attempt": original["attempt"] + 1}) + self.queued(1) + self.stack.launch(self.args.relay) + self.queued(2) + self.stack.launch(self.args.worker) + ready = self.api.wait(ident) + self.drained() + self.verify(ready) + jobs, events = self.stack.sql("jobs", id=ident), self.stack.sql("events", id=ident) + return {"attachment_id": ident, "jobs": jobs, "events": events} + + def published(self, ident, version): + poll( + lambda: self.record(ident)["published_version"] >= version, + self.args.recovery_timeout, + f"published v{version}", + ) + + def reprocess(self, ident): + return self.stack.admin("reprocess", "--attachment-id", ident, "--pipeline-version", "2") + + def reorder_versions(self): + self.stack.stop(self.args.worker, self.args.relay) + ident = self.api.fixture()["id"] + self.reprocess(ident) + self.stack.publish(self.payload(ident, 2)) + self.queued(1) + self.stack.launch(self.args.relay) + self.queued(3) + self.stack.launch(self.args.worker) + self.published(ident, 2) + self.api.wait(ident) + self.drained() + final = self.snapshot(ident) + check(final["published"] == 2, f"published_version rolled back to {final['published']}") + served = {v["name"]: v["sha256"] for v in final["public"]} + stored = {r["name"]: r["sha256"] for r in final["rows"] if r["pipeline_version"] == 2} + check(served == stored, "API serves variants of a version that is not published") + self.verify(self.api.get(ident)) + return {"attachment_id": ident, "jobs": self.stack.sql("jobs", id=ident)} + + def stale_after_new_version(self): + ident = self.api.fixture()["id"] + self.api.wait(ident) + self.reprocess(ident) + self.published(ident, 2) + self.drained() + before = self.snapshot(ident) + self.stack.publish(self.payload(ident, 1)) + self.drained() + time.sleep(1) + after = self.snapshot(ident) + check(before == after and after["published"] == 2, "stale v1 job changed the v2 result") + return {"attachment_id": ident, "published_version": after["published"]} + + def invariants(self, dead_before): + ids = self.api.accepted + array = "{" + ",".join(ids) + "}" + if not ids: + return [{"id": "accepted_work", "result": "FAIL", "details": "no upload got 202"}] + + def empty(name, message, **params): + rows = self.stack.sql(name, ids=array, **params) + check(not rows, f"{message}: {rows[:5]}") + return {"rows": 0} + + def terminal(): + empty("nonterminal", "non-terminal attachments left") + wrong = [i for i in ids if self.api.get(i)["status"] not in self.api.final[i]] + check(not wrong, "unexpected final status: " + ", ".join(wrong)) + return {"accepted": len(ids)} + + def no_rollback(): + traces = self.api.traces.items() + broken = [ + i for i, t in traces if "ready" in t and set(t[t.index("ready") :]) != {"ready"} + ] + check(not broken, "ready rolled back: " + ", ".join(broken)) + return {"traced": len(self.api.traces)} + + def monotonic(): + broken = [i for i, values in self.samples.items() if values != sorted(values)] + check(not broken, "published_version went backwards: " + ", ".join(broken)) + empty("unpublished_version", "published_version differs from the newest done job") + return {"sampled": len(self.samples)} + + def served(): + bodies = [self.api.get(i) for i in ids] + ready = [b for b in bodies if b["status"] == "ready"] + for body in ready: + self.verify(body) + return {"verified": len(ready)} + + def orphans(): + empty("pending_outbox", "unpublished outbox rows") + empty("staging_left", "staging kept after ready") + queues = self.stack.queues() + check(queues[QUEUE] == (0, 0), "messages left in " + QUEUE) + expected = dead_before + self.dead_letters + check(queues[DLQ][0] == expected, f"unexpected dead letters: {queues[DLQ]}") + if not self.args.s3_list: + return {"queues": queues, "s3": "SKIP: --s3-list is not set"} + keys = {r["key"] for r in self.stack.sql("object_keys")} + listing = run(shlex.split(self.args.s3_list)).splitlines() + stray = [k for k in (x.split()[-1] for x in listing if x.strip()) if k not in keys] + check(not stray, f"S3 objects without metadata: {stray[:5]}") + return {"queues": queues, "s3_objects": len(listing)} + + def no_loss(): + known = {r["id"] for r in self.stack.sql("known", ids=array)} + check(known == set(ids), "accepted attachments are missing in PostgreSQL") + return {"accepted": len(ids)} + + claims = [ + ("terminal", "every 202 reaches its expected terminal state", terminal), + ( + "single_effect", + "one variant set per (attachment_id, pipeline_version, variant)", + lambda: empty("duplicate_effects", "duplicate effect rows"), + ), + ("no_ready_rollback", "ready never returns to queued or processing", no_rollback), + ("monotonic_publication", "published_version only moves forward", monotonic), + ("bytes_match_metadata", "sha256, size and real dimensions match served bytes", served), + ( + "bounded_attempts", + "attempt <= JOB_MAX_ATTEMPTS", + lambda: empty("attempts", "attempt above limit", max=self.args.max_attempts), + ), + ("no_orphans", "no pending outbox, messages, dead letters, staging, objects", orphans), + ("no_lost_202", "every 202 is present in PostgreSQL and the API", no_loss), + ] + return [self.claim(name, statement, probe) for name, statement, probe in claims] + + @staticmethod + def claim(name, statement, probe): + try: + return {"id": name, "statement": statement, "result": "PASS", "details": probe()} + except ERRORS as error: + return {"id": name, "statement": statement, "result": "FAIL", "details": str(error)} + + def preflight(self): + def services(): + self.detect_api() + present = set(self.stack.dc("config", "--services").split()) + wanted = {*self.args.api, self.args.relay, self.args.worker, self.args.broker} + wanted |= {self.args.postgres, self.args.s3} + check(wanted <= present, f"missing services {sorted(wanted - present)}") + return sorted(wanted) + + def schema(): + columns = {} + for row in self.stack.sql("columns"): + columns.setdefault(row["table_name"], set()).add(row["column_name"]) + needed = {t: set(c.split()) for t, c in REQUIRED_COLUMNS.items()} + missing = {t: sorted(c - columns.get(t, set())) for t, c in needed.items()} + check(not any(missing.values()), f"missing columns {missing}") + return sorted(needed) + + def broker(): + queues = self.stack.queues() + check({QUEUE, DLQ} <= set(queues), "queues are not declared") + check(QUEUE in self.stack.rabbitmqadmin("list", "queues"), "rabbitmqadmin failed") + return queues + + def faults(): + check("FAULT_POINT" in self.stack.dc("config", faults=True), "no FAULT_POINT in faults") + return self.args.compose_faults + + def api(): + check(http_ready(self.args.base_url), "GET /health/ready is not 200") + return self.args.base_url + + steps = [ + ("compose services", services), + ("api ready", api), + ("postgres schema", schema), + ("broker topology and ops CLI", broker), + ("faults override", faults), + ] + return [self.claim(name, name, probe) for name, probe in steps] + + def main(self): + report = { + "suite": "lab5-pipeline-idempotency", + "started_at": now(), + "config": self.config(), + } + if self.args.preflight: + report["preflight"] = self.preflight() + return self.finish(report, report["preflight"]) + results = [] + try: + self.detect_api() + self.api = Api(self.args) + dead_before = self.stack.queues()[DLQ][0] + self.stack.launch(self.args.relay) + self.stack.launch(self.args.worker) + for name, level, window, scenario in self.scenarios(): + if self.selected(name, level): + results.append(self.attempt(name, level, window, scenario)) + report["invariants"] = self.invariants(dead_before) + except ERRORS as error: + report["error"] = str(error) + finally: + self.stack.up(self.args.s3) + self.stack.restore() + report["scenarios"] = results + return self.finish(report, results + report.get("invariants", [])) + + def attempt(self, name, level, window, scenario): + began = time.monotonic() + print("RUN " + name, flush=True) + try: + outcome = {"result": "PASS", "observed": scenario()} + except ERRORS as error: + outcome = {"result": "FAIL", "error": str(error)} + self.heal() + print(f"{outcome['result']} {name}", flush=True) + seconds = round(time.monotonic() - began, 1) + return {"name": name, "level": level, "window": window, "seconds": seconds, **outcome} + + def heal(self): + try: + self.stack.up(self.args.s3, self.args.broker) + self.stack.launch(self.args.relay) + self.stack.launch(self.args.worker) + except ERRORS as error: + print("heal failed: " + str(error), flush=True) + + def detect_api(self): + if not self.args.api: + names = self.stack.dc("config", "--services").split() + api = [x for x in names if x == "api" or (x[:4] == "api-" and x[4:].isdigit())] + self.args.api = api + check(bool(self.args.api), "no API service: expected api or api-N, or pass --api") + + def selected(self, name, level): + return name in self.args.only.split(",") if self.args.only else level <= self.args.level + + def config(self): + keys = ( + "base_url level only expect_variants max_attempts stall_seconds compose compose_faults" + ) + return {key: getattr(self.args, key) for key in keys.split()} + + def finish(self, report, verdicts): + passed = bool(verdicts) and "error" not in report + report["result"] = ( + "PASS" if passed and all(v["result"] == "PASS" for v in verdicts) else "FAIL" + ) + report["finished_at"] = now() + text = json.dumps(report, ensure_ascii=False, indent=2) + "\n" + if self.args.output: + Path(self.args.output).parent.mkdir(parents=True, exist_ok=True) + Path(self.args.output).write_text(text) + else: + print(text) + print(f"{report['result']}: lab5 pipeline suite, report {self.args.output or 'stdout'}") + return 0 if report["result"] == "PASS" else 1 + + +def arguments(): + env = os.environ.get + faults = "docker compose -f compose.yaml -f infra/compose.faults.yaml" + parser = argparse.ArgumentParser(description="Lab5 pipeline fault injection with invariants") + parser.add_argument("--base-url", default=env("BASE_URL", "http://localhost:8080")) + parser.add_argument("--ca-file", default=env("CA_FILE")) + parser.add_argument("--compose", default=env("COMPOSE", "docker compose")) + parser.add_argument("--compose-faults", default=env("COMPOSE_FAULTS", faults)) + parser.add_argument("--admin", default=env("ADMIN_CLI", "python -m messenger.admin")) + parser.add_argument("--api", nargs="+", default=env("API_SERVICES", "").split()) + parser.add_argument("--worker", default="worker") + parser.add_argument("--relay", default="relay") + parser.add_argument("--broker", default="broker") + parser.add_argument("--postgres", default="postgres") + parser.add_argument("--s3", default="s3") + parser.add_argument("--s3-list", default=env("S3_LIST_CMD", ""), help="prints object keys") + parser.add_argument("--level", type=int, choices=[3, 4, 5], default=5) + parser.add_argument("--only", default="", help="comma separated scenario names") + parser.add_argument("--list", action="store_true") + parser.add_argument("--preflight", action="store_true") + parser.add_argument("--expect-variants", default=None) + parser.add_argument("--max-attempts", type=int, default=int(env("JOB_MAX_ATTEMPTS", "5"))) + parser.add_argument( + "--stall-seconds", type=float, default=float(env("FAULT_STALL_SECONDS", "15")) + ) + parser.add_argument("--recovery-timeout", type=float, default=120) + parser.add_argument( + "--backoff-base", type=float, default=float(env("FAULT_BACKOFF_BASE_SECONDS", "1")) + ) + parser.add_argument("--tight-timeout", type=float, default=0.5, help="render seconds") + parser.add_argument("--tight-memory-mb", type=int, default=200, help="render address space") + parser.add_argument("--images", default=str(ROOT / "build/lab5-images")) + parser.add_argument("--heavy", action="store_true") + parser.add_argument("--output", default="") + args = parser.parse_args() + if args.expect_variants is None: + args.expect_variants = "thumbnail" if args.level == 3 else "thumbnail,crop,optimized" + return args + + +def main(): + suite = Suite(arguments()) + if suite.args.list: + for name, level, window, _ in suite.scenarios(): + print(f"{name:32} level {level} {window}") + return 0 + return suite.main() + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/probe.py b/scripts/probe.py new file mode 100755 index 00000000..a624bd0b --- /dev/null +++ b/scripts/probe.py @@ -0,0 +1,436 @@ +#!/usr/bin/env python3 +import argparse +import base64 +import collections +import functools +import json +import os +import subprocess +import sys +import threading +import time +import urllib.error +import urllib.request +import uuid +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +COUNTERS = ("wretr", "wredis", "econ", "eresp", "hrsp_5xx") + + +@functools.cache +def published(port, fallback): + command = ["docker", "compose", "port", "lb", str(port)] + found = subprocess.run(command, cwd=str(ROOT), capture_output=True, text=True, check=False) + return found.stdout.strip() or fallback + + +def base_url(): + return os.environ.get("BASE_URL") or "http://" + published(8080, "localhost:8080") + + +def stats_url(): + return os.environ.get("LB_STATS_URL") or f"http://{published(8404, '127.0.0.1:8404')}/stats;csv" + + +def call(method, path, body=None, headers=None, timeout=2.0): + data = None if body is None else json.dumps(body, ensure_ascii=False).encode() + head = {"Accept": "application/json"} + if data is not None: + head["Content-Type"] = "application/json" + head.update(headers or {}) + request = urllib.request.Request( + base_url().rstrip("/") + path, data=data, headers=head, method=method + ) + try: + with urllib.request.urlopen(request, timeout=timeout) as response: + payload = json.loads(response.read() or b"null") + return response.status, response.headers.get("X-Instance-Id", ""), payload + except urllib.error.HTTPError as error: + try: + payload = json.loads(error.read() or b"null") + except (OSError, ValueError): + payload = None + return error.code, error.headers.get("X-Instance-Id", ""), payload + except (OSError, ValueError): + return 0, "", None + + +def emit(report, out=None): + text = json.dumps(report, ensure_ascii=False, indent=2) + if out: + Path(out).parent.mkdir(parents=True, exist_ok=True) + Path(out).write_text(text + "\n") + print(text) + + +def created(result, what): + if result[0] != 201: + raise SystemExit(f"{what} failed: HTTP {result[0]}") + return result[2] + + +def actor(label): + username = f"p_{label}_{uuid.uuid4().hex[:12]}" + password = "Probe-" + uuid.uuid4().hex + body = {"username": username, "display_name": "probe " + label, "password": password} + user = created(call("POST", "/api/v1/users", body, timeout=10), "register") + basic = base64.b64encode(f"{username}:{password}".encode()).decode() + login = call( + "POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + basic}, timeout=10 + ) + return { + "id": user["id"], + "auth": {"Authorization": "Bearer " + created(login, "login")["token"]}, + } + + +def chat_of(owner, other): + body = {"title": "probe", "member_ids": [owner["id"], other["id"]]} + return created(call("POST", "/api/v1/chats", body, owner["auth"], timeout=10), "chat")["id"] + + +def send(owner, chat, text, key): + headers = dict(owner["auth"], **{"Idempotency-Key": key}) + return call("POST", f"/api/v1/chats/{chat}/messages", {"text": text}, headers) + + +def lb_rows(backend="api"): + try: + with urllib.request.urlopen(stats_url(), timeout=2) as response: + lines = response.read().decode().splitlines() + except OSError: + return None + header = lines[0].lstrip("# ").split(",") + rows = [dict(zip(header, line.split(","))) for line in lines[1:] if line] + return [r for r in rows if r["pxname"] == backend] + + +def lb_up(): + rows = lb_rows() + if rows is None: + return None + return sum(1 for r in rows if r["svname"] != "BACKEND" and r["status"].startswith("UP")) + + +def lb_counters(): + row = next((r for r in lb_rows() or [] if r["svname"] == "BACKEND"), {}) + return {name: int(row.get(name) or 0) for name in COUNTERS} + + +def run_k6(out, env, on_start=None): + Path(out).mkdir(parents=True, exist_ok=True) + started = [] + process = subprocess.Popen( + [str(ROOT / "scripts/k6.sh"), str(out)], + env=dict(os.environ, **env), + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ) + with (Path(out) / "k6.txt").open("w") as log: + for line in process.stdout: + log.write(line) + if not started and "LOAD-START" in line: + started.append(time.time()) + if on_start: + on_start() + return process.wait(), started[0] if started else None + + +def k6_metrics(path, duration): + metrics = json.loads(Path(path).read_text())["metrics"] + + def value(name, field): + return metrics.get(name + "{scenario:measure}", {}).get("values", {}).get(field, 0.0) + + count = value("http_reqs", "count") + return { + "requests": int(count), + "rps": round(count / duration, 2), + "ok_rps": round(count * (1 - value("http_req_failed", "rate")) / duration, 2), + "failed_ratio": round(value("http_req_failed", "rate"), 5), + "dropped_iterations": int(value("dropped_iterations", "count")), + "p50_ms": round(value("http_req_duration", "med"), 2), + "p95_ms": round(value("http_req_duration", "p(95)"), 2), + "p99_ms": round(value("http_req_duration", "p(99)"), 2), + } + + +def race(alice, chat, parallel): + key, text = uuid.uuid4().hex, "same key " + uuid.uuid4().hex + barrier, results = threading.Barrier(parallel), [] + + def worker(): + barrier.wait() + results.append(send(alice, chat, text, key)) + + threads = [threading.Thread(target=worker) for _ in range(parallel)] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + bodies = [r[2] for r in results if r[0] == 201] + identical = bool(bodies) and all(b == bodies[0] for b in bodies) + return text, results, identical, send(alice, chat, text + " changed", key)[0] + + +def check(args): + alice, bob = actor("alice"), actor("bob") + chat = chat_of(alice, bob) + seen = collections.Counter( + call("GET", "/api/v1/users/me", headers=alice["auth"])[1] for _ in range(args.samples) + ) + rounds = [race(alice, chat, args.parallel) for _ in range(min(args.rounds, 100))] + page = call("GET", f"/api/v1/chats/{chat}/messages?limit=100", headers=bob["auth"])[2] + copies = collections.Counter(m.get("text") for m in (page or {"items": []})["items"]) + statuses = collections.Counter(r[0] for _, results, _, _ in rounds for r in results) + served = collections.Counter(r[1] for _, results, _, _ in rounds for r in results) + same = { + "rounds": len(rounds), + "parallel": args.parallel, + "statuses": dict(statuses), + "instances": dict(served), + "identical_bodies_rounds": sum(1 for _, _, identical, _ in rounds if identical), + "single_copy_rounds": sum(1 for text, _, _, _ in rounds if copies[text] == 1), + "conflict_rounds": sum(1 for _, _, _, other in rounds if other == 409), + } + passed = ( + len(seen) >= 2 + and statuses == {201: len(rounds) * args.parallel} + and len(served) >= 2 + and same["identical_bodies_rounds"] == len(rounds) + and same["single_copy_rounds"] == len(rounds) + and same["conflict_rounds"] == len(rounds) + ) + emit({"instances_one_token": dict(seen), "same_key": same, "pass": passed}, args.out) + return 0 if passed else 1 + + +def record(stop, alice, chat): + probes, trace = [], [] + + def prober(): + key = uuid.uuid4().hex + while not stop.is_set(): + started = time.time() + status, instance, _ = send(alice, chat, "probe " + key, key) + probes.append((started, status, round(1000 * (time.time() - started), 1), instance)) + if status == 201: + key = uuid.uuid4().hex + time.sleep(0.1) + + def poller(): + while not stop.is_set(): + trace.append((time.time(), lb_up())) + time.sleep(0.1) + + threads = [threading.Thread(target=t, daemon=True) for t in (prober, poller)] + for thread in threads: + thread.start() + return probes, trace, threads + + +def write_csv(path, header, rows): + with path.open("w") as handle: + handle.write(header + "\n") + handle.writelines(",".join(str(v) for v in row) + "\n" for row in rows) + + +def summarize(args, probes, trace, action, expected): + t0, end = action["start"], action["end"] + label = args.label or os.environ.get("TARGET_INSTANCE_ID", "action") + failures = [t - t0 for t, s, _, _ in probes if s != 201 and t >= t0] + down = [t for t, up in trace if t >= t0 and up is not None and up < expected] + back = [t for t, up in trace if down and t > down[0] and up is not None and up >= expected] + outage = [t for t in down if not back or t < back[0]] or [end] + served = [(t, i) for t, s, _, i in probes if s == 201] + before = collections.Counter(i for t, i in served if t < t0) + during = collections.Counter(i for t, i in served if outage[0] <= t <= outage[-1]) + after = collections.Counter(i for t, i in served if t > end) + return { + "label": label, + "action": args.command, + "action_exit": action["exit"], + "action_seconds": round(end - t0, 3), + "action_finished_before_load_end": end < action["load_end"], + "lb_up_before": expected, + "lb_up_min": min((up for _, up in trace if up is not None), default=None), + "detection_seconds": round(down[0] - t0, 3) if down else None, + "rejoin_seconds": round(back[0] - t0, 3) if back else None, + "probe": { + "attempts": len(probes), + "failed_attempts": sum(1 for _, s, _, _ in probes if s != 201), + "first_failure_s": round(failures[0], 3) if failures else None, + "recovery_s": round(failures[-1], 3) if failures else 0.0, + "instances_before": dict(before), + "instances_during_outage": dict(during), + "instances_after": dict(after), + }, + } + + +def verdict(args, result): + probe, k6 = result["probe"], result["k6"] + checks = { + "action_ok": result["action_exit"] == 0 and result["action_finished_before_load_end"], + "load_errors_ok": k6["exit"] == 0 and k6["failed_ratio"] <= args.max_failed_ratio, + "write_recovery_ok": probe["recovery_s"] <= args.max_recovery, + } + if args.expect == "dip": + checks["detected"] = result["detection_seconds"] is not None + checks["target_excluded"] = probe["instances_during_outage"].get(result["label"], 0) == 0 + checks["rejoined"] = result["rejoin_seconds"] is not None and bool( + probe["instances_after"].get(result["label"]) + ) + if args.expect == "replace": + checks["capacity_kept"] = (result["lb_up_min"] or 0) >= result["lb_up_before"] + old, new = set(probe["instances_before"]), set(probe["instances_after"]) + checks["all_replaced"] = bool(new) and not old & new + return checks + + +def load(args): + out = Path(args.out) + out.mkdir(parents=True, exist_ok=True) + expected = lb_up() or 0 + alice, bob = actor("alice"), actor("bob") + stop, action = threading.Event(), {} + probes, trace, threads = record(stop, alice, chat_of(alice, bob)) + + def act(): + time.sleep(args.warmup + args.lead) + action["start"] = time.time() + with (out / "action.txt").open("w") as log: + action["exit"] = subprocess.call( + args.command, cwd=str(ROOT), stdout=log, stderr=subprocess.STDOUT + ) + action["end"] = time.time() + + runner = threading.Thread(target=act) + env = {"RATE": str(args.rate), "WARMUP": str(args.warmup), "DURATION": str(args.duration)} + counters = lb_counters() + code, load_start = run_k6(out / "k6", env, runner.start) + if load_start is not None: + runner.join() + action["load_end"] = load_start + args.warmup + args.duration + stop.set() + for thread in threads: + thread.join() + if "start" not in action: + raise SystemExit(f"k6 failed before the action (exit {code}), see {out / 'k6/k6.txt'}") + t0 = action["start"] + write_csv( + out / "probe.csv", + "t_rel_s,status,latency_ms,instance", + ((round(t - t0, 3), *r) for t, *r in probes), + ) + write_csv(out / "lb_up.csv", "t_rel_s,up", ((round(t - t0, 3), up) for t, up in trace)) + result = summarize(args, probes, trace, action, expected) + result["lb_counters"] = {k: v - counters[k] for k, v in lb_counters().items()} + result["k6"] = dict( + k6_metrics(out / "k6/summary.json", args.duration), exit=code, rate=args.rate + ) + result["checks"] = verdict(args, result) + result["pass"] = all(result["checks"].values()) + (out / "result.json").write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n") + print(json.dumps(result, ensure_ascii=False)) + return 0 if result["pass"] else 1 + + +def detect(args): + before = lb_up() or 0 + until = before - 1 if args.until is None else args.until + started = time.time() + process = subprocess.Popen(args.command, cwd=str(ROOT)) + detected = acted = None + while time.time() - started < args.timeout and (detected is None or acted is None): + if acted is None and process.poll() is not None: + acted = time.time() - started + if detected is None and (lb_up() or 0) <= until: + detected = time.time() - started + time.sleep(0.05) + code = process.wait() + acted = time.time() - started if acted is None else acted + status, instance, payload = call("GET", "/health/live") + restored = None + if args.restore: + begun = time.time() + subprocess.call(["sh", "-c", args.restore], cwd=str(ROOT)) + while (lb_up() or 0) < before and time.time() - begun < 3 * args.timeout: + time.sleep(0.1) + restored = round(time.time() - begun, 3) if (lb_up() or 0) >= before else None + checks = { + "action_ok": code == 0, + "detected_in_time": detected is not None and detected <= args.max + args.slack, + "lb_answer_ok": status == args.lb_status, + } + if args.within_action: + checks["excluded_before_action_end"] = detected is not None and detected < acted + if args.restore: + checks["restored"] = restored is not None + report = { + "action": args.command, + "action_exit": code, + "action_seconds": round(acted, 3), + "lb_up_before": before, + "lb_up_counted_as_detected": until, + "detection_seconds": None if detected is None else round(detected, 3), + "declared_max_seconds": args.max, + "measurement_slack_seconds": args.slack, + "lb_answer_after_detection": { + "path": "/health/live", + "status": status, + "x_instance_id_present": bool(instance), + "error_code": ((payload or {}).get("error") or {}).get("code"), + }, + "restore": args.restore, + "restore_seconds": restored, + "lb_up_after_restore": lb_up(), + "checks": checks, + "pass": all(checks.values()), + } + emit(report, args.out) + return 0 if report["pass"] else 1 + + +def main(): + parser = argparse.ArgumentParser(description="LB probes over the public HTTP contract") + sub = parser.add_subparsers(dest="mode", required=True) + one = sub.add_parser("check", help="instances behind one token and same-key races") + one.add_argument("--samples", type=int, default=40) + one.add_argument("--parallel", type=int, default=8) + one.add_argument("--rounds", type=int, default=1) + one.add_argument("--out", help="JSON file, written only after a complete report") + under = sub.add_parser("load", help="run an action under k6 load and measure recovery") + under.add_argument("--out", required=True) + under.add_argument("--label") + under.add_argument("--expect", choices=("dip", "replace"), default="dip") + under.add_argument("--rate", type=int, default=int(os.environ.get("RATE", "50"))) + under.add_argument("--warmup", type=int, default=10) + under.add_argument("--duration", type=int, default=int(os.environ.get("DURATION", "60"))) + under.add_argument("--lead", type=int, default=15) + under.add_argument("--max-failed-ratio", type=float, default=0.01) + under.add_argument("--max-recovery", type=float, default=10.0) + under.add_argument("command", nargs=argparse.REMAINDER) + watch = sub.add_parser("detect", help="time until the LB takes replicas out of rotation") + watch.add_argument("--max", type=float, required=True, help="declared worst detection, s") + watch.add_argument("--slack", type=float, default=0.25, help="docker CLI and polling, s") + watch.add_argument("--until", type=int, help="UP servers counted as detected") + watch.add_argument("--within-action", action="store_true") + watch.add_argument("--lb-status", type=int, default=200) + watch.add_argument("--restore", help="shell command that brings replicas back") + watch.add_argument("--timeout", type=float, default=20) + watch.add_argument("--out", help="JSON file, written only after a complete report") + watch.add_argument("command", nargs=argparse.REMAINDER) + args = parser.parse_args() + if args.mode == "check": + return check(args) + args.command = [c for c in args.command if c != "--"] + if not args.command: + parser.error(f"{args.mode} needs an action command after --") + return load(args) if args.mode == "load" else detect(args) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 00000000..c5b30a67 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,15 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +defined=$(docker compose config --services) +pick() { + for service in $1; do + if printf '%s\n' "$defined" | grep -qx "$service"; then printf ' %s' "$service"; fi + done +} +apps=$(pick "${APP_SERVICES:-lb api relay worker}") +stores=$(pick "${STATE_SERVICES:-postgres valkey s3 broker}") +[ -n "$stores" ] || { echo "no state services found" >&2; exit 3; } +docker compose stop $apps +docker compose restart $stores +docker compose up -d --no-recreate --wait --wait-timeout "${WAIT_TIMEOUT:-75}" diff --git a/scripts/restore.sh b/scripts/restore.sh new file mode 100755 index 00000000..705a092d --- /dev/null +++ b/scripts/restore.sh @@ -0,0 +1,5 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +docker compose up -d --no-deps --no-recreate --wait --wait-timeout "${WAIT_TIMEOUT:-60}" "${API_SERVICE:-api}" +scripts/lb-wait.sh diff --git a/scripts/rolling-update.sh b/scripts/rolling-update.sh new file mode 100755 index 00000000..8bbcb642 --- /dev/null +++ b/scripts/rolling-update.sh @@ -0,0 +1,20 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +service=${API_SERVICE:-api} +show() { docker inspect -f '{{.Config.Hostname}} image={{slice .Image 7 19}}' "$1"; } +echo "$(date -u +%T) migrate once before the first replacement" +docker compose run --rm --no-deps -T migrate 2>&1 +old=$(docker compose ps -q "$service") +count=$(printf '%s\n' "$old" | grep -c .) +for id in $old; do + running=$(docker compose ps -q "$service") + docker compose up -d --no-deps --no-recreate --wait --wait-timeout "${WAIT_TIMEOUT:-90}" --scale "$service=$((count + 1))" "$service" 2>&1 + scripts/lb-wait.sh $((count + 1)) + new=$(docker compose ps -q "$service" | grep -v -F "$running") + echo "$(date -u +%T) replace $(show "$id") -> $(show "$new")" + docker stop -t "${STOP_TIMEOUT:-6}" "$id" >/dev/null + docker rm "$id" >/dev/null +done +scripts/lb-wait.sh "$count" +echo "$(date -u +%T) done: $count replicas replaced" diff --git a/scripts/s3-check.sh b/scripts/s3-check.sh new file mode 100755 index 00000000..fa1bdaec --- /dev/null +++ b/scripts/s3-check.sh @@ -0,0 +1,66 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +cli() { docker compose run --rm --no-deps -T s3-cli "$@"; } +snapshot() { + cli sh -s <<'EOF' | grep -v '_date' +ak=$(cat /run/secrets/s3_access_key_id) +for c in "bucket list root" "bucket anonymous get root/$S3_BUCKET" "admin policy info root messenger-api" "admin user info root $ak" "bucket lifecycle rule export root/$S3_BUCKET"; do + rc $c --json +done | sed "s/$ak//g" +EOF +} +failed=0 +verdict() { if [ "$2" = "$3" ]; then echo "PASS $1: $3"; else echo "FAIL $1: ожидалось $2, получено $3"; failed=1; fi; } +echo "== $(date -u +%FT%TZ) S3 privacy and bootstrap checks" +verdict "s3: опубликованные host-порты" "нет" "$(docker inspect -f '{{json .NetworkSettings.Ports}}' "$(docker compose ps -q s3)" | grep -q HostPort && echo есть || echo нет)" +verdict "s3: консоль :9001 внутри контейнера" "000" "$(docker compose exec -T s3 curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:9001/ || true)" +before=$(snapshot) +first=$(docker compose run --rm --no-deps -T s3-init >/dev/null 2>&1 && echo 0 || echo 1) +second=$(docker compose run --rm --no-deps -T s3-init 2>/dev/null || echo FAILED) +after=$(snapshot) +verdict "s3-init: повторный запуск 1" "0" "$first" +verdict "s3-init: повторный запуск 2" "0" "$(echo "$second" | grep -q '^s3-init:' && echo 0 || echo 1)" +verdict "s3-init: состояние IAM/bucket не изменилось" "same" "$([ "$before" = "$after" ] && echo same || echo changed)" +cli sh -s <<'EOF' || failed=1 +b=$S3_BUCKET +key="attachments/s3-check-$(date +%s)-$$" +other="s3-check-$(date +%s)-$$" +head -c 4096 /dev/urandom > /tmp/probe +result=0 +expect() { + want=$1; name=$2; shift 2 + if "$@" >/tmp/out 2>&1; then got=allow; elif grep -qE '403|AccessDenied|set_bucket_policy: service error' /tmp/out; then got=deny; else got=error; fi + if [ "$got" = "$want" ]; then echo "PASS $name: $got"; else echo "FAIL $name: ожидалось $want, получено $got"; result=1; fi +} +expect allow "api: PutObject" rc put /tmp/probe "api/$b/$key" +expect allow "api: HeadObject" rc object stat "api/$b/$key" +expect allow "api: GetObject" rc object show "api/$b/$key" +expect allow "api: ListBucket" rc object list "api/$b/attachments/" +expect deny "api: CreateBucket" rc bucket create "api/$other" +expect deny "api: DeleteBucket" rc bucket remove "api/$b" +expect deny "api: PutBucketPolicy (public)" rc bucket anonymous set public "api/$b" +expect deny "api: PutBucketLifecycle" rc bucket lifecycle rule import "api/$b" /s3/lifecycle.json +expect deny "api: admin ListUsers" rc admin user list api +rc bucket create --ignore-existing "root/$other" >/dev/null +expect deny "api: PutObject в чужой bucket" rc put /tmp/probe "api/$other/x" +expect deny "api: ListBucket чужого bucket" rc object list "api/$other" +rc bucket remove "root/$other" >/dev/null +expect deny "anonymous: HeadObject" rc object stat "anon/$b/$key" +expect deny "anonymous: ListBucket" rc object list "anon/$b" +expect deny "anonymous: PutObject" rc put /tmp/probe "anon/$b/attachments/anonymous" +status=$(wget -S -O /dev/null "$S3_ENDPOINT/$b/$key" 2>&1 | sed -n 's/^ *HTTP\/[0-9.]* \([0-9]*\).*/\1/p' | head -n 1) +if [ "$status" = 403 ]; then echo "PASS anonymous: прямой HTTP GET объекта из отдельного контейнера: $status"; else echo "FAIL anonymous: прямой HTTP GET: $status"; result=1; fi +permission=$(rc bucket anonymous get "root/$b" --json | sed -n 's/.*"permission": *"\([a-z]*\)".*/\1/p') +if [ "$permission" = private ]; then echo "PASS bucket: anonymous policy: $permission"; else echo "FAIL bucket: anonymous policy: $permission"; result=1; fi +expect allow "api: DeleteObject" rc object remove --force "api/$b/$key" +host=$(echo "$S3_ENDPOINT" | sed 's#^[a-z]*://##; s#:.*##') +if wget -T 2 -O /dev/null "http://$host:9001/" >/tmp/out 2>&1; then echo "FAIL s3: консоль :9001 отвечает из сети стенда"; result=1 +elif grep -q refused /tmp/out; then echo "PASS s3: консоль :9001 из отдельного контейнера сети стенда: connection refused" +else echo "FAIL s3: консоль :9001 из сети стенда: $(tail -n 1 /tmp/out)"; result=1; fi +exit $result +EOF +echo "== IAM/bucket snapshot (без секретов)" +echo "$after" +if [ "$failed" != 0 ]; then echo "RESULT: FAIL"; exit 1; fi +echo "RESULT: PASS" diff --git a/scripts/secrets.sh b/scripts/secrets.sh new file mode 100755 index 00000000..bdc67ec5 --- /dev/null +++ b/scripts/secrets.sh @@ -0,0 +1,24 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +umask 077 +mkdir -p secrets +random() { od -An -N32 -tx1 /dev/urandom | tr -d ' \n'; } +put() { + if [ ! -s "secrets/$1" ]; then + printf '%s' "$2" > "secrets/$1" + chmod 644 "secrets/$1" + echo "создан secrets/$1" + fi +} +put postgres_password "$(random)" +put database_url "postgresql://messenger:$(cat secrets/postgres_password)@postgres:5432/messenger" +put cursor_secret "$(random)" +put valkey_password "$(random)" +put valkey.conf "requirepass $(cat secrets/valkey_password)" +put cache_url "redis://:$(cat secrets/valkey_password)@valkey:6379/0" +put s3_root_user "root$(random | cut -c1-16)" +put s3_root_password "$(random)" +put s3_access_key_id "api$(random | cut -c1-16)" +put s3_secret_access_key "$(random)" +[ -f .env ] || { cp .env.example .env; echo "создан .env из .env.example"; } diff --git a/scripts/sql/explain-history.sql b/scripts/sql/explain-history.sql new file mode 100644 index 00000000..7752625e --- /dev/null +++ b/scripts/sql/explain-history.sql @@ -0,0 +1,59 @@ +\set ON_ERROR_STOP on +\set big '''00000000-0000-7000-8000-00000000c001''' +\set cursor_at '''2026-01-01T02:46:40Z''' +\set cursor_id '''00000000-0000-7000-8001-000000002710''' + +PREPARE history_first (uuid, int) AS +SELECT id, chat_id, sender_id, text, created_at +FROM messages +WHERE chat_id = $1 +ORDER BY created_at, id +LIMIT $2; + +PREPARE history_after (uuid, timestamptz, uuid, int) AS +SELECT id, chat_id, sender_id, text, created_at +FROM messages +WHERE chat_id = $1 AND (created_at, id) > ($2, $3) +ORDER BY created_at, id +LIMIT $4; + +PREPARE history_offset (uuid, int, int) AS +SELECT id, chat_id, sender_id, text, created_at +FROM messages +WHERE chat_id = $1 +ORDER BY created_at, id +LIMIT $2 OFFSET $3; + +SELECT count(*) AS chat_messages FROM messages WHERE chat_id = :big; +SELECT count(*) AS all_messages FROM messages; +SELECT pg_size_pretty(pg_relation_size('messages_history_idx')) AS history_index_size; + +\echo '== Первая страница, limit=50 (запрос HISTORY_FIRST приложения, LIMIT limit+1)' +EXPLAIN (ANALYZE, BUFFERS) EXECUTE history_first(:big, 51); + +\echo '== Глубокая страница: keyset после 10 000-го сообщения (запрос HISTORY_AFTER приложения)' +EXPLAIN (ANALYZE, BUFFERS) EXECUTE history_after(:big, :cursor_at, :cursor_id, 51); + +\echo '== Та же глубокая страница через OFFSET 10000 (не используется приложением, для сравнения)' +EXPLAIN (ANALYZE, BUFFERS) EXECUTE history_offset(:big, 51, 10000); + +\echo '== Проверка: keyset и OFFSET возвращают одну и ту же страницу' +SELECT ( + SELECT array_agg(id ORDER BY created_at, id) FROM ( + SELECT id, created_at FROM messages + WHERE chat_id = :big AND (created_at, id) > (:cursor_at, :cursor_id) + ORDER BY created_at, id LIMIT 51 + ) AS keyset +) = ( + SELECT array_agg(id ORDER BY created_at, id) FROM ( + SELECT id, created_at FROM messages + WHERE chat_id = :big + ORDER BY created_at, id LIMIT 51 OFFSET 10000 + ) AS skipped +) AS same_page; + +\echo '== Контрольный план без индекса истории (DROP INDEX внутри транзакции, затем ROLLBACK)' +BEGIN; +DROP INDEX messages_history_idx; +EXPLAIN (ANALYZE, BUFFERS) EXECUTE history_after(:big, :cursor_at, :cursor_id, 51); +ROLLBACK; diff --git a/scripts/sql/seed-history.sql b/scripts/sql/seed-history.sql new file mode 100644 index 00000000..842b599a --- /dev/null +++ b/scripts/sql/seed-history.sql @@ -0,0 +1,40 @@ +\set ON_ERROR_STOP on +\set author '''00000000-0000-7000-8000-00000000a001''' +\set big '''00000000-0000-7000-8000-00000000c001''' +\set epoch '''2026-01-01T00:00:00Z''' + +INSERT INTO users (id, username, display_name, password_hash, created_at) +VALUES (:author, 'seed_author', 'Синтетический автор', '$argon2id$v=19$m=19456,t=2,p=1$c2VlZA$c2VlZA', :epoch) +ON CONFLICT DO NOTHING; + +INSERT INTO chats (id, title, created_at) +SELECT ('00000000-0000-7000-8000-00000000c' || lpad(to_hex(chat), 3, '0'))::uuid, + 'Синтетический чат ' || chat, :epoch +FROM generate_series(1, 21) AS chat +ON CONFLICT DO NOTHING; + +INSERT INTO chat_members (chat_id, user_id, position, chat_created_at) +SELECT id, :author, 1, created_at FROM chats WHERE title LIKE 'Синтетический чат %' +ON CONFLICT DO NOTHING; + +INSERT INTO messages (id, chat_id, sender_id, text, created_at) +SELECT ('00000000-0000-7000-8001-' || lpad(to_hex(n), 12, '0'))::uuid, :big, :author, + 'синтетическое сообщение ' || n, :epoch::timestamptz + n * interval '1 second' +FROM generate_series(1, 20000) AS n +ON CONFLICT DO NOTHING; + +INSERT INTO messages (id, chat_id, sender_id, text, created_at) +SELECT ('00000000-0000-7000-8002-' || lpad(to_hex(chat * 1000 + n), 12, '0'))::uuid, + ('00000000-0000-7000-8000-00000000c' || lpad(to_hex(chat), 3, '0'))::uuid, :author, + 'фоновое сообщение ' || n, :epoch::timestamptz + n * interval '1 second' +FROM generate_series(2, 21) AS chat, generate_series(1, 500) AS n +ON CONFLICT DO NOTHING; + +UPDATE chats +SET last_message_at = (SELECT max(created_at) FROM messages WHERE messages.chat_id = chats.id) +WHERE title LIKE 'Синтетический чат %'; + +ANALYZE; + +SELECT count(*) FILTER (WHERE chat_id = :big) AS big_chat_messages, count(*) AS all_messages +FROM messages; diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 00000000..222cd887 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,9 @@ +#!/bin/sh +set -eu +: "${TARGET_INSTANCE_ID:?test provides target API instance alias}" +cd "$(dirname "$0")/.." +ids=$(docker compose ps -q "${API_SERVICE:-api}") +[ -n "$ids" ] || { echo "no running ${API_SERVICE:-api} replicas" >&2; exit 3; } +target=$(docker inspect -f '{{.Id}} {{.Config.Hostname}}' $ids | awk -v t="$TARGET_INSTANCE_ID" '$2 == t {print $1}') +[ -n "$target" ] || { echo "no running ${API_SERVICE:-api} replica answers as $TARGET_INSTANCE_ID" >&2; exit 3; } +docker stop -t "${STOP_TIMEOUT:-6}" "$target" >/dev/null diff --git a/scripts/upload_memory.py b/scripts/upload_memory.py new file mode 100755 index 00000000..eb6febb5 --- /dev/null +++ b/scripts/upload_memory.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 +import argparse +import json +import os +import subprocess +import sys +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) +seed = __import__("lab4_seed") +FORMAT = "{:>5} {:>5} {:>4} {:<18} {:>7} {:>7} {:>8} {:>8} {:>7} {:>8} {:>6}" +COLUMNS = FORMAT.format( + "slots", + "size", + "conc", + "codes", + "p50_ms", + "p95_ms", + "api_idle", + "api_peak", + "api_oom", + "lb_peak", + "lb_oom", +) + + +OVERRIDES = ("UPLOAD_CONCURRENCY", "MALLOC_MMAP_THRESHOLD_") + + +def recreate(slots): + env = dict(os.environ, UPLOAD_CONCURRENCY=str(slots)) + if not slots: + env = {k: v for k, v in os.environ.items() if k not in OVERRIDES} + cmd = ["docker", "compose", "up", "-d", "--no-deps", "--force-recreate", "--wait", "api", "lb"] + subprocess.run(cmd, cwd=ROOT, env=env, check=True, capture_output=True) + subprocess.run([str(ROOT / "scripts/lb-wait.sh")], cwd=ROOT, check=True) + + +def balancer(): + cid = seed.run(["docker", "compose", "ps", "-q", "lb"]).strip() + events = dict(row.split() for row in seed.cgroup(cid, "memory.events").splitlines()) + peak = seed.cgroup(cid, "memory.peak") + return { + "cgroup_memory_peak_mib": round(int(peak) / 1048576, 1), + "oom_kill": int(events["oom_kill"]), + } + + +def line(report): + memory = report["memory"].values() + codes = ",".join(f"{k}x{v}" for k, v in sorted(report["status_codes"].items())) + return FORMAT.format( + report["upload_concurrency"], + f"{report['body_bytes'] / 1048576:.1f}M", + report["concurrency"], + codes, + report["latency_ms"]["p50"], + report["latency_ms"]["p95"], + max(m["idle_mib"] for m in memory), + max(m["cgroup_memory_peak_mib"] or 0 for m in memory), + sum(m["oom_kill"] for m in memory), + report["lb"]["cgroup_memory_peak_mib"], + report["lb"]["oom_kill"], + ) + + +def main(): + p = argparse.ArgumentParser(description="Peak memory of API replicas under concurrent uploads") + p.add_argument("--base-url", default=os.environ.get("BASE_URL", "http://localhost:8080")) + p.add_argument("--slots", type=int, nargs="+", default=[64, 3]) + p.add_argument("--sizes", type=int, nargs="+", default=[1 << 20, 10 << 20]) + p.add_argument("--levels", type=int, nargs="+", default=[10, 20, 40]) + p.add_argument("--rounds", type=int, default=2, help="uploads per concurrent client") + p.add_argument("--out", default="evidence/upload-memory") + args = p.parse_args() + out = ROOT / args.out + out.mkdir(parents=True, exist_ok=True) + commit = seed.run(["git", "rev-parse", "--short", "HEAD"]).strip() + threshold = os.environ.get("MALLOC_MMAP_THRESHOLD_", "131072") + lines = [ + f"# {time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())} commit {commit}; api x2: 1 CPU," + f" 512 MiB, 2 workers; MALLOC_MMAP_THRESHOLD_={threshold}; {args.rounds} uploads" + " per client; peaks from cgroup memory.peak, MiB", + COLUMNS, + ] + print("\n".join(lines), flush=True) + try: + for slots in args.slots: + for size in args.sizes: + for level in args.levels: + recreate(slots) + run = argparse.Namespace( + base_url=args.base_url, + timeout=60, + size=size, + count=level * args.rounds, + concurrency=level, + sample=True, + service="api", + link=0, + ids_out=None, + ) + report = {"upload_concurrency": slots, **seed.over_http(run)} + report["lb"] = balancer() + name = f"slots{slots}-{size >> 20}mib-c{level}.json" + (out / name).write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n") + lines.append(line(report)) + print(lines[-1], flush=True) + finally: + recreate(None) + (out / "summary.txt").write_text("\n".join(lines) + "\n") + + +if __name__ == "__main__": + main() diff --git a/scripts/worker.sh b/scripts/worker.sh new file mode 100755 index 00000000..e8f0a8b5 --- /dev/null +++ b/scripts/worker.sh @@ -0,0 +1,12 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +COMPOSE=${COMPOSE:-docker compose} +service=${WORKER_SERVICE:-worker} +case "${1:-}" in + stop) $COMPOSE stop -t 10 "$service" ;; + kill) $COMPOSE kill -s KILL "$service" ;; + start) $COMPOSE up -d --no-deps --wait --wait-timeout 60 "$service" ;; + status) $COMPOSE ps -a --format '{{.Service}} {{.State}} {{.Status}}' "$service" ;; + *) echo "usage: $0 stop|kill|start|status" >&2; exit 2 ;; +esac diff --git a/src/messenger/__init__.py b/src/messenger/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/messenger/admin.py b/src/messenger/admin.py new file mode 100644 index 00000000..4c6f3cbd --- /dev/null +++ b/src/messenger/admin.py @@ -0,0 +1,37 @@ +import asyncio +import sys + +from dishka import make_async_container + +from messenger.attachments.domain.jobs import ReplayResult +from messenger.attachments.entrypoints.admin import rendered, run +from messenger.attachments.entrypoints.ioc import AdminProvider +from messenger.attachments.entrypoints.journal import LOG_FIELDS, subscribe_journal +from messenger.shared.application.messagebus import EventBus +from messenger.shared.config import PlatformSettings +from messenger.shared.entrypoints.ioc import PlatformProvider +from messenger.shared.infrastructure.logging import PLATFORM_FIELDS, StructlogLogger + + +async def serve(argv: list[str]) -> int: + platform = PlatformSettings() + logger = StructlogLogger(level=platform.log_level, allowed_fields=PLATFORM_FIELDS | LOG_FIELDS) + container = make_async_container(PlatformProvider(platform, logger), AdminProvider()) + bus = await container.get(EventBus) + subscribe_journal(bus, logger) + bus.start() + try: + report = await run(container, argv) + finally: + await bus.stop() + await container.close() + sys.stdout.write(rendered(report) + "\n") + return int(report.get("result") == ReplayResult.NOT_FOUND) + + +def main() -> None: + sys.exit(asyncio.run(serve(sys.argv[1:]))) + + +if __name__ == "__main__": + main() diff --git a/src/messenger/attachments/__init__.py b/src/messenger/attachments/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/messenger/attachments/application/__init__.py b/src/messenger/attachments/application/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/messenger/attachments/application/bus.py b/src/messenger/attachments/application/bus.py new file mode 100644 index 00000000..09bcf11a --- /dev/null +++ b/src/messenger/attachments/application/bus.py @@ -0,0 +1,5 @@ +from messenger.shared.application.messagebus import CommandBus + + +class AttachmentsCommandBus(CommandBus): + __slots__ = () diff --git a/src/messenger/attachments/application/commands.py b/src/messenger/attachments/application/commands.py new file mode 100644 index 00000000..af89a4b1 --- /dev/null +++ b/src/messenger/attachments/application/commands.py @@ -0,0 +1,49 @@ +from collections.abc import AsyncIterator +from dataclasses import dataclass +from datetime import timedelta +from uuid import UUID + +from messenger.attachments.domain.jobs import Job +from messenger.shared.application.commands import Command + + +@dataclass(frozen=True, slots=True) +class UploadAttachment(Command): + owner_id: UUID + chat_id: UUID + filename: str | None + content_type: str | None + body: AsyncIterator[bytes] + + +@dataclass(frozen=True, slots=True) +class SweepAbandonedUploads(Command): + grace: timedelta + limit: int + + +@dataclass(frozen=True, slots=True) +class ProcessJob(Command): + job: Job + + +@dataclass(frozen=True, slots=True) +class PublishOutbox(Command): + limit: int + + +@dataclass(frozen=True, slots=True) +class RecoverExpiredLeases(Command): + limit: int + + +@dataclass(frozen=True, slots=True) +class CleanStaging(Command): + limit: int + retention: timedelta + + +@dataclass(frozen=True, slots=True) +class ReplayAttachment(Command): + attachment_id: UUID + operator: str diff --git a/src/messenger/attachments/application/errors.py b/src/messenger/attachments/application/errors.py new file mode 100644 index 00000000..f54eb025 --- /dev/null +++ b/src/messenger/attachments/application/errors.py @@ -0,0 +1,2 @@ +class ObjectMissingError(Exception): + pass diff --git a/src/messenger/attachments/application/handlers.py b/src/messenger/attachments/application/handlers.py new file mode 100644 index 00000000..0ff449aa --- /dev/null +++ b/src/messenger/attachments/application/handlers.py @@ -0,0 +1,158 @@ +import asyncio +from collections.abc import AsyncIterator +from hashlib import sha256 + +from messenger.attachments.application.commands import SweepAbandonedUploads, UploadAttachment +from messenger.attachments.application.ports import ( + AttachmentsUnitOfWork, + ChatMembership, + ImageInspector, + ObjectStorage, +) +from messenger.attachments.domain.errors import ChatNotFound, PayloadTooLarge +from messenger.attachments.domain.jobs import Job +from messenger.attachments.domain.models import Attachment, AttachmentStatus +from messenger.attachments.domain.values import Filename, ImageKind +from messenger.shared.application.ports import Clock, IdGenerator, Logger + + +class UploadSlots(asyncio.Semaphore): + pass + + +async def read_limited(body: AsyncIterator[bytes], limit: int) -> tuple[bytes, str]: + chunks: list[bytes] = [] + digest = sha256() + size = 0 + async for chunk in body: + size += len(chunk) + if size > limit: + raise PayloadTooLarge(f"тело больше {limit} байт") + digest.update(chunk) + chunks.append(chunk) + return b"".join(chunks), digest.hexdigest() + + +async def drop( + uow: AttachmentsUnitOfWork, storage: ObjectStorage, attachment: Attachment, reason: str +) -> None: + if attachment.staging_key is not None: + await storage.delete(attachment.staging_key) + attachment.discard(reason) + await uow.attachments.remove(attachment) + await uow.commit() + + +class UploadAttachmentHandler: + __slots__ = ( + "_clock", + "_ids", + "_images", + "_limit", + "_logger", + "_membership", + "_pipeline_version", + "_slots", + "_storage", + "_uow", + ) + + def __init__( + self, + uow: AttachmentsUnitOfWork, + membership: ChatMembership, + images: ImageInspector, + storage: ObjectStorage, + *, + clock: Clock, + ids: IdGenerator, + limit: int, + slots: UploadSlots, + logger: Logger, + pipeline_version: int = 1, + ) -> None: + self._uow = uow + self._membership = membership + self._images = images + self._storage = storage + self._clock = clock + self._ids = ids + self._limit = limit + self._slots = slots + self._logger = logger + self._pipeline_version = pipeline_version + + async def __call__(self, command: UploadAttachment) -> Attachment: + if not await self._membership.is_member(command.chat_id, command.owner_id): + raise ChatNotFound("чат не найден") + filename = Filename(command.filename or "") + declared = ImageKind.declared(command.content_type) + async with self._slots: + attachment = await self._write(command, filename, declared) + now = self._clock.now() + attachment.enqueue(now) + job = Job(self._ids.new_id(), attachment.id, self._pipeline_version, 1, now) + try: + async with self._uow.transaction(): + await self._uow.attachments.save(attachment) + await self._uow.jobs.add(job) + await self._uow.commit() + except Exception: + await self._compensate(attachment) + raise + return attachment + + async def _write( + self, command: UploadAttachment, filename: Filename, declared: ImageKind + ) -> Attachment: + data, digest = await read_limited(command.body, self._limit) + image = await self._images.inspect(data, declared) + attachment = Attachment.stage( + attachment_id=self._ids.new_id(), + chat_id=command.chat_id, + owner_id=command.owner_id, + filename=filename, + image=image, + size_bytes=len(data), + sha256=digest, + now=self._clock.now(), + ) + async with self._uow.transaction(): + await self._uow.attachments.add(attachment) + await self._uow.commit() + await self._storage.put(str(attachment.staging_key), data, attachment.content_type) + return attachment + + async def _compensate(self, staged: Attachment) -> None: + try: + async with self._uow.transaction(): + current = await self._uow.attachments.lock(staged.id) + if current is None: + await self._storage.delete(str(staged.staging_key)) + elif current.status is AttachmentStatus.PENDING: + await drop(self._uow, self._storage, current, "compensation") + except Exception as error: + self._logger.warning( + "attachment.compensation_failed", + attachment_id=str(staged.id), + error=type(error).__name__, + ) + + +class SweepAbandonedUploadsHandler: + __slots__ = ("_clock", "_storage", "_uow") + + def __init__(self, uow: AttachmentsUnitOfWork, storage: ObjectStorage, clock: Clock) -> None: + self._uow = uow + self._storage = storage + self._clock = clock + + async def __call__(self, command: SweepAbandonedUploads) -> int: + before = self._clock.now() - command.grace + for swept in range(command.limit): + async with self._uow.transaction(): + abandoned = await self._uow.attachments.next_abandoned(before) + if abandoned is None: + return swept + await drop(self._uow, self._storage, abandoned, "abandoned") + return command.limit diff --git a/src/messenger/attachments/application/pipeline.py b/src/messenger/attachments/application/pipeline.py new file mode 100644 index 00000000..acbeebb7 --- /dev/null +++ b/src/messenger/attachments/application/pipeline.py @@ -0,0 +1,301 @@ +import asyncio +from datetime import timedelta +from hashlib import sha256 + +from messenger.attachments.application.commands import ( + CleanStaging, + ProcessJob, + PublishOutbox, + RecoverExpiredLeases, + ReplayAttachment, +) +from messenger.attachments.application.ports import ( + AttachmentsUnitOfWork, + Faults, + Jitter, + JobPublisher, + ObjectStorage, + VariantRenderer, +) +from messenger.attachments.domain.errors import LeaseLost, UnreadableImage +from messenger.attachments.domain.events import AttachmentReplayRequested, JobSettled +from messenger.attachments.domain.jobs import ( + Claim, + FailureCode, + Job, + Lease, + Outcome, + ReplayResult, + RetryPolicy, +) +from messenger.attachments.domain.models import Attachment, AttachmentStatus, original_key +from messenger.attachments.domain.variants import Variant, variant_key +from messenger.shared.application.errors import Unavailable +from messenger.shared.application.ports import IdGenerator + +TRANSIENT_ERROR = "transient_error" +STALL_AFTER_CLAIM = "stall_after_claim" +AFTER_COMMIT_BEFORE_PUBLISH = "after_commit_before_publish" +AFTER_PUBLISH_BEFORE_CONFIRM = "after_publish_before_confirm" +AFTER_CLAIM = "after_claim_before_side_effect" +AFTER_S3_WRITE = "after_s3_write_before_commit" +AFTER_COMMIT_BEFORE_ACK = "after_commit_before_ack" +RENEWALS = 3 + + +def settled( + job: Job, + outcome: Outcome, + *, + error_code: str | None = None, + delay_seconds: float | None = None, +) -> JobSettled: + return JobSettled( + job_id=job.id, + attachment_id=job.attachment_id, + pipeline_version=job.pipeline_version, + attempt=job.attempt, + outcome=outcome.value, + error_code=error_code, + delay_seconds=delay_seconds, + ) + + +class ProcessJobHandler: + __slots__ = ("_faults", "_ids", "_jitter", "_lease", "_policy", "_renderer", "_storage", "_uow") + + def __init__( + self, + uow: AttachmentsUnitOfWork, + storage: ObjectStorage, + renderer: VariantRenderer, + faults: Faults, + *, + ids: IdGenerator, + jitter: Jitter, + policy: RetryPolicy, + lease: timedelta, + ) -> None: + self._uow = uow + self._storage = storage + self._renderer = renderer + self._faults = faults + self._ids = ids + self._jitter = jitter + self._policy = policy + self._lease = lease + + async def __call__(self, command: ProcessJob) -> Outcome: + async with self._uow.transaction(): + claim = await self._uow.jobs.claim(command.job, self._ids.new_id(), self._lease) + if claim is None: + await self._uow.jobs.note(command.job, Outcome.DUPLICATE) + self._uow.record(settled(command.job, Outcome.DUPLICATE)) + await self._uow.commit() + if claim is None: + return Outcome.DUPLICATE + await self._faults.reach(AFTER_CLAIM) + await self._faults.reach(STALL_AFTER_CLAIM) + try: + variants = await self._held(claim) + except LeaseLost: + return await self._fenced(claim.lease) + except UnreadableImage as error: + return await self._fail(claim.lease, FailureCode.INVALID_IMAGE, error) + except Unavailable as error: + return await self._retry(claim.lease, error) + except Exception as error: + return await self._fail(claim.lease, FailureCode.PROCESSING_FAILED, error) + await self._faults.reach(AFTER_S3_WRITE) + outcome = await self._complete(claim.lease, variants) + await self._faults.reach(AFTER_COMMIT_BEFORE_ACK) + return outcome + + async def _held(self, claim: Claim) -> list[Variant]: + await self._renew(claim.lease) + work = asyncio.ensure_future(self._produce(claim)) + try: + while not work.done(): + await asyncio.wait({work}, timeout=self._lease.total_seconds() / RENEWALS) + if not work.done(): + await self._renew(claim.lease) + except BaseException: + work.cancel() + await asyncio.wait({work}) + raise + return work.result() + + async def _renew(self, lease: Lease) -> None: + async with self._uow.transaction(): + held = await self._uow.jobs.renew(lease, self._lease) + await self._uow.commit() + if not held: + raise LeaseLost("аренду задания забрал другой держатель") + + async def _produce(self, claim: Claim) -> list[Variant]: + job = claim.lease.job + await self._faults.reach(TRANSIENT_ERROR) + data = await self._storage.read(claim.source_key) + renditions = await self._renderer.render(data, job.pipeline_version) + if claim.staged: + await self._storage.copy(claim.source_key, original_key(job.attachment_id)) + variants: list[Variant] = [] + for rendition in renditions: + digest = sha256(rendition.data).hexdigest() + key = variant_key( + job.attachment_id, job.pipeline_version, rendition.name, digest, rendition.extension + ) + await self._storage.put(key, rendition.data, rendition.content_type) + variants.append( + Variant( + name=rendition.name, + content_type=rendition.content_type, + size_bytes=len(rendition.data), + width=rendition.width, + height=rendition.height, + sha256=digest, + object_key=key, + ) + ) + return variants + + async def _complete(self, lease: Lease, variants: list[Variant]) -> Outcome: + async with self._uow.transaction(): + outcome = await self._uow.jobs.complete( + lease, variants, original_key(lease.job.attachment_id) + ) + self._uow.record(settled(lease.job, outcome)) + await self._uow.commit() + return outcome + + async def _fenced(self, lease: Lease) -> Outcome: + async with self._uow.transaction(): + await self._uow.jobs.note(lease.job, Outcome.FENCED) + self._uow.record(settled(lease.job, Outcome.FENCED)) + await self._uow.commit() + return Outcome.FENCED + + async def _retry(self, lease: Lease, error: Exception) -> Outcome: + delay = self._policy.delay(lease.job.attempt, self._jitter.spread()) + if delay is None: + return await self._fail(lease, FailureCode.RETRY_EXHAUSTED, error) + async with self._uow.transaction(): + job = await self._uow.jobs.reschedule( + lease, delay, Outcome.RESCHEDULED, type(error).__name__ + ) + outcome = Outcome.FENCED if job is None else Outcome.RESCHEDULED + if job is None: + await self._uow.jobs.note(lease.job, outcome) + self._uow.record(settled(lease.job, outcome, delay_seconds=delay.total_seconds())) + await self._uow.commit() + return outcome + + async def _fail(self, lease: Lease, code: FailureCode, error: Exception) -> Outcome: + async with self._uow.transaction(): + done = await self._uow.jobs.fail(lease, code, type(error).__name__) + outcome = Outcome.FAILED if done else Outcome.FENCED + if not done: + await self._uow.jobs.note(lease.job, outcome) + self._uow.record(settled(lease.job, outcome, error_code=code.value)) + await self._uow.commit() + return outcome + + +class PublishOutboxHandler: + __slots__ = ("_faults", "_publisher", "_uow") + + def __init__(self, uow: AttachmentsUnitOfWork, publisher: JobPublisher, faults: Faults) -> None: + self._uow = uow + self._publisher = publisher + self._faults = faults + + async def __call__(self, command: PublishOutbox) -> int: + failure: Unavailable | None = None + published: list[int] = [] + async with self._uow.transaction(): + for entry in await self._uow.outbox.pending(command.limit): + await self._faults.reach(AFTER_COMMIT_BEFORE_PUBLISH) + try: + await self._publisher.publish(entry.job_id, entry.body) + except Unavailable as error: + failure = error + break + await self._faults.reach(AFTER_PUBLISH_BEFORE_CONFIRM) + published.append(entry.id) + if published: + await self._uow.outbox.published(published) + await self._uow.commit() + if failure is not None: + raise failure + return len(published) + + +class RecoverExpiredLeasesHandler: + __slots__ = ("_policy", "_uow") + + def __init__(self, uow: AttachmentsUnitOfWork, policy: RetryPolicy) -> None: + self._uow = uow + self._policy = policy + + async def __call__(self, command: RecoverExpiredLeases) -> int: + async with self._uow.transaction(): + leases = await self._uow.jobs.expired(command.limit) + for lease in leases: + if self._policy.exhausted(lease.job.attempt): + await self._uow.jobs.fail( + lease, FailureCode.RETRY_EXHAUSTED, Outcome.LEASE_EXPIRED.value + ) + outcome = Outcome.FAILED + else: + await self._uow.jobs.reschedule( + lease, timedelta(0), Outcome.LEASE_EXPIRED, Outcome.LEASE_EXPIRED.value + ) + outcome = Outcome.LEASE_EXPIRED + self._uow.record(settled(lease.job, outcome)) + await self._uow.commit() + return len(leases) + + +class CleanStagingHandler: + __slots__ = ("_storage", "_uow") + + def __init__(self, uow: AttachmentsUnitOfWork, storage: ObjectStorage) -> None: + self._uow = uow + self._storage = storage + + async def __call__(self, command: CleanStaging) -> int: + async with self._uow.transaction(): + staged = await self._uow.attachments.staged(command.limit, command.retention) + await asyncio.gather(*(self._storage.delete(item.key) for item in staged)) + await self._uow.attachments.unstage([item.attachment_id for item in staged]) + await self._uow.commit() + return len(staged) + + +class ReplayAttachmentHandler: + __slots__ = ("_uow",) + + def __init__(self, uow: AttachmentsUnitOfWork) -> None: + self._uow = uow + + async def __call__(self, command: ReplayAttachment) -> ReplayResult: + async with self._uow.transaction(): + attachment = await self._uow.attachments.lock(command.attachment_id) + result = await self._replay(attachment, command.operator) + if attachment is not None and result is not ReplayResult.REPLAYED: + await self._uow.jobs.note_replay(attachment.id, command.operator, result.value) + self._uow.record( + AttachmentReplayRequested(command.attachment_id, command.operator, result.value) + ) + await self._uow.commit() + return result + + async def _replay(self, attachment: Attachment | None, operator: str) -> ReplayResult: + if attachment is None: + return ReplayResult.NOT_FOUND + if attachment.status is not AttachmentStatus.FAILED: + return ReplayResult.NOOP + if attachment.staging_key is None: + return ReplayResult.EXPIRED + job = await self._uow.jobs.replay(attachment, operator) + return ReplayResult.NOOP if job is None else ReplayResult.REPLAYED diff --git a/src/messenger/attachments/application/ports.py b/src/messenger/attachments/application/ports.py new file mode 100644 index 00000000..e32f42cc --- /dev/null +++ b/src/messenger/attachments/application/ports.py @@ -0,0 +1,155 @@ +from collections.abc import AsyncIterator, Sequence +from dataclasses import dataclass +from datetime import datetime, timedelta +from typing import Protocol, runtime_checkable +from uuid import UUID + +from messenger.attachments.domain.jobs import Claim, FailedJob, FailureCode, Job, Lease, Outcome +from messenger.attachments.domain.models import Attachment +from messenger.attachments.domain.values import ImageInfo, ImageKind +from messenger.attachments.domain.variants import Variant +from messenger.shared.application.ports import UnitOfWork + + +@dataclass(frozen=True, slots=True) +class ObjectContent: + size: int + chunks: AsyncIterator[bytes] + + +@dataclass(frozen=True, slots=True) +class Rendition: + name: str + content_type: str + extension: str + data: bytes + width: int + height: int + + +@dataclass(frozen=True, slots=True) +class OutboxEntry: + id: int + job_id: UUID + body: bytes + + +@dataclass(frozen=True, slots=True) +class StagedObject: + attachment_id: UUID + key: str + + +@runtime_checkable +class ObjectStorage(Protocol): + async def put(self, key: str, data: bytes, content_type: str) -> None: ... + + async def open(self, key: str) -> ObjectContent: ... + + async def read(self, key: str) -> bytes: ... + + async def copy(self, source: str, target: str) -> None: ... + + async def delete(self, key: str) -> None: ... + + +@runtime_checkable +class ImageInspector(Protocol): + async def inspect(self, data: bytes, declared: ImageKind) -> ImageInfo: ... + + +@runtime_checkable +class VariantRenderer(Protocol): + async def render(self, data: bytes, pipeline_version: int) -> list[Rendition]: ... + + +@runtime_checkable +class JobPublisher(Protocol): + async def publish(self, job_id: UUID, body: bytes) -> None: ... + + +@runtime_checkable +class Faults(Protocol): + async def reach(self, point: str) -> None: ... + + +@runtime_checkable +class Jitter(Protocol): + def spread(self) -> float: ... + + +@runtime_checkable +class ChatMembership(Protocol): + async def is_member(self, chat_id: UUID, user_id: UUID) -> bool: ... + + +@runtime_checkable +class AttachmentRepository(Protocol): + async def add(self, attachment: Attachment) -> None: ... + + async def save(self, attachment: Attachment) -> None: ... + + async def lock(self, attachment_id: UUID) -> Attachment | None: ... + + async def next_abandoned(self, before: datetime) -> Attachment | None: ... + + async def remove(self, attachment: Attachment) -> None: ... + + async def staged(self, limit: int, retention: timedelta) -> list[StagedObject]: ... + + async def unstage(self, attachment_ids: Sequence[UUID]) -> None: ... + + +@runtime_checkable +class JobRepository(Protocol): + async def add(self, job: Job) -> None: ... + + async def claim(self, job: Job, token: UUID, lease: timedelta) -> Claim | None: ... + + async def complete( + self, lease: Lease, variants: Sequence[Variant], original_key: str + ) -> Outcome: ... + + async def reschedule( + self, lease: Lease, delay: timedelta, kind: Outcome, error: str + ) -> Job | None: ... + + async def renew(self, lease: Lease, duration: timedelta) -> bool: ... + + async def fail(self, lease: Lease, code: FailureCode, error: str) -> bool: ... + + async def note(self, job: Job, kind: Outcome) -> None: ... + + async def expired(self, limit: int) -> list[Lease]: ... + + async def replay(self, attachment: Attachment, operator: str) -> Job | None: ... + + async def note_replay(self, attachment_id: UUID, operator: str, result: str) -> None: ... + + +@runtime_checkable +class OutboxRepository(Protocol): + async def pending(self, limit: int) -> list[OutboxEntry]: ... + + async def published(self, entry_ids: Sequence[int]) -> None: ... + + +@runtime_checkable +class AttachmentsUnitOfWork(UnitOfWork, Protocol): + @property + def attachments(self) -> AttachmentRepository: ... + + @property + def jobs(self) -> JobRepository: ... + + @property + def outbox(self) -> OutboxRepository: ... + + +@runtime_checkable +class AttachmentDirectory(Protocol): + async def visible(self, attachment_id: UUID) -> Attachment | None: ... + + async def failed_among(self, chat_id: UUID, attachment_ids: Sequence[UUID]) -> bool: ... + + async def failed_jobs(self, limit: int) -> list[FailedJob]: ... diff --git a/src/messenger/attachments/application/queries.py b/src/messenger/attachments/application/queries.py new file mode 100644 index 00000000..973d759d --- /dev/null +++ b/src/messenger/attachments/application/queries.py @@ -0,0 +1,57 @@ +from uuid import UUID + +from messenger.attachments.application.errors import ObjectMissingError +from messenger.attachments.application.ports import ( + AttachmentDirectory, + ChatMembership, + ObjectContent, + ObjectStorage, +) +from messenger.attachments.domain.errors import AttachmentNotFound, AttachmentNotReady +from messenger.attachments.domain.models import Attachment +from messenger.shared.application.ports import Logger + +ORIGINAL = "original" + + +class GetAttachment: + __slots__ = ("_attachments", "_membership") + + def __init__(self, attachments: AttachmentDirectory, membership: ChatMembership) -> None: + self._attachments = attachments + self._membership = membership + + async def __call__(self, attachment_id: UUID, user_id: UUID) -> Attachment: + attachment = await self._attachments.visible(attachment_id) + if attachment is None or not await self._membership.is_member(attachment.chat_id, user_id): + raise AttachmentNotFound("вложение не найдено") + return attachment + + +class OpenAttachment: + __slots__ = ("_get", "_logger", "_storage") + + def __init__(self, get: GetAttachment, storage: ObjectStorage, logger: Logger) -> None: + self._get = get + self._storage = storage + self._logger = logger + + async def __call__( + self, attachment_id: UUID, user_id: UUID, variant: str = ORIGINAL + ) -> tuple[str, ObjectContent]: + attachment = await self._get(attachment_id, user_id) + if not attachment.ready: + raise AttachmentNotReady("вложение ещё не обработано или обработка не удалась") + if variant == ORIGINAL: + key, content_type = attachment.original_key, attachment.content_type + else: + found = attachment.variant(variant) + if found is None: + raise AttachmentNotFound("у вложения нет такого варианта") + key, content_type = found.object_key, found.content_type + try: + content = await self._storage.open(str(key)) + except ObjectMissingError as error: + self._logger.warning("attachment.object_missing", attachment_id=str(attachment.id)) + raise AttachmentNotFound("вложение не найдено") from error + return content_type, content diff --git a/src/messenger/attachments/config.py b/src/messenger/attachments/config.py new file mode 100644 index 00000000..e16e768b --- /dev/null +++ b/src/messenger/attachments/config.py @@ -0,0 +1,49 @@ +from pathlib import Path + +from pydantic import Field, SecretStr + +from messenger.shared.config import Settings + +UPLOAD_BYTES_MAX = 10_485_760 +IMAGE_PIXELS_MAX = 40_000_000 + + +class AttachmentsSettings(Settings): + s3_endpoint: str = "http://s3:9000" + s3_region: str = "us-east-1" + s3_bucket: str = Field(default="messenger-attachments", min_length=3) + s3_access_key_id: SecretStr = Field(min_length=3) + s3_secret_access_key: SecretStr = Field(min_length=8) + s3_connect_timeout_seconds: float = Field(default=1.0, gt=0, le=30) + s3_read_timeout_seconds: float = Field(default=2.0, gt=0, le=60) + s3_max_attempts: int = Field(default=2, ge=1, le=5) + max_upload_bytes: int = Field(default=UPLOAD_BYTES_MAX, ge=1, le=UPLOAD_BYTES_MAX) + max_image_pixels: int = Field(default=IMAGE_PIXELS_MAX, ge=1, le=IMAGE_PIXELS_MAX) + image_check_concurrency: int = Field(default=2, ge=1, le=16) + upload_concurrency: int = Field(default=3, ge=1, le=64) + upload_grace_seconds: int = Field(default=900, ge=60) + upload_sweep_interval_seconds: float = Field(default=30.0, gt=0) + upload_sweep_batch: int = Field(default=100, ge=1, le=10_000) + + +class PipelineSettings(AttachmentsSettings): + broker_url: SecretStr = Field(min_length=8) + worker_concurrency: int = Field(default=2, ge=1, le=32) + job_max_attempts: int = Field(default=5, ge=1, le=20) + job_timeout_seconds: float = Field(default=60.0, ge=1, le=290) + job_backoff_base_seconds: float = Field(default=2.0, gt=0, le=60) + job_backoff_cap_seconds: float = Field(default=60.0, gt=0, le=3600) + staging_retention_seconds: int = Field(default=86_400, ge=1) + outbox_poll_seconds: float = Field(default=1.0, gt=0, le=10) + outbox_batch: int = Field(default=50, ge=1, le=1000) + sweep_interval_seconds: float = Field(default=5.0, gt=0, le=300) + sweep_batch: int = Field(default=100, ge=1, le=10_000) + publish_timeout_seconds: float = Field(default=5.0, gt=0, le=60) + shutdown_grace_seconds: float = Field(default=5.0, gt=0, le=9) + heartbeat_file: Path | None = None + heartbeat_interval_seconds: float = Field(default=5.0, gt=0, le=10) + render_timeout_seconds: float = Field(default=30.0, gt=0, le=120) + render_memory_mb: int = Field(default=448, ge=0, le=8192) + render_tasks_per_child: int = Field(default=100, ge=1, le=10_000) + fault_point: str = "" + fault_stall_seconds: float = Field(default=15.0, ge=0, le=300) diff --git a/src/messenger/attachments/domain/__init__.py b/src/messenger/attachments/domain/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/messenger/attachments/domain/errors.py b/src/messenger/attachments/domain/errors.py new file mode 100644 index 00000000..dfb12d8c --- /dev/null +++ b/src/messenger/attachments/domain/errors.py @@ -0,0 +1,45 @@ +from messenger.shared.domain.errors import DomainError + + +class AttachmentError(DomainError): + pass + + +class InvalidFilename(AttachmentError): + pass + + +class UnsupportedImage(AttachmentError): + pass + + +class PayloadTooLarge(AttachmentError): + pass + + +class ChatNotFound(AttachmentError): + pass + + +class AttachmentNotFound(AttachmentError): + pass + + +class AttachmentNotReady(AttachmentError): + pass + + +class UploadAborted(AttachmentError): + pass + + +class UnreadableImage(AttachmentError): + pass + + +class ImageTooDemanding(AttachmentError): + pass + + +class LeaseLost(AttachmentError): + pass diff --git a/src/messenger/attachments/domain/events.py b/src/messenger/attachments/domain/events.py new file mode 100644 index 00000000..4c5eae1d --- /dev/null +++ b/src/messenger/attachments/domain/events.py @@ -0,0 +1,41 @@ +from dataclasses import dataclass, field +from uuid import UUID + +from messenger.shared.domain.events import DomainEvent + + +@dataclass(frozen=True, slots=True) +class AttachmentQueued(DomainEvent): + attachment_id: UUID + chat_id: UUID + owner_id: UUID + size_bytes: int + name: str = field(init=False, default="attachment.queued") + + +@dataclass(frozen=True, slots=True) +class AttachmentDiscarded(DomainEvent): + attachment_id: UUID + chat_id: UUID + reason: str + name: str = field(init=False, default="attachment.discarded") + + +@dataclass(frozen=True, slots=True) +class JobSettled(DomainEvent): + job_id: UUID + attachment_id: UUID + pipeline_version: int + attempt: int + outcome: str + error_code: str | None = None + delay_seconds: float | None = None + name: str = field(init=False, default="job.settled") + + +@dataclass(frozen=True, slots=True) +class AttachmentReplayRequested(DomainEvent): + attachment_id: UUID + operator: str + result: str + name: str = field(init=False, default="attachment.replay_requested") diff --git a/src/messenger/attachments/domain/jobs.py b/src/messenger/attachments/domain/jobs.py new file mode 100644 index 00000000..9c526e55 --- /dev/null +++ b/src/messenger/attachments/domain/jobs.py @@ -0,0 +1,95 @@ +from dataclasses import dataclass, replace +from datetime import UTC, datetime, timedelta +from enum import StrEnum +from typing import Any +from uuid import UUID + +JOB_TYPE = "attachment.process.v1" +SCHEMA_VERSION = 1 + + +class FailureCode(StrEnum): + INVALID_IMAGE = "invalid_image" + PROCESSING_FAILED = "processing_failed" + RETRY_EXHAUSTED = "retry_exhausted" + + +class ReplayResult(StrEnum): + REPLAYED = "replayed" + NOOP = "noop" + EXPIRED = "expired" + NOT_FOUND = "not_found" + + +class Outcome(StrEnum): + DONE = "done" + DUPLICATE = "duplicate_noop" + FENCED = "fenced" + SUPERSEDED = "superseded" + RESCHEDULED = "retry_scheduled" + LEASE_EXPIRED = "lease_expired" + FAILED = "failed" + + +@dataclass(frozen=True, slots=True) +class Job: + id: UUID + attachment_id: UUID + pipeline_version: int + attempt: int + created_at: datetime + + def payload(self) -> dict[str, Any]: + stamp = self.created_at.astimezone(UTC).strftime("%Y-%m-%dT%H:%M:%S.%fZ") + return { + "schema_version": SCHEMA_VERSION, + "job_id": str(self.id), + "type": JOB_TYPE, + "attachment_id": str(self.attachment_id), + "pipeline_version": self.pipeline_version, + "attempt": self.attempt, + "created_at": stamp, + } + + def retried(self, attempt: int) -> Job: + return replace(self, attempt=attempt) + + +@dataclass(frozen=True, slots=True) +class Lease: + job: Job + token: UUID + + +@dataclass(frozen=True, slots=True) +class Claim: + lease: Lease + source_key: str + staged: bool + + +@dataclass(frozen=True, slots=True) +class FailedJob: + job_id: UUID + attachment_id: UUID + pipeline_version: int + attempt: int + error_code: str | None + last_error: str | None + failed_at: datetime + + +@dataclass(frozen=True, slots=True) +class RetryPolicy: + max_attempts: int + base_seconds: float + cap_seconds: float + + def exhausted(self, attempt: int) -> bool: + return attempt >= self.max_attempts + + def delay(self, attempt: int, spread: float) -> timedelta | None: + if self.exhausted(attempt): + return None + ceiling = min(self.cap_seconds, self.base_seconds * 2 ** (attempt - 1)) + return timedelta(seconds=ceiling / 2 * (1 + spread)) diff --git a/src/messenger/attachments/domain/models.py b/src/messenger/attachments/domain/models.py new file mode 100644 index 00000000..cc917040 --- /dev/null +++ b/src/messenger/attachments/domain/models.py @@ -0,0 +1,103 @@ +from dataclasses import dataclass, field +from datetime import datetime +from enum import StrEnum +from uuid import UUID + +from messenger.attachments.domain.events import AttachmentDiscarded, AttachmentQueued +from messenger.attachments.domain.values import Filename, ImageInfo +from messenger.attachments.domain.variants import Variant +from messenger.shared.domain.events import DomainEvent + +STAGING_PREFIX = "staging/" +ORIGINALS_PREFIX = "originals/" + + +class AttachmentStatus(StrEnum): + PENDING = "pending" + QUEUED = "queued" + PROCESSING = "processing" + READY = "ready" + FAILED = "failed" + + +def original_key(attachment_id: UUID) -> str: + return f"{ORIGINALS_PREFIX}{attachment_id}" + + +@dataclass(slots=True) +class Attachment: + id: UUID + chat_id: UUID + owner_id: UUID + filename: Filename + image: ImageInfo + size_bytes: int + sha256: str + status: AttachmentStatus + created_at: datetime + updated_at: datetime + staging_key: str | None = None + original_key: str | None = None + published_version: int = 0 + error_code: str | None = None + variants: tuple[Variant, ...] = () + _events: list[DomainEvent] = field(init=False, default_factory=list, repr=False, compare=False) + + @classmethod + def stage( + cls, + *, + attachment_id: UUID, + chat_id: UUID, + owner_id: UUID, + filename: Filename, + image: ImageInfo, + size_bytes: int, + sha256: str, + now: datetime, + ) -> Attachment: + return cls( + id=attachment_id, + chat_id=chat_id, + owner_id=owner_id, + filename=filename, + image=image, + size_bytes=size_bytes, + sha256=sha256, + status=AttachmentStatus.PENDING, + created_at=now, + updated_at=now, + staging_key=f"{STAGING_PREFIX}{attachment_id}", + ) + + @property + def content_type(self) -> str: + return self.image.kind.value + + @property + def ready(self) -> bool: + return self.status is AttachmentStatus.READY + + def variant(self, name: str) -> Variant | None: + return next((variant for variant in self.variants if variant.name == name), None) + + def enqueue(self, now: datetime) -> None: + self.status = AttachmentStatus.QUEUED + self.updated_at = max(now, self.created_at) + self._events.append( + AttachmentQueued( + attachment_id=self.id, + chat_id=self.chat_id, + owner_id=self.owner_id, + size_bytes=self.size_bytes, + ) + ) + + def discard(self, reason: str) -> None: + self._events.append( + AttachmentDiscarded(attachment_id=self.id, chat_id=self.chat_id, reason=reason) + ) + + def collect_events(self) -> list[DomainEvent]: + events, self._events = self._events, [] + return events diff --git a/src/messenger/attachments/domain/values.py b/src/messenger/attachments/domain/values.py new file mode 100644 index 00000000..4c8a9a8f --- /dev/null +++ b/src/messenger/attachments/domain/values.py @@ -0,0 +1,46 @@ +import unicodedata +from dataclasses import dataclass +from enum import StrEnum + +from messenger.attachments.domain.errors import InvalidFilename, UnsupportedImage + +FILENAME_MAX = 255 +SEPARATORS = frozenset("/\\") + + +@dataclass(frozen=True, slots=True) +class Filename: + value: str + + def __post_init__(self) -> None: + if not self.value.strip(): + raise InvalidFilename("параметр filename обязателен и не может быть пустым") + if len(self.value) > FILENAME_MAX: + raise InvalidFilename(f"параметр filename длиннее {FILENAME_MAX} символов Unicode") + if any(c in SEPARATORS or unicodedata.category(c) == "Cc" for c in self.value): + raise InvalidFilename( + "параметр filename содержит разделитель пути или управляющий символ" + ) + + def __str__(self) -> str: + return self.value + + +class ImageKind(StrEnum): + PNG = "image/png" + JPEG = "image/jpeg" + + @classmethod + def declared(cls, content_type: str | None) -> ImageKind: + media = (content_type or "").partition(";")[0].strip().lower() + try: + return cls(media) + except ValueError as error: + raise UnsupportedImage("Content-Type должен быть image/png или image/jpeg") from error + + +@dataclass(frozen=True, slots=True) +class ImageInfo: + kind: ImageKind + width: int + height: int diff --git a/src/messenger/attachments/domain/variants.py b/src/messenger/attachments/domain/variants.py new file mode 100644 index 00000000..f787d81f --- /dev/null +++ b/src/messenger/attachments/domain/variants.py @@ -0,0 +1,35 @@ +from dataclasses import dataclass +from uuid import UUID + +THUMBNAIL = "thumbnail" +CROP = "crop" +OPTIMIZED = "optimized" +THUMBNAIL_BOX = 256 +CROP_SIDE = 128 +OPTIMIZED_BOX = 1280 +VARIANTS_PREFIX = "variants/" + + +@dataclass(frozen=True, slots=True) +class Variant: + name: str + content_type: str + size_bytes: int + width: int + height: int + sha256: str + object_key: str + + +def fit(width: int, height: int, box: int) -> tuple[int, int]: + if max(width, height) <= box: + return width, height + if width >= height: + return box, max(1, height * box // width) + return max(1, width * box // height), box + + +def variant_key( + attachment_id: UUID, pipeline_version: int, name: str, sha256: str, extension: str +) -> str: + return f"{VARIANTS_PREFIX}{attachment_id}/{pipeline_version}/{name}-{sha256[:16]}.{extension}" diff --git a/src/messenger/attachments/entrypoints/__init__.py b/src/messenger/attachments/entrypoints/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/messenger/attachments/entrypoints/admin.py b/src/messenger/attachments/entrypoints/admin.py new file mode 100644 index 00000000..ca87a9a1 --- /dev/null +++ b/src/messenger/attachments/entrypoints/admin.py @@ -0,0 +1,50 @@ +import argparse +import json +import os +from dataclasses import asdict +from datetime import datetime +from typing import Any +from uuid import UUID + +from dishka import AsyncContainer + +from messenger.attachments.application.bus import AttachmentsCommandBus +from messenger.attachments.application.commands import ReplayAttachment +from messenger.attachments.application.ports import AttachmentDirectory +from messenger.attachments.domain.jobs import ReplayResult + +REGISTRY_LIMIT = 20 + + +def parser() -> argparse.ArgumentParser: + root = argparse.ArgumentParser( + prog="python -m messenger.admin", description="Реестр failed-заданий и явный replay" + ) + commands = root.add_subparsers(dest="command", required=True) + failed = commands.add_parser("failed", help="последние задания в failed") + failed.add_argument("--limit", type=int, default=REGISTRY_LIMIT) + replay = commands.add_parser("replay", help="вернуть failed-вложение в очередь") + replay.add_argument("--attachment-id", type=UUID, required=True) + replay.add_argument("--operator", default=os.environ.get("OPERATOR", "cli")) + return root + + +async def run(container: AsyncContainer, argv: list[str]) -> dict[str, Any]: + args = parser().parse_args(argv) + async with container() as scope: + if args.command == "failed": + directory = await scope.get(AttachmentDirectory) + return {"failed": [asdict(job) for job in await directory.failed_jobs(args.limit)]} + bus = await scope.get(AttachmentsCommandBus) + result: ReplayResult = await bus.execute( + ReplayAttachment(args.attachment_id, args.operator) + ) + return {"result": result.value, "attachment_id": args.attachment_id, "operator": args.operator} + + +def plain(value: object) -> str: + return value.isoformat() if isinstance(value, datetime) else str(value) + + +def rendered(report: dict[str, Any]) -> str: + return json.dumps(report, default=plain, ensure_ascii=False) diff --git a/src/messenger/attachments/entrypoints/errors.py b/src/messenger/attachments/entrypoints/errors.py new file mode 100644 index 00000000..374038cb --- /dev/null +++ b/src/messenger/attachments/entrypoints/errors.py @@ -0,0 +1,20 @@ +from messenger.attachments.domain.errors import ( + AttachmentNotFound, + AttachmentNotReady, + ChatNotFound, + InvalidFilename, + PayloadTooLarge, + UnsupportedImage, + UploadAborted, +) +from messenger.shared.domain.errors import DomainError + +ERROR_CODES: dict[type[DomainError], tuple[str, int]] = { + InvalidFilename: ("invalid_request", 400), + UploadAborted: ("invalid_request", 400), + ChatNotFound: ("not_found", 404), + AttachmentNotFound: ("not_found", 404), + AttachmentNotReady: ("conflict", 409), + PayloadTooLarge: ("payload_too_large", 413), + UnsupportedImage: ("unsupported_media_type", 415), +} diff --git a/src/messenger/attachments/entrypoints/heartbeat.py b/src/messenger/attachments/entrypoints/heartbeat.py new file mode 100644 index 00000000..707ce11d --- /dev/null +++ b/src/messenger/attachments/entrypoints/heartbeat.py @@ -0,0 +1,44 @@ +import asyncio +from collections.abc import Awaitable, Callable +from contextlib import suppress +from pathlib import Path + +from messenger.shared.application.ports import Logger + + +class Heartbeat: + __slots__ = ("_interval", "_logger", "_path", "_probe", "_task") + + def __init__( + self, + path: Path | None, + probe: Callable[[], Awaitable[bool]], + interval: float, + logger: Logger, + ) -> None: + self._path = path + self._probe = probe + self._interval = interval + self._logger = logger + self._task: asyncio.Task[None] | None = None + + def start(self) -> None: + if self._path is not None: + self._task = asyncio.create_task(self._run(self._path)) + + async def stop(self) -> None: + if self._task is None: + return + self._task.cancel() + with suppress(asyncio.CancelledError): + await self._task + self._task = None + + async def _run(self, path: Path) -> None: + while True: + try: + if await self._probe(): + await asyncio.to_thread(path.touch) + except Exception as error: + self._logger.warning("heartbeat.failed", error=type(error).__name__) + await asyncio.sleep(self._interval) diff --git a/src/messenger/attachments/entrypoints/http.py b/src/messenger/attachments/entrypoints/http.py new file mode 100644 index 00000000..9899fecd --- /dev/null +++ b/src/messenger/attachments/entrypoints/http.py @@ -0,0 +1,77 @@ +import re +from collections.abc import AsyncIterator +from typing import Annotated +from uuid import UUID + +from dishka.integrations.fastapi import FromDishka, inject +from fastapi import APIRouter, Query, Request, status +from fastapi.responses import StreamingResponse +from starlette.requests import ClientDisconnect + +from messenger.attachments.application.bus import AttachmentsCommandBus +from messenger.attachments.application.commands import UploadAttachment +from messenger.attachments.application.queries import GetAttachment, OpenAttachment +from messenger.attachments.domain.errors import UploadAborted +from messenger.attachments.entrypoints.schemas import AttachmentOut, VariantName +from messenger.shared.entrypoints.actor import Actor + +UPLOAD_PATH = re.compile(r"/api/v1/chats/[^/]+/attachments/?") +CONTENT_HEADERS = {"X-Content-Type-Options": "nosniff", "Cache-Control": "private, no-store"} + +uploads_router = APIRouter(prefix="/api/v1/chats/{chat_id}/attachments", tags=["attachments"]) +attachments_router = APIRouter(prefix="/api/v1/attachments", tags=["attachments"]) + + +async def body_of(request: Request) -> AsyncIterator[bytes]: + try: + async for chunk in request.stream(): + yield chunk + except ClientDisconnect as error: + raise UploadAborted("клиент оборвал загрузку до конца тела") from error + + +@uploads_router.post("", status_code=status.HTTP_202_ACCEPTED) +@inject +async def upload_attachment( + chat_id: UUID, + actor: Actor, + request: Request, + bus: FromDishka[AttachmentsCommandBus], + filename: Annotated[str | None, Query()] = None, +) -> AttachmentOut: + attachment = await bus.execute( + UploadAttachment( + owner_id=actor, + chat_id=chat_id, + filename=filename, + content_type=request.headers.get("content-type"), + body=body_of(request), + ) + ) + return AttachmentOut.of(attachment) + + +@attachments_router.get("/{attachment_id}") +@inject +async def read_attachment( + attachment_id: UUID, + actor: Actor, + query: FromDishka[GetAttachment], +) -> AttachmentOut: + return AttachmentOut.of(await query(attachment_id, actor)) + + +@attachments_router.get("/{attachment_id}/content") +@inject +async def download_attachment( + attachment_id: UUID, + actor: Actor, + query: FromDishka[OpenAttachment], + variant: Annotated[VariantName, Query()] = "original", +) -> StreamingResponse: + content_type, content = await query(attachment_id, actor, variant) + return StreamingResponse( + content.chunks, + media_type=content_type, + headers={**CONTENT_HEADERS, "Content-Length": str(content.size)}, + ) diff --git a/src/messenger/attachments/entrypoints/ioc.py b/src/messenger/attachments/entrypoints/ioc.py new file mode 100644 index 00000000..c661dd1e --- /dev/null +++ b/src/messenger/attachments/entrypoints/ioc.py @@ -0,0 +1,268 @@ +from collections.abc import AsyncIterator, Iterator +from contextlib import asynccontextmanager +from datetime import timedelta + +from dishka import Provider, Scope, provide +from faststream.rabbit import RabbitBroker + +from messenger.attachments.application.bus import AttachmentsCommandBus +from messenger.attachments.application.commands import ( + CleanStaging, + ProcessJob, + PublishOutbox, + RecoverExpiredLeases, + ReplayAttachment, + SweepAbandonedUploads, + UploadAttachment, +) +from messenger.attachments.application.handlers import ( + SweepAbandonedUploadsHandler, + UploadAttachmentHandler, + UploadSlots, +) +from messenger.attachments.application.pipeline import ( + CleanStagingHandler, + ProcessJobHandler, + PublishOutboxHandler, + RecoverExpiredLeasesHandler, + ReplayAttachmentHandler, +) +from messenger.attachments.application.ports import ( + AttachmentDirectory, + AttachmentsUnitOfWork, + ChatMembership, + Faults, + ImageInspector, + Jitter, + JobPublisher, + ObjectStorage, + VariantRenderer, +) +from messenger.attachments.application.queries import GetAttachment, OpenAttachment +from messenger.attachments.config import AttachmentsSettings, PipelineSettings +from messenger.attachments.domain.jobs import RetryPolicy +from messenger.attachments.infrastructure.faults import EnvironmentFaults +from messenger.attachments.infrastructure.images import ( + PillowImageInspector, + ProcessVariantRenderer, + RenderLimits, +) +from messenger.attachments.infrastructure.jitter import RandomJitter +from messenger.attachments.infrastructure.postgres import ( + PostgresAttachmentDirectory, + PostgresAttachmentsUnitOfWork, +) +from messenger.attachments.infrastructure.rabbitmq import RabbitJobPublisher +from messenger.attachments.infrastructure.s3 import S3ObjectStorage, s3_client +from messenger.shared.application.commands import Command +from messenger.shared.application.messagebus import CommandHandler +from messenger.shared.application.ports import Clock, EventPublisher, IdGenerator, Logger +from messenger.shared.infrastructure.postgres import Database + + +@asynccontextmanager +async def object_storage(settings: AttachmentsSettings) -> AsyncIterator[ObjectStorage]: + async with s3_client( + endpoint=settings.s3_endpoint, + region=settings.s3_region, + access_key_id=settings.s3_access_key_id.get_secret_value(), + secret_access_key=settings.s3_secret_access_key.get_secret_value(), + connect_timeout=settings.s3_connect_timeout_seconds, + read_timeout=settings.s3_read_timeout_seconds, + max_attempts=settings.s3_max_attempts, + ) as client: + yield S3ObjectStorage(client, settings.s3_bucket) + + +class AttachmentsProvider(Provider): + def __init__(self, settings: AttachmentsSettings) -> None: + super().__init__() + self._settings = settings + + @provide(scope=Scope.APP) + def settings(self) -> AttachmentsSettings: + return self._settings + + @provide(scope=Scope.APP) + async def storage(self, settings: AttachmentsSettings) -> AsyncIterator[ObjectStorage]: + async with object_storage(settings) as storage: + yield storage + + @provide(scope=Scope.APP) + def images(self, settings: AttachmentsSettings) -> ImageInspector: + return PillowImageInspector(settings.max_image_pixels, settings.image_check_concurrency) + + @provide(scope=Scope.APP) + def slots(self, settings: AttachmentsSettings) -> UploadSlots: + return UploadSlots(settings.upload_concurrency) + + @provide(scope=Scope.APP) + def directory(self, database: Database) -> AttachmentDirectory: + return PostgresAttachmentDirectory(database) + + @provide(scope=Scope.REQUEST) + def unit_of_work(self, database: Database) -> AttachmentsUnitOfWork: + return PostgresAttachmentsUnitOfWork(database) + + @provide(scope=Scope.REQUEST) + def attachment( + self, directory: AttachmentDirectory, membership: ChatMembership + ) -> GetAttachment: + return GetAttachment(directory, membership) + + @provide(scope=Scope.REQUEST) + def content( + self, attachment: GetAttachment, storage: ObjectStorage, logger: Logger + ) -> OpenAttachment: + return OpenAttachment(attachment, storage, logger) + + @provide(scope=Scope.REQUEST) + def command_bus( + self, + uow: AttachmentsUnitOfWork, + membership: ChatMembership, + images: ImageInspector, + storage: ObjectStorage, + publisher: EventPublisher, + clock: Clock, + ids: IdGenerator, + settings: AttachmentsSettings, + slots: UploadSlots, + logger: Logger, + ) -> AttachmentsCommandBus: + upload = UploadAttachmentHandler( + uow, + membership, + images, + storage, + clock=clock, + ids=ids, + limit=settings.max_upload_bytes, + slots=slots, + logger=logger, + ) + handlers: dict[type[Command], CommandHandler] = { + UploadAttachment: upload, + SweepAbandonedUploads: SweepAbandonedUploadsHandler(uow, storage, clock), + } + return AttachmentsCommandBus(handlers, uow, publisher) + + +class PipelineProvider(Provider): + def __init__(self, settings: PipelineSettings) -> None: + super().__init__() + self._settings = settings + + @provide(scope=Scope.APP) + def settings(self) -> PipelineSettings: + return self._settings + + @provide(scope=Scope.APP) + async def storage(self, settings: PipelineSettings) -> AsyncIterator[ObjectStorage]: + async with object_storage(settings) as storage: + yield storage + + @provide(scope=Scope.APP) + def policy(self, settings: PipelineSettings) -> RetryPolicy: + return RetryPolicy( + settings.job_max_attempts, + settings.job_backoff_base_seconds, + settings.job_backoff_cap_seconds, + ) + + @provide(scope=Scope.APP) + def faults(self, settings: PipelineSettings, logger: Logger) -> Faults: + return EnvironmentFaults(settings.fault_point, logger, settings.fault_stall_seconds) + + @provide(scope=Scope.REQUEST) + def unit_of_work(self, database: Database) -> AttachmentsUnitOfWork: + return PostgresAttachmentsUnitOfWork(database) + + +class WorkerProvider(Provider): + @provide(scope=Scope.APP) + def renderer(self, settings: PipelineSettings) -> Iterator[VariantRenderer]: + renderer = ProcessVariantRenderer( + RenderLimits(settings.max_upload_bytes, settings.max_image_pixels), + concurrency=settings.worker_concurrency, + timeout=settings.render_timeout_seconds, + memory_bytes=settings.render_memory_mb << 20, + tasks_per_child=settings.render_tasks_per_child, + ) + yield renderer + renderer.close() + + @provide(scope=Scope.APP) + def jitter(self) -> Jitter: + return RandomJitter() + + @provide(scope=Scope.REQUEST) + def command_bus( + self, + uow: AttachmentsUnitOfWork, + storage: ObjectStorage, + renderer: VariantRenderer, + faults: Faults, + publisher: EventPublisher, + ids: IdGenerator, + jitter: Jitter, + policy: RetryPolicy, + settings: PipelineSettings, + ) -> AttachmentsCommandBus: + process = ProcessJobHandler( + uow, + storage, + renderer, + faults, + ids=ids, + jitter=jitter, + policy=policy, + lease=timedelta(seconds=settings.job_timeout_seconds), + ) + return AttachmentsCommandBus({ProcessJob: process}, uow, publisher) + + +class RelayProvider(Provider): + def __init__(self, broker: RabbitBroker) -> None: + super().__init__() + self._broker = broker + + @provide(scope=Scope.APP) + def publisher(self, settings: PipelineSettings) -> JobPublisher: + return RabbitJobPublisher(self._broker, settings.publish_timeout_seconds) + + @provide(scope=Scope.REQUEST) + def command_bus( + self, + uow: AttachmentsUnitOfWork, + storage: ObjectStorage, + jobs: JobPublisher, + faults: Faults, + publisher: EventPublisher, + policy: RetryPolicy, + ) -> AttachmentsCommandBus: + handlers: dict[type[Command], CommandHandler] = { + PublishOutbox: PublishOutboxHandler(uow, jobs, faults), + RecoverExpiredLeases: RecoverExpiredLeasesHandler(uow, policy), + CleanStaging: CleanStagingHandler(uow, storage), + } + return AttachmentsCommandBus(handlers, uow, publisher) + + +class AdminProvider(Provider): + @provide(scope=Scope.APP) + def directory(self, database: Database) -> AttachmentDirectory: + return PostgresAttachmentDirectory(database) + + @provide(scope=Scope.REQUEST) + def unit_of_work(self, database: Database) -> AttachmentsUnitOfWork: + return PostgresAttachmentsUnitOfWork(database) + + @provide(scope=Scope.REQUEST) + def command_bus( + self, uow: AttachmentsUnitOfWork, publisher: EventPublisher + ) -> AttachmentsCommandBus: + handlers: dict[type[Command], CommandHandler] = { + ReplayAttachment: ReplayAttachmentHandler(uow) + } + return AttachmentsCommandBus(handlers, uow, publisher) diff --git a/src/messenger/attachments/entrypoints/journal.py b/src/messenger/attachments/entrypoints/journal.py new file mode 100644 index 00000000..014f23fd --- /dev/null +++ b/src/messenger/attachments/entrypoints/journal.py @@ -0,0 +1,78 @@ +from messenger.attachments.domain.events import ( + AttachmentDiscarded, + AttachmentQueued, + AttachmentReplayRequested, + JobSettled, +) +from messenger.shared.application.messagebus import EventBus +from messenger.shared.application.ports import Logger + +LOG_FIELDS = frozenset( + { + "attachment_id", + "chat_id", + "actor_id", + "size_bytes", + "count", + "job_id", + "attempt", + "pipeline_version", + "outcome", + "error_code", + "delay_seconds", + "operator", + "result", + } +) + + +class AttachmentsJournal: + __slots__ = ("_logger",) + + def __init__(self, logger: Logger) -> None: + self._logger = logger + + async def on_attachment_queued(self, event: AttachmentQueued) -> None: + self._logger.info( + "audit.attachment_queued", + attachment_id=str(event.attachment_id), + chat_id=str(event.chat_id), + actor_id=str(event.owner_id), + size_bytes=event.size_bytes, + ) + + async def on_attachment_discarded(self, event: AttachmentDiscarded) -> None: + self._logger.info( + "audit.attachment_discarded", + attachment_id=str(event.attachment_id), + chat_id=str(event.chat_id), + reason=event.reason, + ) + + async def on_job_settled(self, event: JobSettled) -> None: + self._logger.info( + "audit.job_settled", + job_id=str(event.job_id), + attachment_id=str(event.attachment_id), + pipeline_version=event.pipeline_version, + attempt=event.attempt, + outcome=event.outcome, + error_code=event.error_code, + delay_seconds=event.delay_seconds, + ) + + async def on_replay_requested(self, event: AttachmentReplayRequested) -> None: + self._logger.info( + "audit.attachment_replay", + attachment_id=str(event.attachment_id), + operator=event.operator, + result=event.result, + ) + + +def subscribe_journal(bus: EventBus, logger: Logger) -> None: + journal = AttachmentsJournal(logger) + bus.subscribe(AttachmentQueued, journal.on_attachment_queued) + bus.subscribe(AttachmentDiscarded, journal.on_attachment_discarded) + bus.subscribe(JobSettled, journal.on_job_settled) + bus.subscribe(AttachmentReplayRequested, journal.on_replay_requested) diff --git a/src/messenger/attachments/entrypoints/relay.py b/src/messenger/attachments/entrypoints/relay.py new file mode 100644 index 00000000..9c3436fe --- /dev/null +++ b/src/messenger/attachments/entrypoints/relay.py @@ -0,0 +1,73 @@ +import asyncio +from collections.abc import Sequence +from contextlib import suppress +from datetime import timedelta + +from dishka import AsyncContainer + +from messenger.attachments.application.bus import AttachmentsCommandBus +from messenger.attachments.application.commands import ( + CleanStaging, + PublishOutbox, + RecoverExpiredLeases, +) +from messenger.attachments.config import PipelineSettings +from messenger.shared.application.commands import Command +from messenger.shared.application.ports import Logger + + +class Relay: + __slots__ = ("_container", "_logger", "_settings", "_tasks") + + def __init__( + self, container: AsyncContainer, settings: PipelineSettings, logger: Logger + ) -> None: + self._container = container + self._settings = settings + self._logger = logger + self._tasks: list[asyncio.Task[None]] = [] + + def start(self) -> None: + settings = self._settings + loops: tuple[tuple[float, Sequence[Command]], ...] = ( + (settings.outbox_poll_seconds, (PublishOutbox(settings.outbox_batch),)), + ( + settings.sweep_interval_seconds, + ( + RecoverExpiredLeases(settings.sweep_batch), + CleanStaging( + settings.sweep_batch, timedelta(seconds=settings.staging_retention_seconds) + ), + ), + ), + ) + self._tasks = [asyncio.create_task(self._every(*loop)) for loop in loops] + + async def stop(self) -> None: + for task in self._tasks: + task.cancel() + for task in self._tasks: + with suppress(asyncio.CancelledError): + await task + self._tasks = [] + + async def run(self, command: Command) -> int: + async with self._container() as scope: + bus = await scope.get(AttachmentsCommandBus) + return await bus.execute(command) + + async def _every(self, interval: float, commands: Sequence[Command]) -> None: + while True: + for command in commands: + try: + done = await self.run(command) + except Exception as error: + self._logger.warning( + "relay.pass_failed", + reason=type(command).__name__, + error=type(error).__name__, + ) + continue + if done: + self._logger.info("relay.pass", reason=type(command).__name__, count=done) + await asyncio.sleep(interval) diff --git a/src/messenger/attachments/entrypoints/schemas.py b/src/messenger/attachments/entrypoints/schemas.py new file mode 100644 index 00000000..f482d4a5 --- /dev/null +++ b/src/messenger/attachments/entrypoints/schemas.py @@ -0,0 +1,84 @@ +from datetime import datetime +from typing import Annotated, Literal +from uuid import UUID + +from pydantic import AwareDatetime, BaseModel, ConfigDict, Field + +from messenger.attachments.domain.jobs import Job +from messenger.attachments.domain.models import Attachment +from messenger.attachments.domain.variants import Variant + +type VariantName = Literal["original", "thumbnail", "crop", "optimized"] + + +class VariantOut(BaseModel): + name: str + content_type: str + size_bytes: int + width: int + height: int + sha256: str + + @classmethod + def of(cls, variant: Variant) -> VariantOut: + return cls( + name=variant.name, + content_type=variant.content_type, + size_bytes=variant.size_bytes, + width=variant.width, + height=variant.height, + sha256=variant.sha256, + ) + + +class AttachmentOut(BaseModel): + id: UUID + chat_id: UUID + owner_id: UUID + filename: str + content_type: str + size_bytes: int + sha256: str + status: str + variants: list[VariantOut] + error_code: str | None + created_at: datetime + updated_at: datetime + + @classmethod + def of(cls, attachment: Attachment) -> AttachmentOut: + return cls( + id=attachment.id, + chat_id=attachment.chat_id, + owner_id=attachment.owner_id, + filename=attachment.filename.value, + content_type=attachment.content_type, + size_bytes=attachment.size_bytes, + sha256=attachment.sha256, + status=attachment.status.value, + variants=[VariantOut.of(variant) for variant in attachment.variants], + error_code=attachment.error_code, + created_at=attachment.created_at, + updated_at=attachment.updated_at, + ) + + +class JobMessage(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + schema_version: Literal[1] + job_id: UUID + type: Literal["attachment.process.v1"] + attachment_id: UUID + pipeline_version: Annotated[int, Field(ge=1)] + attempt: Annotated[int, Field(ge=1)] + created_at: AwareDatetime + traceparent: Annotated[ + str | None, Field(pattern=r"^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$") + ] = None + tracestate: Annotated[str | None, Field(max_length=512)] = None + + def job(self) -> Job: + return Job( + self.job_id, self.attachment_id, self.pipeline_version, self.attempt, self.created_at + ) diff --git a/src/messenger/attachments/entrypoints/sweeper.py b/src/messenger/attachments/entrypoints/sweeper.py new file mode 100644 index 00000000..63b08da4 --- /dev/null +++ b/src/messenger/attachments/entrypoints/sweeper.py @@ -0,0 +1,53 @@ +import asyncio +from contextlib import suppress +from datetime import timedelta + +from dishka import AsyncContainer + +from messenger.attachments.application.bus import AttachmentsCommandBus +from messenger.attachments.application.commands import SweepAbandonedUploads +from messenger.attachments.config import AttachmentsSettings +from messenger.shared.application.ports import Logger + + +class UploadSweeper: + __slots__ = ("_command", "_container", "_interval", "_logger", "_task") + + def __init__( + self, container: AsyncContainer, settings: AttachmentsSettings, logger: Logger + ) -> None: + self._container = container + self._logger = logger + self._interval = settings.upload_sweep_interval_seconds + self._command = SweepAbandonedUploads( + grace=timedelta(seconds=settings.upload_grace_seconds), + limit=settings.upload_sweep_batch, + ) + self._task: asyncio.Task[None] | None = None + + def start(self) -> None: + self._task = asyncio.create_task(self._run()) + + async def stop(self) -> None: + if self._task is None: + return + self._task.cancel() + with suppress(asyncio.CancelledError): + await self._task + self._task = None + + async def sweep(self) -> int: + async with self._container() as scope: + bus = await scope.get(AttachmentsCommandBus) + return await bus.execute(self._command) + + async def _run(self) -> None: + while True: + await asyncio.sleep(self._interval) + try: + swept = await self.sweep() + except Exception as error: + self._logger.warning("attachments.sweep_failed", error=type(error).__name__) + continue + if swept: + self._logger.info("attachments.swept", count=swept) diff --git a/src/messenger/attachments/entrypoints/worker.py b/src/messenger/attachments/entrypoints/worker.py new file mode 100644 index 00000000..8de721bd --- /dev/null +++ b/src/messenger/attachments/entrypoints/worker.py @@ -0,0 +1,50 @@ +import asyncio + +from dishka import FromDishka +from faststream import AckPolicy +from faststream.rabbit import Channel, QueueType, RabbitBroker, RabbitQueue +from faststream.rabbit.annotations import RabbitMessage +from pydantic import ValidationError + +from messenger.attachments.application.bus import AttachmentsCommandBus +from messenger.attachments.application.commands import ProcessJob +from messenger.attachments.entrypoints.schemas import JobMessage +from messenger.attachments.infrastructure.rabbitmq import QUEUE +from messenger.shared.application.ports import Logger + +REQUEUE_DELAY_SECONDS = 1.0 +JOBS = RabbitQueue(QUEUE, queue_type=QueueType.QUORUM, declare=False) + + +async def raw(message: RabbitMessage) -> bytes: + return message.body + + +async def handle_job( + message: RabbitMessage, + bus: FromDishka[AttachmentsCommandBus], + logger: FromDishka[Logger], +) -> None: + try: + job = JobMessage.model_validate_json(message.body).job() + except ValidationError: + logger.warning("job.rejected", reason="message does not match jobs.schema.json") + await message.reject(requeue=False) + return + try: + await bus.execute(ProcessJob(job)) + except Exception as error: + logger.warning("job.requeued", job_id=str(job.id), error=type(error).__name__) + await asyncio.sleep(REQUEUE_DELAY_SECONDS) + await message.nack(requeue=True) + return + await message.ack() + + +def subscribe(broker: RabbitBroker, concurrency: int) -> None: + broker.subscriber( + JOBS, + channel=Channel(prefetch_count=concurrency), + ack_policy=AckPolicy.MANUAL, + decoder=raw, + )(handle_job) diff --git a/src/messenger/attachments/infrastructure/__init__.py b/src/messenger/attachments/infrastructure/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/messenger/attachments/infrastructure/faults.py b/src/messenger/attachments/infrastructure/faults.py new file mode 100644 index 00000000..9797c731 --- /dev/null +++ b/src/messenger/attachments/infrastructure/faults.py @@ -0,0 +1,32 @@ +import asyncio +import os +import sys + +from messenger.attachments.application.pipeline import STALL_AFTER_CLAIM, TRANSIENT_ERROR +from messenger.shared.application.errors import Unavailable +from messenger.shared.application.ports import Logger + +CRASH_EXIT_CODE = 137 + + +class EnvironmentFaults: + __slots__ = ("_logger", "_point", "_stall_seconds") + + def __init__(self, point: str, logger: Logger, stall_seconds: float = 0.0) -> None: + self._point = point + self._logger = logger + self._stall_seconds = stall_seconds + + async def reach(self, point: str) -> None: + if not self._point or point != self._point: + return + self._logger.warning("fault.injected", reason=point) + if point == TRANSIENT_ERROR: + raise Unavailable("внедрённая временная ошибка") + if point == STALL_AFTER_CLAIM: + self._point = "" + await asyncio.sleep(self._stall_seconds) + return + sys.stdout.flush() + sys.stderr.flush() + os._exit(CRASH_EXIT_CODE) diff --git a/src/messenger/attachments/infrastructure/images.py b/src/messenger/attachments/infrastructure/images.py new file mode 100644 index 00000000..b4d02053 --- /dev/null +++ b/src/messenger/attachments/infrastructure/images.py @@ -0,0 +1,206 @@ +import asyncio +import resource +import sys +import warnings +from collections.abc import Callable +from concurrent.futures import ProcessPoolExecutor +from concurrent.futures.process import BrokenProcessPool +from dataclasses import dataclass +from io import BytesIO + +from PIL import Image, ImageOps + +from messenger.attachments.application.ports import Rendition +from messenger.attachments.domain.errors import ( + ImageTooDemanding, + PayloadTooLarge, + UnreadableImage, + UnsupportedImage, +) +from messenger.attachments.domain.values import ImageInfo, ImageKind +from messenger.attachments.domain.variants import ( + CROP, + CROP_SIDE, + OPTIMIZED, + OPTIMIZED_BOX, + THUMBNAIL, + THUMBNAIL_BOX, + fit, +) +from messenger.shared.application.errors import Unavailable + +KINDS = {"PNG": ImageKind.PNG, "JPEG": ImageKind.JPEG} +BOMBS = (Image.DecompressionBombError, Image.DecompressionBombWarning) +JPEG_SCALE = 8 +WIDE_MODES = frozenset({"I", "I;16", "I;16B", "I;16L", "I;16N"}) +ALPHA_MODES = frozenset({"LA", "La", "PA", "RGBa"}) +JPEG = {"quality": 85, "optimize": True} +WEBP = {"quality": 80, "method": 4} +ENCODINGS = {"JPEG": (ImageKind.JPEG.value, "jpg"), "WEBP": ("image/webp", "webp")} +REDUCING_GAP = 3.0 + + +@dataclass(frozen=True, slots=True) +class RenderLimits: + max_bytes: int + max_pixels: int + + +class PillowImageInspector: + __slots__ = ("_max_pixels", "_slots") + + def __init__(self, max_pixels: int, concurrency: int) -> None: + self._max_pixels = max_pixels + self._slots = asyncio.Semaphore(concurrency) + warnings.simplefilter("error", Image.DecompressionBombWarning) + + async def inspect(self, data: bytes, declared: ImageKind) -> ImageInfo: + async with self._slots: + return await asyncio.to_thread(self._inspect, data, declared) + + def _inspect(self, data: bytes, declared: ImageKind) -> ImageInfo: + try: + image = Image.open(BytesIO(data), formats=tuple(KINDS)) + except BOMBS as error: + raise PayloadTooLarge("изображение больше допустимого числа пикселей") from error + except Exception as error: + raise UnsupportedImage("содержимое не является изображением PNG или JPEG") from error + with image: + kind = KINDS[str(image.format)] + width, height = image.size + if kind is not declared: + raise UnsupportedImage("содержимое не совпадает с заявленным Content-Type") + if width * height > self._max_pixels: + raise PayloadTooLarge("изображение больше допустимого числа пикселей") + try: + if kind is ImageKind.JPEG: + image.draft("RGB", (width // JPEG_SCALE or 1, height // JPEG_SCALE or 1)) + image.load() + else: + image.verify() + except Exception as error: + raise UnsupportedImage("изображение повреждено или обрезано") from error + return ImageInfo(kind, width, height) + + +def flatten(image: Image.Image) -> Image.Image: + if image.mode in WIDE_MODES: + image = image.convert("I").point(lambda value: value / 256, "L") + if image.mode == "P": + image = image.convert("RGBA" if "transparency" in image.info else "RGB") + if image.mode in ALPHA_MODES: + image = image.convert("RGBA") + if image.mode == "RGBA": + background = Image.new("RGB", image.size, "white") + background.paste(image, mask=image.getchannel("A")) + return background + return image if image.mode == "RGB" else image.convert("RGB") + + +def scaled(image: Image.Image, size: tuple[int, int]) -> Image.Image: + if size == image.size: + return image + return image.resize(size, Image.Resampling.LANCZOS, reducing_gap=REDUCING_GAP) + + +def encoded(name: str, image: Image.Image, kind: str, **options: object) -> Rendition: + buffer = BytesIO() + image.save(buffer, kind, **options) + content_type, extension = ENCODINGS[kind] + return Rendition(name, content_type, extension, buffer.getvalue(), *image.size) + + +def decoded(data: bytes, max_pixels: int) -> tuple[Image.Image, int, int]: + try: + image = Image.open(BytesIO(data), formats=tuple(KINDS)) + width, height = image.size + if width * height > max_pixels: + raise UnreadableImage("изображение больше допустимого числа пикселей") + image.draft(None, fit(width, height, OPTIMIZED_BOX)) + stored = image.size + ImageOps.exif_transpose(image, in_place=True) + if image.size != stored: + width, height = height, width + return flatten(image), width, height + except UnreadableImage, MemoryError: + raise + except Exception as error: + raise UnreadableImage("изображение не декодируется") from error + + +def render_variants(data: bytes, limits: RenderLimits) -> list[Rendition]: + if len(data) > limits.max_bytes: + raise ImageTooDemanding("оригинал больше допустимого размера") + upright, width, height = decoded(data, limits.max_pixels) + square = ImageOps.fit(upright, (CROP_SIDE, CROP_SIDE), Image.Resampling.LANCZOS) + return [ + encoded(THUMBNAIL, scaled(upright, fit(width, height, THUMBNAIL_BOX)), "JPEG", **JPEG), + encoded(CROP, square, "JPEG", **JPEG), + encoded(OPTIMIZED, scaled(upright, fit(width, height, OPTIMIZED_BOX)), "WEBP", **WEBP), + ] + + +def limit_memory(limit_bytes: int) -> None: + if limit_bytes and sys.platform == "linux": + resource.setrlimit(resource.RLIMIT_AS, (limit_bytes, limit_bytes)) + + +type Render = Callable[[bytes, RenderLimits], list[Rendition]] + + +class ProcessVariantRenderer: + __slots__ = ("_idle", "_limits", "_memory", "_render", "_tasks", "_timeout") + + def __init__( + self, + limits: RenderLimits, + *, + concurrency: int, + timeout: float, + memory_bytes: int, + tasks_per_child: int, + render: Render = render_variants, + ) -> None: + self._limits = limits + self._timeout = timeout + self._memory = memory_bytes + self._tasks = tasks_per_child + self._render = render + self._idle: asyncio.Queue[ProcessPoolExecutor] = asyncio.Queue() + for _ in range(concurrency): + self._idle.put_nowait(self._spawn()) + + async def render(self, data: bytes, pipeline_version: int) -> list[Rendition]: + pool = await self._idle.get() + try: + future = asyncio.wrap_future(pool.submit(self._render, data, self._limits)) + return await asyncio.wait_for(future, self._timeout) + except TimeoutError as error: + pool = self._replace(pool) + raise ImageTooDemanding("рендер дольше допустимого времени") from error + except MemoryError as error: + raise ImageTooDemanding("рендер больше допустимой памяти") from error + except BrokenProcessPool as error: + pool = self._replace(pool) + raise Unavailable("процесс рендера завершился аварийно") from error + except asyncio.CancelledError: + pool = self._replace(pool) + raise + finally: + self._idle.put_nowait(pool) + + def close(self) -> None: + while not self._idle.empty(): + self._idle.get_nowait().kill_workers() + + def _replace(self, pool: ProcessPoolExecutor) -> ProcessPoolExecutor: + pool.kill_workers() + return self._spawn() + + def _spawn(self) -> ProcessPoolExecutor: + return ProcessPoolExecutor( + 1, + max_tasks_per_child=self._tasks, + initializer=limit_memory, + initargs=(self._memory,), + ) diff --git a/src/messenger/attachments/infrastructure/jitter.py b/src/messenger/attachments/infrastructure/jitter.py new file mode 100644 index 00000000..4207230c --- /dev/null +++ b/src/messenger/attachments/infrastructure/jitter.py @@ -0,0 +1,10 @@ +from secrets import SystemRandom + +SOURCE = SystemRandom() + + +class RandomJitter: + __slots__ = () + + def spread(self) -> float: + return SOURCE.random() diff --git a/src/messenger/attachments/infrastructure/postgres.py b/src/messenger/attachments/infrastructure/postgres.py new file mode 100644 index 00000000..1af88b2d --- /dev/null +++ b/src/messenger/attachments/infrastructure/postgres.py @@ -0,0 +1,461 @@ +import json +from collections.abc import Sequence +from datetime import datetime, timedelta +from uuid import UUID + +from asyncpg import ForeignKeyViolationError, Record + +from messenger.attachments.application.ports import OutboxEntry, StagedObject +from messenger.attachments.domain.errors import ChatNotFound +from messenger.attachments.domain.jobs import Claim, FailedJob, FailureCode, Job, Lease, Outcome +from messenger.attachments.domain.models import Attachment, AttachmentStatus +from messenger.attachments.domain.values import Filename, ImageInfo, ImageKind +from messenger.attachments.domain.variants import Variant +from messenger.shared.application.errors import Unavailable +from messenger.shared.infrastructure.postgres import ( + Connection, + Database, + EventSource, + PostgresUnitOfWork, +) + +INSERT = """ +INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, + width, height, staging_key, status, created_at, updated_at) +VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13) +""" +MARK = "UPDATE attachments SET status = $2, updated_at = $3 WHERE id = $1 AND status = 'pending'" +COLUMNS = """ +SELECT a.id, a.chat_id, a.owner_id, a.filename, a.content_type, a.size_bytes, a.sha256, a.width, + a.height, a.status, a.staging_key, a.original_key, a.published_version, a.error_code, + a.created_at, a.updated_at +FROM attachments a +""" +VISIBLE = """ +SELECT a.id, a.chat_id, a.owner_id, a.filename, a.content_type, a.size_bytes, a.sha256, a.width, + a.height, a.status, a.staging_key, a.original_key, a.published_version, a.error_code, + a.created_at, a.updated_at, + (SELECT coalesce(jsonb_agg(jsonb_build_array(v.name, v.content_type, v.size_bytes, v.width, + v.height, v.sha256, v.object_key) + ORDER BY v.name), '[]') + FROM attachment_variants v + WHERE v.attachment_id = a.id AND v.pipeline_version = a.published_version) AS variants +FROM attachments a +WHERE a.id = $1 AND a.status <> 'pending' +""" +FAILED_AMONG = """ +SELECT EXISTS (SELECT 1 FROM attachments + WHERE id = ANY($1::uuid[]) AND chat_id = $2 AND status = 'failed') +""" +LOCK = COLUMNS + "WHERE a.id = $1 FOR UPDATE" +ABANDONED = ( + COLUMNS + + """WHERE a.status = 'pending' AND a.created_at < $1 +ORDER BY a.created_at LIMIT 1 FOR UPDATE SKIP LOCKED""" +) +REMOVE = "DELETE FROM attachments WHERE id = $1 AND status = 'pending'" +STAGED = """ +SELECT id, staging_key FROM attachments +WHERE staging_key IS NOT NULL + AND (status = 'ready' OR (status = 'failed' AND updated_at < now() - make_interval(secs => $2))) +ORDER BY updated_at LIMIT $1 FOR UPDATE SKIP LOCKED +""" +FAILED_JOBS = """ +SELECT j.job_id, j.attachment_id, j.pipeline_version, j.attempt, a.error_code, j.last_error, + j.updated_at AS failed_at +FROM jobs j JOIN attachments a ON a.id = j.attachment_id +WHERE j.status = 'failed' +ORDER BY j.updated_at DESC LIMIT $1 +""" +UNSTAGE = "UPDATE attachments SET staging_key = NULL WHERE id = ANY($1::uuid[])" + +ADD_JOB = """ +WITH job AS ( + INSERT INTO jobs (job_id, attachment_id, pipeline_version, status, attempt, + created_at, updated_at) + VALUES ($1, $2, $3, 'queued', $4, $5, $5) + RETURNING job_id +), box AS ( + INSERT INTO outbox (job_id, payload, available_at) + SELECT job_id, $6::jsonb, now() FROM job +) +INSERT INTO job_events (job_id, attachment_id, kind, attempt) +SELECT job_id, $2, 'enqueued', $4 FROM job +""" +CLAIM = """ +WITH claimed AS ( + UPDATE jobs + SET status = 'processing', lease_token = $4, claims = claims + 1, updated_at = now(), + lease_until = now() + make_interval(secs => $5) + WHERE job_id = $1 AND attachment_id = $2 AND pipeline_version = $3 + AND ((status = 'queued' AND coalesce(next_attempt_at, '-infinity') <= now()) + OR (status = 'processing' AND lease_until < now())) + RETURNING job_id, attachment_id, attempt, created_at +), moved AS ( + UPDATE attachments a SET status = 'processing', updated_at = greatest(now(), a.created_at) + FROM claimed c WHERE a.id = c.attachment_id AND a.status = 'queued' +), audit AS ( + INSERT INTO job_events (job_id, attachment_id, kind, attempt) + SELECT job_id, attachment_id, 'claimed', attempt FROM claimed +) +SELECT c.attempt, c.created_at, a.staging_key, a.original_key +FROM claimed c JOIN attachments a ON a.id = c.attachment_id +""" +FINISH = """ +UPDATE jobs SET status = 'done', lease_token = NULL, lease_until = NULL, next_attempt_at = NULL, + last_error = NULL, updated_at = now() +WHERE job_id = $1 AND status = 'processing' AND lease_token = $2 +""" +ADD_VARIANTS = """ +INSERT INTO attachment_variants (attachment_id, pipeline_version, name, object_key, content_type, + size_bytes, width, height, sha256, created_at) +SELECT $1, $2, v.name, v.object_key, v.content_type, v.size_bytes, v.width, v.height, v.sha256, + now() +FROM unnest($3::text[], $4::text[], $5::text[], $6::bigint[], $7::int[], $8::int[], $9::text[]) + AS v (name, object_key, content_type, size_bytes, width, height, sha256) +ON CONFLICT DO NOTHING +""" +PUBLISH = """ +UPDATE attachments +SET status = 'ready', published_version = $2, error_code = NULL, original_key = $3, + updated_at = greatest(now(), created_at) +WHERE id = $1 AND published_version < $2 +""" +SUPERSEDE = "UPDATE jobs SET status = 'superseded', updated_at = now() WHERE job_id = $1" +RESCHEDULE = """ +WITH job AS ( + UPDATE jobs + SET status = 'queued', attempt = attempt + 1, lease_token = NULL, lease_until = NULL, + next_attempt_at = now() + make_interval(secs => $3), last_error = $4, updated_at = now() + WHERE job_id = $1 AND status = 'processing' AND lease_token = $2 + RETURNING job_id, attachment_id, attempt, next_attempt_at +), moved AS ( + UPDATE attachments a SET status = 'queued', updated_at = greatest(now(), a.created_at) + FROM job WHERE a.id = job.attachment_id AND a.status = 'processing' +), audit AS ( + INSERT INTO job_events (job_id, attachment_id, kind, attempt, detail) + SELECT job_id, attachment_id, $5, attempt, + jsonb_build_object('error', $4::text, 'next_attempt_at', next_attempt_at, + 'delay_seconds', $3::float8) + FROM job +) +SELECT attempt, next_attempt_at FROM job +""" +RENEW = """ +UPDATE jobs SET lease_until = now() + make_interval(secs => $3), updated_at = now() +WHERE job_id = $1 AND lease_token = $2 AND status = 'processing' +""" +ENQUEUE = "INSERT INTO outbox (job_id, payload, available_at) VALUES ($1, $2::jsonb, $3)" +FAIL = """ +WITH job AS ( + UPDATE jobs SET status = 'failed', lease_token = NULL, lease_until = NULL, last_error = $4, + updated_at = now() + WHERE job_id = $1 AND status = 'processing' AND lease_token = $2 + RETURNING job_id, attachment_id, attempt +), moved AS ( + UPDATE attachments a SET status = 'failed', error_code = $3, + updated_at = greatest(now(), a.created_at) + FROM job WHERE a.id = job.attachment_id AND a.status IN ('queued', 'processing') +), audit AS ( + INSERT INTO job_events (job_id, attachment_id, kind, attempt, detail) + SELECT job_id, attachment_id, 'failed', attempt, + jsonb_build_object('error_code', $3::text, 'error', $4::text) + FROM job +) +SELECT attempt FROM job +""" +NOTE = "INSERT INTO job_events (job_id, attachment_id, kind, attempt) VALUES ($1, $2, $3, $4)" +EXPIRED = """ +SELECT job_id, attachment_id, pipeline_version, attempt, created_at, lease_token FROM jobs +WHERE status = 'processing' AND lease_until < now() +ORDER BY lease_until LIMIT $1 FOR UPDATE SKIP LOCKED +""" +REPLAY = """ +WITH job AS ( + UPDATE jobs + SET status = 'queued', attempt = 1, lease_token = NULL, lease_until = NULL, + next_attempt_at = NULL, last_error = NULL, updated_at = now() + WHERE job_id = (SELECT job_id FROM jobs WHERE attachment_id = $1 AND status = 'failed' + ORDER BY pipeline_version DESC LIMIT 1) + RETURNING job_id, attachment_id, pipeline_version, created_at, updated_at +), moved AS ( + UPDATE attachments a SET status = 'queued', error_code = NULL, + updated_at = greatest(now(), a.created_at) + FROM job WHERE a.id = job.attachment_id AND a.status = 'failed' +), audit AS ( + INSERT INTO job_events (job_id, attachment_id, kind, attempt, detail) + SELECT job_id, attachment_id, 'replayed', 1, + jsonb_build_object('operator', $2::text, 'error_code', $3::text) + FROM job +) +SELECT job_id, pipeline_version, created_at, updated_at FROM job +""" +NOTE_REPLAY = """ +INSERT INTO job_events (job_id, attachment_id, kind, attempt, detail) +SELECT job_id, attachment_id, 'replay_noop', attempt, + jsonb_build_object('operator', $2::text, 'result', $3::text) +FROM jobs WHERE attachment_id = $1 +ORDER BY pipeline_version DESC LIMIT 1 +""" +PENDING = """ +SELECT id, job_id, payload::text AS body FROM outbox +WHERE published_at IS NULL AND available_at <= now() +ORDER BY available_at, id LIMIT $1 FOR UPDATE SKIP LOCKED +""" +PUBLISHED = "UPDATE outbox SET published_at = now() WHERE id = ANY($1::bigint[])" + + +def attachment_of(row: Record, variants: tuple[Variant, ...] = ()) -> Attachment: + return Attachment( + id=row["id"], + chat_id=row["chat_id"], + owner_id=row["owner_id"], + filename=Filename(row["filename"]), + image=ImageInfo(ImageKind(row["content_type"]), row["width"], row["height"]), + size_bytes=row["size_bytes"], + sha256=row["sha256"], + status=AttachmentStatus(row["status"]), + created_at=row["created_at"], + updated_at=row["updated_at"], + staging_key=row["staging_key"], + original_key=row["original_key"], + published_version=row["published_version"], + error_code=row["error_code"], + variants=variants, + ) + + +def payload(job: Job) -> str: + return json.dumps(job.payload(), separators=(",", ":")) + + +class PostgresAttachmentRepository: + __slots__ = ("_connection", "_seen") + + def __init__(self, connection: Connection, seen: list[EventSource]) -> None: + self._connection = connection + self._seen = seen + + async def add(self, attachment: Attachment) -> None: + try: + await self._connection.execute( + INSERT, + attachment.id, + attachment.chat_id, + attachment.owner_id, + attachment.filename.value, + attachment.content_type, + attachment.size_bytes, + attachment.sha256, + attachment.image.width, + attachment.image.height, + attachment.staging_key, + attachment.status.value, + attachment.created_at, + attachment.updated_at, + ) + except ForeignKeyViolationError as error: + raise ChatNotFound("чат не найден") from error + self._seen.append(attachment) + + async def save(self, attachment: Attachment) -> None: + done = await self._connection.execute( + MARK, attachment.id, attachment.status.value, attachment.updated_at + ) + if done == "UPDATE 0": + raise Unavailable("намерение загрузки уже снято уборкой") + self._seen.append(attachment) + + async def lock(self, attachment_id: UUID) -> Attachment | None: + row = await self._connection.fetchrow(LOCK, attachment_id) + return None if row is None else attachment_of(row) + + async def next_abandoned(self, before: datetime) -> Attachment | None: + row = await self._connection.fetchrow(ABANDONED, before) + return None if row is None else attachment_of(row) + + async def remove(self, attachment: Attachment) -> None: + await self._connection.execute(REMOVE, attachment.id) + self._seen.append(attachment) + + async def staged(self, limit: int, retention: timedelta) -> list[StagedObject]: + rows = await self._connection.fetch(STAGED, limit, retention.total_seconds()) + return [StagedObject(row["id"], row["staging_key"]) for row in rows] + + async def unstage(self, attachment_ids: Sequence[UUID]) -> None: + if attachment_ids: + await self._connection.execute(UNSTAGE, list(attachment_ids)) + + +class PostgresJobRepository: + __slots__ = ("_connection",) + + def __init__(self, connection: Connection) -> None: + self._connection = connection + + async def add(self, job: Job) -> None: + await self._connection.execute( + ADD_JOB, + job.id, + job.attachment_id, + job.pipeline_version, + job.attempt, + job.created_at, + payload(job), + ) + + async def claim(self, job: Job, token: UUID, lease: timedelta) -> Claim | None: + row = await self._connection.fetchrow( + CLAIM, + job.id, + job.attachment_id, + job.pipeline_version, + token, + lease.total_seconds(), + ) + if row is None: + return None + claimed = Job( + job.id, job.attachment_id, job.pipeline_version, row["attempt"], row["created_at"] + ) + staged = row["staging_key"] is not None + source = row["staging_key"] if staged else row["original_key"] + return Claim(Lease(claimed, token), source, staged) + + async def complete( + self, lease: Lease, variants: Sequence[Variant], original_key: str + ) -> Outcome: + job = lease.job + if await self._connection.execute(FINISH, job.id, lease.token) == "UPDATE 0": + await self.note(job, Outcome.FENCED) + return Outcome.FENCED + await self._connection.execute( + ADD_VARIANTS, + job.attachment_id, + job.pipeline_version, + [v.name for v in variants], + [v.object_key for v in variants], + [v.content_type for v in variants], + [v.size_bytes for v in variants], + [v.width for v in variants], + [v.height for v in variants], + [v.sha256 for v in variants], + ) + published = await self._connection.execute( + PUBLISH, job.attachment_id, job.pipeline_version, original_key + ) + outcome = Outcome.SUPERSEDED if published == "UPDATE 0" else Outcome.DONE + if outcome is Outcome.SUPERSEDED: + await self._connection.execute(SUPERSEDE, job.id) + await self.note(job, outcome) + return outcome + + async def reschedule( + self, lease: Lease, delay: timedelta, kind: Outcome, error: str + ) -> Job | None: + row = await self._connection.fetchrow( + RESCHEDULE, lease.job.id, lease.token, delay.total_seconds(), error, kind.value + ) + if row is None: + return None + job = lease.job.retried(row["attempt"]) + await self._connection.execute(ENQUEUE, job.id, payload(job), row["next_attempt_at"]) + return job + + async def renew(self, lease: Lease, duration: timedelta) -> bool: + done = await self._connection.execute( + RENEW, lease.job.id, lease.token, duration.total_seconds() + ) + return done == "UPDATE 1" + + async def fail(self, lease: Lease, code: FailureCode, error: str) -> bool: + attempt = await self._connection.fetchval( + FAIL, lease.job.id, lease.token, code.value, error + ) + return attempt is not None + + async def note(self, job: Job, kind: Outcome) -> None: + await self._connection.execute(NOTE, job.id, job.attachment_id, kind.value, job.attempt) + + async def expired(self, limit: int) -> list[Lease]: + rows = await self._connection.fetch(EXPIRED, limit) + return [ + Lease( + Job( + row["job_id"], + row["attachment_id"], + row["pipeline_version"], + row["attempt"], + row["created_at"], + ), + row["lease_token"], + ) + for row in rows + ] + + async def replay(self, attachment: Attachment, operator: str) -> Job | None: + row = await self._connection.fetchrow( + REPLAY, attachment.id, operator, attachment.error_code + ) + if row is None: + return None + job = Job(row["job_id"], attachment.id, row["pipeline_version"], 1, row["created_at"]) + await self._connection.execute(ENQUEUE, job.id, payload(job), row["updated_at"]) + return job + + async def note_replay(self, attachment_id: UUID, operator: str, result: str) -> None: + await self._connection.execute(NOTE_REPLAY, attachment_id, operator, result) + + +class PostgresOutboxRepository: + __slots__ = ("_connection",) + + def __init__(self, connection: Connection) -> None: + self._connection = connection + + async def pending(self, limit: int) -> list[OutboxEntry]: + rows = await self._connection.fetch(PENDING, limit) + return [OutboxEntry(row["id"], row["job_id"], row["body"].encode()) for row in rows] + + async def published(self, entry_ids: Sequence[int]) -> None: + await self._connection.execute(PUBLISHED, list(entry_ids)) + + +class PostgresAttachmentsUnitOfWork(PostgresUnitOfWork): + __slots__ = () + + @property + def attachments(self) -> PostgresAttachmentRepository: + return PostgresAttachmentRepository(self.connection, self.seen) + + @property + def jobs(self) -> PostgresJobRepository: + return PostgresJobRepository(self.connection) + + @property + def outbox(self) -> PostgresOutboxRepository: + return PostgresOutboxRepository(self.connection) + + +def variants_of(raw: str) -> tuple[Variant, ...]: + return tuple(Variant(*item) for item in json.loads(raw)) + + +class PostgresAttachmentDirectory: + __slots__ = ("_database",) + + def __init__(self, database: Database) -> None: + self._database = database + + async def visible(self, attachment_id: UUID) -> Attachment | None: + async with self._database.connection() as connection: + row = await connection.fetchrow(VISIBLE, attachment_id) + return None if row is None else attachment_of(row, variants_of(row["variants"])) + + async def failed_among(self, chat_id: UUID, attachment_ids: Sequence[UUID]) -> bool: + async with self._database.connection() as connection: + return bool(await connection.fetchval(FAILED_AMONG, list(attachment_ids), chat_id)) + + async def failed_jobs(self, limit: int) -> list[FailedJob]: + async with self._database.connection() as connection: + return [FailedJob(**row) for row in await connection.fetch(FAILED_JOBS, limit)] diff --git a/src/messenger/attachments/infrastructure/rabbitmq.py b/src/messenger/attachments/infrastructure/rabbitmq.py new file mode 100644 index 00000000..778c1af9 --- /dev/null +++ b/src/messenger/attachments/infrastructure/rabbitmq.py @@ -0,0 +1,37 @@ +from uuid import UUID + +from faststream.rabbit import RabbitBroker, RabbitExchange +from pamqp.commands import Basic + +from messenger.attachments.domain.jobs import JOB_TYPE +from messenger.shared.application.errors import Unavailable + +EXCHANGE = RabbitExchange("attachments", declare=False) +QUEUE = "attachments.process" +DEAD_LETTERS = "attachments.process.dlq" + + +class RabbitJobPublisher: + __slots__ = ("_broker", "_timeout") + + def __init__(self, broker: RabbitBroker, timeout: float) -> None: + self._broker = broker + self._timeout = timeout + + async def publish(self, job_id: UUID, body: bytes) -> None: + try: + confirmation = await self._broker.publish( + body, + exchange=EXCHANGE, + routing_key=JOB_TYPE, + mandatory=True, + persist=True, + message_id=str(job_id), + message_type=JOB_TYPE, + content_type="application/json", + timeout=self._timeout, + ) + except Exception as error: + raise Unavailable("брокер не принял задание") from error + if not isinstance(confirmation, Basic.Ack): + raise Unavailable("брокер не подтвердил задание") diff --git a/src/messenger/attachments/infrastructure/s3.py b/src/messenger/attachments/infrastructure/s3.py new file mode 100644 index 00000000..3acea375 --- /dev/null +++ b/src/messenger/attachments/infrastructure/s3.py @@ -0,0 +1,104 @@ +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from typing import Any + +from aiobotocore.session import get_session +from botocore.config import Config +from botocore.exceptions import BotoCoreError, ClientError +from botocore.handlers import add_expect_header + +from messenger.attachments.application.errors import ObjectMissingError +from messenger.attachments.application.ports import ObjectContent +from messenger.shared.application.errors import Unavailable + +CHUNK_BYTES = 64 * 1024 +MISSING = frozenset({"NoSuchKey", "404", "NotFound"}) +FAILURES = (BotoCoreError, ClientError, OSError, TimeoutError) + + +def missing(error: Exception) -> bool: + return isinstance(error, ClientError) and error.response.get("Error", {}).get("Code") in MISSING + + +@asynccontextmanager +async def s3_client( + *, + endpoint: str, + region: str, + access_key_id: str, + secret_access_key: str, + connect_timeout: float, + read_timeout: float, + max_attempts: int, +) -> AsyncIterator[Any]: + config = Config( + region_name=region, + signature_version="s3v4", + s3={"addressing_style": "path"}, + connect_timeout=connect_timeout, + read_timeout=read_timeout, + retries={"mode": "standard", "max_attempts": max_attempts}, + max_pool_connections=32, + request_checksum_calculation="when_required", + response_checksum_validation="when_required", + ) + async with get_session().create_client( + "s3", + endpoint_url=endpoint, + aws_access_key_id=access_key_id, + aws_secret_access_key=secret_access_key, + config=config, + ) as client: + client.meta.events.unregister("before-call.s3", add_expect_header) + yield client + + +class S3ObjectStorage: + __slots__ = ("_bucket", "_client") + + def __init__(self, client: Any, bucket: str) -> None: + self._client = client + self._bucket = bucket + + async def put(self, key: str, data: bytes, content_type: str) -> None: + try: + await self._client.put_object( + Bucket=self._bucket, Key=key, Body=data, ContentType=content_type + ) + except FAILURES as error: + raise Unavailable("S3 недоступно") from error + + async def open(self, key: str) -> ObjectContent: + try: + response = await self._client.get_object(Bucket=self._bucket, Key=key) + except FAILURES as error: + if missing(error): + raise ObjectMissingError(key) from error + raise Unavailable("S3 недоступно") from error + return ObjectContent(size=response["ContentLength"], chunks=self._chunks(response["Body"])) + + async def read(self, key: str) -> bytes: + content = await self.open(key) + try: + return b"".join([chunk async for chunk in content.chunks]) + except Exception as error: + raise Unavailable("S3 оборвал чтение объекта") from error + + async def copy(self, source: str, target: str) -> None: + try: + await self._client.copy_object( + Bucket=self._bucket, Key=target, CopySource={"Bucket": self._bucket, "Key": source} + ) + except FAILURES as error: + raise Unavailable("S3 недоступно") from error + + async def delete(self, key: str) -> None: + try: + await self._client.delete_object(Bucket=self._bucket, Key=key) + except FAILURES as error: + raise Unavailable("S3 недоступно") from error + + async def _chunks(self, body: Any) -> AsyncIterator[bytes]: + async with body as stream: + async for chunk in stream.iter_chunks(CHUNK_BYTES): + yield chunk diff --git a/src/messenger/gunicorn_conf.py b/src/messenger/gunicorn_conf.py new file mode 100644 index 00000000..4ccd4983 --- /dev/null +++ b/src/messenger/gunicorn_conf.py @@ -0,0 +1,66 @@ +import asyncio +import logging +import os +import sys +from typing import Any + +from gunicorn.arbiter import Arbiter +from gunicorn.glogging import Logger as GunicornBaseLogger +from uvicorn.server import Server +from uvicorn_worker import UvicornWorker + +from messenger.main import LOG_FIELDS +from messenger.shared.entrypoints.health import DRAINING +from messenger.shared.infrastructure.logging import configure + +LOG_LEVEL = os.environ.get("LOG_LEVEL", "INFO") +SHUTDOWN_SECONDS = int(os.environ.get("SHUTDOWN_TIMEOUT_SECONDS", "5")) +DRAIN_SECONDS = float(os.environ.get("DRAIN_SECONDS", "2")) + + +class StructlogGunicornLogger(GunicornBaseLogger): + def setup(self, cfg: object) -> None: + configure(LOG_LEVEL, LOG_FIELDS) + + def access(self, resp: object, req: object, environ: object, request_time: object) -> None: + return + + +class DrainingServer(Server): + async def main_loop(self) -> None: + await super().main_loop() + if not self.force_exit: + DRAINING.set() + logging.getLogger("uvicorn.error").info("Draining for %.1f s", DRAIN_SECONDS) + await asyncio.sleep(DRAIN_SECONDS) + + +class MessengerWorker(UvicornWorker): + CONFIG_KWARGS: dict[str, Any] = { + "access_log": False, + "server_header": False, + "date_header": False, + "timeout_graceful_shutdown": max(1, int(SHUTDOWN_SECONDS - DRAIN_SECONDS)), + } + + async def _serve(self) -> None: + self.config.app = self.wsgi + server = DrainingServer(config=self.config) + self._install_sigquit_handler() + await server.serve(sockets=self.sockets) + if not server.started: + sys.exit(Arbiter.WORKER_BOOT_ERROR) + + +bind = f"{os.environ.get('HOST', '0.0.0.0')}:{os.environ.get('PORT', '8080')}" +workers = int(os.environ.get("WEB_CONCURRENCY", "2")) +worker_class = "messenger.gunicorn_conf.MessengerWorker" +logger_class = StructlogGunicornLogger +loglevel = LOG_LEVEL.lower() +accesslog = None +errorlog = "-" +timeout = int(float(os.environ.get("REQUEST_TIMEOUT_SECONDS", "8")) + 5) +graceful_timeout = SHUTDOWN_SECONDS +keepalive = 5 +preload_app = False +control_socket_disable = True diff --git a/src/messenger/identity/__init__.py b/src/messenger/identity/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/messenger/identity/application/__init__.py b/src/messenger/identity/application/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/messenger/identity/application/bus.py b/src/messenger/identity/application/bus.py new file mode 100644 index 00000000..1ce4aa89 --- /dev/null +++ b/src/messenger/identity/application/bus.py @@ -0,0 +1,5 @@ +from messenger.shared.application.messagebus import CommandBus + + +class IdentityCommandBus(CommandBus): + __slots__ = () diff --git a/src/messenger/identity/application/commands.py b/src/messenger/identity/application/commands.py new file mode 100644 index 00000000..c98a4b57 --- /dev/null +++ b/src/messenger/identity/application/commands.py @@ -0,0 +1,22 @@ +from dataclasses import dataclass, field +from uuid import UUID + +from messenger.identity.domain.models import Session +from messenger.shared.application.commands import Command + + +@dataclass(frozen=True, slots=True) +class RegisterUser(Command): + username: str + display_name: str + password: str = field(repr=False) + + +@dataclass(frozen=True, slots=True) +class OpenSession(Command): + user_id: UUID + + +@dataclass(frozen=True, slots=True) +class CloseSession(Command): + session: Session diff --git a/src/messenger/identity/application/handlers.py b/src/messenger/identity/application/handlers.py new file mode 100644 index 00000000..b5811ffc --- /dev/null +++ b/src/messenger/identity/application/handlers.py @@ -0,0 +1,76 @@ +from datetime import timedelta + +from messenger.identity.application.commands import CloseSession, OpenSession, RegisterUser +from messenger.identity.application.ports import ( + IdentityUnitOfWork, + PasswordHasher, + SessionDirectory, +) +from messenger.identity.domain.events import SessionClosed, SessionOpened +from messenger.identity.domain.models import Session, User +from messenger.identity.domain.values import DisplayName, Password, Username +from messenger.shared.application.ports import Clock, IdGenerator + + +class RegisterUserHandler: + __slots__ = ("_clock", "_hasher", "_ids", "_uow") + + def __init__( + self, uow: IdentityUnitOfWork, clock: Clock, ids: IdGenerator, hasher: PasswordHasher + ) -> None: + self._uow = uow + self._clock = clock + self._ids = ids + self._hasher = hasher + + async def __call__(self, command: RegisterUser) -> User: + username = Username(command.username) + display_name = DisplayName(command.display_name) + password = Password(command.password) + password_hash = await self._hasher.hash(password.value) + user, event = User.register( + user_id=self._ids.new_id(), + username=username, + display_name=display_name, + password_hash=password_hash, + registered_at=self._clock.now(), + ) + async with self._uow.transaction(): + await self._uow.users.add(user) + self._uow.record(event) + await self._uow.commit() + return user + + +class OpenSessionHandler: + __slots__ = ("_clock", "_ttl", "_uow") + + def __init__(self, uow: IdentityUnitOfWork, clock: Clock, ttl: timedelta) -> None: + self._uow = uow + self._clock = clock + self._ttl = ttl + + async def __call__(self, command: OpenSession) -> tuple[str, Session]: + token, session = Session.issue( + user_id=command.user_id, issued_at=self._clock.now(), ttl=self._ttl + ) + async with self._uow.transaction(): + await self._uow.sessions.add(session) + self._uow.record(SessionOpened(user_id=session.user_id, expires_at=session.expires_at)) + await self._uow.commit() + return token, session + + +class CloseSessionHandler: + __slots__ = ("_sessions", "_uow") + + def __init__(self, uow: IdentityUnitOfWork, sessions: SessionDirectory) -> None: + self._uow = uow + self._sessions = sessions + + async def __call__(self, command: CloseSession) -> None: + await self._sessions.invalidate(command.session) + async with self._uow.transaction(): + await self._uow.sessions.remove(command.session.token_hash) + self._uow.record(SessionClosed(user_id=command.session.user_id)) + await self._uow.commit() diff --git a/src/messenger/identity/application/ports.py b/src/messenger/identity/application/ports.py new file mode 100644 index 00000000..bba207e5 --- /dev/null +++ b/src/messenger/identity/application/ports.py @@ -0,0 +1,48 @@ +from datetime import datetime +from typing import Protocol, runtime_checkable +from uuid import UUID + +from messenger.identity.domain.models import Session, User +from messenger.shared.application.ports import UnitOfWork + + +@runtime_checkable +class UserRepository(Protocol): + async def add(self, user: User) -> None: ... + + +@runtime_checkable +class SessionRepository(Protocol): + async def add(self, session: Session) -> None: ... + + async def remove(self, token_hash: bytes) -> None: ... + + +@runtime_checkable +class IdentityUnitOfWork(UnitOfWork, Protocol): + @property + def users(self) -> UserRepository: ... + + @property + def sessions(self) -> SessionRepository: ... + + +@runtime_checkable +class UserDirectory(Protocol): + async def find(self, user_id: UUID) -> User | None: ... + + async def find_by_username(self, username: str) -> User | None: ... + + +@runtime_checkable +class SessionDirectory(Protocol): + async def find(self, token_hash: bytes, now: datetime) -> Session | None: ... + + async def invalidate(self, session: Session) -> None: ... + + +@runtime_checkable +class PasswordHasher(Protocol): + async def hash(self, password: str) -> str: ... + + async def verify(self, password_hash: str | None, password: str) -> bool: ... diff --git a/src/messenger/identity/application/queries.py b/src/messenger/identity/application/queries.py new file mode 100644 index 00000000..cf0c7d8d --- /dev/null +++ b/src/messenger/identity/application/queries.py @@ -0,0 +1,43 @@ +from uuid import UUID + +from messenger.identity.application.ports import PasswordHasher, SessionDirectory, UserDirectory +from messenger.identity.domain.errors import ActorNotIdentified +from messenger.identity.domain.models import Session, User, token_digest +from messenger.shared.application.ports import Clock + + +class GetCurrentUser: + __slots__ = ("_directory",) + + def __init__(self, directory: UserDirectory) -> None: + self._directory = directory + + async def __call__(self, user_id: UUID) -> User: + user = await self._directory.find(user_id) + if user is None: + raise ActorNotIdentified("идентификация не подтверждена") + return user + + +class VerifyCredentials: + __slots__ = ("_directory", "_hasher") + + def __init__(self, directory: UserDirectory, hasher: PasswordHasher) -> None: + self._directory = directory + self._hasher = hasher + + async def __call__(self, username: str, password: str) -> User | None: + user = await self._directory.find_by_username(username) + valid = await self._hasher.verify(user.password_hash if user else None, password) + return user if valid else None + + +class Authenticate: + __slots__ = ("_clock", "_sessions") + + def __init__(self, sessions: SessionDirectory, clock: Clock) -> None: + self._sessions = sessions + self._clock = clock + + async def __call__(self, token: str) -> Session | None: + return await self._sessions.find(token_digest(token), self._clock.now()) diff --git a/src/messenger/identity/config.py b/src/messenger/identity/config.py new file mode 100644 index 00000000..d1e19705 --- /dev/null +++ b/src/messenger/identity/config.py @@ -0,0 +1,12 @@ +from pydantic import Field, SecretStr + +from messenger.shared.config import Settings + + +class IdentitySettings(Settings): + session_ttl_seconds: int = Field(default=3600, ge=1, le=30 * 24 * 3600) + password_hash_concurrency: int = Field(default=2, ge=1, le=64) + cache_url: SecretStr | None = None + session_cache_ttl_seconds: int = Field(default=300, ge=1, le=24 * 3600) + cache_timeout_seconds: float = Field(default=0.2, gt=0, le=5) + cache_cooldown_seconds: float = Field(default=1.0, ge=0, le=60) diff --git a/src/messenger/identity/domain/__init__.py b/src/messenger/identity/domain/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/messenger/identity/domain/errors.py b/src/messenger/identity/domain/errors.py new file mode 100644 index 00000000..335a6d25 --- /dev/null +++ b/src/messenger/identity/domain/errors.py @@ -0,0 +1,17 @@ +from messenger.shared.domain.errors import DomainError + + +class IdentityError(DomainError): + pass + + +class InvalidUserData(IdentityError): + pass + + +class UsernameAlreadyTaken(IdentityError): + pass + + +class ActorNotIdentified(IdentityError): + pass diff --git a/src/messenger/identity/domain/events.py b/src/messenger/identity/domain/events.py new file mode 100644 index 00000000..429a0b54 --- /dev/null +++ b/src/messenger/identity/domain/events.py @@ -0,0 +1,25 @@ +from dataclasses import dataclass, field +from datetime import datetime +from uuid import UUID + +from messenger.shared.domain.events import DomainEvent + + +@dataclass(frozen=True, slots=True) +class UserRegistered(DomainEvent): + user_id: UUID + registered_at: datetime + name: str = field(init=False, default="user.registered") + + +@dataclass(frozen=True, slots=True) +class SessionOpened(DomainEvent): + user_id: UUID + expires_at: datetime + name: str = field(init=False, default="session.opened") + + +@dataclass(frozen=True, slots=True) +class SessionClosed(DomainEvent): + user_id: UUID + name: str = field(init=False, default="session.closed") diff --git a/src/messenger/identity/domain/models.py b/src/messenger/identity/domain/models.py new file mode 100644 index 00000000..7f505037 --- /dev/null +++ b/src/messenger/identity/domain/models.py @@ -0,0 +1,62 @@ +import secrets +from dataclasses import dataclass, field +from datetime import datetime, timedelta +from hashlib import sha256 +from uuid import UUID + +from messenger.identity.domain.events import UserRegistered +from messenger.identity.domain.values import DisplayName, Username +from messenger.shared.domain.events import DomainEvent + +TOKEN_BYTES = 32 + + +def token_digest(token: str) -> bytes: + return sha256(token.encode()).digest() + + +@dataclass(frozen=True, slots=True) +class User: + id: UUID + username: Username + display_name: DisplayName + created_at: datetime + password_hash: str = field(repr=False) + + @classmethod + def register( + cls, + *, + user_id: UUID, + username: Username, + display_name: DisplayName, + password_hash: str, + registered_at: datetime, + ) -> tuple[User, DomainEvent]: + user = cls( + id=user_id, + username=username, + display_name=display_name, + created_at=registered_at, + password_hash=password_hash, + ) + return user, UserRegistered(user_id=user.id, registered_at=registered_at) + + +@dataclass(frozen=True, slots=True) +class Session: + token_hash: bytes = field(repr=False) + user_id: UUID + created_at: datetime + expires_at: datetime + + @classmethod + def issue(cls, *, user_id: UUID, issued_at: datetime, ttl: timedelta) -> tuple[str, Session]: + token = secrets.token_urlsafe(TOKEN_BYTES) + session = cls( + token_hash=token_digest(token), + user_id=user_id, + created_at=issued_at, + expires_at=issued_at + ttl, + ) + return token, session diff --git a/src/messenger/identity/domain/values.py b/src/messenger/identity/domain/values.py new file mode 100644 index 00000000..8d6170a3 --- /dev/null +++ b/src/messenger/identity/domain/values.py @@ -0,0 +1,54 @@ +import re +from dataclasses import dataclass, field + +from messenger.identity.domain.errors import InvalidUserData + +USERNAME_PATTERN = re.compile(r"\A[a-z0-9_]{3,32}\Z") +DISPLAY_NAME_MAX = 100 +PASSWORD_MIN = 12 +PASSWORD_MAX = 128 + + +@dataclass(frozen=True, slots=True) +class Username: + value: str + + def __post_init__(self) -> None: + if not isinstance(self.value, str): + raise InvalidUserData("поле username должно быть строкой") + if USERNAME_PATTERN.match(self.value) is None: + raise InvalidUserData("поле username должно соответствовать [a-z0-9_]{3,32}") + + def __str__(self) -> str: + return self.value + + +@dataclass(frozen=True, slots=True) +class DisplayName: + value: str + + def __post_init__(self) -> None: + if not isinstance(self.value, str): + raise InvalidUserData("поле display_name должно быть строкой") + if not self.value.strip(): + raise InvalidUserData("поле display_name не может быть пустым или состоять из пробелов") + if len(self.value) > DISPLAY_NAME_MAX: + raise InvalidUserData(f"поле display_name длиннее {DISPLAY_NAME_MAX} символов Unicode") + + def __str__(self) -> str: + return self.value + + +@dataclass(frozen=True, slots=True) +class Password: + value: str = field(repr=False) + + def __post_init__(self) -> None: + if not isinstance(self.value, str): + raise InvalidUserData("поле password должно быть строкой") + if not self.value.strip(): + raise InvalidUserData("поле password не может состоять из пробелов") + if not PASSWORD_MIN <= len(self.value) <= PASSWORD_MAX: + raise InvalidUserData( + f"поле password: от {PASSWORD_MIN} до {PASSWORD_MAX} символов Unicode" + ) diff --git a/src/messenger/identity/entrypoints/__init__.py b/src/messenger/identity/entrypoints/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/messenger/identity/entrypoints/authentication.py b/src/messenger/identity/entrypoints/authentication.py new file mode 100644 index 00000000..1103e3d0 --- /dev/null +++ b/src/messenger/identity/entrypoints/authentication.py @@ -0,0 +1,98 @@ +import re +from base64 import b64decode +from binascii import Error as Base64Error +from typing import Annotated + +from fastapi import Depends, Request +from starlette.types import Scope + +from messenger.identity.application.queries import Authenticate, VerifyCredentials +from messenger.identity.domain.models import Session, User +from messenger.shared.entrypoints.errors import Unauthenticated +from messenger.shared.entrypoints.pipeline import Rejection, header + +LOGIN_PATH = "/api/v1/auth/sessions" +PROTECTED_PREFIX = "/api/v1/" +PUBLIC_ROUTES = frozenset({("POST", "/api/v1/users")}) +BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"' +BEARER_CHALLENGE = 'Bearer realm="messenger"' +AUTHORIZATION = b"authorization" +TOKEN = re.compile(r"[A-Za-z0-9\-._~+/]{1,512}=*") + + +def normalized(path: str) -> str: + return path.rstrip("/") or "/" + + +def challenge_for(path: str) -> str: + return BASIC_CHALLENGE if normalized(path) == LOGIN_PATH else BEARER_CHALLENGE + + +def basic_credentials(value: str) -> tuple[str, str] | None: + scheme, _, encoded = value.partition(" ") + if scheme.lower() != "basic": + return None + try: + decoded = b64decode(encoded.strip(), validate=True).decode("utf-8") + except Base64Error, UnicodeDecodeError: + return None + username, separator, password = decoded.partition(":") + if not separator or not username.isascii(): + return None + return username, password + + +def bearer_token(value: str) -> str | None: + scheme, _, token = value.partition(" ") + token = token.strip() + if scheme.lower() != "bearer" or TOKEN.fullmatch(token) is None: + return None + return token + + +def rejection(path: str) -> Rejection: + return Rejection(401, "unauthorized", (("www-authenticate", challenge_for(path)),)) + + +async def guard(scope: Scope) -> Rejection | None: + method, path = scope["method"], normalized(scope["path"]) + if not path.startswith(PROTECTED_PREFIX) or (method, path) in PUBLIC_ROUTES: + return None + container = scope["app"].state.dishka_container + state = scope.setdefault("state", {}) + raw = header(scope, AUTHORIZATION) + if (method, path) == ("POST", LOGIN_PATH): + credentials = basic_credentials(raw) + verify = await container.get(VerifyCredentials) + user = await verify(*credentials) if credentials else None + if user is None: + return rejection(path) + state["user"] = user + state["actor"] = user.id + return None + token = bearer_token(raw) + authenticate = await container.get(Authenticate) + session = await authenticate(token) if token else None + if session is None: + return rejection(path) + state["session"] = session + state["actor"] = session.user_id + return None + + +async def verified_user(request: Request) -> User: + user = getattr(request.state, "user", None) + if not isinstance(user, User): + raise Unauthenticated("учётные данные не подтверждены") + return user + + +async def current_session(request: Request) -> Session: + session = getattr(request.state, "session", None) + if not isinstance(session, Session): + raise Unauthenticated("сессия не подтверждена") + return session + + +type VerifiedUser = Annotated[User, Depends(verified_user)] +type CurrentSession = Annotated[Session, Depends(current_session)] diff --git a/src/messenger/identity/entrypoints/errors.py b/src/messenger/identity/entrypoints/errors.py new file mode 100644 index 00000000..3b840f1f --- /dev/null +++ b/src/messenger/identity/entrypoints/errors.py @@ -0,0 +1,12 @@ +from messenger.identity.domain.errors import ( + ActorNotIdentified, + InvalidUserData, + UsernameAlreadyTaken, +) +from messenger.shared.domain.errors import DomainError + +ERROR_CODES: dict[type[DomainError], tuple[str, int]] = { + InvalidUserData: ("invalid_request", 400), + UsernameAlreadyTaken: ("conflict", 409), + ActorNotIdentified: ("unauthorized", 401), +} diff --git a/src/messenger/identity/entrypoints/http.py b/src/messenger/identity/entrypoints/http.py new file mode 100644 index 00000000..e67ec4e6 --- /dev/null +++ b/src/messenger/identity/entrypoints/http.py @@ -0,0 +1,57 @@ +from dishka.integrations.fastapi import FromDishka, inject +from fastapi import APIRouter, Request, Response, status +from fastapi.exceptions import RequestValidationError + +from messenger.identity.application.bus import IdentityCommandBus +from messenger.identity.application.commands import CloseSession, OpenSession, RegisterUser +from messenger.identity.application.queries import GetCurrentUser +from messenger.identity.entrypoints.authentication import CurrentSession, VerifiedUser +from messenger.identity.entrypoints.schemas import CreateUserRequest, SessionOut, UserOut +from messenger.shared.entrypoints.actor import Actor + +router = APIRouter(prefix="/api/v1/users", tags=["users"]) +sessions_router = APIRouter(prefix="/api/v1/auth/sessions", tags=["sessions"]) + + +@router.post("", status_code=status.HTTP_201_CREATED) +@inject +async def create_user( + body: CreateUserRequest, + bus: FromDishka[IdentityCommandBus], +) -> UserOut: + user = await bus.execute( + RegisterUser(username=body.username, display_name=body.display_name, password=body.password) + ) + return UserOut.of(user) + + +@router.get("/me") +@inject +async def current_user( + actor: Actor, + query: FromDishka[GetCurrentUser], +) -> UserOut: + return UserOut.of(await query(actor)) + + +@sessions_router.post("", status_code=status.HTTP_201_CREATED) +@inject +async def open_session( + request: Request, + user: VerifiedUser, + bus: FromDishka[IdentityCommandBus], +) -> SessionOut: + if await request.body(): + raise RequestValidationError([]) + token, session = await bus.execute(OpenSession(user_id=user.id)) + return SessionOut.of(token, session, user) + + +@sessions_router.delete("/current", status_code=status.HTTP_204_NO_CONTENT) +@inject +async def close_session( + session: CurrentSession, + bus: FromDishka[IdentityCommandBus], +) -> Response: + await bus.execute(CloseSession(session=session)) + return Response(status_code=status.HTTP_204_NO_CONTENT) diff --git a/src/messenger/identity/entrypoints/ioc.py b/src/messenger/identity/entrypoints/ioc.py new file mode 100644 index 00000000..01ae21cc --- /dev/null +++ b/src/messenger/identity/entrypoints/ioc.py @@ -0,0 +1,112 @@ +from collections.abc import AsyncIterator +from datetime import timedelta + +from dishka import Provider, Scope, provide +from redis.asyncio import Redis +from redis.asyncio.retry import Retry +from redis.backoff import NoBackoff + +from messenger.identity.application.bus import IdentityCommandBus +from messenger.identity.application.commands import CloseSession, OpenSession, RegisterUser +from messenger.identity.application.handlers import ( + CloseSessionHandler, + OpenSessionHandler, + RegisterUserHandler, +) +from messenger.identity.application.ports import ( + IdentityUnitOfWork, + PasswordHasher, + SessionDirectory, + UserDirectory, +) +from messenger.identity.application.queries import Authenticate, GetCurrentUser, VerifyCredentials +from messenger.identity.config import IdentitySettings +from messenger.identity.infrastructure.argon2 import Argon2PasswordHasher +from messenger.identity.infrastructure.postgres import ( + PostgresIdentityUnitOfWork, + PostgresSessionDirectory, + PostgresUserDirectory, +) +from messenger.identity.infrastructure.valkey import CachedSessionDirectory +from messenger.shared.application.commands import Command +from messenger.shared.application.messagebus import CommandHandler +from messenger.shared.application.ports import Clock, EventPublisher, IdGenerator, Logger +from messenger.shared.infrastructure.postgres import Database + + +class IdentityProvider(Provider): + def __init__(self, settings: IdentitySettings) -> None: + super().__init__() + self._settings = settings + + @provide(scope=Scope.APP) + def settings(self) -> IdentitySettings: + return self._settings + + @provide(scope=Scope.APP) + def hasher(self, settings: IdentitySettings) -> PasswordHasher: + return Argon2PasswordHasher(settings.password_hash_concurrency) + + @provide(scope=Scope.APP) + def users(self, database: Database) -> UserDirectory: + return PostgresUserDirectory(database) + + @provide(scope=Scope.APP) + async def sessions( + self, database: Database, settings: IdentitySettings, logger: Logger + ) -> AsyncIterator[SessionDirectory]: + origin = PostgresSessionDirectory(database) + if settings.cache_url is None: + yield origin + return + client = Redis.from_url( + settings.cache_url.get_secret_value(), + socket_connect_timeout=settings.cache_timeout_seconds, + socket_timeout=settings.cache_timeout_seconds, + retry=Retry(NoBackoff(), 1), + ) + try: + yield CachedSessionDirectory( + origin, + client, + logger, + limit=timedelta(seconds=settings.session_cache_ttl_seconds), + cooldown=settings.cache_cooldown_seconds, + ) + finally: + await client.aclose() + + @provide(scope=Scope.APP) + def verify(self, users: UserDirectory, hasher: PasswordHasher) -> VerifyCredentials: + return VerifyCredentials(users, hasher) + + @provide(scope=Scope.APP) + def authenticate(self, sessions: SessionDirectory, clock: Clock) -> Authenticate: + return Authenticate(sessions, clock) + + @provide(scope=Scope.REQUEST) + def unit_of_work(self, database: Database) -> IdentityUnitOfWork: + return PostgresIdentityUnitOfWork(database) + + @provide(scope=Scope.REQUEST) + def current_user(self, users: UserDirectory) -> GetCurrentUser: + return GetCurrentUser(users) + + @provide(scope=Scope.REQUEST) + def command_bus( + self, + uow: IdentityUnitOfWork, + publisher: EventPublisher, + clock: Clock, + ids: IdGenerator, + hasher: PasswordHasher, + sessions: SessionDirectory, + settings: IdentitySettings, + ) -> IdentityCommandBus: + ttl = timedelta(seconds=settings.session_ttl_seconds) + handlers: dict[type[Command], CommandHandler] = { + RegisterUser: RegisterUserHandler(uow, clock, ids, hasher), + OpenSession: OpenSessionHandler(uow, clock, ttl), + CloseSession: CloseSessionHandler(uow, sessions), + } + return IdentityCommandBus(handlers, uow, publisher) diff --git a/src/messenger/identity/entrypoints/journal.py b/src/messenger/identity/entrypoints/journal.py new file mode 100644 index 00000000..9d51aeee --- /dev/null +++ b/src/messenger/identity/entrypoints/journal.py @@ -0,0 +1,20 @@ +from messenger.identity.domain.events import SessionClosed, SessionOpened, UserRegistered +from messenger.shared.application.ports import Logger + +LOG_FIELDS = frozenset({"actor_id"}) + + +class IdentityJournal: + __slots__ = ("_logger",) + + def __init__(self, logger: Logger) -> None: + self._logger = logger + + async def on_user_registered(self, event: UserRegistered) -> None: + self._logger.info("audit.user_registered", actor_id=str(event.user_id)) + + async def on_session_opened(self, event: SessionOpened) -> None: + self._logger.info("audit.session_opened", actor_id=str(event.user_id)) + + async def on_session_closed(self, event: SessionClosed) -> None: + self._logger.info("audit.session_closed", actor_id=str(event.user_id)) diff --git a/src/messenger/identity/entrypoints/schemas.py b/src/messenger/identity/entrypoints/schemas.py new file mode 100644 index 00000000..7560cfd2 --- /dev/null +++ b/src/messenger/identity/entrypoints/schemas.py @@ -0,0 +1,42 @@ +from datetime import datetime +from typing import Literal +from uuid import UUID + +from pydantic import BaseModel, ConfigDict + +from messenger.identity.domain.models import Session, User + + +class CreateUserRequest(BaseModel): + model_config = ConfigDict(extra="forbid", strict=True) + + username: str + display_name: str + password: str + + +class UserOut(BaseModel): + id: UUID + username: str + display_name: str + created_at: datetime + + @classmethod + def of(cls, user: User) -> UserOut: + return cls( + id=user.id, + username=user.username.value, + display_name=user.display_name.value, + created_at=user.created_at, + ) + + +class SessionOut(BaseModel): + token: str + token_type: Literal["Bearer"] = "Bearer" + expires_at: datetime + user: UserOut + + @classmethod + def of(cls, token: str, session: Session, user: User) -> SessionOut: + return cls(token=token, expires_at=session.expires_at, user=UserOut.of(user)) diff --git a/src/messenger/identity/infrastructure/__init__.py b/src/messenger/identity/infrastructure/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/messenger/identity/infrastructure/argon2.py b/src/messenger/identity/infrastructure/argon2.py new file mode 100644 index 00000000..159dba76 --- /dev/null +++ b/src/messenger/identity/infrastructure/argon2.py @@ -0,0 +1,35 @@ +import asyncio +import secrets + +from argon2 import PasswordHasher, Type +from argon2.exceptions import InvalidHashError, VerificationError + +TIME_COST = 2 +MEMORY_COST_KIB = 19_456 +PARALLELISM = 1 + + +class Argon2PasswordHasher: + __slots__ = ("_dummy", "_hasher", "_slots") + + def __init__(self, concurrency: int) -> None: + self._hasher = PasswordHasher( + time_cost=TIME_COST, + memory_cost=MEMORY_COST_KIB, + parallelism=PARALLELISM, + type=Type.ID, + ) + self._slots = asyncio.Semaphore(concurrency) + self._dummy = self._hasher.hash(secrets.token_urlsafe(16)) + + async def hash(self, password: str) -> str: + async with self._slots: + return await asyncio.to_thread(self._hasher.hash, password) + + async def verify(self, password_hash: str | None, password: str) -> bool: + async with self._slots: + try: + await asyncio.to_thread(self._hasher.verify, password_hash or self._dummy, password) + except VerificationError, InvalidHashError: + return False + return password_hash is not None diff --git a/src/messenger/identity/infrastructure/postgres.py b/src/messenger/identity/infrastructure/postgres.py new file mode 100644 index 00000000..ab8ddf3a --- /dev/null +++ b/src/messenger/identity/infrastructure/postgres.py @@ -0,0 +1,124 @@ +from datetime import datetime +from uuid import UUID + +from asyncpg import Record + +from messenger.identity.domain.errors import UsernameAlreadyTaken +from messenger.identity.domain.models import Session, User +from messenger.identity.domain.values import DisplayName, Username +from messenger.shared.infrastructure.postgres import Connection, Database, PostgresUnitOfWork + +INSERT_USER = """ +INSERT INTO users (id, username, display_name, password_hash, created_at) +VALUES ($1, $2, $3, $4, $5) +ON CONFLICT (username) DO NOTHING +RETURNING true +""" +USER_BY_ID = """ +SELECT id, username, display_name, password_hash, created_at FROM users WHERE id = $1 +""" +USER_BY_USERNAME = """ +SELECT id, username, display_name, password_hash, created_at FROM users WHERE username = $1 +""" +INSERT_SESSION = """ +WITH expired AS (DELETE FROM sessions WHERE user_id = $2 AND expires_at <= $3) +INSERT INTO sessions (token_hash, user_id, created_at, expires_at) VALUES ($1, $2, $3, $4) +""" +DELETE_SESSION = "DELETE FROM sessions WHERE token_hash = $1" +ACTIVE_SESSION = """ +SELECT token_hash, user_id, created_at, expires_at +FROM sessions +WHERE token_hash = $1 AND expires_at > $2 +""" + + +def user_of(row: Record | None) -> User | None: + if row is None: + return None + return User( + id=row["id"], + username=Username(row["username"]), + display_name=DisplayName(row["display_name"]), + created_at=row["created_at"], + password_hash=row["password_hash"], + ) + + +class PostgresUserRepository: + __slots__ = ("_connection",) + + def __init__(self, connection: Connection) -> None: + self._connection = connection + + async def add(self, user: User) -> None: + inserted = await self._connection.fetchval( + INSERT_USER, + user.id, + user.username.value, + user.display_name.value, + user.password_hash, + user.created_at, + ) + if inserted is None: + raise UsernameAlreadyTaken("username уже занят") + + +class PostgresSessionRepository: + __slots__ = ("_connection",) + + def __init__(self, connection: Connection) -> None: + self._connection = connection + + async def add(self, session: Session) -> None: + await self._connection.execute( + INSERT_SESSION, + session.token_hash, + session.user_id, + session.created_at, + session.expires_at, + ) + + async def remove(self, token_hash: bytes) -> None: + await self._connection.execute(DELETE_SESSION, token_hash) + + +class PostgresIdentityUnitOfWork(PostgresUnitOfWork): + __slots__ = () + + @property + def users(self) -> PostgresUserRepository: + return PostgresUserRepository(self.connection) + + @property + def sessions(self) -> PostgresSessionRepository: + return PostgresSessionRepository(self.connection) + + +class PostgresUserDirectory: + __slots__ = ("_database",) + + def __init__(self, database: Database) -> None: + self._database = database + + async def find(self, user_id: UUID) -> User | None: + async with self._database.connection() as connection: + return user_of(await connection.fetchrow(USER_BY_ID, user_id)) + + async def find_by_username(self, username: str) -> User | None: + async with self._database.connection() as connection: + return user_of(await connection.fetchrow(USER_BY_USERNAME, username)) + + +class PostgresSessionDirectory: + __slots__ = ("_database",) + + def __init__(self, database: Database) -> None: + self._database = database + + async def find(self, token_hash: bytes, now: datetime) -> Session | None: + async with self._database.connection() as connection: + row = await connection.fetchrow(ACTIVE_SESSION, token_hash, now) + return None if row is None else Session(**dict(row)) + + async def invalidate(self, session: Session) -> None: + return None diff --git a/src/messenger/identity/infrastructure/valkey.py b/src/messenger/identity/infrastructure/valkey.py new file mode 100644 index 00000000..c3d80200 --- /dev/null +++ b/src/messenger/identity/infrastructure/valkey.py @@ -0,0 +1,102 @@ +import time +from collections.abc import Awaitable +from datetime import datetime, timedelta +from uuid import UUID + +from redis.asyncio import Redis +from redis.exceptions import RedisError + +from messenger.identity.application.ports import SessionDirectory +from messenger.identity.domain.models import Session +from messenger.shared.application.errors import Unavailable +from messenger.shared.application.ports import Logger + +REVOKED = "-" +FAILURES = (RedisError, OSError) + + +def key_of(token_hash: bytes) -> str: + return f"session:{token_hash.hex()}" + + +def milliseconds(moment: datetime) -> int: + return int(moment.timestamp() * 1000) + + +def encode(session: Session) -> str: + return f"{session.user_id} {session.created_at.isoformat()} {session.expires_at.isoformat()}" + + +def decode(token_hash: bytes, value: str) -> Session: + user_id, created_at, expires_at = value.split(" ") + return Session( + token_hash=token_hash, + user_id=UUID(user_id), + created_at=datetime.fromisoformat(created_at), + expires_at=datetime.fromisoformat(expires_at), + ) + + +class CachedSessionDirectory: + __slots__ = ("_client", "_cooldown", "_limit", "_logger", "_origin", "_resume_at") + + def __init__( + self, + origin: SessionDirectory, + client: Redis, + logger: Logger, + *, + limit: timedelta, + cooldown: float, + ) -> None: + self._origin = origin + self._client = client + self._logger = logger + self._limit = limit + self._cooldown = cooldown + self._resume_at = 0.0 + + async def find(self, token_hash: bytes, now: datetime) -> Session | None: + key = key_of(token_hash) + cached = await self._read(key) + if cached == REVOKED: + return None + if cached is not None: + session = decode(token_hash, cached) + return session if session.expires_at > now else None + session = await self._origin.find(token_hash, now) + if session is not None and self._closed(): + until = milliseconds(min(session.expires_at, now + self._limit)) + await self._quietly(self._client.set(key, encode(session), pxat=until, nx=True)) + return session + + async def invalidate(self, session: Session) -> None: + if not self._closed(): + raise Unavailable("кэш сессий недоступен") + try: + await self._client.set( + key_of(session.token_hash), REVOKED, pxat=milliseconds(session.expires_at) + ) + except FAILURES as error: + self._trip(error) + raise Unavailable("кэш сессий недоступен") from error + + async def _read(self, key: str) -> str | None: + if not self._closed(): + return None + value = await self._quietly(self._client.get(key)) + return value.decode() if isinstance(value, bytes) else value + + async def _quietly[ResultT](self, call: Awaitable[ResultT]) -> ResultT | None: + try: + return await call + except FAILURES as error: + self._trip(error) + return None + + def _closed(self) -> bool: + return time.monotonic() >= self._resume_at + + def _trip(self, error: Exception) -> None: + self._resume_at = time.monotonic() + self._cooldown + self._logger.warning("cache.bypassed", error=type(error).__name__, limit=self._cooldown) diff --git a/src/messenger/main.py b/src/messenger/main.py new file mode 100644 index 00000000..ae65997e --- /dev/null +++ b/src/messenger/main.py @@ -0,0 +1,162 @@ +from collections.abc import AsyncIterator, Sequence +from contextlib import asynccontextmanager +from uuid import UUID + +from dishka import Provider, Scope, make_async_container, provide +from dishka.integrations.fastapi import FastapiProvider, setup_dishka +from fastapi import FastAPI + +from messenger.attachments.application.ports import AttachmentDirectory, ChatMembership +from messenger.attachments.config import AttachmentsSettings +from messenger.attachments.entrypoints.errors import ERROR_CODES as ATTACHMENT_ERRORS +from messenger.attachments.entrypoints.http import UPLOAD_PATH, attachments_router, uploads_router +from messenger.attachments.entrypoints.ioc import AttachmentsProvider +from messenger.attachments.entrypoints.journal import LOG_FIELDS as ATTACHMENT_FIELDS +from messenger.attachments.entrypoints.journal import subscribe_journal +from messenger.attachments.entrypoints.sweeper import UploadSweeper +from messenger.identity.config import IdentitySettings +from messenger.identity.domain.events import SessionClosed, SessionOpened, UserRegistered +from messenger.identity.entrypoints.authentication import challenge_for, guard +from messenger.identity.entrypoints.errors import ERROR_CODES as IDENTITY_ERRORS +from messenger.identity.entrypoints.http import router as users_router +from messenger.identity.entrypoints.http import sessions_router +from messenger.identity.entrypoints.ioc import IdentityProvider +from messenger.identity.entrypoints.journal import LOG_FIELDS as IDENTITY_FIELDS +from messenger.identity.entrypoints.journal import IdentityJournal +from messenger.messaging.application.ports import AttachmentStates, ChatDirectory +from messenger.messaging.config import MessagingSettings +from messenger.messaging.domain.events import ChatCreated, MessagePosted +from messenger.messaging.entrypoints.errors import ERROR_CODES as MESSAGING_ERRORS +from messenger.messaging.entrypoints.http import chats_router, messages_router +from messenger.messaging.entrypoints.ioc import MessagingProvider +from messenger.messaging.entrypoints.journal import LOG_FIELDS as MESSAGING_FIELDS +from messenger.messaging.entrypoints.journal import MessagingJournal +from messenger.shared.application.messagebus import EventBus +from messenger.shared.application.ports import Logger +from messenger.shared.config import PlatformSettings +from messenger.shared.entrypoints import errors, health +from messenger.shared.entrypoints.config import TransportSettings +from messenger.shared.entrypoints.ioc import PlatformProvider +from messenger.shared.entrypoints.pipeline import RequestPipeline +from messenger.shared.infrastructure.logging import PLATFORM_FIELDS, StructlogLogger + +TRANSPORT_FIELDS = frozenset({"dropped_total", "queue", "instance"}) + +LOG_FIELDS = ( + PLATFORM_FIELDS | TRANSPORT_FIELDS | IDENTITY_FIELDS | MESSAGING_FIELDS | ATTACHMENT_FIELDS +) +ERROR_CODES = {**IDENTITY_ERRORS, **MESSAGING_ERRORS, **ATTACHMENT_ERRORS} + + +class MessagingMembership: + __slots__ = ("_chats",) + + def __init__(self, chats: ChatDirectory) -> None: + self._chats = chats + + async def is_member(self, chat_id: UUID, user_id: UUID) -> bool: + return await self._chats.chat_for_member(chat_id, user_id) is not None + + +class AttachmentGate: + __slots__ = ("_attachments",) + + def __init__(self, attachments: AttachmentDirectory) -> None: + self._attachments = attachments + + async def unusable(self, chat_id: UUID, attachment_ids: Sequence[UUID]) -> bool: + return await self._attachments.failed_among(chat_id, attachment_ids) + + +class ZoneLinks(Provider): + @provide(scope=Scope.APP) + def membership(self, chats: ChatDirectory) -> ChatMembership: + return MessagingMembership(chats) + + @provide(scope=Scope.APP) + def attachment_states(self, attachments: AttachmentDirectory) -> AttachmentStates: + return AttachmentGate(attachments) + + +def wire_events(bus: EventBus, logger: Logger) -> None: + identity = IdentityJournal(logger) + bus.subscribe(UserRegistered, identity.on_user_registered) + bus.subscribe(SessionOpened, identity.on_session_opened) + bus.subscribe(SessionClosed, identity.on_session_closed) + + messaging = MessagingJournal(logger) + bus.subscribe(ChatCreated, messaging.on_chat_created) + bus.subscribe(MessagePosted, messaging.on_message_posted) + + subscribe_journal(bus, logger) + + +def create_app( + *, + platform: PlatformSettings | None = None, + transport: TransportSettings | None = None, + identity: IdentitySettings | None = None, + messaging: MessagingSettings | None = None, + attachments: AttachmentsSettings | None = None, + overrides: Sequence[Provider] = (), +) -> FastAPI: + platform = platform or PlatformSettings() + transport = transport or TransportSettings() + identity = identity or IdentitySettings() + messaging = messaging or MessagingSettings() + attachments = attachments or AttachmentsSettings() + logger = StructlogLogger(level=platform.log_level, allowed_fields=LOG_FIELDS) + container = make_async_container( + PlatformProvider(platform, logger), + IdentityProvider(identity), + MessagingProvider(messaging), + AttachmentsProvider(attachments), + ZoneLinks(), + FastapiProvider(), + *overrides, + ) + sweeper = UploadSweeper(container, attachments, logger) + + @asynccontextmanager + async def lifespan(_app: FastAPI) -> AsyncIterator[None]: + bus = await container.get(EventBus) + wire_events(bus, logger) + bus.start() + sweeper.start() + logger.info("service.started") + try: + yield + finally: + await sweeper.stop() + await bus.stop() + await container.close() + logger.info("service.stopped") + + app = FastAPI( + title="MTUSI Messenger", + version="1.6.0", + lifespan=lifespan, + docs_url="/docs" if transport.docs else None, + redoc_url=None, + openapi_url="/openapi.json" if transport.docs else None, + ) + app.add_middleware( + RequestPipeline, + logger=logger, + max_bytes=transport.max_body_bytes, + timeout=transport.request_timeout_seconds, + guard=guard, + instance=platform.instance_id, + limits=((UPLOAD_PATH, attachments.max_upload_bytes),), + ) + errors.install(app, logger, ERROR_CODES, challenge_for, platform.instance_id) + app.include_router(health.router) + app.include_router(users_router) + app.include_router(sessions_router) + app.include_router(chats_router) + app.include_router(messages_router) + app.include_router(uploads_router) + app.include_router(attachments_router) + setup_dishka(container, app) + app.state.sweeper = sweeper + return app diff --git a/src/messenger/messaging/__init__.py b/src/messenger/messaging/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/messenger/messaging/application/__init__.py b/src/messenger/messaging/application/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/messenger/messaging/application/bus.py b/src/messenger/messaging/application/bus.py new file mode 100644 index 00000000..26fb2e6e --- /dev/null +++ b/src/messenger/messaging/application/bus.py @@ -0,0 +1,5 @@ +from messenger.shared.application.messagebus import CommandBus + + +class MessagingCommandBus(CommandBus): + __slots__ = () diff --git a/src/messenger/messaging/application/commands.py b/src/messenger/messaging/application/commands.py new file mode 100644 index 00000000..104b37eb --- /dev/null +++ b/src/messenger/messaging/application/commands.py @@ -0,0 +1,22 @@ +from collections.abc import Sequence +from dataclasses import dataclass +from uuid import UUID + +from messenger.shared.application.commands import Command + + +@dataclass(frozen=True, slots=True) +class CreateChat(Command): + creator_id: UUID + title: str + member_ids: Sequence[UUID] + + +@dataclass(frozen=True, slots=True) +class PostMessage(Command): + sender_id: UUID + chat_id: UUID + text: str + key: str + fingerprint: bytes + attachment_ids: tuple[UUID, ...] = () diff --git a/src/messenger/messaging/application/handlers.py b/src/messenger/messaging/application/handlers.py new file mode 100644 index 00000000..948fea8b --- /dev/null +++ b/src/messenger/messaging/application/handlers.py @@ -0,0 +1,73 @@ +from datetime import timedelta + +from messenger.messaging.application.commands import CreateChat, PostMessage +from messenger.messaging.application.ports import AttachmentStates, MessagingUnitOfWork +from messenger.messaging.domain.errors import AttachmentUnavailable, ChatNotFound +from messenger.messaging.domain.models import Chat, Message, Receipt +from messenger.shared.application.ports import Clock, IdGenerator + + +class CreateChatHandler: + __slots__ = ("_clock", "_ids", "_uow") + + def __init__(self, uow: MessagingUnitOfWork, clock: Clock, ids: IdGenerator) -> None: + self._uow = uow + self._clock = clock + self._ids = ids + + async def __call__(self, command: CreateChat) -> Chat: + chat = Chat.create( + chat_id=self._ids.new_id(), + title=command.title, + member_ids=command.member_ids, + creator_id=command.creator_id, + created_at=self._clock.now(), + ) + async with self._uow.transaction(): + await self._uow.chats.add(chat) + await self._uow.commit() + return chat + + +class PostMessageHandler: + __slots__ = ("_attachments", "_clock", "_ids", "_ttl", "_uow") + + def __init__( + self, + uow: MessagingUnitOfWork, + attachments: AttachmentStates, + clock: Clock, + ids: IdGenerator, + ttl: timedelta, + ) -> None: + self._uow = uow + self._attachments = attachments + self._clock = clock + self._ids = ids + self._ttl = ttl + + async def __call__(self, command: PostMessage) -> Message: + async with self._uow.transaction(): + chat = await self._uow.chats.get(command.chat_id) + if chat is None: + raise ChatNotFound("чат не найден") + now = self._clock.now() + message = chat.post( + message_id=self._ids.new_id(), + sender_id=command.sender_id, + text=command.text, + now=now, + attachment_ids=command.attachment_ids, + ) + known = await self._uow.chats.receipt( + chat.id, command.sender_id, command.key, now - self._ttl + ) + if known is not None: + return known.replay(command.fingerprint) + if message.attachment_ids and await self._attachments.unusable( + chat.id, message.attachment_ids + ): + raise AttachmentUnavailable("вложение не обработано и не может быть отправлено") + await self._uow.chats.append(Receipt(command.key, command.fingerprint, message)) + await self._uow.commit() + return message diff --git a/src/messenger/messaging/application/ports.py b/src/messenger/messaging/application/ports.py new file mode 100644 index 00000000..bf7b6a80 --- /dev/null +++ b/src/messenger/messaging/application/ports.py @@ -0,0 +1,52 @@ +from collections.abc import Sequence +from datetime import datetime +from typing import Protocol, runtime_checkable +from uuid import UUID + +from messenger.messaging.domain.keys import HistoryKey +from messenger.messaging.domain.models import Chat, Message, Receipt +from messenger.shared.application.ports import UnitOfWork + + +@runtime_checkable +class CursorCodec(Protocol): + def encode(self, scope: str, key: HistoryKey) -> str: ... + + def decode(self, scope: str, raw: str) -> HistoryKey: ... + + +@runtime_checkable +class ChatRepository(Protocol): + async def add(self, chat: Chat) -> None: ... + + async def get(self, chat_id: UUID) -> Chat | None: ... + + async def receipt( + self, chat_id: UUID, sender_id: UUID, key: str, since: datetime + ) -> Receipt | None: ... + + async def append(self, receipt: Receipt) -> None: ... + + +@runtime_checkable +class MessagingUnitOfWork(UnitOfWork, Protocol): + @property + def chats(self) -> ChatRepository: ... + + +@runtime_checkable +class AttachmentStates(Protocol): + async def unusable(self, chat_id: UUID, attachment_ids: Sequence[UUID]) -> bool: ... + + +@runtime_checkable +class ChatDirectory(Protocol): + async def chat_for_member(self, chat_id: UUID, user_id: UUID) -> Chat | None: ... + + async def chats_of( + self, user_id: UUID, *, after: HistoryKey | None, limit: int + ) -> tuple[list[Chat], bool]: ... + + async def history( + self, chat_id: UUID, *, after: HistoryKey | None, limit: int + ) -> tuple[list[Message], bool]: ... diff --git a/src/messenger/messaging/application/queries.py b/src/messenger/messaging/application/queries.py new file mode 100644 index 00000000..8e16e947 --- /dev/null +++ b/src/messenger/messaging/application/queries.py @@ -0,0 +1,73 @@ +from dataclasses import dataclass +from uuid import UUID + +from messenger.messaging.application.ports import ChatDirectory, CursorCodec +from messenger.messaging.domain.errors import ChatNotFound +from messenger.messaging.domain.keys import HistoryKey +from messenger.messaging.domain.models import Chat, Message + + +@dataclass(frozen=True, slots=True) +class Page[ItemT]: + items: list[ItemT] + next_cursor: str | None + + +def chats_scope(user_id: UUID) -> str: + return f"chats:{user_id}" + + +def messages_scope(chat_id: UUID) -> str: + return f"messages:{chat_id}" + + +class GetChat: + __slots__ = ("_chats",) + + def __init__(self, chats: ChatDirectory) -> None: + self._chats = chats + + async def __call__(self, chat_id: UUID, user_id: UUID) -> Chat: + chat = await self._chats.chat_for_member(chat_id, user_id) + if chat is None: + raise ChatNotFound("чат не найден") + return chat + + +class ListChats: + __slots__ = ("_chats", "_cursors") + + def __init__(self, chats: ChatDirectory, cursors: CursorCodec) -> None: + self._chats = chats + self._cursors = cursors + + async def __call__(self, user_id: UUID, *, limit: int, cursor: str | None) -> Page[Chat]: + scope = chats_scope(user_id) + after: HistoryKey | None = self._cursors.decode(scope, cursor) if cursor else None + items, has_more = await self._chats.chats_of(user_id, after=after, limit=limit) + next_cursor = self._cursors.encode(scope, items[-1].key) if has_more and items else None + return Page(items=items, next_cursor=next_cursor) + + +class ReadHistory: + __slots__ = ("_chats", "_cursors") + + def __init__(self, chats: ChatDirectory, cursors: CursorCodec) -> None: + self._chats = chats + self._cursors = cursors + + async def __call__( + self, + chat_id: UUID, + user_id: UUID, + *, + limit: int, + cursor: str | None, + ) -> Page[Message]: + if await self._chats.chat_for_member(chat_id, user_id) is None: + raise ChatNotFound("чат не найден") + scope = messages_scope(chat_id) + after: HistoryKey | None = self._cursors.decode(scope, cursor) if cursor else None + items, has_more = await self._chats.history(chat_id, after=after, limit=limit) + next_cursor = self._cursors.encode(scope, items[-1].key) if has_more and items else None + return Page(items=items, next_cursor=next_cursor) diff --git a/src/messenger/messaging/config.py b/src/messenger/messaging/config.py new file mode 100644 index 00000000..71cd0c2a --- /dev/null +++ b/src/messenger/messaging/config.py @@ -0,0 +1,8 @@ +from pydantic import Field, SecretStr + +from messenger.shared.config import Settings + + +class MessagingSettings(Settings): + cursor_secret: SecretStr = Field(min_length=16) + idempotency_ttl_seconds: int = Field(default=86_400, ge=1) diff --git a/src/messenger/messaging/domain/__init__.py b/src/messenger/messaging/domain/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/messenger/messaging/domain/errors.py b/src/messenger/messaging/domain/errors.py new file mode 100644 index 00000000..fde8c050 --- /dev/null +++ b/src/messenger/messaging/domain/errors.py @@ -0,0 +1,37 @@ +from messenger.shared.domain.errors import DomainError + + +class MessagingError(DomainError): + pass + + +class InvalidChatData(MessagingError): + pass + + +class InvalidMessageText(MessagingError): + pass + + +class InvalidPageRequest(MessagingError): + pass + + +class ChatNotFound(MessagingError): + pass + + +class IdempotencyConflict(MessagingError): + pass + + +class InvalidAttachmentList(MessagingError): + pass + + +class UnknownAttachment(MessagingError): + pass + + +class AttachmentUnavailable(MessagingError): + pass diff --git a/src/messenger/messaging/domain/events.py b/src/messenger/messaging/domain/events.py new file mode 100644 index 00000000..1b0709d3 --- /dev/null +++ b/src/messenger/messaging/domain/events.py @@ -0,0 +1,24 @@ +from dataclasses import dataclass, field +from datetime import datetime +from uuid import UUID + +from messenger.shared.domain.events import DomainEvent + + +@dataclass(frozen=True, slots=True) +class ChatCreated(DomainEvent): + chat_id: UUID + creator_id: UUID + member_count: int + created_at: datetime + name: str = field(init=False, default="chat.created") + + +@dataclass(frozen=True, slots=True) +class MessagePosted(DomainEvent): + message_id: UUID + chat_id: UUID + sender_id: UUID + text_length: int + created_at: datetime + name: str = field(init=False, default="message.posted") diff --git a/src/messenger/messaging/domain/keys.py b/src/messenger/messaging/domain/keys.py new file mode 100644 index 00000000..2e432477 --- /dev/null +++ b/src/messenger/messaging/domain/keys.py @@ -0,0 +1,3 @@ +from datetime import datetime + +type HistoryKey = tuple[datetime, int] diff --git a/src/messenger/messaging/domain/models.py b/src/messenger/messaging/domain/models.py new file mode 100644 index 00000000..f98b8430 --- /dev/null +++ b/src/messenger/messaging/domain/models.py @@ -0,0 +1,177 @@ +from collections.abc import Sequence +from dataclasses import dataclass +from datetime import datetime, timedelta +from uuid import UUID + +from messenger.messaging.domain.errors import ( + ChatNotFound, + IdempotencyConflict, + InvalidAttachmentList, + InvalidChatData, + InvalidMessageText, +) +from messenger.messaging.domain.events import ChatCreated, MessagePosted +from messenger.messaging.domain.keys import HistoryKey +from messenger.messaging.domain.values import ( + CHAT_MEMBERS_MAX, + MESSAGE_ATTACHMENTS_MAX, + ChatTitle, + MessageText, +) +from messenger.shared.domain.events import DomainEvent + +TICK = timedelta(microseconds=1) + + +@dataclass(frozen=True, slots=True) +class Message: + id: UUID + chat_id: UUID + sender_id: UUID + text: MessageText + created_at: datetime + attachment_ids: tuple[UUID, ...] = () + + @property + def key(self) -> HistoryKey: + return (self.created_at, self.id.int) + + +def attachment_list(text: MessageText, attachment_ids: Sequence[UUID]) -> tuple[UUID, ...]: + if len(attachment_ids) > MESSAGE_ATTACHMENTS_MAX: + raise InvalidAttachmentList( + f"поле attachment_ids содержит больше {MESSAGE_ATTACHMENTS_MAX} вложений" + ) + if len(set(attachment_ids)) != len(attachment_ids): + raise InvalidAttachmentList("поле attachment_ids содержит дубликаты") + if not text.value and not attachment_ids: + raise InvalidMessageText("пустой text допустим только с вложением") + return tuple(attachment_ids) + + +@dataclass(frozen=True, slots=True) +class Receipt: + key: str + fingerprint: bytes + message: Message + + def replay(self, fingerprint: bytes) -> Message: + if fingerprint != self.fingerprint: + raise IdempotencyConflict("ключ идемпотентности уже использован с другим телом") + return self.message + + +class Chat: + __slots__ = ( + "_events", + "_members", + "created_at", + "id", + "last_message_at", + "member_ids", + "title", + ) + + def __init__( + self, + *, + chat_id: UUID, + title: ChatTitle, + member_ids: Sequence[UUID], + created_at: datetime, + last_message_at: datetime | None = None, + ) -> None: + self.id = chat_id + self.title = title + self.member_ids: tuple[UUID, ...] = tuple(member_ids) + self.created_at = created_at + self.last_message_at = last_message_at + self._members: frozenset[UUID] = frozenset(member_ids) + self._events: list[DomainEvent] = [] + + @classmethod + def create( + cls, + *, + chat_id: UUID, + title: str, + member_ids: Sequence[UUID], + creator_id: UUID, + created_at: datetime, + ) -> Chat: + if not member_ids: + raise InvalidChatData("поле member_ids не может быть пустым") + if len(member_ids) > CHAT_MEMBERS_MAX: + raise InvalidChatData(f"поле member_ids содержит больше {CHAT_MEMBERS_MAX} участников") + unique = set(member_ids) + if len(unique) != len(member_ids): + raise InvalidChatData("поле member_ids содержит дубликаты") + if creator_id not in unique: + raise InvalidChatData("создатель чата обязан входить в список member_ids") + chat = cls( + chat_id=chat_id, + title=ChatTitle(title), + member_ids=member_ids, + created_at=created_at, + ) + chat.record( + ChatCreated( + chat_id=chat.id, + creator_id=creator_id, + member_count=len(unique), + created_at=created_at, + ) + ) + return chat + + @property + def key(self) -> HistoryKey: + return (self.created_at, self.id.int) + + def record(self, event: DomainEvent) -> None: + self._events.append(event) + + def is_member(self, user_id: UUID) -> bool: + return user_id in self._members + + def require_member(self, user_id: UUID) -> None: + if not self.is_member(user_id): + raise ChatNotFound("чат не найден") + + def post( + self, + *, + message_id: UUID, + sender_id: UUID, + text: str, + now: datetime, + attachment_ids: Sequence[UUID] = (), + ) -> Message: + self.require_member(sender_id) + body = MessageText(text) + attachments = attachment_list(body, attachment_ids) + last = self.last_message_at + created_at = now if last is None or now > last else last + TICK + message = Message( + id=message_id, + chat_id=self.id, + sender_id=sender_id, + text=body, + created_at=created_at, + attachment_ids=attachments, + ) + self.last_message_at = created_at + self.record( + MessagePosted( + message_id=message.id, + chat_id=self.id, + sender_id=sender_id, + text_length=len(message.text.value), + created_at=created_at, + ) + ) + return message + + def collect_events(self) -> list[DomainEvent]: + events, self._events = self._events, [] + return events diff --git a/src/messenger/messaging/domain/values.py b/src/messenger/messaging/domain/values.py new file mode 100644 index 00000000..5a2f31b9 --- /dev/null +++ b/src/messenger/messaging/domain/values.py @@ -0,0 +1,40 @@ +from dataclasses import dataclass + +from messenger.messaging.domain.errors import InvalidChatData, InvalidMessageText + +CHAT_TITLE_MAX = 100 +MESSAGE_TEXT_MAX = 4000 +CHAT_MEMBERS_MAX = 100 +MESSAGE_ATTACHMENTS_MAX = 10 + + +@dataclass(frozen=True, slots=True) +class ChatTitle: + value: str + + def __post_init__(self) -> None: + if not isinstance(self.value, str): + raise InvalidChatData("поле title должно быть строкой") + if not self.value.strip(): + raise InvalidChatData("поле title не может быть пустым или состоять из пробелов") + if len(self.value) > CHAT_TITLE_MAX: + raise InvalidChatData(f"поле title длиннее {CHAT_TITLE_MAX} символов Unicode") + + def __str__(self) -> str: + return self.value + + +@dataclass(frozen=True, slots=True) +class MessageText: + value: str + + def __post_init__(self) -> None: + if not isinstance(self.value, str): + raise InvalidMessageText("поле text должно быть строкой") + if self.value and not self.value.strip(): + raise InvalidMessageText("поле text не может состоять из пробелов") + if len(self.value) > MESSAGE_TEXT_MAX: + raise InvalidMessageText(f"поле text длиннее {MESSAGE_TEXT_MAX} символов Unicode") + + def __str__(self) -> str: + return self.value diff --git a/src/messenger/messaging/entrypoints/__init__.py b/src/messenger/messaging/entrypoints/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/messenger/messaging/entrypoints/errors.py b/src/messenger/messaging/entrypoints/errors.py new file mode 100644 index 00000000..fe818b9c --- /dev/null +++ b/src/messenger/messaging/entrypoints/errors.py @@ -0,0 +1,22 @@ +from messenger.messaging.domain.errors import ( + AttachmentUnavailable, + ChatNotFound, + IdempotencyConflict, + InvalidAttachmentList, + InvalidChatData, + InvalidMessageText, + InvalidPageRequest, + UnknownAttachment, +) +from messenger.shared.domain.errors import DomainError + +ERROR_CODES: dict[type[DomainError], tuple[str, int]] = { + InvalidChatData: ("invalid_request", 400), + InvalidMessageText: ("invalid_request", 400), + InvalidPageRequest: ("invalid_request", 400), + InvalidAttachmentList: ("invalid_request", 400), + ChatNotFound: ("not_found", 404), + UnknownAttachment: ("not_found", 404), + IdempotencyConflict: ("conflict", 409), + AttachmentUnavailable: ("conflict", 409), +} diff --git a/src/messenger/messaging/entrypoints/http.py b/src/messenger/messaging/entrypoints/http.py new file mode 100644 index 00000000..ad0a692d --- /dev/null +++ b/src/messenger/messaging/entrypoints/http.py @@ -0,0 +1,90 @@ +from typing import Annotated +from uuid import UUID + +from dishka.integrations.fastapi import FromDishka, inject +from fastapi import APIRouter, Query, status + +from messenger.messaging.application.bus import MessagingCommandBus +from messenger.messaging.application.commands import CreateChat, PostMessage +from messenger.messaging.application.queries import GetChat, ListChats, ReadHistory +from messenger.messaging.entrypoints.schemas import ( + ChatOut, + CreateChatRequest, + CreateMessageRequest, + IdempotencyKey, + MessageOut, + PageOut, + PageQuery, +) +from messenger.shared.entrypoints.actor import Actor + +chats_router = APIRouter(prefix="/api/v1/chats", tags=["chats"]) +messages_router = APIRouter(prefix="/api/v1/chats/{chat_id}/messages", tags=["messages"]) + + +@chats_router.post("", status_code=status.HTTP_201_CREATED) +@inject +async def create_chat( + actor: Actor, + body: CreateChatRequest, + bus: FromDishka[MessagingCommandBus], +) -> ChatOut: + chat = await bus.execute( + CreateChat(creator_id=actor, title=body.title, member_ids=body.member_ids) + ) + return ChatOut.of(chat) + + +@chats_router.get("") +@inject +async def list_chats( + actor: Actor, + page: Annotated[PageQuery, Query()], + query: FromDishka[ListChats], +) -> PageOut[ChatOut]: + chats = await query(actor, limit=page.limit, cursor=page.cursor) + return PageOut[ChatOut].of(chats, ChatOut.of) + + +@chats_router.get("/{chat_id}") +@inject +async def read_chat( + chat_id: UUID, + actor: Actor, + query: FromDishka[GetChat], +) -> ChatOut: + return ChatOut.of(await query(chat_id, actor)) + + +@messages_router.post("", status_code=status.HTTP_201_CREATED) +@inject +async def post_message( + chat_id: UUID, + actor: Actor, + body: CreateMessageRequest, + key: IdempotencyKey, + bus: FromDishka[MessagingCommandBus], +) -> MessageOut: + message = await bus.execute( + PostMessage( + sender_id=actor, + chat_id=chat_id, + text=body.text, + key=key, + fingerprint=body.fingerprint(), + attachment_ids=tuple(body.attachment_ids), + ) + ) + return MessageOut.of(message) + + +@messages_router.get("") +@inject +async def read_history( + chat_id: UUID, + actor: Actor, + page: Annotated[PageQuery, Query()], + query: FromDishka[ReadHistory], +) -> PageOut[MessageOut]: + messages = await query(chat_id, actor, limit=page.limit, cursor=page.cursor) + return PageOut[MessageOut].of(messages, MessageOut.of) diff --git a/src/messenger/messaging/entrypoints/ioc.py b/src/messenger/messaging/entrypoints/ioc.py new file mode 100644 index 00000000..a2e21dd5 --- /dev/null +++ b/src/messenger/messaging/entrypoints/ioc.py @@ -0,0 +1,75 @@ +from datetime import timedelta + +from dishka import Provider, Scope, provide + +from messenger.messaging.application.bus import MessagingCommandBus +from messenger.messaging.application.commands import CreateChat, PostMessage +from messenger.messaging.application.handlers import CreateChatHandler, PostMessageHandler +from messenger.messaging.application.ports import ( + AttachmentStates, + ChatDirectory, + CursorCodec, + MessagingUnitOfWork, +) +from messenger.messaging.application.queries import GetChat, ListChats, ReadHistory +from messenger.messaging.config import MessagingSettings +from messenger.messaging.infrastructure.cursor import SignedCursorCodec +from messenger.messaging.infrastructure.postgres import ( + PostgresChatDirectory, + PostgresMessagingUnitOfWork, +) +from messenger.shared.application.commands import Command +from messenger.shared.application.messagebus import CommandHandler +from messenger.shared.application.ports import Clock, EventPublisher, IdGenerator +from messenger.shared.infrastructure.postgres import Database + + +class MessagingProvider(Provider): + def __init__(self, settings: MessagingSettings) -> None: + super().__init__() + self._settings = settings + + @provide(scope=Scope.APP) + def settings(self) -> MessagingSettings: + return self._settings + + @provide(scope=Scope.APP) + def directory(self, database: Database) -> ChatDirectory: + return PostgresChatDirectory(database) + + @provide(scope=Scope.APP) + def cursors(self, settings: MessagingSettings) -> CursorCodec: + return SignedCursorCodec(settings.cursor_secret.get_secret_value().encode()) + + @provide(scope=Scope.REQUEST) + def unit_of_work(self, database: Database) -> MessagingUnitOfWork: + return PostgresMessagingUnitOfWork(database) + + @provide(scope=Scope.REQUEST) + def chat(self, directory: ChatDirectory) -> GetChat: + return GetChat(directory) + + @provide(scope=Scope.REQUEST) + def chat_list(self, directory: ChatDirectory, cursors: CursorCodec) -> ListChats: + return ListChats(directory, cursors) + + @provide(scope=Scope.REQUEST) + def history(self, directory: ChatDirectory, cursors: CursorCodec) -> ReadHistory: + return ReadHistory(directory, cursors) + + @provide(scope=Scope.REQUEST) + def command_bus( + self, + uow: MessagingUnitOfWork, + attachments: AttachmentStates, + publisher: EventPublisher, + clock: Clock, + ids: IdGenerator, + settings: MessagingSettings, + ) -> MessagingCommandBus: + ttl = timedelta(seconds=settings.idempotency_ttl_seconds) + handlers: dict[type[Command], CommandHandler] = { + CreateChat: CreateChatHandler(uow, clock, ids), + PostMessage: PostMessageHandler(uow, attachments, clock, ids, ttl), + } + return MessagingCommandBus(handlers, uow, publisher) diff --git a/src/messenger/messaging/entrypoints/journal.py b/src/messenger/messaging/entrypoints/journal.py new file mode 100644 index 00000000..17c7e675 --- /dev/null +++ b/src/messenger/messaging/entrypoints/journal.py @@ -0,0 +1,27 @@ +from messenger.messaging.domain.events import ChatCreated, MessagePosted +from messenger.shared.application.ports import Logger + +LOG_FIELDS = frozenset({"chat_id", "message_id", "count"}) + + +class MessagingJournal: + __slots__ = ("_logger",) + + def __init__(self, logger: Logger) -> None: + self._logger = logger + + async def on_chat_created(self, event: ChatCreated) -> None: + self._logger.info( + "audit.chat_created", + chat_id=str(event.chat_id), + actor_id=str(event.creator_id), + count=event.member_count, + ) + + async def on_message_posted(self, event: MessagePosted) -> None: + self._logger.info( + "audit.message_posted", + chat_id=str(event.chat_id), + actor_id=str(event.sender_id), + message_id=str(event.message_id), + ) diff --git a/src/messenger/messaging/entrypoints/schemas.py b/src/messenger/messaging/entrypoints/schemas.py new file mode 100644 index 00000000..6b7f1a35 --- /dev/null +++ b/src/messenger/messaging/entrypoints/schemas.py @@ -0,0 +1,85 @@ +from collections.abc import Callable +from datetime import datetime +from hashlib import sha256 +from typing import Annotated, Self +from uuid import UUID + +from fastapi import Header +from pydantic import BaseModel, ConfigDict, Field + +from messenger.messaging.application.queries import Page +from messenger.messaging.domain.models import Chat, Message + +STRICT = ConfigDict(extra="forbid", strict=True) + +type IdempotencyKey = Annotated[ + str, + Header(alias="Idempotency-Key", min_length=1, max_length=128, pattern=r"^[A-Za-z0-9._:-]+$"), +] + + +class CreateChatRequest(BaseModel): + model_config = STRICT + + title: str + member_ids: list[Annotated[UUID, Field(strict=False)]] + + +class CreateMessageRequest(BaseModel): + model_config = STRICT + + text: str + attachment_ids: list[Annotated[UUID, Field(strict=False)]] = Field(default_factory=list) + + def fingerprint(self) -> bytes: + return sha256(self.model_dump_json().encode()).digest() + + +class PageQuery(BaseModel): + limit: int = Field(default=50, ge=1, le=100) + cursor: str | None = None + + +class ChatOut(BaseModel): + id: UUID + title: str + member_ids: tuple[UUID, ...] + created_at: datetime + + @classmethod + def of(cls, chat: Chat) -> ChatOut: + return cls( + id=chat.id, + title=chat.title.value, + member_ids=chat.member_ids, + created_at=chat.created_at, + ) + + +class MessageOut(BaseModel): + id: UUID + chat_id: UUID + sender_id: UUID + text: str + created_at: datetime + attachment_ids: tuple[UUID, ...] + + @classmethod + def of(cls, message: Message) -> MessageOut: + return cls( + id=message.id, + chat_id=message.chat_id, + sender_id=message.sender_id, + text=message.text.value, + created_at=message.created_at, + attachment_ids=message.attachment_ids, + ) + + +class PageOut[ItemT](BaseModel): + items: list[ItemT] + next_cursor: str | None + + @classmethod + def of[SourceT](cls, page: Page[SourceT], view: Callable[[SourceT], ItemT]) -> Self: + return cls(items=[view(item) for item in page.items], next_cursor=page.next_cursor) diff --git a/src/messenger/messaging/infrastructure/__init__.py b/src/messenger/messaging/infrastructure/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/messenger/messaging/infrastructure/cursor.py b/src/messenger/messaging/infrastructure/cursor.py new file mode 100644 index 00000000..40044241 --- /dev/null +++ b/src/messenger/messaging/infrastructure/cursor.py @@ -0,0 +1,59 @@ +import hmac +from base64 import urlsafe_b64decode, urlsafe_b64encode +from datetime import UTC, datetime, timedelta +from hashlib import sha256 + +from messenger.messaging.domain.errors import InvalidPageRequest +from messenger.messaging.domain.keys import HistoryKey + +VERSION = "1" +SIGNATURE_BYTES = 12 +EPOCH = datetime(1970, 1, 1, tzinfo=UTC) +PARTS = 4 + + +def _encode_part(value: bytes) -> str: + return urlsafe_b64encode(value).decode("ascii").rstrip("=") + + +def _decode_part(value: str) -> bytes: + padding = "=" * (-len(value) % 4) + try: + return urlsafe_b64decode(value + padding) + except (ValueError, TypeError) as error: + raise InvalidPageRequest("параметр cursor повреждён") from error + + +class SignedCursorCodec: + __slots__ = ("_secret",) + + def __init__(self, secret: bytes) -> None: + self._secret = secret + + def encode(self, scope: str, key: HistoryKey) -> str: + moment, identifier = key + micros = (moment - EPOCH) // timedelta(microseconds=1) + body = f"{VERSION}.{micros}.{identifier:032x}" + return f"{body}.{_encode_part(self._sign(scope, body))}" + + def decode(self, scope: str, raw: str) -> HistoryKey: + parts = raw.split(".") + if len(parts) != PARTS or parts[0] != VERSION: + raise InvalidPageRequest("параметр cursor не распознан") + body = ".".join(parts[:3]) + if not hmac.compare_digest(self._sign(scope, body), _decode_part(parts[3])): + raise InvalidPageRequest("параметр cursor недействителен для этого запроса") + try: + micros = int(parts[1]) + identifier = int(parts[2], 16) + except ValueError as error: + raise InvalidPageRequest("параметр cursor повреждён") from error + try: + moment = EPOCH + timedelta(microseconds=micros) + except (OverflowError, ValueError) as error: + raise InvalidPageRequest("параметр cursor повреждён") from error + return (moment, identifier) + + def _sign(self, scope: str, body: str) -> bytes: + payload = f"{scope}|{body}".encode() + return hmac.new(self._secret, payload, sha256).digest()[:SIGNATURE_BYTES] diff --git a/src/messenger/messaging/infrastructure/postgres.py b/src/messenger/messaging/infrastructure/postgres.py new file mode 100644 index 00000000..3c8b94b7 --- /dev/null +++ b/src/messenger/messaging/infrastructure/postgres.py @@ -0,0 +1,219 @@ +from collections.abc import Sequence +from datetime import datetime +from uuid import UUID + +from asyncpg import ForeignKeyViolationError, Record + +from messenger.messaging.domain.errors import InvalidChatData, UnknownAttachment +from messenger.messaging.domain.keys import HistoryKey +from messenger.messaging.domain.models import Chat, Message, Receipt +from messenger.messaging.domain.values import ChatTitle, MessageText +from messenger.shared.infrastructure.postgres import ( + Connection, + Database, + EventSource, + PostgresUnitOfWork, +) + +INSERT_CHAT = "INSERT INTO chats (id, title, created_at) VALUES ($1, $2, $3)" +INSERT_MEMBERS = """ +INSERT INTO chat_members (chat_id, user_id, position, chat_created_at) +SELECT $1, member.user_id, member.position, $3 +FROM unnest($2::uuid[]) WITH ORDINALITY AS member (user_id, position) +""" +LOCK_CHAT = """ +SELECT c.id, c.title, c.created_at, c.last_message_at, + ARRAY(SELECT user_id FROM chat_members WHERE chat_id = c.id ORDER BY position) AS member_ids +FROM chats c +WHERE c.id = $1 +FOR UPDATE +""" +ATTACHMENT_FKEY = "message_attachments_attachment_fkey" +RECEIPT = """ +SELECT k.fingerprint, m.id, m.chat_id, m.sender_id, m.text, m.created_at, + ARRAY(SELECT a.attachment_id FROM message_attachments a + WHERE a.message_id = m.id ORDER BY a.position) AS attachment_ids +FROM idempotency_keys k JOIN messages m ON m.id = k.message_id +WHERE k.chat_id = $1 AND k.user_id = $2 AND k.key = $3 AND k.created_at > $4 +""" +APPEND_MESSAGE = """ +WITH message AS ( + INSERT INTO messages (id, chat_id, sender_id, text, created_at) + VALUES ($1, $2, $3, $4, $5) + RETURNING id, chat_id, sender_id, created_at +), links AS ( + INSERT INTO message_attachments (message_id, position, chat_id, attachment_id) + SELECT message.id, link.position - 1, message.chat_id, link.attachment_id + FROM message, unnest($8::uuid[]) WITH ORDINALITY AS link (attachment_id, position) +), receipt AS ( + INSERT INTO idempotency_keys (chat_id, user_id, key, fingerprint, message_id, created_at) + SELECT chat_id, sender_id, $6, $7, id, created_at FROM message + ON CONFLICT (chat_id, user_id, key) DO UPDATE + SET fingerprint = EXCLUDED.fingerprint, + message_id = EXCLUDED.message_id, + created_at = EXCLUDED.created_at +) +UPDATE chats SET last_message_at = message.created_at +FROM message +WHERE chats.id = message.chat_id +""" +CHAT_OF_MEMBER = """ +SELECT c.id, c.title, c.created_at, + ARRAY(SELECT user_id FROM chat_members WHERE chat_id = c.id ORDER BY position) AS member_ids +FROM chat_members m JOIN chats c ON c.id = m.chat_id +WHERE m.chat_id = $1 AND m.user_id = $2 +""" +CHATS_FIRST = """ +SELECT c.id, c.title, c.created_at, + ARRAY(SELECT user_id FROM chat_members WHERE chat_id = c.id ORDER BY position) AS member_ids +FROM chat_members m JOIN chats c ON c.id = m.chat_id +WHERE m.user_id = $1 +ORDER BY m.chat_created_at, m.chat_id +LIMIT $2 +""" +CHATS_AFTER = """ +SELECT c.id, c.title, c.created_at, + ARRAY(SELECT user_id FROM chat_members WHERE chat_id = c.id ORDER BY position) AS member_ids +FROM chat_members m JOIN chats c ON c.id = m.chat_id +WHERE m.user_id = $1 AND (m.chat_created_at, m.chat_id) > ($2, $3) +ORDER BY m.chat_created_at, m.chat_id +LIMIT $4 +""" +HISTORY_FIRST = """ +SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at, + ARRAY(SELECT a.attachment_id FROM message_attachments a + WHERE a.message_id = m.id ORDER BY a.position) AS attachment_ids +FROM messages m +WHERE m.chat_id = $1 +ORDER BY m.created_at, m.id +LIMIT $2 +""" +HISTORY_AFTER = """ +SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at, + ARRAY(SELECT a.attachment_id FROM message_attachments a + WHERE a.message_id = m.id ORDER BY a.position) AS attachment_ids +FROM messages m +WHERE m.chat_id = $1 AND (m.created_at, m.id) > ($2, $3) +ORDER BY m.created_at, m.id +LIMIT $4 +""" + + +def chat_of(row: Record) -> Chat: + return Chat( + chat_id=row["id"], + title=ChatTitle(row["title"]), + member_ids=row["member_ids"], + created_at=row["created_at"], + last_message_at=row.get("last_message_at"), + ) + + +def message_of(row: Record) -> Message: + return Message( + id=row["id"], + chat_id=row["chat_id"], + sender_id=row["sender_id"], + text=MessageText(row["text"]), + created_at=row["created_at"], + attachment_ids=tuple(row["attachment_ids"]), + ) + + +def page[ItemT](rows: Sequence[ItemT], limit: int) -> tuple[list[ItemT], bool]: + return list(rows[:limit]), len(rows) > limit + + +class PostgresChatRepository: + __slots__ = ("_connection", "_seen") + + def __init__(self, connection: Connection, seen: list[EventSource]) -> None: + self._connection = connection + self._seen = seen + + async def add(self, chat: Chat) -> None: + await self._connection.execute(INSERT_CHAT, chat.id, chat.title.value, chat.created_at) + try: + await self._connection.execute( + INSERT_MEMBERS, chat.id, list(chat.member_ids), chat.created_at + ) + except ForeignKeyViolationError as error: + raise InvalidChatData("поле member_ids содержит неизвестного пользователя") from error + self._seen.append(chat) + + async def get(self, chat_id: UUID) -> Chat | None: + row = await self._connection.fetchrow(LOCK_CHAT, chat_id) + if row is None: + return None + chat = chat_of(row) + self._seen.append(chat) + return chat + + async def receipt( + self, chat_id: UUID, sender_id: UUID, key: str, since: datetime + ) -> Receipt | None: + row = await self._connection.fetchrow(RECEIPT, chat_id, sender_id, key, since) + return None if row is None else Receipt(key, row["fingerprint"], message_of(row)) + + async def append(self, receipt: Receipt) -> None: + message = receipt.message + try: + await self._connection.execute( + APPEND_MESSAGE, + message.id, + message.chat_id, + message.sender_id, + message.text.value, + message.created_at, + receipt.key, + receipt.fingerprint, + list(message.attachment_ids), + ) + except ForeignKeyViolationError as error: + if getattr(error, "constraint_name", None) != ATTACHMENT_FKEY: + raise + raise UnknownAttachment("вложение не найдено в этом чате") from error + + +class PostgresMessagingUnitOfWork(PostgresUnitOfWork): + __slots__ = () + + @property + def chats(self) -> PostgresChatRepository: + return PostgresChatRepository(self.connection, self.seen) + + +class PostgresChatDirectory: + __slots__ = ("_database",) + + def __init__(self, database: Database) -> None: + self._database = database + + async def chat_for_member(self, chat_id: UUID, user_id: UUID) -> Chat | None: + async with self._database.connection() as connection: + row = await connection.fetchrow(CHAT_OF_MEMBER, chat_id, user_id) + return None if row is None else chat_of(row) + + async def chats_of( + self, user_id: UUID, *, after: HistoryKey | None, limit: int + ) -> tuple[list[Chat], bool]: + async with self._database.connection() as connection: + if after is None: + rows = await connection.fetch(CHATS_FIRST, user_id, limit + 1) + else: + rows = await connection.fetch( + CHATS_AFTER, user_id, after[0], UUID(int=after[1]), limit + 1 + ) + return page([chat_of(row) for row in rows], limit) + + async def history( + self, chat_id: UUID, *, after: HistoryKey | None, limit: int + ) -> tuple[list[Message], bool]: + async with self._database.connection() as connection: + if after is None: + rows = await connection.fetch(HISTORY_FIRST, chat_id, limit + 1) + else: + rows = await connection.fetch( + HISTORY_AFTER, chat_id, after[0], UUID(int=after[1]), limit + 1 + ) + return page([message_of(row) for row in rows], limit) diff --git a/src/messenger/migrate.py b/src/messenger/migrate.py new file mode 100644 index 00000000..27037f6d --- /dev/null +++ b/src/messenger/migrate.py @@ -0,0 +1,70 @@ +import asyncio +import hashlib +from importlib import resources + +import asyncpg + +from messenger.shared.config import PlatformSettings +from messenger.shared.infrastructure.logging import PLATFORM_FIELDS, StructlogLogger + +LOCK_KEY = 20_260_002 +JOURNAL = """ +CREATE TABLE IF NOT EXISTS schema_migrations ( + version text PRIMARY KEY, + checksum text NOT NULL, + applied_at timestamptz NOT NULL DEFAULT now() +) +""" + + +class MigrationChangedError(Exception): + pass + + +def migrations() -> list[tuple[str, str]]: + folder = resources.files("messenger") / "migrations" + found = (item for item in folder.iterdir() if item.name.endswith(".sql")) + return sorted((item.name.removesuffix(".sql"), item.read_text()) for item in found) + + +async def migrate(dsn: str) -> list[str]: + connection = await asyncpg.connect(dsn, timeout=10) + try: + await connection.execute("SELECT pg_advisory_lock($1)", LOCK_KEY) + await connection.execute(JOURNAL) + applied = dict(await connection.fetch("SELECT version, checksum FROM schema_migrations")) + done: list[str] = [] + for version, sql in migrations(): + checksum = hashlib.sha256(sql.encode()).hexdigest() + if version in applied: + if applied[version] != checksum: + raise MigrationChangedError(version) + continue + async with connection.transaction(): + await connection.execute(sql) + await connection.execute( + "INSERT INTO schema_migrations (version, checksum) VALUES ($1, $2)", + version, + checksum, + ) + done.append(version) + return done + finally: + await connection.close() + + +def main() -> None: + settings = PlatformSettings() + logger = StructlogLogger(level=settings.log_level, allowed_fields=PLATFORM_FIELDS | {"version"}) + try: + done = asyncio.run(migrate(settings.database_url.get_secret_value())) + except MigrationChangedError as error: + logger.error("migration.changed", version=str(error)) + raise SystemExit(1) from None + for version in done: + logger.info("migration.applied", version=version) + logger.info("migration.completed", reason="up to date" if not done else "applied") + + +if __name__ == "__main__": + main() diff --git a/src/messenger/migrations/0001_init.sql b/src/messenger/migrations/0001_init.sql new file mode 100644 index 00000000..d8ccac01 --- /dev/null +++ b/src/messenger/migrations/0001_init.sql @@ -0,0 +1,46 @@ +CREATE TABLE users ( + id uuid PRIMARY KEY, + username text NOT NULL CONSTRAINT users_username_key UNIQUE + CONSTRAINT users_username_format CHECK (username ~ '^[a-z0-9_]{3,32}$'), + display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100), + password_hash text NOT NULL CHECK (password_hash LIKE '$argon2id$%'), + created_at timestamptz NOT NULL +); + +CREATE TABLE sessions ( + token_hash bytea PRIMARY KEY CHECK (octet_length(token_hash) = 32), + user_id uuid NOT NULL REFERENCES users (id) ON DELETE CASCADE, + created_at timestamptz NOT NULL, + expires_at timestamptz NOT NULL, + CHECK (expires_at > created_at) +); + +CREATE INDEX sessions_user_idx ON sessions (user_id); + +CREATE TABLE chats ( + id uuid PRIMARY KEY, + title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100), + created_at timestamptz NOT NULL, + last_message_at timestamptz +); + +CREATE TABLE chat_members ( + chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE, + user_id uuid NOT NULL REFERENCES users (id), + position smallint NOT NULL CHECK (position BETWEEN 1 AND 100), + chat_created_at timestamptz NOT NULL, + PRIMARY KEY (chat_id, user_id) +); + +CREATE INDEX chat_members_user_idx ON chat_members (user_id, chat_created_at, chat_id); + +CREATE TABLE messages ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL, + sender_id uuid NOT NULL, + text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000), + created_at timestamptz NOT NULL, + FOREIGN KEY (chat_id, sender_id) REFERENCES chat_members (chat_id, user_id) +); + +CREATE INDEX messages_history_idx ON messages (chat_id, created_at, id); diff --git a/src/messenger/migrations/0002_idempotency_keys.sql b/src/messenger/migrations/0002_idempotency_keys.sql new file mode 100644 index 00000000..a556f0fd --- /dev/null +++ b/src/messenger/migrations/0002_idempotency_keys.sql @@ -0,0 +1,10 @@ +CREATE TABLE idempotency_keys ( + chat_id uuid NOT NULL, + user_id uuid NOT NULL, + key text NOT NULL CHECK (key ~ '^[A-Za-z0-9._:-]{1,128}$'), + fingerprint bytea NOT NULL CHECK (octet_length(fingerprint) = 32), + message_id uuid NOT NULL REFERENCES messages (id) ON DELETE CASCADE, + created_at timestamptz NOT NULL, + PRIMARY KEY (chat_id, user_id, key), + FOREIGN KEY (chat_id, user_id) REFERENCES chat_members (chat_id, user_id) +); diff --git a/src/messenger/migrations/0003_attachments.sql b/src/messenger/migrations/0003_attachments.sql new file mode 100644 index 00000000..91259da9 --- /dev/null +++ b/src/messenger/migrations/0003_attachments.sql @@ -0,0 +1,37 @@ +ALTER TABLE messages + DROP CONSTRAINT messages_text_check, + ADD CONSTRAINT messages_text_check CHECK (char_length(text) <= 4000); + +CREATE TABLE attachments ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL, + owner_id uuid NOT NULL, + filename text NOT NULL CHECK (char_length(filename) BETWEEN 1 AND 255), + content_type text NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg')), + size_bytes bigint NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760), + sha256 text NOT NULL CHECK (sha256 ~ '^[0-9a-f]{64}$'), + width integer NOT NULL CHECK (width > 0), + height integer NOT NULL CHECK (height > 0), + object_key text COLLATE "C" NOT NULL CONSTRAINT attachments_object_key_key UNIQUE, + status text NOT NULL CHECK (status IN ('pending', 'ready')), + created_at timestamptz NOT NULL, + updated_at timestamptz NOT NULL, + CONSTRAINT attachments_chat_key UNIQUE (id, chat_id), + CONSTRAINT attachments_owner_fkey FOREIGN KEY (chat_id, owner_id) + REFERENCES chat_members (chat_id, user_id) +); + +CREATE INDEX attachments_pending_idx ON attachments (created_at) WHERE status = 'pending'; + +CREATE TABLE message_attachments ( + message_id uuid NOT NULL REFERENCES messages (id) ON DELETE CASCADE, + position smallint NOT NULL CHECK (position BETWEEN 0 AND 9), + chat_id uuid NOT NULL, + attachment_id uuid NOT NULL, + PRIMARY KEY (message_id, position), + UNIQUE (message_id, attachment_id), + CONSTRAINT message_attachments_attachment_fkey FOREIGN KEY (attachment_id, chat_id) + REFERENCES attachments (id, chat_id) ON DELETE RESTRICT +); + +CREATE INDEX message_attachments_attachment_idx ON message_attachments (attachment_id); diff --git a/src/messenger/migrations/0004_jobs.sql b/src/messenger/migrations/0004_jobs.sql new file mode 100644 index 00000000..41e2b832 --- /dev/null +++ b/src/messenger/migrations/0004_jobs.sql @@ -0,0 +1,70 @@ +ALTER TABLE attachments + DROP CONSTRAINT attachments_status_check, + ADD CONSTRAINT attachments_status_check + CHECK (status IN ('pending', 'queued', 'processing', 'ready', 'failed')), + ADD COLUMN staging_key text COLLATE "C" CONSTRAINT attachments_staging_key_key UNIQUE, + ADD COLUMN original_key text COLLATE "C" CONSTRAINT attachments_original_key_key UNIQUE, + ADD COLUMN published_version integer NOT NULL DEFAULT 0 CHECK (published_version >= 0), + ADD COLUMN error_code text + CHECK (error_code IN ('invalid_image', 'processing_failed', 'retry_exhausted')), + ALTER COLUMN object_key DROP NOT NULL; + +UPDATE attachments SET original_key = object_key, published_version = 1 WHERE status = 'ready'; +UPDATE attachments SET staging_key = object_key WHERE status = 'pending'; + +CREATE INDEX attachments_staged_idx ON attachments (updated_at) WHERE staging_key IS NOT NULL; + +CREATE TABLE attachment_variants ( + attachment_id uuid NOT NULL REFERENCES attachments (id), + pipeline_version integer NOT NULL CHECK (pipeline_version >= 1), + name text NOT NULL CHECK (name IN ('thumbnail', 'crop', 'optimized')), + object_key text COLLATE "C" NOT NULL CONSTRAINT attachment_variants_object_key_key UNIQUE, + content_type text NOT NULL CHECK (content_type IN ('image/jpeg', 'image/png', 'image/webp')), + size_bytes bigint NOT NULL CHECK (size_bytes >= 1), + width integer NOT NULL CHECK (width >= 1), + height integer NOT NULL CHECK (height >= 1), + sha256 text NOT NULL CHECK (sha256 ~ '^[0-9a-f]{64}$'), + created_at timestamptz NOT NULL, + PRIMARY KEY (attachment_id, pipeline_version, name) +); + +CREATE TABLE jobs ( + job_id uuid PRIMARY KEY, + attachment_id uuid NOT NULL REFERENCES attachments (id), + pipeline_version integer NOT NULL CHECK (pipeline_version >= 1), + status text NOT NULL CHECK (status IN ('queued', 'processing', 'done', 'failed', 'superseded')), + attempt integer NOT NULL CHECK (attempt >= 1), + claims integer NOT NULL DEFAULT 0 CHECK (claims >= 0), + lease_token uuid, + lease_until timestamptz, + next_attempt_at timestamptz, + last_error text, + created_at timestamptz NOT NULL, + updated_at timestamptz NOT NULL, + CONSTRAINT jobs_effect_key UNIQUE (attachment_id, pipeline_version) +); + +CREATE INDEX jobs_lease_idx ON jobs (lease_until) WHERE status = 'processing'; + +CREATE TABLE outbox ( + id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + job_id uuid NOT NULL REFERENCES jobs (job_id), + payload jsonb NOT NULL, + available_at timestamptz NOT NULL, + published_at timestamptz +); + +CREATE INDEX outbox_pending_idx ON outbox (available_at, id) WHERE published_at IS NULL; +CREATE INDEX outbox_job_idx ON outbox (job_id); + +CREATE TABLE job_events ( + id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + job_id uuid NOT NULL, + attachment_id uuid NOT NULL, + kind text NOT NULL, + attempt integer, + detail jsonb NOT NULL DEFAULT '{}', + at timestamptz NOT NULL DEFAULT now() +); + +CREATE INDEX job_events_attachment_idx ON job_events (attachment_id, id); diff --git a/src/messenger/migrations/0005_failed_registry.sql b/src/messenger/migrations/0005_failed_registry.sql new file mode 100644 index 00000000..e3244d5e --- /dev/null +++ b/src/messenger/migrations/0005_failed_registry.sql @@ -0,0 +1 @@ +CREATE INDEX jobs_failed_idx ON jobs (updated_at DESC) WHERE status = 'failed'; diff --git a/src/messenger/relay.py b/src/messenger/relay.py new file mode 100644 index 00000000..7db49563 --- /dev/null +++ b/src/messenger/relay.py @@ -0,0 +1,66 @@ +import asyncio +import signal + +from dishka import make_async_container +from faststream.rabbit import Channel, RabbitBroker + +from messenger.attachments.config import PipelineSettings +from messenger.attachments.entrypoints.heartbeat import Heartbeat +from messenger.attachments.entrypoints.ioc import PipelineProvider, RelayProvider +from messenger.attachments.entrypoints.journal import LOG_FIELDS, subscribe_journal +from messenger.attachments.entrypoints.relay import Relay +from messenger.shared.application.messagebus import EventBus +from messenger.shared.config import PlatformSettings +from messenger.shared.entrypoints.ioc import PlatformProvider +from messenger.shared.infrastructure.logging import PLATFORM_FIELDS, StructlogLogger + +PING_TIMEOUT_SECONDS = 2.0 + + +async def serve() -> None: + platform = PlatformSettings() + pipeline = PipelineSettings() + logger = StructlogLogger(level=platform.log_level, allowed_fields=PLATFORM_FIELDS | LOG_FIELDS) + broker = RabbitBroker( + pipeline.broker_url.get_secret_value(), + default_channel=Channel(publisher_confirms=True, on_return_raises=True), + logger=None, + ) + container = make_async_container( + PlatformProvider(platform, logger), PipelineProvider(pipeline), RelayProvider(broker) + ) + stop = asyncio.Event() + loop = asyncio.get_running_loop() + for signum in (signal.SIGTERM, signal.SIGINT): + loop.add_signal_handler(signum, stop.set) + await broker.connect() + bus = await container.get(EventBus) + subscribe_journal(bus, logger) + bus.start() + relay = Relay(container, pipeline, logger) + heartbeat = Heartbeat( + pipeline.heartbeat_file, + lambda: broker.ping(PING_TIMEOUT_SECONDS), + pipeline.heartbeat_interval_seconds, + logger, + ) + relay.start() + heartbeat.start() + logger.info("relay.started") + try: + await stop.wait() + finally: + await heartbeat.stop() + await relay.stop() + await bus.stop() + await broker.stop() + await container.close() + logger.info("relay.stopped") + + +def main() -> None: + asyncio.run(serve()) + + +if __name__ == "__main__": + main() diff --git a/src/messenger/shared/__init__.py b/src/messenger/shared/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/messenger/shared/application/__init__.py b/src/messenger/shared/application/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/messenger/shared/application/commands.py b/src/messenger/shared/application/commands.py new file mode 100644 index 00000000..9eaa1e2e --- /dev/null +++ b/src/messenger/shared/application/commands.py @@ -0,0 +1,2 @@ +class Command: + pass diff --git a/src/messenger/shared/application/errors.py b/src/messenger/shared/application/errors.py new file mode 100644 index 00000000..dee87401 --- /dev/null +++ b/src/messenger/shared/application/errors.py @@ -0,0 +1,2 @@ +class Unavailable(Exception): + pass diff --git a/src/messenger/shared/application/messagebus.py b/src/messenger/shared/application/messagebus.py new file mode 100644 index 00000000..f6cb0152 --- /dev/null +++ b/src/messenger/shared/application/messagebus.py @@ -0,0 +1,124 @@ +import asyncio +from collections.abc import Awaitable, Callable, Mapping, Sequence +from contextlib import suppress +from typing import Any + +from messenger.shared.application.commands import Command +from messenger.shared.application.ports import EventPublisher, Logger, UnitOfWork +from messenger.shared.domain.events import DomainEvent + +type CommandHandler = Callable[[Any], Awaitable[Any]] +type EventHandler[EventT: DomainEvent] = Callable[[EventT], Awaitable[None]] + + +class CommandBus: + __slots__ = ("_handlers", "_publisher", "_uow") + + def __init__( + self, + handlers: Mapping[type[Command], CommandHandler], + uow: UnitOfWork, + publisher: EventPublisher, + ) -> None: + self._handlers = handlers + self._uow = uow + self._publisher = publisher + + async def execute(self, command: Command) -> Any: + handler = self._handlers.get(type(command)) + if handler is None: + raise LookupError(f"нет обработчика для команды {type(command).__name__}") + try: + return await handler(command) + finally: + events = self._uow.collect_new_events() + if events: + await self._publisher.publish(events) + + +class EventBus: + __slots__ = ("_deferred", "_dropped", "_logger", "_projections", "_queue", "_worker") + + def __init__(self, logger: Logger, *, max_queue: int = 10_000) -> None: + self._logger = logger + self._queue: asyncio.Queue[DomainEvent] = asyncio.Queue(maxsize=max_queue) + self._projections: dict[type[DomainEvent], list[Any]] = {} + self._deferred: dict[type[DomainEvent], list[Any]] = {} + self._worker: asyncio.Task[None] | None = None + self._dropped = 0 + + def project[EventT: DomainEvent]( + self, event_type: type[EventT], handler: EventHandler[EventT] + ) -> None: + self._projections.setdefault(event_type, []).append(handler) + + def subscribe[EventT: DomainEvent]( + self, event_type: type[EventT], handler: EventHandler[EventT] + ) -> None: + self._deferred.setdefault(event_type, []).append(handler) + + async def emit(self, event: DomainEvent) -> bool: + await self._run(self._projections.get(type(event), []), event, "projection") + if not self._deferred.get(type(event)): + return True + try: + self._queue.put_nowait(event) + except asyncio.QueueFull: + self._dropped += 1 + self._logger.warning( + "event.dropped", event_name=event.name, dropped_total=self._dropped + ) + return False + return True + + @property + def dropped(self) -> int: + return self._dropped + + @property + def pending(self) -> int: + return self._queue.qsize() + + def start(self) -> None: + if self._worker is None: + self._worker = asyncio.create_task(self._drain()) + + async def stop(self) -> None: + if self._worker is None: + return + await self._queue.join() + self._worker.cancel() + with suppress(asyncio.CancelledError): + await self._worker + self._worker = None + + async def _drain(self) -> None: + while True: + event = await self._queue.get() + try: + await self._run(self._deferred.get(type(event), []), event, "subscriber") + finally: + self._queue.task_done() + + async def _run(self, handlers: Sequence[Any], event: DomainEvent, kind: str) -> None: + for handler in handlers: + try: + await handler(event) + except Exception as error: + self._logger.error( + "event.handler_failed", + event_name=event.name, + reason=kind, + error=type(error).__name__, + ) + + +class InMemoryEventPublisher: + __slots__ = ("_bus",) + + def __init__(self, bus: EventBus) -> None: + self._bus = bus + + async def publish(self, events: Sequence[DomainEvent]) -> None: + for event in events: + await self._bus.emit(event) diff --git a/src/messenger/shared/application/ports.py b/src/messenger/shared/application/ports.py new file mode 100644 index 00000000..16586867 --- /dev/null +++ b/src/messenger/shared/application/ports.py @@ -0,0 +1,42 @@ +from collections.abc import Sequence +from contextlib import AbstractAsyncContextManager +from datetime import datetime +from typing import Any, Protocol, runtime_checkable +from uuid import UUID + +from messenger.shared.domain.events import DomainEvent + + +@runtime_checkable +class Clock(Protocol): + def now(self) -> datetime: ... + + +@runtime_checkable +class IdGenerator(Protocol): + def new_id(self) -> UUID: ... + + +@runtime_checkable +class Logger(Protocol): + def info(self, event: str, /, **fields: Any) -> None: ... + + def warning(self, event: str, /, **fields: Any) -> None: ... + + def error(self, event: str, /, **fields: Any) -> None: ... + + +@runtime_checkable +class UnitOfWork(Protocol): + def transaction(self) -> AbstractAsyncContextManager[None]: ... + + async def commit(self) -> None: ... + + def record(self, event: DomainEvent) -> None: ... + + def collect_new_events(self) -> list[DomainEvent]: ... + + +@runtime_checkable +class EventPublisher(Protocol): + async def publish(self, events: Sequence[DomainEvent]) -> None: ... diff --git a/src/messenger/shared/config.py b/src/messenger/shared/config.py new file mode 100644 index 00000000..8eff8736 --- /dev/null +++ b/src/messenger/shared/config.py @@ -0,0 +1,49 @@ +import os +import socket +from pathlib import Path +from typing import Any + +from pydantic import Field, SecretStr +from pydantic.fields import FieldInfo +from pydantic_settings import BaseSettings, PydanticBaseSettingsSource, SettingsConfigDict + +CONFIG = SettingsConfigDict(extra="ignore", frozen=True) + + +class FileVariables(PydanticBaseSettingsSource): + def get_field_value(self, field: FieldInfo, field_name: str) -> tuple[Any, str, bool]: + path = os.environ.get(f"{field_name.upper()}_FILE") + return (Path(path).read_text().strip() if path else None), field_name, False + + def __call__(self) -> dict[str, Any]: + found = { + name: self.get_field_value(field, name)[0] + for name, field in self.settings_cls.model_fields.items() + } + return {name: value for name, value in found.items() if value is not None} + + +class Settings(BaseSettings): + model_config = CONFIG + + @classmethod + def settings_customise_sources( + cls, + settings_cls: type[BaseSettings], + init_settings: PydanticBaseSettingsSource, + env_settings: PydanticBaseSettingsSource, + dotenv_settings: PydanticBaseSettingsSource, + file_secret_settings: PydanticBaseSettingsSource, + ) -> tuple[PydanticBaseSettingsSource, ...]: + return init_settings, env_settings, FileVariables(settings_cls) + + +class PlatformSettings(Settings): + log_level: str = "INFO" + instance_id: str = Field(default_factory=socket.gethostname, min_length=1) + event_queue_max: int = Field(default=10_000, ge=16) + database_url: SecretStr + db_pool_max_size: int = Field(default=8, ge=1, le=50) + db_connect_timeout_seconds: float = Field(default=1.0, gt=0, le=30) + db_acquire_timeout_seconds: float = Field(default=1.0, gt=0, le=30) + db_command_timeout_seconds: float = Field(default=2.0, gt=0, le=60) diff --git a/src/messenger/shared/domain/__init__.py b/src/messenger/shared/domain/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/messenger/shared/domain/errors.py b/src/messenger/shared/domain/errors.py new file mode 100644 index 00000000..f7d33023 --- /dev/null +++ b/src/messenger/shared/domain/errors.py @@ -0,0 +1,4 @@ +class DomainError(Exception): + def __init__(self, detail: str) -> None: + super().__init__(detail) + self.detail = detail diff --git a/src/messenger/shared/domain/events.py b/src/messenger/shared/domain/events.py new file mode 100644 index 00000000..a295e356 --- /dev/null +++ b/src/messenger/shared/domain/events.py @@ -0,0 +1,6 @@ +from dataclasses import dataclass, field + + +@dataclass(frozen=True, slots=True) +class DomainEvent: + name: str = field(init=False, default="domain.event") diff --git a/src/messenger/shared/entrypoints/__init__.py b/src/messenger/shared/entrypoints/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/messenger/shared/entrypoints/actor.py b/src/messenger/shared/entrypoints/actor.py new file mode 100644 index 00000000..0a81d11b --- /dev/null +++ b/src/messenger/shared/entrypoints/actor.py @@ -0,0 +1,16 @@ +from typing import Annotated +from uuid import UUID + +from fastapi import Depends, Request + +from messenger.shared.entrypoints.errors import Unauthenticated + + +async def current_actor(request: Request) -> UUID: + actor = getattr(request.state, "actor", None) + if not isinstance(actor, UUID): + raise Unauthenticated("заголовок идентификации обязателен и должен быть действительным") + return actor + + +type Actor = Annotated[UUID, Depends(current_actor)] diff --git a/src/messenger/shared/entrypoints/config.py b/src/messenger/shared/entrypoints/config.py new file mode 100644 index 00000000..c3e77663 --- /dev/null +++ b/src/messenger/shared/entrypoints/config.py @@ -0,0 +1,10 @@ +from pydantic import Field + +from messenger.shared.config import Settings + + +class TransportSettings(Settings): + docs: bool = False + max_body_bytes: int = Field(default=1_048_576, ge=4_096) + request_timeout_seconds: float = Field(default=8.0, gt=0, le=300) + shutdown_timeout_seconds: int = Field(default=5, ge=1, le=10) diff --git a/src/messenger/shared/entrypoints/errors.py b/src/messenger/shared/entrypoints/errors.py new file mode 100644 index 00000000..e32c6e72 --- /dev/null +++ b/src/messenger/shared/entrypoints/errors.py @@ -0,0 +1,136 @@ +from collections.abc import Callable, Mapping + +from fastapi import FastAPI, Request, Response, status +from fastapi.exceptions import RequestValidationError +from starlette.exceptions import HTTPException + +from messenger.shared.application.errors import Unavailable +from messenger.shared.application.ports import Logger +from messenger.shared.domain.errors import DomainError +from messenger.shared.entrypoints.schemas import ErrorEnvelope + +REQUEST_ID_HEADER = "X-Request-Id" +INSTANCE_ID_HEADER = "X-Instance-Id" + +type ErrorCodes = Mapping[type[DomainError], tuple[str, int]] +type Challenge = Callable[[str], str] + +MESSAGES = { + "invalid_request": "Запрос не соответствует контракту", + "unauthorized": "Аутентификация не предоставлена или недействительна", + "not_found": "Объект не найден", + "conflict": "Конфликт состояния", + "payload_too_large": "Тело запроса больше допустимого", + "unsupported_media_type": "Неподдерживаемый тип содержимого", + "rate_limited": "Слишком много запросов", + "unavailable": "Сервис временно недоступен", + "internal_error": "Внутренняя ошибка", +} + +STATUS_CODES = { + status.HTTP_400_BAD_REQUEST: "invalid_request", + status.HTTP_401_UNAUTHORIZED: "unauthorized", + status.HTTP_404_NOT_FOUND: "not_found", + status.HTTP_409_CONFLICT: "conflict", + status.HTTP_413_CONTENT_TOO_LARGE: "payload_too_large", + status.HTTP_415_UNSUPPORTED_MEDIA_TYPE: "unsupported_media_type", + status.HTTP_422_UNPROCESSABLE_CONTENT: "invalid_request", + status.HTTP_429_TOO_MANY_REQUESTS: "rate_limited", + status.HTTP_503_SERVICE_UNAVAILABLE: "unavailable", +} + + +class Unauthenticated(Exception): + pass + + +def classify(error: DomainError, codes: ErrorCodes) -> tuple[str, int]: + for kind in type(error).__mro__: + known = codes.get(kind) # type: ignore[arg-type] + if known is not None: + return known + return "internal_error", status.HTTP_500_INTERNAL_SERVER_ERROR + + +def route_of(request: Request) -> str: + route = request.scope.get("route") + return getattr(route, "path", request.url.path) + + +def install( + app: FastAPI, logger: Logger, codes: ErrorCodes, challenge: Challenge, instance: str +) -> None: + def error_response(request: Request, code: str, detail: str, http_status: int) -> Response: + request_id = getattr(request.state, "request_id", "") or "unknown" + headers = {REQUEST_ID_HEADER: request_id, INSTANCE_ID_HEADER: instance} + if http_status == status.HTTP_401_UNAUTHORIZED: + headers["WWW-Authenticate"] = challenge(request.url.path) + return Response( + status_code=http_status, + content=ErrorEnvelope.of(code, detail, request_id).model_dump_json(), + media_type="application/json", + headers=headers, + ) + + @app.exception_handler(DomainError) + async def _domain(request: Request, exc: DomainError) -> Response: + code, http_status = classify(exc, codes) + return error_response(request, code, exc.detail, http_status) + + @app.exception_handler(Unavailable) + async def _unavailable(request: Request, exc: Unavailable) -> Response: + logger.warning( + "dependency.unavailable", + route=route_of(request), + error=type(exc.__cause__ or exc).__name__, + request_id=getattr(request.state, "request_id", None), + ) + return error_response( + request, + "unavailable", + MESSAGES["unavailable"], + status.HTTP_503_SERVICE_UNAVAILABLE, + ) + + @app.exception_handler(Unauthenticated) + async def _unauthenticated(request: Request, _exc: Unauthenticated) -> Response: + return error_response( + request, + "unauthorized", + MESSAGES["unauthorized"], + status.HTTP_401_UNAUTHORIZED, + ) + + @app.exception_handler(RequestValidationError) + async def _validation(request: Request, _exc: RequestValidationError) -> Response: + return error_response( + request, + "invalid_request", + MESSAGES["invalid_request"], + status.HTTP_400_BAD_REQUEST, + ) + + @app.exception_handler(HTTPException) + async def _http(request: Request, exc: HTTPException) -> Response: + http_status = exc.status_code + if http_status == status.HTTP_405_METHOD_NOT_ALLOWED: + http_status = status.HTTP_404_NOT_FOUND + code = STATUS_CODES.get(http_status, "internal_error") + return error_response( + request, code, MESSAGES.get(code, MESSAGES["internal_error"]), http_status + ) + + @app.exception_handler(Exception) + async def _unexpected(request: Request, exc: Exception) -> Response: + logger.error( + "request.failed", + route=route_of(request), + error=type(exc).__name__, + request_id=getattr(request.state, "request_id", None), + ) + return error_response( + request, + "internal_error", + MESSAGES["internal_error"], + status.HTTP_500_INTERNAL_SERVER_ERROR, + ) diff --git a/src/messenger/shared/entrypoints/health.py b/src/messenger/shared/entrypoints/health.py new file mode 100644 index 00000000..b2da72c9 --- /dev/null +++ b/src/messenger/shared/entrypoints/health.py @@ -0,0 +1,25 @@ +from threading import Event + +from dishka.integrations.fastapi import FromDishka, inject +from fastapi import APIRouter + +from messenger.shared.application.errors import Unavailable +from messenger.shared.entrypoints.schemas import HealthOut +from messenger.shared.infrastructure.postgres import Database + +router = APIRouter(tags=["health"]) +DRAINING = Event() + + +@router.get("/health/live") +async def live() -> HealthOut: + return HealthOut() + + +@router.get("/health/ready") +@inject +async def ready(database: FromDishka[Database]) -> HealthOut: + if DRAINING.is_set(): + raise Unavailable("Реплика выходит из ротации") + await database.ping() + return HealthOut() diff --git a/src/messenger/shared/entrypoints/ioc.py b/src/messenger/shared/entrypoints/ioc.py new file mode 100644 index 00000000..2c99ab1b --- /dev/null +++ b/src/messenger/shared/entrypoints/ioc.py @@ -0,0 +1,54 @@ +from collections.abc import AsyncIterator + +from dishka import Provider, Scope, provide + +from messenger.shared.application.messagebus import EventBus, InMemoryEventPublisher +from messenger.shared.application.ports import Clock, EventPublisher, IdGenerator, Logger +from messenger.shared.config import PlatformSettings +from messenger.shared.infrastructure.clock import MonotonicUtcClock +from messenger.shared.infrastructure.ids import Uuid7Generator +from messenger.shared.infrastructure.postgres import Database + + +class PlatformProvider(Provider): + def __init__(self, settings: PlatformSettings, logger: Logger) -> None: + super().__init__() + self._settings = settings + self._logger = logger + + @provide(scope=Scope.APP) + def settings(self) -> PlatformSettings: + return self._settings + + @provide(scope=Scope.APP) + def logger(self) -> Logger: + return self._logger + + @provide(scope=Scope.APP) + def clock(self) -> Clock: + return MonotonicUtcClock() + + @provide(scope=Scope.APP) + def ids(self) -> IdGenerator: + return Uuid7Generator() + + @provide(scope=Scope.APP) + async def database(self, settings: PlatformSettings) -> AsyncIterator[Database]: + database = Database( + settings.database_url.get_secret_value(), + max_size=settings.db_pool_max_size, + connect_timeout=settings.db_connect_timeout_seconds, + acquire_timeout=settings.db_acquire_timeout_seconds, + command_timeout=settings.db_command_timeout_seconds, + ) + await database.open() + yield database + await database.close() + + @provide(scope=Scope.APP) + def event_bus(self, logger: Logger, settings: PlatformSettings) -> EventBus: + return EventBus(logger, max_queue=settings.event_queue_max) + + @provide(scope=Scope.APP) + def publisher(self, bus: EventBus) -> EventPublisher: + return InMemoryEventPublisher(bus) diff --git a/src/messenger/shared/entrypoints/pipeline.py b/src/messenger/shared/entrypoints/pipeline.py new file mode 100644 index 00000000..2acde7c1 --- /dev/null +++ b/src/messenger/shared/entrypoints/pipeline.py @@ -0,0 +1,175 @@ +import asyncio +import re +import time +from collections.abc import Awaitable, Callable +from dataclasses import dataclass +from uuid import uuid4 + +from starlette.types import ASGIApp, Message, Receive, Scope, Send + +from messenger.shared.application.errors import Unavailable +from messenger.shared.application.ports import Logger +from messenger.shared.entrypoints.errors import INSTANCE_ID_HEADER, MESSAGES, REQUEST_ID_HEADER +from messenger.shared.entrypoints.schemas import ErrorEnvelope + +type Headers = tuple[tuple[str, str], ...] + + +@dataclass(frozen=True, slots=True) +class Rejection: + status: int + code: str + headers: Headers = () + + +type Guard = Callable[[Scope], Awaitable[Rejection | None]] +type BodyLimits = tuple[tuple[re.Pattern[str], int], ...] + +HEADER_BYTES = REQUEST_ID_HEADER.lower().encode() +INSTANCE_BYTES = INSTANCE_ID_HEADER.lower().encode() +STAMPED = frozenset({HEADER_BYTES, INSTANCE_BYTES}) +MAX_REQUEST_ID = 200 + + +def header(scope: Scope, name: bytes) -> str: + for key, value in scope["headers"]: + if key == name: + return value.decode("latin-1").strip() + return "" + + +async def send_error( + send: Send, status: int, code: str, request_id: str, headers: Headers = () +) -> None: + body = ErrorEnvelope.of(code, MESSAGES[code], request_id).model_dump_json().encode() + await send( + { + "type": "http.response.start", + "status": status, + "headers": [ + (b"content-type", b"application/json"), + (b"content-length", str(len(body)).encode()), + (HEADER_BYTES, request_id.encode("latin-1", "replace")), + *((name.encode(), value.encode("latin-1")) for name, value in headers), + ], + } + ) + await send({"type": "http.response.body", "body": body}) + + +class RequestPipeline: + __slots__ = ("_app", "_guard", "_instance", "_limits", "_logger", "_max_bytes", "_timeout") + + def __init__( + self, + app: ASGIApp, + logger: Logger, + max_bytes: int, + timeout: float, + guard: Guard, + instance: str, + limits: BodyLimits = (), + ) -> None: + self._app = app + self._logger = logger + self._max_bytes = max_bytes + self._limits = limits + self._timeout = timeout + self._guard = guard + self._instance = instance.encode() + + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: + if scope["type"] != "http": + await self._app(scope, receive, send) + return + + incoming = header(scope, HEADER_BYTES) + request_id = incoming[:MAX_REQUEST_ID] if incoming else uuid4().hex + state = scope.setdefault("state", {}) + state["request_id"] = request_id + state["actor"] = None + started = time.perf_counter() + status_holder = [500] + answered = [False] + + async def wrapped_send(message: Message) -> None: + if message["type"] == "http.response.start": + status_holder[0] = message["status"] + answered[0] = True + headers = [ + (key, value) + for key, value in message.get("headers", []) + if key.lower() not in STAMPED + ] + headers.append((HEADER_BYTES, request_id.encode("latin-1", "replace"))) + headers.append((INSTANCE_BYTES, self._instance)) + message = {**message, "headers": headers} + await send(message) + + declared = header(scope, b"content-length") + if declared.isdigit() and int(declared) > self._limit(scope["path"]): + await send_error(wrapped_send, 413, "payload_too_large", request_id) + self._log(scope, status_holder[0], request_id, started) + return + + rejection: Rejection | None = None + try: + async with asyncio.timeout(self._timeout): + rejection = await self._guard(scope) + if rejection is None: + await self._app(scope, receive, wrapped_send) + except Unavailable as error: + self._logger.warning( + "dependency.unavailable", + route=self._route(scope), + error=type(error.__cause__ or error).__name__, + request_id=request_id, + ) + rejection = Rejection(503, "unavailable") + except TimeoutError: + self._logger.warning( + "request.timeout", + method=scope["method"], + route=self._route(scope), + request_id=request_id, + limit=self._timeout, + duration_ms=round((time.perf_counter() - started) * 1000, 3), + ) + if not answered[0]: + await send_error(wrapped_send, 503, "unavailable", request_id) + self._log(scope, status_holder[0], request_id, started) + return + except Exception: + self._logger.error( + "request.unhandled", + method=scope["method"], + route=self._route(scope), + request_id=request_id, + duration_ms=round((time.perf_counter() - started) * 1000, 3), + ) + raise + if rejection is not None and not answered[0]: + await send_error( + wrapped_send, rejection.status, rejection.code, request_id, rejection.headers + ) + self._log(scope, status_holder[0], request_id, started) + + def _limit(self, path: str) -> int: + for pattern, limit in self._limits: + if pattern.fullmatch(path): + return limit + return self._max_bytes + + def _route(self, scope: Scope) -> str: + route = scope.get("route") + return getattr(route, "path", scope["path"]) + + def _log(self, scope: Scope, status: int, request_id: str, started: float) -> None: + self._logger.info( + "request.handled", + method=scope["method"], + route=self._route(scope), + status=status, + request_id=request_id, + duration_ms=round((time.perf_counter() - started) * 1000, 3), + ) diff --git a/src/messenger/shared/entrypoints/schemas.py b/src/messenger/shared/entrypoints/schemas.py new file mode 100644 index 00000000..f27d12b5 --- /dev/null +++ b/src/messenger/shared/entrypoints/schemas.py @@ -0,0 +1,21 @@ +from typing import Literal + +from pydantic import BaseModel + + +class ErrorDetail(BaseModel): + code: str + message: str + request_id: str + + +class ErrorEnvelope(BaseModel): + error: ErrorDetail + + @classmethod + def of(cls, code: str, message: str, request_id: str) -> ErrorEnvelope: + return cls(error=ErrorDetail(code=code, message=message, request_id=request_id)) + + +class HealthOut(BaseModel): + status: Literal["ok"] = "ok" diff --git a/src/messenger/shared/infrastructure/__init__.py b/src/messenger/shared/infrastructure/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/messenger/shared/infrastructure/clock.py b/src/messenger/shared/infrastructure/clock.py new file mode 100644 index 00000000..808a3bf7 --- /dev/null +++ b/src/messenger/shared/infrastructure/clock.py @@ -0,0 +1,17 @@ +from datetime import UTC, datetime, timedelta + +TICK = timedelta(microseconds=1) + + +class MonotonicUtcClock: + __slots__ = ("_last",) + + def __init__(self) -> None: + self._last: datetime | None = None + + def now(self) -> datetime: + current = datetime.now(UTC) + if self._last is not None and current <= self._last: + current = self._last + TICK + self._last = current + return current diff --git a/src/messenger/shared/infrastructure/ids.py b/src/messenger/shared/infrastructure/ids.py new file mode 100644 index 00000000..95998244 --- /dev/null +++ b/src/messenger/shared/infrastructure/ids.py @@ -0,0 +1,8 @@ +from uuid import UUID, uuid7 + + +class Uuid7Generator: + __slots__ = () + + def new_id(self) -> UUID: + return uuid7() diff --git a/src/messenger/shared/infrastructure/logging.py b/src/messenger/shared/infrastructure/logging.py new file mode 100644 index 00000000..a3936546 --- /dev/null +++ b/src/messenger/shared/infrastructure/logging.py @@ -0,0 +1,113 @@ +import logging +import sys +from typing import Any + +import orjson +import structlog +from structlog.typing import EventDict, Processor, WrappedLogger + +PLATFORM_FIELDS = frozenset( + { + "method", + "route", + "status", + "duration_ms", + "request_id", + "event_name", + "error", + "reason", + "limit", + } +) + + +def render(obj: object, **_kwargs: object) -> str: + return orjson.dumps(obj).decode() + + +RESERVED = frozenset({"event", "level", "ts", "logger"}) + +FOREIGN_LOGGERS = ( + "uvicorn", + "uvicorn.error", + "gunicorn", + "gunicorn.error", + "asyncio", +) + +SILENCED_LOGGERS = ("uvicorn.access", "gunicorn.access") + + +def field_filter(allowed: frozenset[str]) -> Processor: + def drop_unlisted_fields( + _logger: WrappedLogger, _method: str, event_dict: EventDict + ) -> EventDict: + return { + key: value + for key, value in event_dict.items() + if (key in RESERVED or key in allowed) and value is not None + } + + return drop_unlisted_fields + + +def configure(level: str = "INFO", allowed_fields: frozenset[str] = PLATFORM_FIELDS) -> None: + shared = [ + structlog.contextvars.merge_contextvars, + structlog.processors.add_log_level, + structlog.processors.TimeStamper(fmt="iso", utc=True, key="ts"), + ] + structlog.configure( + processors=[*shared, structlog.stdlib.ProcessorFormatter.wrap_for_formatter], + logger_factory=structlog.stdlib.LoggerFactory(), + wrapper_class=structlog.stdlib.BoundLogger, + cache_logger_on_first_use=False, + ) + handler = logging.StreamHandler(sys.stdout) + handler.setFormatter( + structlog.stdlib.ProcessorFormatter( + foreign_pre_chain=shared, + processors=[ + structlog.stdlib.ProcessorFormatter.remove_processors_meta, + field_filter(allowed_fields), + structlog.processors.JSONRenderer(serializer=render), + ], + ) + ) + root = logging.getLogger() + for existing in list(root.handlers): + root.removeHandler(existing) + root.addHandler(handler) + root.setLevel(level.upper()) + for name in FOREIGN_LOGGERS: + foreign = logging.getLogger(name) + for existing in list(foreign.handlers): + foreign.removeHandler(existing) + foreign.propagate = True + for name in SILENCED_LOGGERS: + silenced = logging.getLogger(name) + for existing in list(silenced.handlers): + silenced.removeHandler(existing) + silenced.propagate = False + + +class StructlogLogger: + __slots__ = ("_logger",) + + def __init__( + self, + name: str = "messenger", + level: str = "INFO", + allowed_fields: frozenset[str] = PLATFORM_FIELDS, + ) -> None: + configure(level, allowed_fields) + self._logger = structlog.get_logger(name) + + def info(self, event: str, /, **fields: Any) -> None: + self._logger.info(event, **fields) + + def warning(self, event: str, /, **fields: Any) -> None: + self._logger.warning(event, **fields) + + def error(self, event: str, /, **fields: Any) -> None: + self._logger.error(event, **fields) diff --git a/src/messenger/shared/infrastructure/postgres.py b/src/messenger/shared/infrastructure/postgres.py new file mode 100644 index 00000000..ba28cd67 --- /dev/null +++ b/src/messenger/shared/infrastructure/postgres.py @@ -0,0 +1,119 @@ +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from typing import Any, Protocol + +import asyncpg + +from messenger.shared.application.errors import Unavailable +from messenger.shared.domain.events import DomainEvent + +FAILURES = ( + OSError, + asyncpg.InterfaceError, + asyncpg.PostgresConnectionError, + asyncpg.exceptions.OperatorInterventionError, + asyncpg.exceptions.InsufficientResourcesError, + asyncpg.exceptions.LockNotAvailableError, +) + +PING_TIMEOUT_SECONDS = 0.4 + +type Connection = asyncpg.Connection + + +class EventSource(Protocol): + def collect_events(self) -> list[DomainEvent]: ... + + +class Database: + __slots__ = ("_acquire_timeout", "_pool") + + def __init__( + self, + dsn: str, + *, + max_size: int, + connect_timeout: float, + acquire_timeout: float, + command_timeout: float, + ) -> None: + self._acquire_timeout = acquire_timeout + self._pool = asyncpg.create_pool( + dsn, + min_size=0, + max_size=max_size, + timeout=connect_timeout, + command_timeout=command_timeout, + max_inactive_connection_lifetime=60, + server_settings={ + "statement_timeout": str(int(command_timeout * 1000)), + "lock_timeout": str(int(command_timeout * 1000)), + "idle_in_transaction_session_timeout": "10000", + }, + ) + + async def open(self) -> None: + await self._pool + + async def close(self) -> None: + await self._pool.close() + + @asynccontextmanager + async def connection(self, wait: float | None = None) -> AsyncIterator[Connection]: + try: + async with self._pool.acquire(timeout=wait or self._acquire_timeout) as connection: + yield connection + except FAILURES as error: + raise Unavailable("PostgreSQL недоступен") from error + + async def ping(self) -> None: + async with self.connection(PING_TIMEOUT_SECONDS) as connection: + await connection.fetchval("SELECT 1", timeout=PING_TIMEOUT_SECONDS) + + +class PostgresUnitOfWork: + __slots__ = ("_committed", "_connection", "_database", "_events", "_transaction", "seen") + + def __init__(self, database: Database) -> None: + self._database = database + self._connection: Connection | None = None + self._transaction: Any = None + self._committed = False + self._events: list[DomainEvent] = [] + self.seen: list[EventSource] = [] + + @property + def connection(self) -> Connection: + if self._connection is None: + raise RuntimeError("репозиторий используется вне транзакции") + return self._connection + + @asynccontextmanager + async def transaction(self) -> AsyncIterator[None]: + async with self._database.connection() as connection: + self._connection = connection + self._transaction = connection.transaction() + await self._transaction.start() + self._committed = False + mark = len(self._events) + try: + yield + finally: + if not self._committed: + del self._events[mark:] + await self._transaction.rollback() + self.seen.clear() + self._connection = None + + async def commit(self) -> None: + await self._transaction.commit() + self._committed = True + for source in self.seen: + self._events.extend(source.collect_events()) + + def record(self, event: DomainEvent) -> None: + self._events.append(event) + + def collect_new_events(self) -> list[DomainEvent]: + events, self._events = self._events, [] + return events diff --git a/src/messenger/worker.py b/src/messenger/worker.py new file mode 100644 index 00000000..a18902c1 --- /dev/null +++ b/src/messenger/worker.py @@ -0,0 +1,78 @@ +import asyncio +from collections.abc import Sequence + +from dishka import Provider, make_async_container +from dishka_faststream import setup_dishka +from faststream import FastStream +from faststream.rabbit import RabbitBroker + +from messenger.attachments.config import PipelineSettings +from messenger.attachments.entrypoints.heartbeat import Heartbeat +from messenger.attachments.entrypoints.ioc import PipelineProvider, WorkerProvider +from messenger.attachments.entrypoints.journal import LOG_FIELDS, subscribe_journal +from messenger.attachments.entrypoints.worker import subscribe +from messenger.shared.application.messagebus import EventBus +from messenger.shared.config import PlatformSettings +from messenger.shared.entrypoints.ioc import PlatformProvider +from messenger.shared.infrastructure.logging import PLATFORM_FIELDS, StructlogLogger + +PING_TIMEOUT_SECONDS = 2.0 + + +def create_worker( + *, + platform: PlatformSettings | None = None, + pipeline: PipelineSettings | None = None, + overrides: Sequence[Provider] = (), +) -> FastStream: + platform = platform or PlatformSettings() + pipeline = pipeline or PipelineSettings() + logger = StructlogLogger(level=platform.log_level, allowed_fields=PLATFORM_FIELDS | LOG_FIELDS) + broker = RabbitBroker( + pipeline.broker_url.get_secret_value(), + graceful_timeout=pipeline.shutdown_grace_seconds, + logger=None, + ) + subscribe(broker, pipeline.worker_concurrency) + container = make_async_container( + PlatformProvider(platform, logger), + PipelineProvider(pipeline), + WorkerProvider(), + *overrides, + ) + heartbeat = Heartbeat( + pipeline.heartbeat_file, + lambda: broker.ping(PING_TIMEOUT_SECONDS), + pipeline.heartbeat_interval_seconds, + logger, + ) + app = FastStream(broker, logger=None) + + @app.on_startup + async def start_bus() -> None: + bus = await container.get(EventBus) + subscribe_journal(bus, logger) + bus.start() + + @app.after_startup + async def started() -> None: + heartbeat.start() + logger.info("worker.started", limit=pipeline.worker_concurrency) + + @app.after_shutdown + async def stopped() -> None: + await heartbeat.stop() + await (await container.get(EventBus)).stop() + await container.close() + logger.info("worker.stopped") + + setup_dishka(container, app, auto_inject=True) + return app + + +def main() -> None: + asyncio.run(create_worker().run()) + + +if __name__ == "__main__": + main() diff --git a/student-tests/adapters/test_rabbitmq.py b/student-tests/adapters/test_rabbitmq.py new file mode 100644 index 00000000..fb62b845 --- /dev/null +++ b/student-tests/adapters/test_rabbitmq.py @@ -0,0 +1,259 @@ +import asyncio +import base64 +import hashlib +import json +import os +import re +import subprocess +import time +from collections.abc import AsyncIterator, Iterator +from contextlib import asynccontextmanager +from datetime import UTC, datetime +from typing import Any +from uuid import uuid4 + +import pytest +from dishka import Provider, Scope, make_async_container, provide +from dishka_faststream import setup_dishka +from faststream.rabbit import Channel, RabbitBroker + +from conftest import ROOT, STARTUP_SECONDS, container, image, published +from fakes import NullLogger +from messenger.attachments.application.bus import AttachmentsCommandBus +from messenger.attachments.application.commands import ProcessJob +from messenger.attachments.domain.jobs import Job, Outcome +from messenger.attachments.entrypoints.worker import subscribe +from messenger.attachments.infrastructure.rabbitmq import RabbitJobPublisher +from messenger.shared.application.errors import Unavailable +from messenger.shared.application.ports import Logger + +RABBITMQ_IMAGE = re.compile(r"image: (rabbitmq:\S+)") +PASSWORDS = {role: os.urandom(12).hex() for role in ("api", "worker", "ops")} +GRANTS = { + "api": ("^$", "^attachments$", "^$"), + "worker": ("^$", "^$", "^attachments\\.process$"), + "ops": (".*", ".*", ".*"), +} + + +def user(role: str) -> dict[str, Any]: + salt = os.urandom(4) + digest = hashlib.sha256(salt + PASSWORDS[role].encode()).digest() + return { + "name": f"messenger-{role}", + "password_hash": base64.b64encode(salt + digest).decode(), + "hashing_algorithm": "rabbit_password_hashing_sha256", + "tags": ["administrator"] if role == "ops" else [], + } + + +def definitions() -> dict[str, Any]: + return { + "users": [user(role) for role in PASSWORDS], + "permissions": [ + { + "user": f"messenger-{role}", + "vhost": "messenger", + **dict(zip(("configure", "write", "read"), grant, strict=True)), + } + for role, grant in GRANTS.items() + ], + } + + +def ready(name: str) -> bool: + probe = ["docker", "exec", name, "rabbitmq-diagnostics", "-q", "check_port_connectivity"] + return subprocess.run(probe, capture_output=True, check=False).returncode == 0 + + +@pytest.fixture(scope="session") +def rabbitmq() -> Iterator[dict[str, Any]]: + name = f"mtusi-lab6-test-rabbitmq-{uuid4().hex[:8]}" + folder = ROOT / "build" / name + folder.mkdir(parents=True) + users = folder / "20-users.json" + users.write_text(json.dumps(definitions())) + users.chmod(0o644) + infra = ROOT / "infra" / "rabbitmq" + mounts = [ + f"{infra / 'rabbitmq.conf'}:/etc/rabbitmq/conf.d/20-messenger.conf:ro", + f"{infra / 'enabled_plugins'}:/etc/rabbitmq/enabled_plugins:ro", + f"{infra / 'definitions.json'}:/etc/rabbitmq/definitions.d/10-topology.json:ro", + f"{users}:/etc/rabbitmq/definitions.d/20-users.json:ro", + ] + volumes = [part for mount in mounts for part in ("-v", mount)] + container( + name, + "--hostname", + "broker", + "--user", + "100:101", + *volumes, + "-p", + "127.0.0.1::5672", + image(RABBITMQ_IMAGE), + ) + try: + deadline = time.monotonic() + STARTUP_SECONDS * 2 + while not ready(name): + assert time.monotonic() < deadline, "RabbitMQ не стал готов" + time.sleep(1) + port = published(name, 5672) + urls = { + role: f"amqp://messenger-{role}:{secret}@127.0.0.1:{port}/messenger" + for role, secret in PASSWORDS.items() + } + yield {"name": name, "urls": urls} + finally: + subprocess.run(["docker", "rm", "-f", name], capture_output=True, check=False) + users.unlink() + folder.rmdir() + + +def depth(broker: dict[str, Any], queue: str) -> int: + listing = subprocess.run( + [ + "docker", + "exec", + broker["name"], + "rabbitmqctl", + "-q", + "list_queues", + "-p", + "messenger", + "name", + "messages", + "--formatter", + "json", + ], + capture_output=True, + check=True, + text=True, + ).stdout + return {row["name"]: row["messages"] for row in json.loads(listing)}[queue] + + +async def wait_for(probe: Any, seconds: float = 20) -> None: + deadline = time.monotonic() + seconds + while not probe(): + assert time.monotonic() < deadline + await asyncio.sleep(0.1) + + +class NoEvents: + @asynccontextmanager + async def transaction(self) -> AsyncIterator[None]: + yield + + async def commit(self) -> None: ... + + def record(self, event: object) -> None: ... + + def collect_new_events(self) -> list[Any]: + return [] + + +class Recorder: + def __init__(self) -> None: + self.jobs: list[Job] = [] + self.failures = 0 + + async def __call__(self, command: ProcessJob) -> Outcome: + if self.failures: + self.failures -= 1 + raise Unavailable("PostgreSQL недоступен") + self.jobs.append(command.job) + return Outcome.DONE + + +class FakeBus(Provider): + def __init__(self, recorder: Recorder) -> None: + super().__init__() + self._recorder = recorder + + @provide(scope=Scope.APP) + def logger(self) -> Logger: + return NullLogger() + + @provide(scope=Scope.REQUEST) + def bus(self) -> AttachmentsCommandBus: + return AttachmentsCommandBus({ProcessJob: self._recorder}, NoEvents(), NoPublisher()) + + +class NoPublisher: + async def publish(self, events: Any) -> None: ... + + +@asynccontextmanager +async def worker(rabbitmq: dict[str, Any], recorder: Recorder) -> AsyncIterator[None]: + broker = RabbitBroker(rabbitmq["urls"]["worker"], logger=None, graceful_timeout=2) + subscribe(broker, 2) + setup_dishka(make_async_container(FakeBus(recorder)), broker=broker, auto_inject=True) + await broker.start() + try: + yield + finally: + await broker.stop() + + +@asynccontextmanager +async def publisher( + rabbitmq: dict[str, Any], role: str = "api" +) -> AsyncIterator[RabbitJobPublisher]: + broker = RabbitBroker( + rabbitmq["urls"][role], + default_channel=Channel(publisher_confirms=True, on_return_raises=True), + logger=None, + ) + await broker.connect() + try: + yield RabbitJobPublisher(broker, 5) + finally: + await broker.stop() + + +def job() -> Job: + return Job(uuid4(), uuid4(), 1, 1, datetime.now(UTC)) + + +def body(payload: dict[str, Any]) -> bytes: + return json.dumps(payload).encode() + + +async def test_confirmed_job_is_consumed_and_acked(rabbitmq: dict[str, Any]) -> None: + recorder, sent = Recorder(), job() + async with publisher(rabbitmq) as jobs: + await jobs.publish(sent.id, body(sent.payload())) + await wait_for(lambda: depth(rabbitmq, "attachments.process") == 1) + async with worker(rabbitmq, recorder): + await wait_for(lambda: recorder.jobs) + await wait_for(lambda: depth(rabbitmq, "attachments.process") == 0) + assert recorder.jobs == [sent] + + +async def test_message_off_the_schema_goes_to_the_dead_letter_queue( + rabbitmq: dict[str, Any], +) -> None: + before = depth(rabbitmq, "attachments.process.dlq") + async with publisher(rabbitmq) as jobs: + await jobs.publish(uuid4(), body({"schema_version": 2, "job_id": str(uuid4())})) + async with worker(rabbitmq, Recorder()): + await wait_for(lambda: depth(rabbitmq, "attachments.process.dlq") == before + 1) + assert depth(rabbitmq, "attachments.process") == 0 + + +async def test_failed_processing_is_requeued_and_retried(rabbitmq: dict[str, Any]) -> None: + recorder, sent = Recorder(), job() + recorder.failures = 1 + async with publisher(rabbitmq) as jobs: + await jobs.publish(sent.id, body(sent.payload())) + async with worker(rabbitmq, recorder): + await wait_for(lambda: recorder.jobs) + assert recorder.jobs == [sent] + + +async def test_worker_account_cannot_publish(rabbitmq: dict[str, Any]) -> None: + sent = job() + with pytest.raises(Unavailable): + async with publisher(rabbitmq, "worker") as jobs: + await jobs.publish(sent.id, body(sent.payload())) diff --git a/student-tests/adapters/test_s3_storage.py b/student-tests/adapters/test_s3_storage.py new file mode 100644 index 00000000..8722a269 --- /dev/null +++ b/student-tests/adapters/test_s3_storage.py @@ -0,0 +1,95 @@ +import os +import re +import subprocess +import time +from collections.abc import AsyncIterator, Iterator +from contextlib import asynccontextmanager +from uuid import uuid4 + +import httpx +import pytest + +from conftest import STARTUP_SECONDS, container, image, published +from messenger.attachments.application.errors import ObjectMissingError +from messenger.attachments.infrastructure.s3 import S3ObjectStorage, s3_client +from messenger.shared.application.errors import Unavailable + +RUSTFS_IMAGE = re.compile(r"image: (rustfs/rustfs:\S+)") +ROOT_KEY, ROOT_SECRET = "testroot", "testroot-secret-key" +BUCKET = "messenger-attachments" + + +@pytest.fixture(scope="session") +def rustfs() -> Iterator[str]: + name = f"mtusi-lab6-test-rustfs-{uuid4().hex[:8]}" + keys = ["-e", f"RUSTFS_ACCESS_KEY={ROOT_KEY}", "-e", f"RUSTFS_SECRET_KEY={ROOT_SECRET}"] + container(name, *keys, "-p", "127.0.0.1::9000", image(RUSTFS_IMAGE)) + try: + endpoint = f"http://127.0.0.1:{published(name, 9000)}" + deadline = time.monotonic() + STARTUP_SECONDS + while True: + try: + if httpx.get(f"{endpoint}/health/ready", timeout=1).status_code == 200: + break + except httpx.TransportError: + pass + assert time.monotonic() < deadline, "RustFS не стал готов" + time.sleep(0.3) + yield endpoint + finally: + subprocess.run(["docker", "rm", "-f", name], capture_output=True, check=False) + + +@asynccontextmanager +async def storage(endpoint: str) -> AsyncIterator[S3ObjectStorage]: + async with s3_client( + endpoint=endpoint, + region="us-east-1", + access_key_id=ROOT_KEY, + secret_access_key=ROOT_SECRET, + connect_timeout=1, + read_timeout=2, + max_attempts=2, + ) as client: + buckets = await client.list_buckets() + if BUCKET not in {bucket["Name"] for bucket in buckets.get("Buckets", [])}: + await client.create_bucket(Bucket=BUCKET) + yield S3ObjectStorage(client, BUCKET) + + +async def test_round_trip_of_the_largest_object(rustfs: str) -> None: + data = os.urandom(10_485_760) + key = f"attachments/{uuid4()}" + async with storage(rustfs) as objects: + await objects.put(key, data, "image/png") + content = await objects.open(key) + assert content.size == len(data) + assert b"".join([chunk async for chunk in content.chunks]) == data + await objects.delete(key) + with pytest.raises(ObjectMissingError): + await objects.open(key) + await objects.delete(key) + + +async def test_unreachable_endpoint_is_unavailable_in_bounded_time() -> None: + started = time.monotonic() + async with storage_without_bucket("http://127.0.0.1:1") as objects: + with pytest.raises(Unavailable): + await objects.put("attachments/x", b"x", "image/png") + with pytest.raises(Unavailable): + await objects.open("attachments/x") + assert time.monotonic() - started < 8 + + +@asynccontextmanager +async def storage_without_bucket(endpoint: str) -> AsyncIterator[S3ObjectStorage]: + async with s3_client( + endpoint=endpoint, + region="us-east-1", + access_key_id=ROOT_KEY, + secret_access_key=ROOT_SECRET, + connect_timeout=1, + read_timeout=2, + max_attempts=2, + ) as client: + yield S3ObjectStorage(client, BUCKET) diff --git a/student-tests/api/test_access.py b/student-tests/api/test_access.py new file mode 100644 index 00000000..8a623623 --- /dev/null +++ b/student-tests/api/test_access.py @@ -0,0 +1,84 @@ +from uuid import uuid4 + +from httpx import AsyncClient + +from conftest import actor, keyed, make_chat, register + + +async def test_outsider_gets_404_on_chat(client: AsyncClient) -> None: + alice = await register(client, "alice") + bob = await register(client, "bob") + eve = await register(client, "eve") + chat = await make_chat(client, alice, bob) + response = await client.get(f"/api/v1/chats/{chat['id']}", headers=actor(eve)) + assert response.status_code == 404 + assert response.json()["error"]["code"] == "not_found" + + +async def test_unknown_chat_is_indistinguishable_from_foreign(client: AsyncClient) -> None: + alice = await register(client, "alice") + eve = await register(client, "eve") + chat = await make_chat(client, alice) + foreign = await client.get(f"/api/v1/chats/{chat['id']}", headers=actor(eve)) + missing = await client.get(f"/api/v1/chats/{uuid4()}", headers=actor(eve)) + assert foreign.status_code == missing.status_code == 404 + assert foreign.json()["error"]["code"] == missing.json()["error"]["code"] + assert foreign.json()["error"]["message"] == missing.json()["error"]["message"] + + +async def test_outsider_cannot_write(client: AsyncClient) -> None: + alice = await register(client, "alice") + eve = await register(client, "eve") + chat = await make_chat(client, alice) + response = await client.post( + f"/api/v1/chats/{chat['id']}/messages", + json={"text": "чужое"}, + headers=keyed(eve), + ) + assert response.status_code == 404 + history = await client.get(f"/api/v1/chats/{chat['id']}/messages", headers=actor(alice)) + assert history.json()["items"] == [] + + +async def test_outsider_cannot_read_history(client: AsyncClient) -> None: + alice = await register(client, "alice") + eve = await register(client, "eve") + chat = await make_chat(client, alice) + response = await client.get(f"/api/v1/chats/{chat['id']}/messages", headers=actor(eve)) + assert response.status_code == 404 + + +async def test_access_checked_before_body(client: AsyncClient) -> None: + alice = await register(client, "alice") + eve = await register(client, "eve") + chat = await make_chat(client, alice) + response = await client.post( + f"/api/v1/chats/{chat['id']}/messages", + json={"text": ""}, + headers=keyed(eve), + ) + assert response.status_code == 404 + + +async def test_chat_list_hides_foreign_chats(client: AsyncClient) -> None: + alice = await register(client, "alice") + bob = await register(client, "bob") + eve = await register(client, "eve") + chat = await make_chat(client, alice, bob) + for member in (alice, bob): + listed = await client.get("/api/v1/chats?limit=100", headers=actor(member)) + assert any(item["id"] == chat["id"] for item in listed.json()["items"]) + hidden = await client.get("/api/v1/chats?limit=100", headers=actor(eve)) + assert hidden.json()["items"] == [] + + +async def test_403_is_never_used(client: AsyncClient) -> None: + alice = await register(client, "alice") + eve = await register(client, "eve") + chat = await make_chat(client, alice) + paths = [ + f"/api/v1/chats/{chat['id']}", + f"/api/v1/chats/{chat['id']}/messages", + ] + for path in paths: + assert (await client.get(path, headers=actor(eve))).status_code != 403 diff --git a/student-tests/api/test_attachments.py b/student-tests/api/test_attachments.py new file mode 100644 index 00000000..a4d3339a --- /dev/null +++ b/student-tests/api/test_attachments.py @@ -0,0 +1,326 @@ +from collections.abc import AsyncIterator +from hashlib import sha256 +from io import BytesIO +from typing import Any +from urllib.parse import quote +from uuid import UUID, uuid4 + +import pytest +from httpx import AsyncClient, Response +from PIL import Image + +from conftest import actor, make_chat, register, resolve, scalar +from fakes import MemoryObjectStorage +from messenger.attachments.application.ports import ObjectStorage +from pictures import GIF, JPEG, SAMPLE, WEBP, png +from pipeline import settle + +LIMIT = 10_485_760 +PNG = {"Content-Type": "image/png"} + + +async def upload( + http: AsyncClient, + user: dict[str, Any], + chat: dict[str, Any], + content: bytes | AsyncIterator[bytes] = SAMPLE, + *, + filename: str | None = "sample.png", + headers: dict[str, str] | None = None, +) -> Response: + query = "" if filename is None else f"?filename={quote(filename, safe='')}" + return await http.post( + f"/api/v1/chats/{chat['id']}/attachments{query}", + content=content, + headers={**actor(user), **(PNG if headers is None else headers)}, + ) + + +async def storage(http: AsyncClient) -> MemoryObjectStorage: + found = await resolve(http, ObjectStorage) + assert isinstance(found, MemoryObjectStorage) + return found + + +async def trio(http: AsyncClient) -> tuple[dict[str, Any], ...]: + alice, bob, eve = [await register(http, label) for label in ("alice", "bob", "eve")] + return alice, bob, eve, await make_chat(http, alice, bob) + + +async def test_member_uploads_and_another_member_downloads(client: AsyncClient) -> None: + alice, bob, eve, chat = await trio(client) + response = await upload(client, alice, chat) + assert response.status_code == 202, response.text + attachment = response.json() + assert set(attachment) == { + "id", + "chat_id", + "owner_id", + "filename", + "content_type", + "size_bytes", + "sha256", + "status", + "variants", + "error_code", + "created_at", + "updated_at", + } + assert attachment["chat_id"] == chat["id"] + assert attachment["owner_id"] == alice["id"] + assert attachment["filename"] == "sample.png" + assert attachment["content_type"] == "image/png" + assert attachment["size_bytes"] == len(SAMPLE) == 111817 + assert attachment["sha256"] == sha256(SAMPLE).hexdigest() + assert (attachment["status"], attachment["variants"], attachment["error_code"]) == ( + "queued", + [], + None, + ) + assert attachment["created_at"].endswith("Z") + path = f"/api/v1/attachments/{attachment['id']}" + assert (await client.get(path, headers=actor(bob))).json() == attachment + await settle(client) + ready = (await client.get(path, headers=actor(bob))).json() + assert (ready["status"], ready["error_code"], ready["sha256"]) == ( + "ready", + None, + attachment["sha256"], + ) + assert [variant["name"] for variant in ready["variants"]] == ["crop", "optimized", "thumbnail"] + content = await client.get( + f"{path}/content", headers={**actor(bob), "Accept": "application/json"} + ) + assert content.status_code == 200 + assert content.content == SAMPLE + assert content.headers["content-type"] == "image/png" + assert content.headers["content-length"] == str(len(SAMPLE)) + assert content.headers["x-content-type-options"] == "nosniff" + assert content.headers["x-request-id"] + assert content.headers["x-instance-id"] + for suffix in ("", "/content"): + stranger = await client.get(f"{path}{suffix}", headers=actor(eve)) + assert stranger.status_code == 404 + assert stranger.json()["error"]["code"] == "not_found" + + +async def test_object_keys_are_assigned_by_server(client: AsyncClient) -> None: + alice, _, _, chat = await trio(client) + attachment = (await upload(client, alice, chat, filename="a.png")).json() + staging = await scalar("SELECT staging_key FROM attachments WHERE id = $1", attachment["id"]) + assert staging == f"staging/{attachment['id']}" + assert set((await storage(client)).objects) >= {staging} + await settle(client) + original = await scalar("SELECT original_key FROM attachments WHERE id = $1", attachment["id"]) + assert original == f"originals/{attachment['id']}" + assert (await storage(client)).objects[original][0] == SAMPLE + + +async def test_jpeg_is_accepted_by_content(client: AsyncClient) -> None: + alice, bob, _, chat = await trio(client) + response = await upload(client, alice, chat, JPEG, headers={"Content-Type": "IMAGE/JPEG"}) + assert response.status_code == 202, response.text + assert response.json()["content_type"] == "image/jpeg" + await settle(client) + download = await client.get( + f"/api/v1/attachments/{response.json()['id']}/content", headers=actor(bob) + ) + assert (download.content, download.headers["content-type"]) == (JPEG, "image/jpeg") + + +async def test_access_is_checked_before_content(client: AsyncClient) -> None: + alice, _, eve, chat = await trio(client) + assert (await upload(client, eve, chat, b"not an image", filename="../x")).status_code == 404 + missing = {"id": str(uuid4())} + assert (await upload(client, alice, missing)).status_code == 404 + anonymous = await client.post( + f"/api/v1/chats/{chat['id']}/attachments?filename=a.png", content=SAMPLE, headers=PNG + ) + assert anonymous.status_code == 401 + assert anonymous.headers["www-authenticate"] == 'Bearer realm="messenger"' + + +async def test_exact_limit_is_accepted(client: AsyncClient) -> None: + alice, _, _, chat = await trio(client) + response = await upload(client, alice, chat, png(16, 16, size=LIMIT)) + assert response.status_code == 202, response.text + assert response.json()["size_bytes"] == LIMIT + + +async def test_one_byte_over_with_content_length(client: AsyncClient) -> None: + alice, _, _, chat = await trio(client) + response = await upload(client, alice, chat, png(16, 16, size=LIMIT + 1)) + assert response.status_code == 413 + assert response.json()["error"]["code"] == "payload_too_large" + + +async def test_one_byte_over_chunked(client: AsyncClient) -> None: + alice, _, _, chat = await trio(client) + body = png(16, 16, size=LIMIT + 1) + + async def chunked() -> AsyncIterator[bytes]: + for start in range(0, len(body), 1 << 20): + yield body[start : start + (1 << 20)] + + before = await scalar("SELECT count(*) FROM attachments") + response = await upload(client, alice, chat, chunked()) + assert response.status_code == 413 + assert await scalar("SELECT count(*) FROM attachments") == before + + +@pytest.mark.parametrize( + ("width", "height", "status"), + [(8000, 5000, 202), (8001, 5000, 413), (100_000, 100_000, 413)], +) +async def test_pixel_limit_is_read_from_header( + client: AsyncClient, width: int, height: int, status: int +) -> None: + alice, _, _, chat = await trio(client) + body = png(width, height, rows=width * height <= 40_000_000) + assert len(body) < 64_000 + response = await upload(client, alice, chat, body) + assert response.status_code == status, response.text + + +@pytest.mark.parametrize( + ("content", "header"), + [ + (JPEG, "image/png"), + (GIF, "image/png"), + (WEBP, "image/png"), + (b"plain text, not an image", "image/png"), + (SAMPLE, "image/jpeg"), + (SAMPLE[: len(SAMPLE) // 2], "image/png"), + (JPEG[: len(JPEG) // 2], "image/jpeg"), + (b"", "image/png"), + (SAMPLE, "application/octet-stream"), + (SAMPLE, "image/gif"), + ], + ids=[ + "jpeg-as-png", + "gif-as-png", + "webp-as-png", + "text-as-png", + "png-as-jpeg", + "truncated-png", + "truncated-jpeg", + "empty", + "octet-stream", + "gif-type", + ], +) +async def test_type_is_checked_by_content(client: AsyncClient, content: bytes, header: str) -> None: + alice, _, _, chat = await trio(client) + response = await upload(client, alice, chat, content, headers={"Content-Type": header}) + assert response.status_code == 415, response.text + assert response.json()["error"]["code"] == "unsupported_media_type" + + +async def test_missing_content_type_is_unsupported(client: AsyncClient) -> None: + alice, _, _, chat = await trio(client) + response = await upload(client, alice, chat, headers={}) + assert response.status_code == 415 + + +@pytest.mark.parametrize( + "filename", + [ + *("../a.png", "a/b.png", "a\\b.png", "a\x00.png", "a\x01.png", "a\x7f.png"), + *("", " ", "x" * 256, None), + ], +) +async def test_dangerous_filename_is_rejected(client: AsyncClient, filename: str | None) -> None: + alice, _, _, chat = await trio(client) + response = await upload(client, alice, chat, filename=filename) + assert response.status_code == 400, response.text + assert response.json()["error"]["code"] == "invalid_request" + + +@pytest.mark.parametrize("filename", ["фото 1.png", "я" * 255, "%41.png"]) +async def test_display_name_is_kept(client: AsyncClient, filename: str) -> None: + alice, _, _, chat = await trio(client) + response = await upload(client, alice, chat, filename=filename) + assert response.status_code == 202, response.text + assert response.json()["filename"] == filename + + +async def test_content_waits_for_ready_and_serves_every_variant(client: AsyncClient) -> None: + alice, bob, _, chat = await trio(client) + attachment = (await upload(client, alice, chat)).json() + path = f"/api/v1/attachments/{attachment['id']}/content" + for variant in ("original", "thumbnail", "crop", "optimized"): + early = await client.get(f"{path}?variant={variant}", headers=actor(bob)) + assert early.status_code == 409, early.text + assert early.json()["error"]["code"] == "conflict" + await settle(client) + assert (await client.get(f"{path}?variant=original", headers=actor(alice))).status_code == 200 + meta = (await client.get(path.removesuffix("/content"), headers=actor(bob))).json() + expected = { + "crop": ("image/jpeg", "JPEG", (128, 128)), + "optimized": ("image/webp", "WEBP", (320, 200)), + "thumbnail": ("image/jpeg", "JPEG", (256, 160)), + } + for variant in meta["variants"]: + served = await client.get(f"{path}?variant={variant['name']}", headers=actor(bob)) + content_type, kind, size = expected[variant["name"]] + assert (served.headers["content-type"], variant["content_type"]) == (content_type,) * 2 + assert len(served.content) == variant["size_bytes"] + assert sha256(served.content).hexdigest() == variant["sha256"] + image = Image.open(BytesIO(served.content)) + assert (image.format, image.size, (variant["width"], variant["height"])) == ( + kind, + size, + size, + ) + await scalar( + "DELETE FROM attachment_variants WHERE attachment_id = $1 AND name = 'crop'", + UUID(attachment["id"]), + ) + missing = await client.get(f"{path}?variant=crop", headers=actor(alice)) + assert missing.status_code == 404 + assert (await client.get(f"{path}?variant=preview", headers=actor(alice))).status_code == 400 + assert ( + await client.get("/api/v1/attachments/x/content", headers=actor(alice)) + ).status_code == 400 + unknown = await client.get(f"/api/v1/attachments/{uuid4()}", headers=actor(alice)) + assert unknown.status_code == 404 + + +async def test_storage_failure_leaves_nothing_visible(client: AsyncClient) -> None: + alice, _, _, chat = await trio(client) + ready = await upload(client, alice, chat) + await settle(client) + store = await storage(client) + store.failing = True + try: + failed = await upload(client, alice, chat, filename="failed.png") + download = await client.get( + f"/api/v1/attachments/{ready.json()['id']}/content", headers=actor(alice) + ) + metadata = await client.get( + f"/api/v1/attachments/{ready.json()['id']}", headers=actor(alice) + ) + finally: + store.failing = False + assert failed.status_code == 503 + assert failed.json()["error"]["code"] == "unavailable" + assert download.status_code == 503 + assert metadata.status_code == 200 + status = await scalar( + "SELECT string_agg(status, ',') FROM attachments WHERE chat_id = $1 AND filename = $2", + chat["id"], + "failed.png", + ) + assert status == "pending" + + +async def test_lost_object_is_not_found(client: AsyncClient) -> None: + alice, _, _, chat = await trio(client) + attachment = (await upload(client, alice, chat)).json() + await settle(client) + (await storage(client)).objects.clear() + response = await client.get( + f"/api/v1/attachments/{attachment['id']}/content", headers=actor(alice) + ) + assert response.status_code == 404 + assert response.json()["error"]["code"] == "not_found" diff --git a/student-tests/api/test_bounded_contexts.py b/student-tests/api/test_bounded_contexts.py new file mode 100644 index 00000000..b588703c --- /dev/null +++ b/student-tests/api/test_bounded_contexts.py @@ -0,0 +1,289 @@ +import ast +import re +from pathlib import Path + +SOURCE = Path(__file__).resolve().parents[2] / "src" / "messenger" +IDENTITY = "messenger.identity" +MESSAGING = "messenger.messaging" +ATTACHMENTS = "messenger.attachments" +ZONES = ("identity", "messaging", "attachments") + + +def imports_of(path: Path) -> list[str]: + tree = ast.parse(path.read_text(), filename=str(path)) + found: list[str] = [] + for node in ast.walk(tree): + if isinstance(node, ast.ImportFrom) and node.module: + found.append(node.module) + elif isinstance(node, ast.Import): + found.extend(alias.name for alias in node.names) + return found + + +def modules_under(*parts: str) -> list[Path]: + root = SOURCE.joinpath(*parts) + return [path for path in root.rglob("*.py") if path.name != "__init__.py"] + + +def test_identity_does_not_know_about_messaging() -> None: + offenders = { + str(path.relative_to(SOURCE)): [m for m in imports_of(path) if m.startswith(MESSAGING)] + for path in modules_under("identity") + } + assert not {k: v for k, v in offenders.items() if v} + + +def test_messaging_touches_identity_only_through_events() -> None: + allowed = f"{IDENTITY}.domain.events" + offenders: dict[str, list[str]] = {} + for path in modules_under("messaging"): + borrowed = [ + module + for module in imports_of(path) + if module.startswith(IDENTITY) and module != allowed + ] + if borrowed: + offenders[str(path.relative_to(SOURCE))] = borrowed + assert not offenders + + +def test_attachments_and_other_zones_do_not_import_each_other() -> None: + offenders: dict[str, list[str]] = {} + for zone, foreign in ( + ("attachments", (IDENTITY, MESSAGING)), + ("identity", (ATTACHMENTS,)), + ("messaging", (ATTACHMENTS,)), + ): + for path in modules_under(zone): + borrowed = [m for m in imports_of(path) if m.startswith(foreign)] + if borrowed: + offenders[str(path.relative_to(SOURCE))] = borrowed + assert not offenders + + +def test_domains_do_not_import_infrastructure() -> None: + offenders: dict[str, list[str]] = {} + for zone in (*ZONES, "shared"): + for layer in ("domain", "application"): + for path in modules_under(zone, layer): + borrowed = [m for m in imports_of(path) if ".infrastructure" in m] + if borrowed: + offenders[str(path.relative_to(SOURCE))] = borrowed + assert not offenders + + +def test_core_layers_do_not_import_pydantic() -> None: + offenders: dict[str, list[str]] = {} + for zone in (*ZONES, "shared"): + for layer in ("domain", "application"): + for path in modules_under(zone, layer): + borrowed = [m for m in imports_of(path) if m.split(".")[0] == "pydantic"] + if borrowed: + offenders[str(path.relative_to(SOURCE))] = borrowed + assert not offenders, "схемы pydantic живут на границе: в entrypoints/ и config.py" + + +def test_shared_knows_nothing_about_zones() -> None: + offenders: dict[str, list[str]] = {} + for path in modules_under("shared"): + borrowed = [ + module + for module in imports_of(path) + if module.startswith((IDENTITY, MESSAGING, ATTACHMENTS)) + ] + if borrowed: + offenders[str(path.relative_to(SOURCE))] = borrowed + assert not offenders + + +TRANSPORT = ("fastapi", "starlette", "uvicorn", "gunicorn") + + +def test_shared_core_is_free_of_transport() -> None: + offenders: dict[str, list[str]] = {} + for layer in ("domain", "application", "infrastructure"): + for path in modules_under("shared", layer): + borrowed = [module for module in imports_of(path) if module.split(".")[0] in TRANSPORT] + if borrowed: + offenders[str(path.relative_to(SOURCE))] = borrowed + assert not offenders, "ядро shared обязано жить без веб-фреймворка" + + +def test_shared_core_holds_only_platform() -> None: + names = { + path.name + for layer in ("domain", "application", "infrastructure") + for path in modules_under("shared", layer) + } + assert names == { + "errors.py", + "events.py", + "commands.py", + "messagebus.py", + "ports.py", + "clock.py", + "ids.py", + "logging.py", + "postgres.py", + } + + +def test_shared_entrypoints_hold_only_transport() -> None: + names = {path.name for path in modules_under("shared", "entrypoints")} + assert names == { + "actor.py", + "config.py", + "errors.py", + "health.py", + "ioc.py", + "pipeline.py", + "schemas.py", + } + + +def test_every_zone_owns_its_errors_events_and_commands() -> None: + for zone in ZONES: + assert (SOURCE / zone / "domain" / "errors.py").is_file() + assert (SOURCE / zone / "domain" / "events.py").is_file() + assert (SOURCE / zone / "application" / "commands.py").is_file() + + +def test_every_zone_owns_its_http_layer() -> None: + for zone in ZONES: + for part in ("http.py", "ioc.py", "errors.py", "journal.py"): + assert (SOURCE / zone / "entrypoints" / part).is_file(), f"{zone}: нет {part}" + + +PROCESSES = {"relay.py": (ATTACHMENTS,), "worker.py": (ATTACHMENTS,), "admin.py": (ATTACHMENTS,)} + + +def test_only_main_knows_several_zones() -> None: + offenders: dict[str, list[str]] = {} + for path in SOURCE.rglob("*.py"): + relative = path.relative_to(SOURCE) + if path.name in {"__init__.py", "main.py"} or relative.parts[0] in ZONES: + continue + allowed = PROCESSES.get(str(relative), ()) + borrowed = [ + module + for module in imports_of(path) + if module.startswith((IDENTITY, MESSAGING, ATTACHMENTS)) + and not module.startswith(allowed or ("-",)) + ] + if borrowed: + offenders[str(relative)] = borrowed + assert not offenders + + +def test_broker_framework_stays_out_of_shared_and_core_layers() -> None: + offenders: dict[str, list[str]] = {} + for path in [ + *modules_under("shared"), + *( + p + for zone in ZONES + for layer in ("domain", "application") + for p in modules_under(zone, layer) + ), + ]: + borrowed = [ + m for m in imports_of(path) if m.split(".")[0] in {"faststream", "aio_pika", "aiormq"} + ] + if borrowed: + offenders[str(path.relative_to(SOURCE))] = borrowed + assert not offenders + + +def test_application_has_no_separate_entrypoints_root() -> None: + assert not (SOURCE / "entrypoints").exists() + assert (SOURCE / "main.py").is_file() + + +ZONE_WORDS = ( + "username", + "display_name", + "chat", + "member", + "history", + "cursor", + "sender", + "message_id", + "message_text", + "attachment", + "filename", +) + + +def test_shared_source_mentions_no_domain_vocabulary() -> None: + offenders: dict[str, list[str]] = {} + for path in modules_under("shared"): + source = path.read_text().lower() + hits = sorted({word for word in ZONE_WORDS if word in source}) + if hits: + offenders[str(path.relative_to(SOURCE))] = hits + assert not offenders + + +def test_shared_declares_no_concrete_commands_or_events() -> None: + for path in modules_under("shared"): + tree = ast.parse(path.read_text(), filename=str(path)) + declared = [n.name for n in ast.walk(tree) if isinstance(n, ast.ClassDef)] + concrete = [ + name + for name in declared + if name not in {"Command", "DomainEvent", "DomainError"} + and ( + name.endswith("Event") + or name in {"RegisterUser", "CreateChat", "PostMessage", "UploadAttachment"} + ) + ] + assert not concrete, f"{path.name}: {concrete}" + + +def test_shared_has_no_test_doubles() -> None: + for path in modules_under("shared"): + source = path.read_text() + for marker in ("Frozen", "Fake", "Stub", "Null", "Dummy"): + assert marker not in source, f"{path.name} содержит подделку {marker}" + + +def test_zone_errors_do_not_leak_transport_codes() -> None: + for zone in (*ZONES, "shared"): + source = (SOURCE / zone / "domain" / "errors.py").read_text() + assert "http_status" not in source + assert "invalid_request" not in source + assert "404" not in source + + +IDENTITY_TABLES = ("users", "sessions") +MESSAGING_TABLES = ("chats", "chat_members", "messages", "idempotency_keys", "message_attachments") +ATTACHMENT_TABLES = ("attachments", "attachment_variants", "jobs", "outbox", "job_events") + + +def sql_tables(zone: str, tables: tuple[str, ...]) -> set[str]: + words: set[str] = set() + for path in modules_under(zone): + tree = ast.parse(path.read_text(), filename=str(path)) + for node in ast.walk(tree): + if isinstance(node, ast.Constant) and isinstance(node.value, str): + words.update(re.findall(r"\b(?:FROM|JOIN|INTO|UPDATE)\s+(\w+)", node.value)) + return words & set(tables) + + +def test_sql_of_each_zone_reads_only_its_tables() -> None: + owned = { + "identity": IDENTITY_TABLES, + "messaging": MESSAGING_TABLES, + "attachments": ATTACHMENT_TABLES, + } + every = {table for tables in owned.values() for table in tables} + for zone, tables in owned.items(): + assert sql_tables(zone, tuple(every)) == set(tables), zone + + +def test_messaging_does_not_import_identity_at_all() -> None: + borrowed = { + str(path.relative_to(SOURCE)): [m for m in imports_of(path) if m.startswith(IDENTITY)] + for path in modules_under("messaging") + } + assert not {key: value for key, value in borrowed.items() if value} diff --git a/student-tests/api/test_cache.py b/student-tests/api/test_cache.py new file mode 100644 index 00000000..63f845d1 --- /dev/null +++ b/student-tests/api/test_cache.py @@ -0,0 +1,118 @@ +import asyncio +import time +from datetime import datetime +from hashlib import sha256 + +from redis.asyncio import Redis + +from conftest import actor, bearer, register, resolve, running_app, scalar +from messenger.identity.application.ports import SessionDirectory +from messenger.identity.domain.models import Session +from messenger.identity.infrastructure.valkey import CachedSessionDirectory + +BEARER = 'Bearer realm="messenger"' + + +def key_of(token: str) -> str: + return f"session:{sha256(token.encode()).hexdigest()}" + + +def token_of(user: dict) -> str: + return actor(user)["Authorization"].split()[1] + + +async def test_second_check_is_served_from_cache(valkey: dict[str, str]) -> None: + cache = Redis.from_url(valkey["url"]) + async with running_app(cache_url=valkey["url"]) as http: + alice = await register(http, "alice") + assert (await http.get("/api/v1/users/me", headers=actor(alice))).status_code == 200 + hits = (await cache.info("stats"))["keyspace_hits"] + assert (await http.get("/api/v1/users/me", headers=actor(alice))).status_code == 200 + assert (await cache.info("stats"))["keyspace_hits"] == hits + 1 + key = key_of(token_of(alice)) + stored = str(await cache.get(key)) + sql = "SELECT expires_at FROM sessions WHERE user_id = $1::uuid" + expires_at: datetime = await scalar(sql, alice["id"]) + assert alice["id"] in stored and token_of(alice) not in stored + assert await cache.pexpiretime(key) <= int(expires_at.timestamp() * 1000) + await cache.aclose() + + +async def test_logout_writes_tombstone_before_database(valkey: dict[str, str]) -> None: + cache = Redis.from_url(valkey["url"]) + async with running_app(cache_url=valkey["url"]) as http: + alice = await register(http, "alice") + assert (await http.get("/api/v1/users/me", headers=actor(alice))).status_code == 200 + assert ( + await http.delete("/api/v1/auth/sessions/current", headers=actor(alice)) + ).status_code == 204 + assert await cache.get(key_of(token_of(alice))) == b"-" + response = await http.get("/api/v1/users/me", headers=actor(alice)) + assert response.status_code == 401 + assert response.headers["WWW-Authenticate"] == BEARER + await cache.aclose() + + +async def test_fill_racing_with_logout_does_not_revive_token(valkey: dict[str, str]) -> None: + async with running_app(cache_url=valkey["url"]) as http: + alice = await register(http, "alice") + directory = await resolve(http, SessionDirectory) + assert isinstance(directory, CachedSessionDirectory) + origin = directory._origin + stale: list[Session | None] = [] + entered, released = asyncio.Event(), asyncio.Event() + + class SlowOrigin: + async def find(self, token_hash: bytes, now: datetime) -> Session | None: + stale.append(await origin.find(token_hash, now)) + entered.set() + await released.wait() + return stale[-1] + + async def invalidate(self, session: Session) -> None: + await origin.invalidate(session) + + directory._origin = SlowOrigin() + reader = asyncio.create_task(http.get("/api/v1/users/me", headers=actor(alice))) + await entered.wait() + directory._origin = origin + assert ( + await http.delete("/api/v1/auth/sessions/current", headers=actor(alice)) + ).status_code == 204 + released.set() + assert (await reader).status_code == 200 + assert (await http.get("/api/v1/users/me", headers=actor(alice))).status_code == 401 + + +async def test_flushall_loses_nothing(valkey: dict[str, str]) -> None: + cache = Redis.from_url(valkey["url"]) + async with running_app(cache_url=valkey["url"]) as http: + alice = await register(http, "alice") + bob = await register(http, "bob") + assert ( + await http.delete("/api/v1/auth/sessions/current", headers=actor(bob)) + ).status_code == 204 + await cache.flushall() + assert (await http.get("/api/v1/users/me", headers=actor(alice))).status_code == 200 + assert (await http.get("/api/v1/users/me", headers=actor(bob))).status_code == 401 + await cache.aclose() + + +async def test_unreachable_cache_is_bypassed_without_anonymous_access() -> None: + async with running_app(cache_url="redis://127.0.0.1:1/0") as http: + alice = await register(http, "alice") + started = time.perf_counter() + me = await http.get("/api/v1/users/me", headers=actor(alice)) + stranger = await http.get("/api/v1/users/me", headers=bearer("x" * 43)) + elapsed = time.perf_counter() - started + ready = await http.get("/health/ready") + logout = await http.delete("/api/v1/auth/sessions/current", headers=actor(alice)) + still = await http.get("/api/v1/users/me", headers=actor(alice)) + assert me.json() == alice + assert stranger.status_code == 401 + assert stranger.headers["WWW-Authenticate"] == BEARER + assert elapsed < 1 + assert ready.status_code == 200 + assert logout.status_code == 503 + assert logout.json()["error"]["code"] == "unavailable" + assert still.status_code == 200 diff --git a/student-tests/api/test_concurrency.py b/student-tests/api/test_concurrency.py new file mode 100644 index 00000000..aeeeb4e5 --- /dev/null +++ b/student-tests/api/test_concurrency.py @@ -0,0 +1,88 @@ +import asyncio +from urllib.parse import quote +from uuid import uuid4 + +from httpx import AsyncClient + +from conftest import PASSWORD, actor, keyed, make_chat, register + +CLIENTS = 10 +PER_CLIENT = 10 +TOTAL = CLIENTS * PER_CLIENT + + +async def drain(client: AsyncClient, chat: dict, reader: dict) -> list[dict]: + collected: list[dict] = [] + cursor: str | None = None + while True: + query = "?limit=100" + (f"&cursor={quote(cursor, safe='')}" if cursor else "") + page = ( + await client.get(f"/api/v1/chats/{chat['id']}/messages{query}", headers=actor(reader)) + ).json() + collected.extend(page["items"]) + cursor = page["next_cursor"] + if cursor is None: + return collected + + +async def flood(client: AsyncClient, chat: dict, sender: dict) -> list[int]: + async def writer(worker: int) -> list[int]: + codes = [] + for index in range(PER_CLIENT): + response = await client.post( + f"/api/v1/chats/{chat['id']}/messages", + json={"text": f"нагрузка-{worker}-{index}"}, + headers=keyed(sender), + ) + codes.append(response.status_code) + return codes + + batches = await asyncio.gather(*(writer(worker) for worker in range(CLIENTS))) + return [code for batch in batches for code in batch] + + +async def test_hundred_messages_from_ten_clients(client: AsyncClient) -> None: + alice = await register(client, "alice") + bob = await register(client, "bob") + chat = await make_chat(client, alice, bob) + + codes = await flood(client, chat, alice) + assert codes.count(201) == TOTAL, f"неуспешные ответы: {set(codes) - {201}}" + + history = await drain(client, chat, bob) + assert len(history) == TOTAL + assert len({item["id"] for item in history}) == TOTAL + assert len({item["text"] for item in history}) == TOTAL + + keys = [(item["created_at"], item["id"]) for item in history] + assert keys == sorted(keys), "история не упорядочена по (created_at, id)" + stamps = [item["created_at"] for item in history] + assert len(set(stamps)) == TOTAL, "created_at внутри чата обязан строго возрастать" + + +async def test_parallel_registration_keeps_username_unique(client: AsyncClient) -> None: + body = {"username": f"race_{uuid4().hex[:12]}", "display_name": "Гонка", "password": PASSWORD} + responses = await asyncio.gather(*(client.post("/api/v1/users", json=body) for _ in range(20))) + codes = [response.status_code for response in responses] + assert codes.count(201) == 1 + assert codes.count(409) == len(codes) - 1 + + +async def test_parallel_chat_creation_is_atomic(client: AsyncClient) -> None: + alice = await register(client, "alice") + bob = await register(client, "bob") + responses = await asyncio.gather( + *( + client.post( + "/api/v1/chats", + json={"title": f"Чат {index}", "member_ids": [alice["id"], bob["id"]]}, + headers=actor(alice), + ) + for index in range(20) + ) + ) + assert all(response.status_code == 201 for response in responses) + listed = (await client.get("/api/v1/chats?limit=100", headers=actor(bob))).json() + assert len(listed["items"]) == 20 + keys = [(item["created_at"], item["id"]) for item in listed["items"]] + assert keys == sorted(keys) diff --git a/student-tests/api/test_consistency.py b/student-tests/api/test_consistency.py new file mode 100644 index 00000000..5a616a2a --- /dev/null +++ b/student-tests/api/test_consistency.py @@ -0,0 +1,188 @@ +import asyncio +from urllib.parse import quote +from uuid import uuid4 + +from httpx import AsyncClient + +from conftest import PASSWORD, actor, keyed, make_chat, new_user, register, running_app, scalar + +REGISTRATIONS = 25 + + +async def drain_history(client: AsyncClient, chat: dict, reader: dict) -> list[dict]: + collected: list[dict] = [] + cursor: str | None = None + while True: + query = "?limit=100" + (f"&cursor={quote(cursor, safe='')}" if cursor else "") + page = ( + await client.get(f"/api/v1/chats/{chat['id']}/messages{query}", headers=actor(reader)) + ).json() + collected.extend(page["items"]) + cursor = page["next_cursor"] + if cursor is None: + return collected + + +async def test_registration_is_visible_to_chats_immediately() -> None: + async with running_app() as client: + for _ in range(REGISTRATIONS): + user = await register(client, "race") + response = await client.post( + "/api/v1/chats", + json={"title": "Сразу после регистрации", "member_ids": [user["id"]]}, + headers=actor(user), + ) + assert response.status_code == 201, response.text + + +async def test_parallel_registrations_are_all_usable_as_members() -> None: + async with running_app() as client: + alice = await register(client, "alice") + bodies = [new_user(f"par{index}") for index in range(REGISTRATIONS)] + responses = await asyncio.gather( + *(client.post("/api/v1/users", json=body) for body in bodies) + ) + assert all(response.status_code == 201 for response in responses) + created = [response.json()["id"] for response in responses] + chat = await client.post( + "/api/v1/chats", + json={"title": "Все параллельные", "member_ids": [alice["id"], *created]}, + headers=actor(alice), + ) + assert chat.status_code == 201, chat.text + assert len(chat.json()["member_ids"]) == REGISTRATIONS + 1 + + +async def test_rejected_registration_leaves_no_trace() -> None: + async with running_app() as client: + alice = await register(client, "alice") + duplicate = await client.post( + "/api/v1/users", + json={"username": alice["username"], "display_name": "Двойник", "password": PASSWORD}, + ) + assert duplicate.status_code == 409 + invalid = await client.post( + "/api/v1/users", json={"username": "AB", "display_name": "", "password": PASSWORD} + ) + assert invalid.status_code == 400 + sql = "SELECT count(*) FROM users WHERE username = $1" + assert await scalar(sql, alice["username"]) == 1 + + +async def test_failed_chat_creation_rolls_back_both_tables() -> None: + title = f"Откат {uuid4().hex}" + async with running_app() as client: + alice = await register(client, "alice") + before = (await client.get("/api/v1/chats?limit=100", headers=actor(alice))).json() + rejected = await client.post( + "/api/v1/chats", + json={"title": title, "member_ids": [alice["id"], str(uuid4())]}, + headers=actor(alice), + ) + assert rejected.status_code == 400 + assert rejected.json()["error"]["code"] == "invalid_request" + after = (await client.get("/api/v1/chats?limit=100", headers=actor(alice))).json() + assert after["items"] == before["items"] + assert await scalar("SELECT count(*) FROM chats WHERE title = $1", title) == 0 + members = "SELECT count(*) FROM chat_members WHERE user_id = $1::uuid" + assert await scalar(members, alice["id"]) == 0 + + +async def test_rejected_message_does_not_enter_history() -> None: + async with running_app() as client: + alice = await register(client, "alice") + eve = await register(client, "eve") + chat = await make_chat(client, alice) + + assert ( + await client.post( + f"/api/v1/chats/{chat['id']}/messages", + json={"text": " "}, + headers=keyed(alice), + ) + ).status_code == 400 + assert ( + await client.post( + f"/api/v1/chats/{chat['id']}/messages", + json={"text": "чужое"}, + headers=keyed(eve), + ) + ).status_code == 404 + + assert await drain_history(client, chat, alice) == [] + + +async def test_confirmed_write_is_visible_to_next_read() -> None: + async with running_app() as client: + alice = await register(client, "alice") + bob = await register(client, "bob") + chat = await make_chat(client, alice, bob) + + for index in range(30): + posted = await client.post( + f"/api/v1/chats/{chat['id']}/messages", + json={"text": f"подтверждение {index}"}, + headers=keyed(alice), + ) + assert posted.status_code == 201 + history = await drain_history(client, chat, bob) + assert history[-1]["id"] == posted.json()["id"] + assert len(history) == index + 1 + + +async def test_parallel_writes_across_zones_keep_counts_consistent() -> None: + async with running_app() as client: + alice = await register(client, "alice") + chats = [await make_chat(client, alice) for _ in range(4)] + + async def writer(chat: dict, worker: int) -> None: + for index in range(10): + response = await client.post( + f"/api/v1/chats/{chat['id']}/messages", + json={"text": f"поток-{worker}-{index}"}, + headers=keyed(alice), + ) + assert response.status_code == 201 + + async def registrar(index: int) -> str: + return (await register(client, f"parallel{index}"))["id"] + + results = await asyncio.gather( + *(writer(chat, number) for number, chat in enumerate(chats)), + *(registrar(index) for index in range(10)), + ) + + registered = [item for item in results if isinstance(item, str)] + assert len(set(registered)) == 10 + for chat in chats: + history = await drain_history(client, chat, alice) + assert len(history) == 10 + assert len({item["id"] for item in history}) == 10 + keys = [(item["created_at"], item["id"]) for item in history] + assert keys == sorted(keys) + + +async def test_chat_membership_survives_parallel_reads_and_writes() -> None: + async with running_app() as client: + alice = await register(client, "alice") + bob = await register(client, "bob") + eve = await register(client, "eve") + chat = await make_chat(client, alice, bob) + + async def write(index: int) -> None: + await client.post( + f"/api/v1/chats/{chat['id']}/messages", + json={"text": f"параллельно {index}"}, + headers=keyed(alice), + ) + + async def intruder() -> int: + response = await client.get(f"/api/v1/chats/{chat['id']}/messages", headers=actor(eve)) + return response.status_code + + results = await asyncio.gather( + *(write(index) for index in range(20)), + *(intruder() for _ in range(20)), + ) + assert {code for code in results if isinstance(code, int)} == {404} + assert len(await drain_history(client, chat, bob)) == 20 diff --git a/student-tests/api/test_drain.py b/student-tests/api/test_drain.py new file mode 100644 index 00000000..299addf5 --- /dev/null +++ b/student-tests/api/test_drain.py @@ -0,0 +1,110 @@ +import json +import os +import signal +import socket +import subprocess +import sys +import time + +import httpx + +from conftest import DATABASE, new_user +from messenger.shared.entrypoints.health import DRAINING + +DRAIN = 1.5 +SHUTDOWN = 4 +GUNICORN = ( + "-m", + "gunicorn", + "--config", + "python:messenger.gunicorn_conf", + "messenger.main:create_app()", +) + + +async def test_draining_replica_is_not_ready_but_alive(client: httpx.AsyncClient) -> None: + DRAINING.set() + try: + ready = await client.get("/health/ready") + live = await client.get("/health/live") + finally: + DRAINING.clear() + assert ready.status_code == 503 + assert ready.json()["error"]["code"] == "unavailable" + assert ready.headers["X-Instance-Id"] + assert live.status_code == 200 + assert (await client.get("/health/ready")).status_code == 200 + + +def free_port() -> int: + with socket.socket() as probe: + probe.bind(("127.0.0.1", 0)) + return probe.getsockname()[1] + + +def status(port: int, path: str) -> int: + try: + return httpx.get(f"http://127.0.0.1:{port}{path}", timeout=1).status_code + except httpx.TransportError: + return 0 + + +def begin_registration(port: int) -> tuple[socket.socket, bytes]: + body = json.dumps(new_user("drain")).encode() + head = ( + "POST /api/v1/users HTTP/1.1\r\nHost: replica\r\nContent-Type: application/json\r\n" + f"Content-Length: {len(body)}\r\nConnection: close\r\n\r\n" + ) + connection = socket.create_connection(("127.0.0.1", port), timeout=5) + connection.sendall(head.encode() + body[:8]) + return connection, body[8:] + + +def test_sigterm_leaves_rotation_then_closes_and_exits() -> None: + port = free_port() + settings = { + "HOST": "127.0.0.1", + "PORT": str(port), + "WEB_CONCURRENCY": "2", + "DATABASE_URL": DATABASE["url"], + "CURSOR_SECRET": "test-secret-for-cursors", + "S3_ACCESS_KEY_ID": "test-key", + "S3_SECRET_ACCESS_KEY": "test-secret-key", + "LOG_LEVEL": "WARNING", + "DRAIN_SECONDS": str(DRAIN), + "SHUTDOWN_TIMEOUT_SECONDS": str(SHUTDOWN), + } + server = subprocess.Popen( + [sys.executable, *GUNICORN], + env={**os.environ, **settings}, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + try: + deadline = time.monotonic() + 30 + while status(port, "/health/ready") != 200: + assert time.monotonic() < deadline, "gunicorn не стал готов" + time.sleep(0.2) + pending, rest = begin_registration(port) + sent = time.monotonic() + server.send_signal(signal.SIGTERM) + time.sleep(0.3) + during = status(port, "/health/ready"), status(port, "/health/live") + while status(port, "/health/live") == 200: + time.sleep(0.05) + closed = time.monotonic() - sent + pending.sendall(rest) + answer = pending.recv(64).split(b"\r\n")[0].decode() + pending.close() + code = server.wait(timeout=SHUTDOWN + 2) + exited = time.monotonic() - sent + finally: + if server.poll() is None: + server.kill() + print(f"окно: ready/live {during}, порт закрыт через {closed:.2f} с, выход {exited:.2f} с") + print(f"запрос, принятый до SIGTERM и дописанный после закрытия порта: {answer}") + assert during == (503, 200) + assert answer == "HTTP/1.1 201 Created" + assert DRAIN - 0.2 <= closed < SHUTDOWN + assert exited < SHUTDOWN + assert code == 0 diff --git a/student-tests/api/test_events.py b/student-tests/api/test_events.py new file mode 100644 index 00000000..a6975565 --- /dev/null +++ b/student-tests/api/test_events.py @@ -0,0 +1,124 @@ +import asyncio +from dataclasses import fields +from datetime import UTC, datetime +from uuid import uuid4 + +from httpx import ASGITransport, AsyncClient + +from conftest import keyed, make_chat, register, split_settings +from fakes import NullLogger +from messenger.identity.domain.events import UserRegistered +from messenger.main import create_app +from messenger.messaging.domain.events import MessagePosted +from messenger.shared.application.messagebus import EventBus, InMemoryEventPublisher + + +def posted() -> MessagePosted: + return MessagePosted( + message_id=uuid4(), + chat_id=uuid4(), + sender_id=uuid4(), + text_length=5, + created_at=datetime.now(UTC), + ) + + +async def test_event_has_no_message_text() -> None: + names = {field.name for field in fields(posted())} + assert names == {"name", "message_id", "chat_id", "sender_id", "text_length", "created_at"} + + +async def test_every_subscriber_receives_event() -> None: + bus = EventBus(NullLogger()) + seen: list[str] = [] + + async def first(_event: MessagePosted) -> None: + seen.append("first") + + async def second(_event: MessagePosted) -> None: + seen.append("second") + + bus.subscribe(MessagePosted, first) + bus.subscribe(MessagePosted, second) + bus.start() + await InMemoryEventPublisher(bus).publish([posted()]) + await bus.stop() + assert sorted(seen) == ["first", "second"] + + +async def test_failing_subscriber_does_not_block_others() -> None: + bus = EventBus(NullLogger()) + survived: list[str] = [] + + async def broken(_event: MessagePosted) -> None: + raise RuntimeError("обработчик упал") + + async def healthy(_event: MessagePosted) -> None: + survived.append("ok") + + bus.subscribe(MessagePosted, broken) + bus.subscribe(MessagePosted, healthy) + bus.start() + await InMemoryEventPublisher(bus).publish([posted()]) + await bus.stop() + assert survived == ["ok"] + + +async def test_queue_overflow_drops_events_without_raising() -> None: + bus = EventBus(NullLogger(), max_queue=16) + + async def never_called(_event: MessagePosted) -> None: + raise AssertionError("обработчик не должен вызываться без запущенной шины") + + bus.subscribe(MessagePosted, never_called) + publisher = InMemoryEventPublisher(bus) + await publisher.publish([posted() for _ in range(64)]) + assert bus.dropped > 0 + assert bus.pending <= 16 + + +async def test_projection_runs_before_publish_returns() -> None: + bus = EventBus(NullLogger()) + seen: list[str] = [] + + async def projection(_event: MessagePosted) -> None: + seen.append("projected") + + bus.project(MessagePosted, projection) + await InMemoryEventPublisher(bus).publish([posted()]) + assert seen == ["projected"] + + +async def test_publication_does_not_delay_response() -> None: + app = create_app(**split_settings(log_level="CRITICAL")) + async with ( + app.router.lifespan_context(app), + AsyncClient(transport=ASGITransport(app=app), base_url="http://testserver") as http, + ): + alice = await register(http, "alice") + chat = await make_chat(http, alice) + bus = await app.state.dishka_container.get(EventBus) + slow = asyncio.Event() + + async def blocker(_event: MessagePosted) -> None: + await slow.wait() + + bus.subscribe(MessagePosted, blocker) + response = await asyncio.wait_for( + http.post( + f"/api/v1/chats/{chat['id']}/messages", + json={"text": "не ждём обработчик"}, + headers=keyed(alice), + ), + timeout=2, + ) + assert response.status_code == 201 + slow.set() + + +async def test_null_logger_hides_everything() -> None: + logger = NullLogger() + logger.info("x", route="/") + logger.warning("x") + logger.error("x") + assert isinstance(UserRegistered, type) diff --git a/student-tests/api/test_idempotency.py b/student-tests/api/test_idempotency.py new file mode 100644 index 00000000..02f94a90 --- /dev/null +++ b/student-tests/api/test_idempotency.py @@ -0,0 +1,145 @@ +import asyncio +from typing import Any +from uuid import uuid4 + +import pytest +from httpx import AsyncClient, Response + +from conftest import actor, keyed, make_chat, register, running_app, scalar + +INVALID_KEYS = ["", "k" * 129, "два слова", "a b", "a/b", "ключ"] + + +async def send(http: AsyncClient, chat: dict, user: dict, text: str, key: str) -> Response: + return await http.post( + f"/api/v1/chats/{chat['id']}/messages", json={"text": text}, headers=keyed(user, key) + ) + + +async def history(http: AsyncClient, chat: dict, user: dict) -> list[dict[str, Any]]: + response = await http.get(f"/api/v1/chats/{chat['id']}/messages?limit=100", headers=actor(user)) + return response.json()["items"] + + +async def test_replay_returns_the_same_message(client: AsyncClient) -> None: + alice = await register(client, "alice") + chat = await make_chat(client, alice) + first = await send(client, chat, alice, "один раз", "replay-1") + again = await send(client, chat, alice, "один раз", "replay-1") + assert first.status_code == again.status_code == 201 + assert again.json() == first.json() + assert [item["id"] for item in await history(client, chat, alice)] == [first.json()["id"]] + + +async def test_same_json_in_other_layout_is_a_replay(client: AsyncClient) -> None: + alice = await register(client, "alice") + chat = await make_chat(client, alice) + first = await send(client, chat, alice, "форма", "layout-1") + again = await client.post( + f"/api/v1/chats/{chat['id']}/messages", + content='{ "text" : "форма" }'.encode(), + headers={**keyed(alice, "layout-1"), "Content-Type": "application/json"}, + ) + assert again.status_code == 201 + assert again.json() == first.json() + + +async def test_other_body_with_same_key_conflicts(client: AsyncClient) -> None: + alice = await register(client, "alice") + chat = await make_chat(client, alice) + await send(client, chat, alice, "первое", "conflict-1") + response = await send(client, chat, alice, "другое", "conflict-1") + assert response.status_code == 409 + assert response.json()["error"]["code"] == "conflict" + assert len(await history(client, chat, alice)) == 1 + + +async def test_missing_key_is_rejected(client: AsyncClient) -> None: + alice = await register(client, "alice") + chat = await make_chat(client, alice) + response = await client.post( + f"/api/v1/chats/{chat['id']}/messages", json={"text": "без ключа"}, headers=actor(alice) + ) + assert response.status_code == 400 + assert response.json()["error"]["code"] == "invalid_request" + assert await history(client, chat, alice) == [] + + +@pytest.mark.parametrize("key", INVALID_KEYS, ids=range(len(INVALID_KEYS))) +async def test_malformed_key_is_rejected(client: AsyncClient, key: str) -> None: + alice = await register(client, "alice") + chat = await make_chat(client, alice) + headers = {**actor(alice), "Idempotency-Key": key} + response = await client.post( + f"/api/v1/chats/{chat['id']}/messages", + json={"text": "ключ плохой"}, + headers={name.encode(): value.encode() for name, value in headers.items()}, + ) + assert response.status_code == 400 + assert await history(client, chat, alice) == [] + + +async def test_rejected_attempt_does_not_take_the_key(client: AsyncClient) -> None: + alice = await register(client, "alice") + eve = await register(client, "eve") + chat = await make_chat(client, alice) + assert (await send(client, chat, alice, "", "spare-1")).status_code == 400 + assert (await send(client, chat, eve, "чужое", "spare-1")).status_code == 404 + assert (await send(client, chat, alice, "теперь можно", "spare-1")).status_code == 201 + + +async def test_key_is_scoped_by_user_and_chat(client: AsyncClient) -> None: + alice = await register(client, "alice") + bob = await register(client, "bob") + chat = await make_chat(client, alice, bob) + other = await make_chat(client, alice) + ids = { + (await send(client, chat, alice, "алиса", "shared-1")).json()["id"], + (await send(client, chat, bob, "боб", "shared-1")).json()["id"], + (await send(client, other, alice, "другой чат", "shared-1")).json()["id"], + } + assert len(ids) == 3 + + +async def test_replay_checks_session_and_access_again(client: AsyncClient) -> None: + alice = await register(client, "alice") + eve = await register(client, "eve") + chat = await make_chat(client, alice) + assert (await send(client, chat, alice, "до выхода", "recheck-1")).status_code == 201 + assert (await send(client, chat, eve, "до выхода", "recheck-1")).status_code == 404 + logout = await client.delete("/api/v1/auth/sessions/current", headers=actor(alice)) + assert logout.status_code == 204 + assert (await send(client, chat, alice, "до выхода", "recheck-1")).status_code == 401 + + +async def test_expired_key_starts_over() -> None: + async with running_app(idempotency_ttl_seconds=1) as http: + alice = await register(http, "alice") + chat = await make_chat(http, alice) + first = await send(http, chat, alice, "давно", "expire-1") + await asyncio.sleep(1.1) + later = await send(http, chat, alice, "заново", "expire-1") + assert later.status_code == 201 + assert later.json()["id"] != first.json()["id"] + assert len(await history(http, chat, alice)) == 2 + + +async def test_concurrent_retries_through_two_replicas_store_one_message() -> None: + async with ( + running_app(instance_id="replica-a") as first, + running_app(instance_id="replica-b") as second, + ): + alice = await register(first, "alice") + chat = await make_chat(first, alice) + key, text = uuid4().hex, "гонка ключа" + responses = await asyncio.gather( + *(send(http, chat, alice, text, key) for http in (first, second) * 4) + ) + assert {response.status_code for response in responses} == {201} + assert {response.headers["X-Instance-Id"] for response in responses} == { + "replica-a", + "replica-b", + } + assert all(response.json() == responses[0].json() for response in responses) + assert [item["text"] for item in await history(second, chat, alice)] == [text] + assert await scalar("SELECT count(*) FROM idempotency_keys WHERE key = $1", key) == 1 diff --git a/student-tests/api/test_instance.py b/student-tests/api/test_instance.py new file mode 100644 index 00000000..b76fe4fb --- /dev/null +++ b/student-tests/api/test_instance.py @@ -0,0 +1,64 @@ +import socket +from uuid import uuid4 + +from httpx import ASGITransport, AsyncClient, Response + +from conftest import actor, register, running_app, split_settings +from messenger.main import create_app +from messenger.shared.config import PlatformSettings + +REPLICA = "replica-7f3a" + + +def stamped(response: Response) -> bool: + return bool(response.headers.get("X-Request-Id")) and ( + response.headers.get("X-Instance-Id") == REPLICA + ) + + +async def test_every_response_names_the_replica() -> None: + async with running_app(instance_id=REPLICA) as http: + alice = await register(http, "alice") + probes = [ + await http.get("/health/live"), + await http.get("/health/ready"), + await http.get("/api/v1/users/me", headers=actor(alice)), + await http.get("/api/v1/users/me"), + await http.get(f"/api/v1/chats/{uuid4()}", headers=actor(alice)), + await http.get("/nothing-here"), + await http.delete("/health/live"), + await http.post("/api/v1/users", json={"username": "x"}), + await http.delete("/api/v1/auth/sessions/current", headers=actor(alice)), + ] + assert [probe.status_code for probe in probes] == [200, 200, 200, 401, 404, 404, 404, 400, 204] + assert all(stamped(probe) for probe in probes) + + +async def test_rejected_body_names_the_replica() -> None: + async with running_app(instance_id=REPLICA, max_body_bytes=4096) as http: + response = await http.post("/api/v1/users", content=b"x" * 5000) + assert response.status_code == 413 + assert stamped(response) + + +async def test_unhandled_error_names_the_replica() -> None: + app = create_app(**split_settings(instance_id=REPLICA)) + + async def boom() -> None: + raise RuntimeError("сбой") + + app.add_api_route("/health/boom", boom) + transport = ASGITransport(app=app, raise_app_exceptions=False) + async with ( + app.router.lifespan_context(app), + AsyncClient(transport=transport, base_url="http://testserver") as http, + ): + response = await http.get("/health/boom") + assert response.status_code == 500 + assert response.json()["error"]["code"] == "internal_error" + assert stamped(response) + + +def test_replica_defaults_to_container_hostname() -> None: + settings = PlatformSettings.model_validate({"database_url": "postgresql://u@h/d"}) + assert settings.instance_id == socket.gethostname() diff --git a/student-tests/api/test_message_attachments.py b/student-tests/api/test_message_attachments.py new file mode 100644 index 00000000..b0ecc7f7 --- /dev/null +++ b/student-tests/api/test_message_attachments.py @@ -0,0 +1,149 @@ +from typing import Any +from uuid import uuid4 + +import pytest +from httpx import AsyncClient + +from conftest import actor, keyed, make_chat, register, resolve +from fakes import MemoryObjectStorage +from messenger.attachments.application.ports import ObjectStorage +from pictures import SAMPLE +from pipeline import settle + + +async def attach(http: AsyncClient, user: dict[str, Any], chat: dict[str, Any]) -> str: + response = await http.post( + f"/api/v1/chats/{chat['id']}/attachments?filename=a.png", + content=SAMPLE, + headers={**actor(user), "Content-Type": "image/png"}, + ) + assert response.status_code == 202, response.text + return response.json()["id"] + + +async def send( + http: AsyncClient, user: dict[str, Any], chat: dict[str, Any], body: Any, key: str | None = None +) -> Any: + return await http.post( + f"/api/v1/chats/{chat['id']}/messages", json=body, headers=keyed(user, key) + ) + + +async def scene(http: AsyncClient) -> tuple[dict[str, Any], ...]: + alice, bob, eve = [await register(http, label) for label in ("alice", "bob", "eve")] + return alice, bob, eve, await make_chat(http, alice, bob), await make_chat(http, alice) + + +async def test_text_message_carries_empty_list(client: AsyncClient) -> None: + alice, _, _, chat, _ = await scene(client) + response = await send(client, alice, chat, {"text": "привет"}) + assert response.status_code == 201 + assert response.json()["attachment_ids"] == [] + + +async def test_empty_text_with_attachment_keeps_order(client: AsyncClient) -> None: + alice, bob, _, chat, _ = await scene(client) + ids = [await attach(client, alice, chat) for _ in range(3)] + ordered = [ids[2], ids[0], ids[1]] + response = await send(client, bob, chat, {"text": "", "attachment_ids": ordered}) + assert response.status_code == 201, response.text + message = response.json() + assert message["attachment_ids"] == ordered + again = await send(client, alice, chat, {"text": "ещё раз", "attachment_ids": [ids[0]]}) + assert again.status_code == 201 + history = await client.get(f"/api/v1/chats/{chat['id']}/messages", headers=actor(bob)) + assert history.json()["items"] == [message, again.json()] + + +async def test_replay_with_attachments_and_default_list(client: AsyncClient) -> None: + alice, _, _, chat, _ = await scene(client) + first_id, second_id = await attach(client, alice, chat), await attach(client, alice, chat) + body = {"text": "", "attachment_ids": [first_id, second_id]} + first = await send(client, alice, chat, body, "with-files") + assert (await send(client, alice, chat, body, "with-files")).json() == first.json() + swapped = {"text": "", "attachment_ids": [second_id, first_id]} + assert (await send(client, alice, chat, swapped, "with-files")).status_code == 409 + plain = await send(client, alice, chat, {"text": "x"}, "plain") + explicit = await send(client, alice, chat, {"attachment_ids": [], "text": "x"}, "plain") + assert (explicit.status_code, explicit.json()) == (201, plain.json()) + + +async def test_attachment_of_another_chat_or_unknown(client: AsyncClient) -> None: + alice, _, _, chat, solo = await scene(client) + foreign = await attach(client, alice, solo) + for ids in ([foreign], [str(uuid4())]): + response = await send(client, alice, chat, {"text": "x", "attachment_ids": ids}) + assert response.status_code == 404 + assert response.json()["error"]["code"] == "not_found" + history = await client.get(f"/api/v1/chats/{chat['id']}/messages", headers=actor(alice)) + assert history.json()["items"] == [] + + +@pytest.mark.parametrize("kind", ["eleven", "duplicate", "not-uuid", "empty", "blank"]) +async def test_invalid_attachment_lists(client: AsyncClient, kind: str) -> None: + alice, _, _, chat, _ = await scene(client) + file = await attach(client, alice, chat) + body = { + "eleven": {"text": "x", "attachment_ids": [file] + [str(uuid4()) for _ in range(10)]}, + "duplicate": {"text": "x", "attachment_ids": [file, file]}, + "not-uuid": {"text": "x", "attachment_ids": ["not-a-uuid"]}, + "empty": {"text": "", "attachment_ids": []}, + "blank": {"text": " ", "attachment_ids": [file]}, + }[kind] + response = await send(client, alice, chat, body) + assert response.status_code == 400, response.text + assert response.json()["error"]["code"] == "invalid_request" + + +async def test_outsider_gets_not_found_before_validation(client: AsyncClient) -> None: + alice, _, eve, chat, _ = await scene(client) + file = await attach(client, alice, chat) + for body in ({"text": "", "attachment_ids": []}, {"text": "x", "attachment_ids": [file] * 11}): + assert (await send(client, eve, chat, body)).status_code == 404 + + +async def test_one_attachment_in_several_messages(client: AsyncClient) -> None: + alice, bob, _, chat, _ = await scene(client) + file = await attach(client, alice, chat) + for user in (alice, bob): + response = await send(client, user, chat, {"text": "", "attachment_ids": [file]}) + assert response.status_code == 201 + assert response.json()["attachment_ids"] == [file] + + +async def broken(http: AsyncClient, user: dict[str, Any], chat: dict[str, Any]) -> str: + attachment_id = await attach(http, user, chat) + store = await resolve(http, ObjectStorage) + assert isinstance(store, MemoryObjectStorage) + store.objects[f"staging/{attachment_id}"] = (b"\x89PNG\r\n\x1a\n broken", "image/png") + await settle(http) + return attachment_id + + +async def test_queued_attachment_is_sent_and_failed_is_conflict(client: AsyncClient) -> None: + alice, bob, _, chat, _ = await scene(client) + queued = await attach(client, alice, chat) + early = await send(client, alice, chat, {"text": "", "attachment_ids": [queued]}) + assert early.status_code == 201, early.text + failed = await broken(client, bob, chat) + meta = (await client.get(f"/api/v1/attachments/{failed}", headers=actor(alice))).json() + assert (meta["status"], meta["error_code"]) == ("failed", "invalid_image") + refused = await send(client, alice, chat, {"text": "x", "attachment_ids": [queued, failed]}) + assert refused.status_code == 409 + assert refused.json()["error"]["code"] == "conflict" + content = await client.get(f"/api/v1/attachments/{failed}/content", headers=actor(bob)) + assert content.status_code == 409 + + +async def test_replay_survives_a_later_failure(client: AsyncClient) -> None: + alice, _, _, chat, _ = await scene(client) + attachment_id = await attach(client, alice, chat) + body = {"text": "", "attachment_ids": [attachment_id]} + first = await send(client, alice, chat, body, key="replay-after-failure") + store = await resolve(client, ObjectStorage) + assert isinstance(store, MemoryObjectStorage) + store.objects[f"staging/{attachment_id}"] = (b"garbage", "image/png") + await settle(client) + again = await send(client, alice, chat, body, key="replay-after-failure") + assert (first.status_code, again.status_code) == (201, 201) + assert again.json() == first.json() diff --git a/student-tests/api/test_migrations.py b/student-tests/api/test_migrations.py new file mode 100644 index 00000000..50af1a5d --- /dev/null +++ b/student-tests/api/test_migrations.py @@ -0,0 +1,240 @@ +from uuid import uuid4 + +import asyncpg +import pytest + +from conftest import DATABASE, scalar +from messenger.migrate import MigrationChangedError, migrate, migrations + +COUNTS = """ +SELECT (SELECT count(*) FROM users), (SELECT count(*) FROM sessions), + (SELECT count(*) FROM chats), (SELECT count(*) FROM chat_members), + (SELECT count(*) FROM messages), (SELECT count(*) FROM idempotency_keys), + (SELECT count(*) FROM attachments), (SELECT count(*) FROM message_attachments), + (SELECT count(*) FROM attachment_variants), (SELECT count(*) FROM jobs), + (SELECT count(*) FROM outbox), (SELECT count(*) FROM job_events) +""" +PIPELINE_COLUMNS = { + "attachments": { + "id", + "status", + "published_version", + "error_code", + "staging_key", + "original_key", + }, + "attachment_variants": { + "attachment_id", + "pipeline_version", + "name", + "sha256", + "size_bytes", + "object_key", + }, + "jobs": {"job_id", "attachment_id", "pipeline_version", "status", "attempt"}, + "outbox": {"job_id", "payload", "published_at"}, + "job_events": {"job_id", "attachment_id", "kind", "attempt"}, +} + + +async def counts() -> list[int]: + connection = await asyncpg.connect(DATABASE["url"]) + try: + return list(await connection.fetchrow(COUNTS)) + finally: + await connection.close() + + +async def test_migrations_are_ordered_and_versioned() -> None: + versions = [version for version, _ in migrations()] + assert versions == sorted(versions) + assert versions[:5] == [ + "0001_init", + "0002_idempotency_keys", + "0003_attachments", + "0004_jobs", + "0005_failed_registry", + ] + plan = await scalar("SELECT indexdef FROM pg_indexes WHERE indexname = 'jobs_failed_idx'") + assert "WHERE (status = 'failed'::text)" in plan + applied = await scalar("SELECT array_agg(version ORDER BY version) FROM schema_migrations") + assert applied == versions + + +async def test_rerun_changes_nothing() -> None: + before = await counts() + assert await migrate(DATABASE["url"]) == [] + assert await counts() == before + + +async def test_changed_migration_is_refused() -> None: + connection = await asyncpg.connect(DATABASE["url"]) + try: + original = await connection.fetchval( + "SELECT checksum FROM schema_migrations WHERE version = '0001_init'" + ) + await connection.execute( + "UPDATE schema_migrations SET checksum = 'tampered' WHERE version = '0001_init'" + ) + with pytest.raises(MigrationChangedError): + await migrate(DATABASE["url"]) + await connection.execute( + "UPDATE schema_migrations SET checksum = $1 WHERE version = '0001_init'", original + ) + finally: + await connection.close() + + +async def test_sender_must_be_member_even_bypassing_api() -> None: + connection = await asyncpg.connect(DATABASE["url"]) + try: + with pytest.raises(asyncpg.ForeignKeyViolationError): + await connection.execute( + "INSERT INTO messages (id, chat_id, sender_id, text, created_at) " + "VALUES (gen_random_uuid(), gen_random_uuid(), gen_random_uuid(), 'x', now())" + ) + finally: + await connection.close() + + +async def test_key_needs_a_member_and_a_message() -> None: + connection = await asyncpg.connect(DATABASE["url"]) + try: + with pytest.raises(asyncpg.ForeignKeyViolationError): + await connection.execute( + "INSERT INTO idempotency_keys " + "(chat_id, user_id, key, fingerprint, message_id, created_at) " + "VALUES (gen_random_uuid(), gen_random_uuid(), 'k-1', " + "sha256('x'), gen_random_uuid(), now())" + ) + finally: + await connection.close() + + +async def test_attachment_owner_must_be_member() -> None: + connection = await asyncpg.connect(DATABASE["url"]) + try: + with pytest.raises(asyncpg.ForeignKeyViolationError, match="attachments_owner_fkey"): + await connection.execute( + "INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, " + "size_bytes, sha256, width, height, object_key, status, created_at, updated_at) " + "VALUES (gen_random_uuid(), gen_random_uuid(), gen_random_uuid(), 'a.png', " + "'image/png', 1, repeat('0', 64), 1, 1, 'attachments/x', 'pending', now(), now())" + ) + finally: + await connection.close() + + +async def test_message_links_only_attachments_of_its_chat() -> None: + user, chat_a, chat_b, attachment, message = (uuid4() for _ in range(5)) + seed = [ + ( + "INSERT INTO users (id, username, display_name, password_hash, created_at) " + "VALUES ($1, $2, 'A', '$argon2id$x', now())", + user, + f"u_{user.hex[:12]}", + ), + ("INSERT INTO chats (id, title, created_at) VALUES ($1, 'A', now())", chat_a), + ("INSERT INTO chats (id, title, created_at) VALUES ($1, 'B', now())", chat_b), + ( + "INSERT INTO chat_members (chat_id, user_id, position, chat_created_at) " + "VALUES ($1, $3, 1, now()), ($2, $3, 1, now())", + chat_a, + chat_b, + user, + ), + ( + "INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, " + "sha256, width, height, object_key, status, created_at, updated_at) VALUES ($1, $2, " + "$3, 'a.png', 'image/png', 1, repeat('0', 64), 1, 1, $4, 'ready', now(), now())", + attachment, + chat_a, + user, + f"attachments/{attachment}", + ), + ( + "INSERT INTO messages (id, chat_id, sender_id, text, created_at) " + "VALUES ($1, $2, $3, '', now())", + message, + chat_b, + user, + ), + ] + connection = await asyncpg.connect(DATABASE["url"]) + transaction = connection.transaction() + await transaction.start() + try: + for sql, *arguments in seed: + await connection.execute(sql, *arguments) + assert await connection.fetchval("SELECT text FROM messages WHERE id = $1", message) == "" + with pytest.raises( + asyncpg.ForeignKeyViolationError, match="message_attachments_attachment_fkey" + ): + await connection.execute( + "INSERT INTO message_attachments (message_id, position, chat_id, attachment_id) " + "VALUES ($1, 0, $2, $3)", + message, + chat_b, + attachment, + ) + finally: + await transaction.rollback() + await connection.close() + + +async def test_pipeline_tables_have_the_columns_the_suite_reads() -> None: + connection = await asyncpg.connect(DATABASE["url"]) + try: + rows = await connection.fetch( + "SELECT table_name, column_name FROM information_schema.columns " + "WHERE table_schema = current_schema()" + ) + finally: + await connection.close() + found: dict[str, set[str]] = {} + for row in rows: + found.setdefault(row["table_name"], set()).add(row["column_name"]) + assert all(columns <= found.get(table, set()) for table, columns in PIPELINE_COLUMNS.items()) + + +async def test_one_variant_row_per_effect_key() -> None: + user, chat, attachment = uuid4(), uuid4(), uuid4() + variant = ( + "INSERT INTO attachment_variants (attachment_id, pipeline_version, name, object_key, " + "content_type, size_bytes, width, height, sha256, created_at) " + "VALUES ($1, 1, 'thumbnail', $2, 'image/jpeg', 1, 1, 1, repeat('a', 64), now())" + ) + connection = await asyncpg.connect(DATABASE["url"]) + transaction = connection.transaction() + await transaction.start() + try: + await connection.execute( + "INSERT INTO users (id, username, display_name, password_hash, created_at) " + "VALUES ($1, $2, 'A', '$argon2id$x', now())", + user, + f"u_{user.hex[:12]}", + ) + await connection.execute( + "INSERT INTO chats (id, title, created_at) VALUES ($1, 'A', now())", chat + ) + await connection.execute( + "INSERT INTO chat_members (chat_id, user_id, position, chat_created_at) " + "VALUES ($1, $2, 1, now())", + chat, + user, + ) + await connection.execute( + "INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, " + "sha256, width, height, staging_key, status, created_at, updated_at) VALUES ($1, $2, " + "$3, 'a.png', 'image/png', 1, repeat('0', 64), 1, 1, $4, 'queued', now(), now())", + attachment, + chat, + user, + f"staging/{attachment}", + ) + await connection.execute(variant, attachment, f"variants/{attachment}/1/a") + with pytest.raises(asyncpg.UniqueViolationError, match="attachment_variants_pkey"): + await connection.execute(variant, attachment, f"variants/{attachment}/1/b") + finally: + await transaction.rollback() + await connection.close() diff --git a/student-tests/api/test_negative.py b/student-tests/api/test_negative.py new file mode 100644 index 00000000..a1dc7ccf --- /dev/null +++ b/student-tests/api/test_negative.py @@ -0,0 +1,196 @@ +from uuid import uuid4 + +import pytest +from httpx import AsyncClient + +from conftest import PASSWORD, actor, keyed, make_chat, register + +ERROR_CODES = { + 400: "invalid_request", + 401: "unauthorized", + 404: "not_found", + 409: "conflict", +} + + +def assert_error(response, expected_status: int) -> None: + assert response.status_code == expected_status, response.text + assert response.headers.get("content-type", "").startswith("application/json") + body = response.json() + assert set(body) == {"error"} + assert set(body["error"]) == {"code", "message", "request_id"} + assert body["error"]["code"] == ERROR_CODES[expected_status] + assert body["error"]["message"] + assert body["error"]["request_id"] == response.headers["X-Request-Id"] + + +async def test_malformed_json(client: AsyncClient) -> None: + response = await client.post( + "/api/v1/users", + content=b'{"username": ', + headers={"Content-Type": "application/json"}, + ) + assert_error(response, 400) + + +async def test_unexpected_field(client: AsyncClient) -> None: + response = await client.post( + "/api/v1/users", + json={"username": "abc_def", "display_name": "X", "password": PASSWORD, "role": "admin"}, + ) + assert_error(response, 400) + + +async def test_wrong_types(client: AsyncClient) -> None: + response = await client.post( + "/api/v1/users", json={"username": 1, "display_name": True, "password": 12345678901234} + ) + assert_error(response, 400) + + +@pytest.mark.parametrize( + "body", + [ + {"username": "Alice", "display_name": "X", "password": PASSWORD}, + {"username": "ab", "display_name": "X", "password": PASSWORD}, + {"username": "alice_01\n", "display_name": "X", "password": PASSWORD}, + {"username": "abc_def", "display_name": "", "password": PASSWORD}, + {"username": "abc_def", "display_name": " ", "password": PASSWORD}, + {"username": "abc_def", "display_name": "x" * 101, "password": PASSWORD}, + {"username": "abc_def", "display_name": "X", "password": "short"}, + {"username": "abc_def", "display_name": "X", "password": "x" * 129}, + {"username": "abc_def", "display_name": "X", "password": " " * 20}, + {"username": "abc_def", "display_name": "X"}, + {"username": "abc_def", "password": PASSWORD}, + ], +) +async def test_invalid_registration(client: AsyncClient, body: dict) -> None: + assert_error(await client.post("/api/v1/users", json=body), 400) + + +async def test_duplicate_username(client: AsyncClient) -> None: + alice = await register(client, "alice") + repeat = await client.post( + "/api/v1/users", + json={"username": alice["username"], "display_name": "Другая", "password": PASSWORD}, + ) + assert_error(repeat, 409) + + +async def test_missing_identity(client: AsyncClient) -> None: + assert_error(await client.get("/api/v1/users/me"), 401) + + +BAD_AUTHORIZATIONS = [ + "", + "Bearer", + "Bearer ", + "Bearer not a token", + "Bearer " + "x" * 43, + "Token abcdef", + "11111111-1111-4111-8111-111111111111", +] + + +@pytest.mark.parametrize("raw", BAD_AUTHORIZATIONS) +async def test_invalid_bearer(client: AsyncClient, raw: str) -> None: + response = await client.get("/api/v1/users/me", headers={"Authorization": raw}) + assert_error(response, 401) + assert response.headers["WWW-Authenticate"] == 'Bearer realm="messenger"' + + +async def test_user_id_header_alone_is_ignored(client: AsyncClient) -> None: + alice = await register(client, "alice") + response = await client.get("/api/v1/users/me", headers={"X-User-Id": alice["id"]}) + assert_error(response, 401) + + +async def test_identity_checked_before_body(client: AsyncClient) -> None: + response = await client.post( + "/api/v1/chats", + content=b"{broken", + headers={"Content-Type": "application/json"}, + ) + assert_error(response, 401) + + +async def test_empty_message_rejected(client: AsyncClient) -> None: + alice = await register(client, "alice") + bob = await register(client, "bob") + chat = await make_chat(client, alice, bob) + for text in ("", " ", "\n\t"): + response = await client.post( + f"/api/v1/chats/{chat['id']}/messages", + json={"text": text}, + headers=keyed(alice), + ) + assert_error(response, 400) + + +async def test_long_message_rejected(client: AsyncClient) -> None: + alice = await register(client, "alice") + chat = await make_chat(client, alice) + response = await client.post( + f"/api/v1/chats/{chat['id']}/messages", + json={"text": "я" * 4001}, + headers=keyed(alice), + ) + assert_error(response, 400) + + +async def test_sender_id_not_accepted(client: AsyncClient) -> None: + alice = await register(client, "alice") + bob = await register(client, "bob") + chat = await make_chat(client, alice, bob) + response = await client.post( + f"/api/v1/chats/{chat['id']}/messages", + json={"text": "подмена", "sender_id": bob["id"]}, + headers=keyed(alice), + ) + assert_error(response, 400) + + +@pytest.mark.parametrize( + "body", + [ + {"title": "Чат", "member_ids": []}, + {"title": "", "member_ids": None}, + {"title": " ", "member_ids": None}, + {"title": "Чат", "member_ids": ["не-uuid"]}, + {"title": "Чат", "member_ids": [1]}, + {"title": "Чат", "member_ids": "не-список"}, + ], +) +async def test_invalid_chat_creation(client: AsyncClient, body: dict) -> None: + alice = await register(client, "alice") + payload = dict(body) + if payload["member_ids"] is None: + payload["member_ids"] = [alice["id"]] + response = await client.post("/api/v1/chats", json=payload, headers=actor(alice)) + assert_error(response, 400) + + +async def test_creator_must_be_member(client: AsyncClient) -> None: + alice = await register(client, "alice") + bob = await register(client, "bob") + response = await client.post( + "/api/v1/chats", + json={"title": "Чат", "member_ids": [bob["id"]]}, + headers=actor(alice), + ) + assert_error(response, 400) + + +async def test_unknown_member_rejected(client: AsyncClient) -> None: + alice = await register(client, "alice") + response = await client.post( + "/api/v1/chats", + json={"title": "Чат", "member_ids": [alice["id"], str(uuid4())]}, + headers=actor(alice), + ) + assert_error(response, 400) + + +async def test_bad_chat_id_in_path(client: AsyncClient) -> None: + alice = await register(client, "alice") + assert_error(await client.get("/api/v1/chats/не-uuid", headers=actor(alice)), 400) diff --git a/student-tests/api/test_operability.py b/student-tests/api/test_operability.py new file mode 100644 index 00000000..91adaeb4 --- /dev/null +++ b/student-tests/api/test_operability.py @@ -0,0 +1,144 @@ +import json +import time +from uuid import uuid4 + +from httpx import ASGITransport, AsyncClient + +from conftest import PASSWORD, actor, keyed, make_chat, register, running_app, split_settings +from messenger.main import create_app + +MARKER_TEXT = "маркер-который-не-должен-попасть-в-логи-9f3a" + + +async def test_health_needs_no_identity(client: AsyncClient) -> None: + for path in ("/health/live", "/health/ready"): + response = await client.get(path) + assert response.status_code == 200 + assert response.json() == {"status": "ok"} + assert response.headers["X-Request-Id"] + + +async def test_every_response_carries_request_id(client: AsyncClient) -> None: + alice = await register(client, "alice") + probes = [ + await client.get("/health/live"), + await client.get("/api/v1/users/me", headers=actor(alice)), + await client.get("/api/v1/users/me"), + await client.get(f"/api/v1/chats/{uuid4()}", headers=actor(alice)), + await client.get("/api/v1/unknown-path"), + ] + assert all(probe.headers.get("X-Request-Id") for probe in probes) + + +async def test_client_request_id_is_preserved(client: AsyncClient) -> None: + response = await client.get("/health/live", headers={"X-Request-Id": "trace-42"}) + assert response.headers["X-Request-Id"] == "trace-42" + + +async def test_unknown_public_route_is_404(client: AsyncClient) -> None: + response = await client.get("/nothing-here") + assert response.status_code == 404 + assert response.json()["error"]["code"] == "not_found" + + +async def test_unknown_protected_route_needs_identity(client: AsyncClient) -> None: + anonymous = await client.get("/api/v1/unknown-path") + assert anonymous.status_code == 401 + assert set(anonymous.json()["error"]) == {"code", "message", "request_id"} + alice = await register(client, "alice") + identified = await client.get("/api/v1/unknown-path", headers=actor(alice)) + assert identified.status_code == 404 + + +async def test_method_not_allowed_is_hidden(client: AsyncClient) -> None: + response = await client.delete("/health/live") + assert response.status_code == 404 + assert response.json()["error"]["code"] == "not_found" + + +async def test_logs_do_not_contain_message_text(capsys) -> None: + app = create_app(**split_settings(log_level="INFO")) + async with ( + app.router.lifespan_context(app), + AsyncClient(transport=ASGITransport(app=app), base_url="http://testserver") as http, + ): + alice = await register(http, "alice") + chat = await make_chat(http, alice) + response = await http.post( + f"/api/v1/chats/{chat['id']}/messages", + json={"text": MARKER_TEXT}, + headers=keyed(alice), + ) + assert response.status_code == 201 + captured = capsys.readouterr() + assert MARKER_TEXT not in captured.out + assert MARKER_TEXT not in captured.err + assert alice["display_name"] not in captured.out + assert any("request.handled" in line for line in captured.out.splitlines()) + + +async def test_log_lines_are_json(capsys) -> None: + app = create_app(**split_settings(log_level="INFO")) + async with ( + app.router.lifespan_context(app), + AsyncClient(transport=ASGITransport(app=app), base_url="http://testserver") as http, + ): + await http.get("/health/live") + for line in capsys.readouterr().out.splitlines(): + record = json.loads(line) + assert {"ts", "level", "event"} <= set(record) + + +async def test_body_limit_returns_413() -> None: + app = create_app(**split_settings(max_body_bytes=4096)) + async with ( + app.router.lifespan_context(app), + AsyncClient(transport=ASGITransport(app=app), base_url="http://testserver") as http, + ): + response = await http.post("/api/v1/users", content=b"x" * 5000) + assert response.status_code == 413 + assert response.json()["error"]["code"] == "payload_too_large" + + +async def test_ready_turns_unavailable_without_database() -> None: + async with running_app(database_url="postgresql://u:p@127.0.0.1:1/none") as http: + started = time.perf_counter() + ready = await http.get("/health/ready") + live = await http.get("/health/live") + elapsed = time.perf_counter() - started + assert ready.status_code == 503 + assert ready.json()["error"]["code"] == "unavailable" + assert live.status_code == 200 + assert elapsed < 2 + + +async def test_protected_request_is_503_not_401_without_database() -> None: + async with running_app(database_url="postgresql://u:p@127.0.0.1:1/none") as http: + response = await http.get( + "/api/v1/users/me", headers={"Authorization": "Bearer " + "x" * 43} + ) + assert response.status_code == 503 + assert response.json()["error"]["code"] == "unavailable" + assert response.json()["error"]["request_id"] == response.headers["X-Request-Id"] + + +async def test_state_survives_new_process() -> None: + async with running_app() as http: + alice = await register(http, "alice") + chat = await make_chat(http, alice) + async with running_app() as http: + me = await http.get("/api/v1/users/me", headers=actor(alice)) + listed = await http.get("/api/v1/chats?limit=100", headers=actor(alice)) + assert me.json() == alice + assert [item["id"] for item in listed.json()["items"]] == [chat["id"]] + + +async def test_logs_do_not_contain_credentials(capsys) -> None: + async with running_app(log_level="INFO") as http: + alice = await register(http, "secret") + assert (await http.get("/api/v1/users/me", headers=actor(alice))).status_code == 200 + output = capsys.readouterr() + captured = output.out + output.err + assert PASSWORD not in captured + assert actor(alice)["Authorization"].split()[1] not in captured + assert "audit.session_opened" in captured diff --git a/student-tests/api/test_pagination.py b/student-tests/api/test_pagination.py new file mode 100644 index 00000000..980aa559 --- /dev/null +++ b/student-tests/api/test_pagination.py @@ -0,0 +1,155 @@ +from urllib.parse import quote + +import pytest +from httpx import AsyncClient + +from conftest import actor, keyed, make_chat, register + +TOTAL = 12 + + +async def fill(client: AsyncClient, chat: dict, sender: dict, count: int) -> list[str]: + texts = [] + for index in range(count): + text = f"сообщение {index:02d}" + response = await client.post( + f"/api/v1/chats/{chat['id']}/messages", + json={"text": text}, + headers=keyed(sender), + ) + assert response.status_code == 201, response.text + texts.append(text) + return texts + + +async def walk(client: AsyncClient, chat: dict, reader: dict, limits: list[int]) -> list[str]: + collected: list[str] = [] + cursor: str | None = None + step = 0 + while True: + limit = limits[min(step, len(limits) - 1)] + query = f"?limit={limit}" + if cursor: + query += f"&cursor={quote(cursor, safe='')}" + response = await client.get( + f"/api/v1/chats/{chat['id']}/messages{query}", headers=actor(reader) + ) + assert response.status_code == 200, response.text + page = response.json() + assert len(page["items"]) <= limit + collected.extend(item["text"] for item in page["items"]) + cursor = page["next_cursor"] + step += 1 + if cursor is None: + return collected + + +async def test_full_walk_has_no_gaps_or_repeats(client: AsyncClient) -> None: + alice = await register(client, "alice") + bob = await register(client, "bob") + chat = await make_chat(client, alice, bob) + expected = await fill(client, chat, alice, TOTAL) + assert await walk(client, chat, bob, [5]) == expected + + +async def test_limit_may_change_between_pages(client: AsyncClient) -> None: + alice = await register(client, "alice") + chat = await make_chat(client, alice) + expected = await fill(client, chat, alice, TOTAL) + assert await walk(client, chat, alice, [1, 7, 3, 100]) == expected + + +async def test_last_page_closes_history(client: AsyncClient) -> None: + alice = await register(client, "alice") + chat = await make_chat(client, alice) + await fill(client, chat, alice, 2) + first = ( + await client.get(f"/api/v1/chats/{chat['id']}/messages?limit=2", headers=actor(alice)) + ).json() + assert len(first["items"]) == 2 + assert first["next_cursor"] is None + + +async def test_cursor_is_shared_between_members(client: AsyncClient) -> None: + alice = await register(client, "alice") + bob = await register(client, "bob") + chat = await make_chat(client, alice, bob) + await fill(client, chat, alice, 3) + page = ( + await client.get(f"/api/v1/chats/{chat['id']}/messages?limit=1", headers=actor(alice)) + ).json() + cursor = quote(page["next_cursor"], safe="") + response = await client.get( + f"/api/v1/chats/{chat['id']}/messages?limit=1&cursor={cursor}", headers=actor(bob) + ) + assert response.status_code == 200 + + +async def test_cursor_from_another_chat_is_rejected(client: AsyncClient) -> None: + alice = await register(client, "alice") + first = await make_chat(client, alice) + second = await make_chat(client, alice) + await fill(client, first, alice, 3) + await fill(client, second, alice, 3) + page = ( + await client.get(f"/api/v1/chats/{first['id']}/messages?limit=1", headers=actor(alice)) + ).json() + cursor = quote(page["next_cursor"], safe="") + response = await client.get( + f"/api/v1/chats/{second['id']}/messages?limit=1&cursor={cursor}", headers=actor(alice) + ) + assert response.status_code == 400 + assert response.json()["error"]["code"] == "invalid_request" + + +@pytest.mark.parametrize("cursor", ["garbage", "1.2.3.4", "%20"]) +async def test_broken_cursor_is_rejected(client: AsyncClient, cursor: str) -> None: + alice = await register(client, "alice") + chat = await make_chat(client, alice) + response = await client.get( + f"/api/v1/chats/{chat['id']}/messages?cursor={cursor}", headers=actor(alice) + ) + assert response.status_code == 400 + + +@pytest.mark.parametrize("limit", ["0", "-1", "101", "abc", "1.5", ""]) +async def test_limit_out_of_range(client: AsyncClient, limit: str) -> None: + alice = await register(client, "alice") + chat = await make_chat(client, alice) + for path in (f"/api/v1/chats/{chat['id']}/messages", "/api/v1/chats"): + response = await client.get(f"{path}?limit={limit}", headers=actor(alice)) + assert response.status_code == 400, f"{path}?limit={limit} → {response.status_code}" + + +async def test_default_limit_is_50(client: AsyncClient) -> None: + alice = await register(client, "alice") + chat = await make_chat(client, alice) + await fill(client, chat, alice, 51) + page = (await client.get(f"/api/v1/chats/{chat['id']}/messages", headers=actor(alice))).json() + assert len(page["items"]) == 50 + assert page["next_cursor"] is not None + + +async def test_unknown_query_parameter_is_ignored(client: AsyncClient) -> None: + alice = await register(client, "alice") + chat = await make_chat(client, alice) + await fill(client, chat, alice, 2) + for path in (f"/api/v1/chats/{chat['id']}/messages", "/api/v1/chats"): + response = await client.get(f"{path}?limit=1&unknown=x", headers=actor(alice)) + assert response.status_code == 200, response.text + assert len(response.json()["items"]) == 1 + + +async def test_chat_list_pagination(client: AsyncClient) -> None: + alice = await register(client, "alice") + created = [(await make_chat(client, alice))["id"] for _ in range(5)] + collected: list[str] = [] + cursor: str | None = None + while True: + query = "?limit=2" + (f"&cursor={quote(cursor, safe='')}" if cursor else "") + page = (await client.get(f"/api/v1/chats{query}", headers=actor(alice))).json() + collected.extend(item["id"] for item in page["items"]) + cursor = page["next_cursor"] + if cursor is None: + break + assert collected == created diff --git a/student-tests/api/test_pipeline.py b/student-tests/api/test_pipeline.py new file mode 100644 index 00000000..811abcfd --- /dev/null +++ b/student-tests/api/test_pipeline.py @@ -0,0 +1,303 @@ +import asyncio +import json +import re +from typing import Any +from uuid import uuid4 + +import asyncpg +import pytest +from httpx import AsyncClient +from test_attachments import storage, trio, upload + +from conftest import DATABASE, actor +from messenger.attachments.application.pipeline import TRANSIENT_ERROR +from messenger.attachments.application.ports import Rendition +from messenger.attachments.domain.errors import ImageTooDemanding +from messenger.attachments.domain.jobs import Job, Outcome, RetryPolicy +from messenger.attachments.entrypoints.schemas import JobMessage +from messenger.shared.application.errors import Unavailable +from pictures import SAMPLE +from pipeline import pipeline_of + + +async def rows(sql: str, *args: Any) -> list[dict[str, Any]]: + connection = await asyncpg.connect(DATABASE["url"]) + try: + return [dict(row) for row in await connection.fetch(sql, *args)] + finally: + await connection.close() + + +async def accepted(http: AsyncClient) -> tuple[dict[str, Any], dict[str, Any], str]: + alice, bob, _, chat = await trio(http) + response = await upload(http, alice, chat) + assert response.status_code == 202, response.text + return alice, bob, response.json()["id"] + + +async def state(attachment_id: str) -> dict[str, Any]: + (row,) = await rows( + "SELECT a.status, a.error_code, a.published_version, a.staging_key, a.original_key, " + "j.status AS job_status, j.attempt, j.claims FROM attachments a " + "JOIN jobs j ON j.attachment_id = a.id WHERE a.id = $1", + attachment_id, + ) + return row + + +async def variants(attachment_id: str) -> list[dict[str, Any]]: + return await rows( + "SELECT pipeline_version, name, sha256, size_bytes, object_key FROM attachment_variants " + "WHERE attachment_id = $1 ORDER BY name", + attachment_id, + ) + + +async def events(attachment_id: str) -> list[str]: + found = await rows( + "SELECT kind FROM job_events WHERE attachment_id = $1 ORDER BY id", attachment_id + ) + return [row["kind"] for row in found] + + +async def job_of(attachment_id: str) -> Job: + (row,) = await rows( + "SELECT payload::text AS body FROM outbox o JOIN jobs j USING (job_id) " + "WHERE j.attachment_id = $1 ORDER BY o.id LIMIT 1", + attachment_id, + ) + return JobMessage.model_validate_json(row["body"]).job() + + +async def test_job_is_processed_once_and_staging_is_cleaned(client: AsyncClient) -> None: + _, bob, attachment_id = await accepted(client) + pipeline = await pipeline_of(client) + outcomes = await pipeline.drain() + assert Outcome.DONE in outcomes + done = await state(attachment_id) + assert (done["status"], done["job_status"], done["published_version"]) == ("ready", "done", 1) + assert done["original_key"] == f"originals/{attachment_id}" + assert done["staging_key"] == f"staging/{attachment_id}" + pattern = rf"variants/{attachment_id}/1/(crop|optimized|thumbnail)-[0-9a-f]{{16}}\.(jpg|webp)" + found = [re.fullmatch(pattern, row["object_key"]) for row in await variants(attachment_id)] + assert [match.groups() for match in found if match] == [ + ("crop", "jpg"), + ("optimized", "webp"), + ("thumbnail", "jpg"), + ] + assert await pipeline.clean() >= 1 + assert (await state(attachment_id))["staging_key"] is None + objects = (await storage(client)).objects + assert f"staging/{attachment_id}" not in objects + assert objects[f"originals/{attachment_id}"][0] == SAMPLE + content = await client.get(f"/api/v1/attachments/{attachment_id}/content", headers=actor(bob)) + assert content.content == SAMPLE + assert await events(attachment_id) == ["enqueued", "claimed", "done"] + + +async def test_message_is_a_contract_job(client: AsyncClient) -> None: + _, _, attachment_id = await accepted(client) + pipeline = await pipeline_of(client) + await pipeline.publish() + mine = [ + json.loads(body) + for body in pipeline.publisher.bodies + if json.loads(body)["attachment_id"] == attachment_id + ] + assert len(mine) == 1 + assert set(mine[0]) == { + "schema_version", + "job_id", + "type", + "attachment_id", + "pipeline_version", + "attempt", + "created_at", + } + assert (mine[0]["pipeline_version"], mine[0]["attempt"]) == (1, 1) + assert await pipeline.publish() == [] + + +async def test_redelivery_after_ready_changes_nothing(client: AsyncClient) -> None: + _, _, attachment_id = await accepted(client) + pipeline = await pipeline_of(client) + await pipeline.drain() + before, job = await variants(attachment_id), await job_of(attachment_id) + assert await pipeline.process(job) is Outcome.DUPLICATE + assert await variants(attachment_id) == before + assert (await state(attachment_id))["status"] == "ready" + assert (await events(attachment_id))[-1] == "duplicate_noop" + + +async def test_two_copies_in_flight_give_one_result(client: AsyncClient) -> None: + _, _, attachment_id = await accepted(client) + pipeline = await pipeline_of(client) + await pipeline.publish() + job = await job_of(attachment_id) + outcomes = await asyncio.gather(pipeline.process(job), pipeline.process(job)) + assert sorted(outcomes) == sorted([Outcome.DONE, Outcome.DUPLICATE]) + assert len(await variants(attachment_id)) == 3 + assert (await state(attachment_id))["claims"] == 1 + + +async def test_partial_variant_set_is_never_published(client: AsyncClient) -> None: + _, _, attachment_id = await accepted(client) + pipeline = await pipeline_of(client) + await pipeline.publish() + store = await storage(client) + + async def refuse(key: str) -> None: + if "/optimized-" in key: + raise Unavailable("S3 недоступно") + + store.after_put = refuse + try: + assert await pipeline.process(await job_of(attachment_id)) is Outcome.RESCHEDULED + finally: + store.after_put = None + assert (await state(attachment_id))["status"] == "queued" + assert await variants(attachment_id) == [] + + +async def test_transient_error_reschedules_with_the_same_job(client: AsyncClient) -> None: + _, _, attachment_id = await accepted(client) + pipeline = await pipeline_of(client) + await pipeline.publish() + job = await job_of(attachment_id) + store = await storage(client) + store.failing = True + try: + assert await pipeline.process(job) is Outcome.RESCHEDULED + finally: + store.failing = False + queued = await state(attachment_id) + assert (queued["status"], queued["job_status"], queued["attempt"]) == ("queued", "queued", 2) + (retry,) = [j for j in await pipeline.publish() if str(j.attachment_id) == attachment_id] + assert (retry.id, retry.attempt) == (job.id, 2) + assert await pipeline.process(retry) is Outcome.DONE + assert (await state(attachment_id))["status"] == "ready" + + +async def test_attempts_are_bounded(client: AsyncClient) -> None: + alice, _, attachment_id = await accepted(client) + pipeline = await pipeline_of(client) + pipeline.faults = FailingFaults() + await pipeline.drain() + failed = await state(attachment_id) + assert (failed["status"], failed["error_code"], failed["job_status"]) == ( + "failed", + "retry_exhausted", + "failed", + ) + assert failed["attempt"] == pipeline.policy.max_attempts + assert (await events(attachment_id)).count("retry_scheduled") == 4 + content = await client.get(f"/api/v1/attachments/{attachment_id}/content", headers=actor(alice)) + assert content.status_code == 409 + + +async def test_undecodable_image_fails_without_retries(client: AsyncClient) -> None: + _, _, attachment_id = await accepted(client) + (await storage(client)).objects[f"staging/{attachment_id}"] = (b"not a png", "image/png") + pipeline = await pipeline_of(client) + await pipeline.drain() + failed = await state(attachment_id) + assert (failed["status"], failed["error_code"], failed["attempt"]) == ( + "failed", + "invalid_image", + 1, + ) + + +class DemandingRenderer: + async def render(self, data: bytes, pipeline_version: int) -> list[Rendition]: + raise ImageTooDemanding("рендер дольше допустимого времени") + + +@pytest.mark.parametrize("cause", ["limit", "missing"]) +async def test_limits_and_lost_original_fail_without_retries( + client: AsyncClient, cause: str +) -> None: + _, _, attachment_id = await accepted(client) + pipeline = await pipeline_of(client) + if cause == "limit": + pipeline.renderer = DemandingRenderer() + else: + del (await storage(client)).objects[f"staging/{attachment_id}"] + await pipeline.drain() + failed = await state(attachment_id) + assert (failed["status"], failed["error_code"], failed["attempt"]) == ( + "failed", + "processing_failed", + 1, + ) + + +async def test_expired_lease_is_recovered_and_old_holder_is_fenced(client: AsyncClient) -> None: + _, _, attachment_id = await accepted(client) + pipeline = await pipeline_of(client) + await pipeline.publish() + job = await job_of(attachment_id) + uow = pipeline.uow() + async with uow.transaction(): + claim = await uow.jobs.claim(job, uuid4(), pipeline.lease) + await uow.commit() + assert claim is not None + assert (await state(attachment_id))["status"] == "processing" + await rows( + "UPDATE jobs SET lease_until = now() - interval '1 second' WHERE attachment_id = $1", + attachment_id, + ) + assert await pipeline.recover() >= 1 + assert (await state(attachment_id))["status"] == "queued" + late = pipeline.uow() + async with late.transaction(): + outcome = await late.jobs.complete(claim.lease, [], f"originals/{attachment_id}") + await late.commit() + assert outcome is Outcome.FENCED + assert await variants(attachment_id) == [] + await pipeline.drain() + recovered = await state(attachment_id) + assert (recovered["status"], recovered["attempt"]) == ("ready", 2) + assert {"lease_expired", "fenced"} <= set(await events(attachment_id)) + + +async def test_unpublished_rows_stay_when_the_broker_refuses(client: AsyncClient) -> None: + _, _, attachment_id = await accepted(client) + pipeline = await pipeline_of(client) + pipeline.publisher.failing = True + with pytest.raises(Unavailable): + await pipeline.publish() + pending = await rows( + "SELECT o.published_at FROM outbox o JOIN jobs j USING (job_id) WHERE j.attachment_id = $1", + attachment_id, + ) + assert pending == [{"published_at": None}] + pipeline.publisher.failing = False + assert any(str(job.attachment_id) == attachment_id for job in await pipeline.publish()) + + +async def test_backoff_delays_the_next_attempt(client: AsyncClient) -> None: + _, _, attachment_id = await accepted(client) + pipeline = await pipeline_of(client, policy=RetryPolicy(5, 30, 60)) + pipeline.faults = FailingFaults() + await pipeline.drain() + queued = await state(attachment_id) + assert (queued["status"], queued["attempt"]) == ("queued", 2) + (row,) = await rows( + "SELECT extract(epoch FROM next_attempt_at - now()) AS wait FROM jobs " + "WHERE attachment_id = $1", + attachment_id, + ) + assert 20 < row["wait"] <= 22.5 + (event,) = await rows( + "SELECT (detail->>'delay_seconds')::float8 AS delay FROM job_events " + "WHERE attachment_id = $1 AND kind = 'retry_scheduled'", + attachment_id, + ) + assert event["delay"] == 22.5 + + +class FailingFaults: + async def reach(self, point: str) -> None: + if point == TRANSIENT_ERROR: + raise Unavailable(point) diff --git a/student-tests/api/test_pool.py b/student-tests/api/test_pool.py new file mode 100644 index 00000000..e27586f6 --- /dev/null +++ b/student-tests/api/test_pool.py @@ -0,0 +1,27 @@ +import time + +from conftest import actor, register, resolve, running_app +from messenger.shared.infrastructure.postgres import Database + + +async def test_exhausted_pool_answers_503_in_bounded_time() -> None: + async with running_app(db_pool_max_size=1, db_acquire_timeout_seconds=0.3) as http: + alice = await register(http, "alice") + database = await resolve(http, Database) + async with database.connection(): + started = time.perf_counter() + starved = await http.get("/api/v1/users/me", headers=actor(alice)) + elapsed = time.perf_counter() - started + recovered = await http.get("/api/v1/users/me", headers=actor(alice)) + assert starved.status_code == 503 + assert starved.json()["error"]["code"] == "unavailable" + assert 0.25 < elapsed < 1 + assert recovered.json() == alice + + +async def test_statement_timeout_is_set_on_connections() -> None: + async with running_app(db_command_timeout_seconds=1.5) as http: + database = await resolve(http, Database) + async with database.connection() as connection: + assert await connection.fetchval("SHOW statement_timeout") == "1500ms" + assert await connection.fetchval("SHOW lock_timeout") == "1500ms" diff --git a/student-tests/api/test_recovery.py b/student-tests/api/test_recovery.py new file mode 100644 index 00000000..e6c2a951 --- /dev/null +++ b/student-tests/api/test_recovery.py @@ -0,0 +1,307 @@ +import asyncio +import json +import subprocess +import sys +from datetime import datetime, timedelta +from uuid import UUID, uuid4 + +import pytest +from httpx import AsyncClient +from test_attachments import storage +from test_pipeline import FailingFaults, accepted, events, job_of, rows, state, variants + +from conftest import DATABASE +from fakes import NullLogger +from messenger.admin import serve +from messenger.attachments.application.pipeline import ( + AFTER_CLAIM, + AFTER_COMMIT_BEFORE_ACK, + AFTER_COMMIT_BEFORE_PUBLISH, + AFTER_PUBLISH_BEFORE_CONFIRM, + AFTER_S3_WRITE, + STALL_AFTER_CLAIM, + TRANSIENT_ERROR, +) +from messenger.attachments.application.ports import Rendition +from messenger.attachments.domain.jobs import Outcome, ReplayResult +from messenger.attachments.infrastructure.faults import EnvironmentFaults +from messenger.shared.application.errors import Unavailable +from pipeline import InlinePipeline, InlineRenderer, pipeline_of + +WORKER_POINTS = [ + AFTER_CLAIM, + STALL_AFTER_CLAIM, + TRANSIENT_ERROR, + AFTER_S3_WRITE, + AFTER_COMMIT_BEFORE_ACK, +] + + +class Crash(BaseException): + pass + + +class RecordingFaults: + __slots__ = ("crash_at", "reached") + + def __init__(self, crash_at: str = "") -> None: + self.crash_at = crash_at + self.reached: list[str] = [] + + async def reach(self, point: str) -> None: + self.reached.append(point) + if point == self.crash_at: + raise Crash(point) + + +class SlowRenderer: + __slots__ = ("seconds",) + + def __init__(self, seconds: float) -> None: + self.seconds = seconds + + async def render(self, data: bytes, pipeline_version: int) -> list[Rendition]: + await asyncio.sleep(self.seconds) + return await InlineRenderer().render(data, pipeline_version) + + +class ExpiringFaults: + __slots__ = ("attachment_id", "pipeline") + + def __init__(self, pipeline: InlinePipeline, attachment_id: str) -> None: + self.pipeline = pipeline + self.attachment_id = attachment_id + + async def reach(self, point: str) -> None: + if point == STALL_AFTER_CLAIM: + await rows( + "UPDATE jobs SET lease_until = now() - interval '1 second' " + "WHERE attachment_id = $1", + self.attachment_id, + ) + await self.pipeline.recover() + + +async def outbox(attachment_id: str) -> list[bool]: + found = await rows( + "SELECT o.published_at IS NOT NULL AS published FROM outbox o JOIN jobs j USING (job_id) " + "WHERE j.attachment_id = $1 ORDER BY o.id", + attachment_id, + ) + return [row["published"] for row in found] + + +def stored(objects: dict[str, tuple[bytes, str]], attachment_id: str) -> list[str]: + return sorted(key for key in objects if key.startswith(f"variants/{attachment_id}/")) + + +async def test_fault_points_are_reached_in_order(client: AsyncClient) -> None: + _, _, attachment_id = await accepted(client) + pipeline = await pipeline_of(client) + faults = RecordingFaults() + pipeline.faults = faults + published = await pipeline.publish() + assert faults.reached[:2] == [AFTER_COMMIT_BEFORE_PUBLISH, AFTER_PUBLISH_BEFORE_CONFIRM] + faults.reached.clear() + (job,) = [job for job in published if str(job.attachment_id) == attachment_id] + assert await pipeline.process(job) is Outcome.DONE + assert faults.reached == WORKER_POINTS + + +@pytest.mark.parametrize("point", [AFTER_COMMIT_BEFORE_PUBLISH, AFTER_PUBLISH_BEFORE_CONFIRM]) +async def test_relay_crash_leaves_the_row_for_the_next_relay( + client: AsyncClient, point: str +) -> None: + pipeline = await pipeline_of(client) + await pipeline.drain() + _, _, attachment_id = await accepted(client) + pipeline.faults = RecordingFaults(point) + with pytest.raises(Crash): + await pipeline.publish() + assert await outbox(attachment_id) == [False] + sent = [body for body in pipeline.publisher.bodies if attachment_id.encode() in body] + assert len(sent) == (point == AFTER_PUBLISH_BEFORE_CONFIRM) + pipeline.faults = RecordingFaults() + outcomes = await pipeline.drain() + assert Outcome.DONE in outcomes + assert await outbox(attachment_id) == [True] + assert (await state(attachment_id))["status"] == "ready" + + +async def test_crash_after_s3_write_is_finished_by_the_next_attempt(client: AsyncClient) -> None: + _, _, attachment_id = await accepted(client) + pipeline = await pipeline_of(client, lease=timedelta(seconds=60)) + await pipeline.publish() + job = await job_of(attachment_id) + pipeline.faults = RecordingFaults(AFTER_S3_WRITE) + with pytest.raises(Crash): + await pipeline.process(job) + objects = (await storage(client)).objects + written = stored(objects, attachment_id) + assert len(written) == 3 + assert await variants(attachment_id) == [] + assert (await state(attachment_id))["status"] == "processing" + await rows( + "UPDATE jobs SET lease_until = now() - interval '1 second' WHERE attachment_id = $1", + attachment_id, + ) + pipeline.faults = RecordingFaults() + assert await pipeline.recover() >= 1 + await pipeline.drain() + done = await state(attachment_id) + assert (done["status"], done["attempt"]) == ("ready", 2) + assert stored(objects, attachment_id) == written + assert sorted(row["object_key"] for row in await variants(attachment_id)) == written + + +async def test_crash_after_commit_is_a_duplicate_on_redelivery(client: AsyncClient) -> None: + _, _, attachment_id = await accepted(client) + pipeline = await pipeline_of(client) + await pipeline.publish() + job = await job_of(attachment_id) + pipeline.faults = RecordingFaults(AFTER_COMMIT_BEFORE_ACK) + with pytest.raises(Crash): + await pipeline.process(job) + before = await variants(attachment_id) + assert (await state(attachment_id))["status"] == "ready" + pipeline.faults = RecordingFaults() + assert await pipeline.process(job) is Outcome.DUPLICATE + assert await variants(attachment_id) == before + + +async def test_long_render_keeps_its_lease(client: AsyncClient) -> None: + _, _, attachment_id = await accepted(client) + pipeline = await pipeline_of(client, lease=timedelta(seconds=1)) + pipeline.renderer = SlowRenderer(2.5) + await pipeline.publish() + job = await job_of(attachment_id) + + async def sweep() -> None: + for _ in range(4): + await asyncio.sleep(0.6) + await pipeline.recover() + + outcome, _ = await asyncio.gather(pipeline.process(job), sweep()) + assert outcome is Outcome.DONE + done = await state(attachment_id) + assert (done["status"], done["attempt"], done["claims"]) == ("ready", 1, 1) + assert "lease_expired" not in await events(attachment_id) + + +async def test_lost_lease_stops_before_side_effects(client: AsyncClient) -> None: + _, _, attachment_id = await accepted(client) + pipeline = await pipeline_of(client) + await pipeline.publish() + job = await job_of(attachment_id) + pipeline.faults = ExpiringFaults(pipeline, attachment_id) + assert await pipeline.process(job) is Outcome.FENCED + objects = (await storage(client)).objects + assert stored(objects, attachment_id) == [] + assert f"originals/{attachment_id}" not in objects + assert (await state(attachment_id))["status"] == "queued" + assert (await events(attachment_id))[-2:] == ["lease_expired", "fenced"] + pipeline.faults = RecordingFaults() + await pipeline.drain() + assert (await state(attachment_id))["status"] == "ready" + + +async def test_environment_faults_stall_once_and_fail_transiently() -> None: + stall = EnvironmentFaults(STALL_AFTER_CLAIM, NullLogger(), stall_seconds=0.3) + started = asyncio.get_running_loop().time() + await stall.reach(AFTER_CLAIM) + await stall.reach(STALL_AFTER_CLAIM) + await stall.reach(STALL_AFTER_CLAIM) + assert 0.3 <= asyncio.get_running_loop().time() - started < 0.6 + with pytest.raises(Unavailable): + await EnvironmentFaults(TRANSIENT_ERROR, NullLogger()).reach(TRANSIENT_ERROR) + + +def test_environment_fault_crashes_the_process_with_137() -> None: + script = ( + "import asyncio\n" + "from fakes import NullLogger\n" + "from messenger.attachments.infrastructure.faults import EnvironmentFaults\n" + "asyncio.run(EnvironmentFaults('after_commit_before_ack', NullLogger())" + ".reach('after_commit_before_ack'))\n" + "print('survived')\n" + ) + done = subprocess.run( + [sys.executable, "-c", script], + capture_output=True, + text=True, + check=False, + env={"PYTHONPATH": ":".join(sys.path)}, + ) + assert (done.returncode, done.stdout) == (137, "") + + +async def failed(client: AsyncClient) -> tuple[InlinePipeline, str]: + _, _, attachment_id = await accepted(client) + pipeline = await pipeline_of(client) + pipeline.faults = FailingFaults() + await pipeline.drain() + pipeline.faults = RecordingFaults() + assert (await state(attachment_id))["error_code"] == "retry_exhausted" + return pipeline, attachment_id + + +async def test_replay_of_failed_reaches_ready_once(client: AsyncClient) -> None: + pipeline, attachment_id = await failed(client) + job = await job_of(attachment_id) + assert await pipeline.replay(UUID(attachment_id)) is ReplayResult.REPLAYED + queued = await state(attachment_id) + assert (queued["status"], queued["job_status"], queued["attempt"], queued["error_code"]) == ( + "queued", + "queued", + 1, + None, + ) + assert await outbox(attachment_id) == [True] * 5 + [False] + await pipeline.drain() + done = await state(attachment_id) + assert (done["status"], done["attempt"]) == ("ready", 1) + assert (await job_of(attachment_id)).id == job.id + before = await variants(attachment_id) + assert await pipeline.replay(UUID(attachment_id)) is ReplayResult.NOOP + assert await outbox(attachment_id) == [True] * 6 + assert await variants(attachment_id) == before + assert (await events(attachment_id))[-1] == "replay_noop" + assert "replayed" in await events(attachment_id) + assert await pipeline.replay(uuid4()) is ReplayResult.NOT_FOUND + + +async def test_failed_staging_is_kept_for_the_retention_period(client: AsyncClient) -> None: + pipeline, attachment_id = await failed(client) + _, _, waiting = await accepted(client) + objects = (await storage(client)).objects + await pipeline.clean(timedelta(days=1)) + assert (await state(attachment_id))["staging_key"] == f"staging/{attachment_id}" + assert f"staging/{attachment_id}" in objects + await pipeline.clean(timedelta(0)) + assert (await state(attachment_id))["staging_key"] is None + assert f"staging/{attachment_id}" not in objects + assert (await state(waiting))["staging_key"] == f"staging/{waiting}" + assert f"staging/{waiting}" in objects + assert await pipeline.replay(UUID(attachment_id)) is ReplayResult.EXPIRED + assert (await state(attachment_id))["status"] == "failed" + + +async def test_operator_cli_lists_failed_jobs_and_replays( + client: AsyncClient, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + _, attachment_id = await failed(client) + monkeypatch.setenv("DATABASE_URL", DATABASE["url"]) + monkeypatch.setenv("LOG_LEVEL", "WARNING") + assert await serve(["failed", "--limit", "500"]) == 0 + registry = json.loads(capsys.readouterr().out.splitlines()[-1])["failed"] + (mine,) = [row for row in registry if row["attachment_id"] == attachment_id] + assert datetime.fromisoformat(mine["failed_at"]).utcoffset() == timedelta(0) + assert (mine["error_code"], mine["attempt"], mine["last_error"]) == ( + "retry_exhausted", + 5, + "Unavailable", + ) + assert await serve(["replay", "--attachment-id", attachment_id, "--operator", "alice"]) == 0 + report = json.loads(capsys.readouterr().out.splitlines()[-1]) + assert report == {"result": "replayed", "attachment_id": attachment_id, "operator": "alice"} + assert await serve(["replay", "--attachment-id", str(uuid4())]) == 1 diff --git a/student-tests/api/test_sessions.py b/student-tests/api/test_sessions.py new file mode 100644 index 00000000..6397ad1f --- /dev/null +++ b/student-tests/api/test_sessions.py @@ -0,0 +1,172 @@ +import asyncio +from datetime import datetime, timedelta + +import pytest +from httpx import AsyncClient + +from conftest import ( + PASSWORD, + actor, + basic, + bearer, + login, + make_chat, + new_user, + register, + running_app, + scalar, +) + +BASIC = 'Basic realm="messenger", charset="UTF-8"' +BEARER = 'Bearer realm="messenger"' + + +def challenged(response, challenge: str) -> None: + assert response.status_code == 401, response.text + assert response.headers["WWW-Authenticate"] == challenge + assert response.json()["error"]["code"] == "unauthorized" + assert response.json()["error"]["request_id"] == response.headers["X-Request-Id"] + + +async def test_login_returns_session_for_registered_user(client: AsyncClient) -> None: + body = new_user("alice") + user = (await client.post("/api/v1/users", json=body)).json() + session = await login(client, body["username"]) + assert set(session) == {"token", "token_type", "expires_at", "user"} + assert session["token_type"] == "Bearer" + assert session["user"] == user + assert len(session["token"]) >= 43 + assert body["username"] not in session["token"] and user["id"] not in session["token"] + assert session["expires_at"].endswith("Z") + expires = datetime.fromisoformat(session["expires_at"]) + created = datetime.fromisoformat(user["created_at"]) + assert timedelta(seconds=3500) < expires - created < timedelta(seconds=3700) + + +async def test_registration_response_hides_password(client: AsyncClient) -> None: + body = new_user("alice") + response = await client.post("/api/v1/users", json=body) + assert set(response.json()) == {"id", "username", "display_name", "created_at"} + assert PASSWORD not in response.text + + +async def test_password_is_stored_as_argon2id(client: AsyncClient) -> None: + alice = await register(client, "alice") + stored = await scalar("SELECT password_hash FROM users WHERE id = $1::uuid", alice["id"]) + assert stored.startswith("$argon2id$v=19$m=19456,t=2,p=1$") + assert PASSWORD not in stored + + +async def test_session_stores_only_token_digest(client: AsyncClient) -> None: + alice = await register(client, "alice") + token = actor(alice)["Authorization"].split()[1] + sql = "SELECT count(*) FROM sessions WHERE token_hash = sha256(convert_to($1, 'UTF8'))" + assert await scalar(sql, token) == 1 + raw = "SELECT count(*) FROM sessions WHERE encode(token_hash, 'escape') = $1" + assert await scalar(raw, token) == 0 + + +async def test_wrong_password_and_unknown_user_look_the_same(client: AsyncClient) -> None: + alice = await register(client, "alice") + wrong = await client.post("/api/v1/auth/sessions", headers=basic(alice["username"], "x" * 20)) + unknown = await client.post("/api/v1/auth/sessions", headers=basic("u_ghost_000", PASSWORD)) + challenged(wrong, BASIC) + challenged(unknown, BASIC) + assert wrong.json()["error"]["message"] == unknown.json()["error"]["message"] + + +@pytest.mark.parametrize( + "headers", + [{}, {"Authorization": "Basic !!!"}, {"Authorization": "Bearer abc"}, {"X-User-Id": "x"}], +) +async def test_login_without_valid_basic(client: AsyncClient, headers: dict) -> None: + challenged(await client.post("/api/v1/auth/sessions", headers=headers), BASIC) + + +async def test_password_with_colon_logs_in(client: AsyncClient) -> None: + body = new_user("colon", password="пароль:с:двоеточием") + assert (await client.post("/api/v1/users", json=body)).status_code == 201 + assert (await login(client, body["username"], body["password"]))["token"] + + +async def test_login_with_body_is_rejected(client: AsyncClient) -> None: + body = new_user("body") + await client.post("/api/v1/users", json=body) + response = await client.post( + "/api/v1/auth/sessions", headers=basic(body["username"], PASSWORD), json={"x": 1} + ) + assert response.status_code == 400 + assert response.json()["error"]["code"] == "invalid_request" + + +async def test_basic_on_business_endpoint_gets_bearer_challenge(client: AsyncClient) -> None: + body = new_user("alice") + await client.post("/api/v1/users", json=body) + challenged( + await client.get("/api/v1/users/me", headers=basic(body["username"], PASSWORD)), BEARER + ) + + +async def test_user_id_header_does_not_override_owner(client: AsyncClient) -> None: + alice = await register(client, "alice") + eve = await register(client, "eve") + response = await client.get( + "/api/v1/users/me", headers={**actor(alice), "X-User-Id": eve["id"]} + ) + assert response.json() == alice + + +async def test_authentication_precedes_validation(client: AsyncClient) -> None: + response = await client.post( + "/api/v1/chats/not-a-uuid/messages", + content=b"{broken", + headers={"Content-Type": "application/json"}, + ) + challenged(response, BEARER) + + +async def test_logout_revokes_only_current_session(client: AsyncClient) -> None: + body = new_user("alice") + await client.post("/api/v1/users", json=body) + first = (await login(client, body["username"]))["token"] + second = (await login(client, body["username"]))["token"] + logout = await client.delete("/api/v1/auth/sessions/current", headers=bearer(first)) + assert logout.status_code == 204 + assert logout.content == b"" + assert logout.headers["X-Request-Id"] + challenged(await client.get("/api/v1/users/me", headers=bearer(first)), BEARER) + challenged(await client.delete("/api/v1/auth/sessions/current", headers=bearer(first)), BEARER) + assert (await client.get("/api/v1/users/me", headers=bearer(second))).status_code == 200 + + +async def test_short_ttl_session_expires() -> None: + async with running_app(session_ttl_seconds=1) as client: + alice = await register(client, "alice") + assert (await client.get("/api/v1/users/me", headers=actor(alice))).status_code == 200 + await asyncio.sleep(1.2) + challenged(await client.get("/api/v1/users/me", headers=actor(alice)), BEARER) + + +async def test_expired_sessions_are_purged_on_next_login() -> None: + async with running_app(session_ttl_seconds=1) as client: + body = new_user("purge") + user = (await client.post("/api/v1/users", json=body)).json() + await login(client, body["username"]) + await asyncio.sleep(1.2) + await login(client, body["username"]) + sql = "SELECT count(*) FROM sessions WHERE user_id = $1::uuid" + assert await scalar(sql, user["id"]) == 1 + + +async def test_chat_routes_require_bearer(client: AsyncClient) -> None: + alice = await register(client, "alice") + chat = await make_chat(client, alice) + for method, path in [ + ("GET", "/api/v1/chats"), + ("POST", "/api/v1/chats"), + ("GET", f"/api/v1/chats/{chat['id']}"), + ("GET", f"/api/v1/chats/{chat['id']}/messages"), + ("POST", f"/api/v1/chats/{chat['id']}/messages"), + ("DELETE", "/api/v1/auth/sessions/current"), + ]: + challenged(await client.request(method, path, headers={"X-User-Id": alice["id"]}), BEARER) diff --git a/student-tests/api/test_timeout.py b/student-tests/api/test_timeout.py new file mode 100644 index 00000000..9118d660 --- /dev/null +++ b/student-tests/api/test_timeout.py @@ -0,0 +1,97 @@ +import asyncio +from collections.abc import AsyncIterator, Callable +from contextlib import asynccontextmanager + +from httpx import AsyncClient + +from conftest import keyed, make_chat, register, resolve, running_app +from messenger.messaging.application import handlers +from messenger.messaging.domain.events import MessagePosted +from messenger.shared.application.messagebus import EventBus + +MARKER = "текст-который-не-должен-утечь-при-таймауте" + + +@asynccontextmanager +async def slow_post(delay: float) -> AsyncIterator[None]: + original: Callable = handlers.PostMessageHandler.__call__ + + async def stalled(self, command): + await asyncio.sleep(delay) + return await original(self, command) + + handlers.PostMessageHandler.__call__ = stalled + try: + yield + finally: + handlers.PostMessageHandler.__call__ = original + + +async def post(client: AsyncClient, chat: dict, sender: dict, text: str): + return await client.post( + f"/api/v1/chats/{chat['id']}/messages", json={"text": text}, headers=keyed(sender) + ) + + +async def prepared(client: AsyncClient) -> tuple[dict, dict]: + alice = await register(client, "alice") + return alice, await make_chat(client, alice) + + +async def test_timeout_returns_unavailable_error_body() -> None: + async with running_app(request_timeout_seconds=0.15) as client: + alice, chat = await prepared(client) + async with slow_post(1.0): + response = await post(client, chat, alice, "медленная запись") + assert response.status_code == 503 + body = response.json() + assert body["error"]["code"] == "unavailable" + assert body["error"]["request_id"] == response.headers["X-Request-Id"] + + +async def test_timeout_does_not_break_following_requests() -> None: + async with running_app(request_timeout_seconds=0.15) as client: + alice, chat = await prepared(client) + async with slow_post(1.0): + assert (await post(client, chat, alice, "зависшая запись")).status_code == 503 + assert (await post(client, chat, alice, "нормальная запись")).status_code == 201 + assert (await client.get("/health/ready")).status_code == 200 + + +async def test_timeout_is_logged_without_message_text(capsys) -> None: + async with running_app(request_timeout_seconds=0.15, log_level="INFO") as client: + alice, chat = await prepared(client) + async with slow_post(1.0): + assert (await post(client, chat, alice, MARKER)).status_code == 503 + captured = capsys.readouterr() + assert "request.timeout" in captured.out + assert MARKER not in captured.out + assert MARKER not in captured.err + + +async def test_normal_request_is_not_affected_by_timeout() -> None: + async with running_app(request_timeout_seconds=5.0) as client: + alice, chat = await prepared(client) + assert (await post(client, chat, alice, "обычная запись")).status_code == 201 + + +async def test_slow_event_handler_does_not_trigger_timeout() -> None: + async with running_app(request_timeout_seconds=0.3) as client: + alice, chat = await prepared(client) + blocked = asyncio.Event() + bus = await resolve(client, EventBus) + + async def stall(_event: MessagePosted) -> None: + await blocked.wait() + + bus.subscribe(MessagePosted, stall) + response = await post(client, chat, alice, "ответ не ждёт обработчик") + blocked.set() + assert response.status_code == 201 + + +async def test_healthcheck_stays_fast(client: AsyncClient) -> None: + loop = asyncio.get_running_loop() + started = loop.time() + assert (await client.get("/health/live")).status_code == 200 + assert loop.time() - started < 1.0 diff --git a/student-tests/api/test_upload_consistency.py b/student-tests/api/test_upload_consistency.py new file mode 100644 index 00000000..b65bb084 --- /dev/null +++ b/student-tests/api/test_upload_consistency.py @@ -0,0 +1,254 @@ +import asyncio +from collections.abc import AsyncIterator +from typing import Any +from uuid import uuid4 + +import asyncpg +from httpx import AsyncClient +from starlette.types import Message +from test_attachments import storage, trio, upload + +from conftest import APPS, DATABASE, actor, keyed, resolve, running_app, scalar +from messenger.attachments.domain.events import AttachmentDiscarded +from messenger.attachments.entrypoints.sweeper import UploadSweeper +from messenger.shared.application.messagebus import EventBus +from pictures import SAMPLE +from pipeline import settle + +AGED = "UPDATE attachments SET created_at = created_at - interval '1 hour' WHERE id = $1" + + +async def execute(sql: str, *args: Any) -> None: + connection = await asyncpg.connect(DATABASE["url"]) + try: + await connection.execute(sql, *args) + finally: + await connection.close() + + +def sweeper(http: AsyncClient) -> UploadSweeper: + return APPS[id(http)].state.sweeper + + +async def discarded(http: AsyncClient) -> list[AttachmentDiscarded]: + seen: list[AttachmentDiscarded] = [] + + async def keep(event: AttachmentDiscarded) -> None: + seen.append(event) + + (await resolve(http, EventBus)).project(AttachmentDiscarded, keep) + return seen + + +async def intent(http: AsyncClient, user: dict[str, Any], chat: dict[str, Any]) -> str: + store = await storage(http) + store.failing = True + name = f"intent-{uuid4().hex}.png" + try: + assert (await upload(http, user, chat, filename=name)).status_code == 503 + finally: + store.failing = False + found = await scalar( + "SELECT id::text FROM attachments WHERE filename = $1 AND status = 'pending'", name + ) + assert found is not None + store.objects[f"staging/{found}"] = (SAMPLE, "image/png") + return found + + +async def exists(attachment_id: str) -> bool: + return await scalar("SELECT count(*) FROM attachments WHERE id = $1", attachment_id) == 1 + + +async def test_commit_failure_after_put_is_compensated(client: AsyncClient) -> None: + alice, _, _, chat = await trio(client) + store = await storage(client) + name = f"fail-{uuid4().hex}.png" + trigger = f"lab4_fail_{uuid4().hex[:8]}" + await execute( + f"""CREATE FUNCTION {trigger}() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN RAISE EXCEPTION 'injected' USING ERRCODE = '55P03'; END $$; + CREATE TRIGGER {trigger} BEFORE UPDATE ON attachments FOR EACH ROW + WHEN (NEW.filename = '{name}' AND NEW.status = 'queued') EXECUTE FUNCTION {trigger}()""" + ) + before, seen = set(store.objects), await discarded(client) + try: + response = await upload(client, alice, chat, filename=name) + finally: + await execute(f"DROP TRIGGER {trigger} ON attachments; DROP FUNCTION {trigger}()") + assert response.status_code == 503 + assert response.json()["error"]["code"] == "unavailable" + assert await scalar("SELECT count(*) FROM attachments WHERE filename = $1", name) == 0 + assert ( + await scalar( + "SELECT count(*) FROM jobs j JOIN attachments a ON a.id = j.attachment_id " + "WHERE a.filename = $1", + name, + ) + == 0 + ) + assert set(store.objects) == before + assert [event.reason for event in seen] == ["compensation"] + + +async def test_lost_intent_gives_503_and_no_object(client: AsyncClient) -> None: + alice, _, _, chat = await trio(client) + store = await storage(client) + + async def swept_meanwhile(key: str) -> None: + await execute("DELETE FROM attachments WHERE staging_key = $1", key) + + store.after_put = swept_meanwhile + before = set(store.objects) + try: + response = await upload(client, alice, chat, filename="late.png") + finally: + store.after_put = None + assert response.status_code == 503 + assert set(store.objects) == before + + +async def test_failed_put_leaves_an_intent_until_grace_ends(client: AsyncClient) -> None: + alice, _, _, chat = await trio(client) + pending = await intent(client, alice, chat) + await sweeper(client).sweep() + assert await exists(pending) + await execute(AGED, pending) + assert await sweeper(client).sweep() >= 1 + assert not await exists(pending) + assert f"staging/{pending}" not in (await storage(client)).objects + + +async def test_sweep_keeps_fresh_intents_and_ready_attachments(client: AsyncClient) -> None: + alice, bob, _, chat = await trio(client) + ready = (await upload(client, alice, chat)).json() + await settle(client) + sent = await client.post( + f"/api/v1/chats/{chat['id']}/messages", + json={"text": "", "attachment_ids": [ready["id"]]}, + headers=keyed(alice), + ) + assert sent.status_code == 201 + old, fresh = await intent(client, alice, chat), await intent(client, alice, chat) + await execute(AGED, old) + await execute(AGED, ready["id"]) + assert await sweeper(client).sweep() == 1 + objects = (await storage(client)).objects + assert not await exists(old) + assert f"staging/{old}" not in objects + assert await exists(fresh) + assert f"staging/{fresh}" in objects + download = await client.get(f"/api/v1/attachments/{ready['id']}/content", headers=actor(bob)) + assert download.status_code == 200 + assert download.content == SAMPLE + assert await sweeper(client).sweep() == 0 + + +async def test_parallel_sweeps_share_the_work(client: AsyncClient) -> None: + alice, _, _, chat = await trio(client) + pending = [await intent(client, alice, chat) for _ in range(5)] + for attachment_id in pending: + await execute(AGED, attachment_id) + seen = await discarded(client) + first, second = await asyncio.gather(sweeper(client).sweep(), sweeper(client).sweep()) + assert first + second == 5 + assert sorted(str(event.attachment_id) for event in seen) == sorted(pending) + assert {event.reason for event in seen} == {"abandoned"} + assert await sweeper(client).sweep() == 0 + objects = (await storage(client)).objects + assert not any(f"staging/{attachment_id}" in objects for attachment_id in pending) + + +async def test_background_sweeper_removes_abandoned_intents() -> None: + async with running_app(upload_sweep_interval_seconds=0.05) as http: + alice, _, _, chat = await trio(http) + pending = await intent(http, alice, chat) + await execute(AGED, pending) + for _ in range(100): + if not await exists(pending): + break + await asyncio.sleep(0.05) + assert not await exists(pending) + + +async def test_upload_waits_for_a_free_slot() -> None: + async with running_app(upload_concurrency=1) as http: + alice, _, _, chat = await trio(http) + gate, holding, read = asyncio.Event(), asyncio.Event(), [] + + async def held() -> AsyncIterator[bytes]: + read.append("first") + holding.set() + yield SAMPLE[:1000] + await gate.wait() + yield SAMPLE[1000:] + + async def queued() -> AsyncIterator[bytes]: + read.append("second") + yield SAMPLE + + first = asyncio.create_task(upload(http, alice, chat, held())) + await holding.wait() + second = asyncio.create_task(upload(http, alice, chat, queued())) + await asyncio.sleep(0.3) + assert read == ["first"] + gate.set() + assert [(await first).status_code, (await second).status_code] == [202, 202] + assert read == ["first", "second"] + + +async def test_aborted_upload_is_a_client_error(client: AsyncClient) -> None: + alice, _, _, chat = await trio(client) + name = f"abort-{uuid4().hex}.png" + headers = [(k.lower().encode(), v.encode()) for k, v in actor(alice).items()] + scope = { + "type": "http", + "asgi": {"version": "3.0"}, + "http_version": "1.1", + "method": "POST", + "scheme": "http", + "path": f"/api/v1/chats/{chat['id']}/attachments", + "raw_path": f"/api/v1/chats/{chat['id']}/attachments".encode(), + "query_string": f"filename={name}".encode(), + "root_path": "", + "headers": [ + *headers, + (b"content-type", b"image/png"), + (b"content-length", b"1000000"), + ], + "client": ("127.0.0.1", 50000), + "server": ("testserver", 80), + } + messages = iter( + [ + {"type": "http.request", "body": SAMPLE[:1000], "more_body": True}, + {"type": "http.disconnect"}, + ] + ) + sent: list[Message] = [] + + async def receive() -> dict[str, Any]: + return next(messages, {"type": "http.disconnect"}) + + async def send(message: Message) -> None: + sent.append(message) + + await APPS[id(client)](scope, receive, send) + assert sent[0]["status"] == 400 + assert await scalar("SELECT count(*) FROM attachments WHERE filename = $1", name) == 0 + + +async def test_stalled_upload_times_out_and_frees_the_slot() -> None: + async with running_app(upload_concurrency=1, request_timeout_seconds=0.5) as http: + alice, _, _, chat = await trio(http) + name = f"stalled-{uuid4().hex}.png" + + async def stalled() -> AsyncIterator[bytes]: + yield SAMPLE[:1000] + await asyncio.Event().wait() + + response = await upload(http, alice, chat, stalled(), filename=name) + assert response.status_code == 503 + assert response.json()["error"]["code"] == "unavailable" + assert await scalar("SELECT count(*) FROM attachments WHERE filename = $1", name) == 0 + assert (await upload(http, alice, chat)).status_code == 202 diff --git a/student-tests/api/test_wire_format.py b/student-tests/api/test_wire_format.py new file mode 100644 index 00000000..0167837d --- /dev/null +++ b/student-tests/api/test_wire_format.py @@ -0,0 +1,69 @@ +import json +import re +from datetime import UTC, datetime +from typing import Any + +import pytest +from httpx import Response + +from conftest import bearer, login, new_user, running_app +from messenger.shared.infrastructure.clock import MonotonicUtcClock + +RFC3339_UTC = re.compile(r"\A\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d{1,6})?Z\Z") +MOMENTS = [ + datetime(2026, 10, 12, 8, 49, 2, tzinfo=UTC), + datetime(2026, 10, 12, 8, 49, 2, 120, tzinfo=UTC), +] + + +def assert_wire(response: Response, expected: dict[str, Any]) -> None: + assert ( + response.content == json.dumps(expected, ensure_ascii=False, separators=(",", ":")).encode() + ) + + +@pytest.mark.parametrize("moment", MOMENTS, ids=["whole-second", "microseconds"]) +async def test_created_at_is_rfc3339_utc(monkeypatch: pytest.MonkeyPatch, moment: datetime) -> None: + monkeypatch.setattr(MonotonicUtcClock, "now", lambda _self: moment) + stamp = moment.isoformat().replace("+00:00", "Z") + assert RFC3339_UTC.match(stamp) + async with running_app() as http: + body = new_user("wire") + response = await http.post("/api/v1/users", json=body) + user_id = response.json()["id"] + user = { + "id": user_id, + "username": body["username"], + "display_name": body["display_name"], + "created_at": stamp, + } + assert_wire(response, user) + session = await login(http, body["username"]) + headers = bearer(session["token"]) + assert session["user"] == user + assert_wire(await http.get("/api/v1/users/me", headers=headers), user) + + response = await http.post( + "/api/v1/chats", json={"title": "Чат", "member_ids": [user_id]}, headers=headers + ) + chat_id = response.json()["id"] + chat = {"id": chat_id, "title": "Чат", "member_ids": [user_id], "created_at": stamp} + assert_wire(response, chat) + assert_wire(await http.get(f"/api/v1/chats/{chat_id}", headers=headers), chat) + listed = await http.get("/api/v1/chats", headers=headers) + assert_wire(listed, {"items": [chat], "next_cursor": None}) + + path = f"/api/v1/chats/{chat_id}/messages" + sent = {**headers, "Idempotency-Key": "wire-1"} + response = await http.post(path, json={"text": "привет"}, headers=sent) + message = { + "id": response.json()["id"], + "chat_id": chat_id, + "sender_id": user_id, + "text": "привет", + "created_at": stamp, + "attachment_ids": [], + } + assert_wire(response, message) + history = await http.get(path, headers=headers) + assert_wire(history, {"items": [message], "next_cursor": None}) diff --git a/student-tests/conftest.py b/student-tests/conftest.py new file mode 100644 index 00000000..8cb762f0 --- /dev/null +++ b/student-tests/conftest.py @@ -0,0 +1,213 @@ +import asyncio +import re +import subprocess +import time +from base64 import b64encode +from collections.abc import AsyncIterator, Iterator +from contextlib import asynccontextmanager +from pathlib import Path +from typing import Any +from uuid import uuid4 + +import asyncpg +import pytest +from fastapi import FastAPI +from httpx import ASGITransport, AsyncClient + +from fakes import MemoryStorageProvider +from messenger.attachments.config import AttachmentsSettings +from messenger.identity.config import IdentitySettings +from messenger.main import create_app +from messenger.messaging.config import MessagingSettings +from messenger.migrate import migrate +from messenger.shared.config import PlatformSettings +from messenger.shared.entrypoints.config import TransportSettings + +ROOT = Path(__file__).resolve().parents[1] +POSTGRES_IMAGE = re.compile(r"image: (postgres:\S+)") +VALKEY_IMAGE = re.compile(r"image: (valkey/valkey:\S+)") +STARTUP_SECONDS = 60 +PASSWORD = "Пароль-для-тестов-01" +APPS: dict[int, FastAPI] = {} +TOKENS: dict[str, str] = {} +DATABASE: dict[str, str] = {} + + +def docker(*args: str) -> str: + done = subprocess.run(["docker", *args], check=True, capture_output=True, text=True) + return done.stdout.strip() + + +async def migrated(dsn: str) -> None: + deadline = time.monotonic() + STARTUP_SECONDS + while True: + try: + await migrate(dsn) + return + except OSError, asyncpg.PostgresError, asyncpg.InterfaceError: + if time.monotonic() > deadline: + raise + await asyncio.sleep(0.5) + + +@pytest.fixture(autouse=True) +def database_for_api(request: pytest.FixtureRequest) -> None: + if "api" in request.node.path.parts: + request.getfixturevalue("database") + + +def image(pattern: re.Pattern[str]) -> str: + found = pattern.search((ROOT / "compose.yaml").read_text()) + assert found is not None + return found[1] + + +def published(name: str, port: int) -> int: + return int(docker("port", name, f"{port}/tcp").splitlines()[0].rsplit(":", 1)[1]) + + +def container(name: str, *args: str) -> None: + docker("run", "-d", "--rm", "--name", name, *args) + + +def answering(name: str, *probe: str) -> bool: + done = subprocess.run(["docker", "exec", name, *probe], capture_output=True, check=False) + return done.returncode == 0 and b"PONG" in done.stdout + + +@pytest.fixture(scope="session") +def valkey() -> Iterator[dict[str, str]]: + name = f"mtusi-lab6-test-valkey-{uuid4().hex[:8]}" + container(name, "-p", "127.0.0.1::6379", image(VALKEY_IMAGE)) + try: + deadline = time.monotonic() + STARTUP_SECONDS + while not answering(name, "valkey-cli", "ping"): + assert time.monotonic() < deadline + time.sleep(0.2) + yield {"name": name, "url": f"redis://127.0.0.1:{published(name, 6379)}/0"} + finally: + subprocess.run(["docker", "rm", "-f", name], capture_output=True, check=False) + + +@pytest.fixture(scope="session") +def database() -> Iterator[str]: + name = f"mtusi-lab6-test-{uuid4().hex[:8]}" + tuning = ["-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] + environment = ["-e", "POSTGRES_PASSWORD=test", "-e", "POSTGRES_DB=messenger"] + container(name, *environment, "-p", "127.0.0.1::5432", image(POSTGRES_IMAGE), *tuning) + try: + dsn = f"postgresql://postgres:test@127.0.0.1:{published(name, 5432)}/messenger" + asyncio.run(migrated(dsn)) + DATABASE["url"] = dsn + yield dsn + finally: + subprocess.run(["docker", "rm", "-f", name], capture_output=True, check=False) + + +def split_settings(**overrides: Any) -> dict[str, Any]: + base: dict[str, Any] = { + "log_level": "CRITICAL", + "cursor_secret": "test-secret-for-cursors", + "database_url": DATABASE.get("url", "postgresql://unused@127.0.0.1:1/unused"), + "s3_access_key_id": "test-key", + "s3_secret_access_key": "test-secret-key", + } + base.update(overrides) + groups = { + "platform": PlatformSettings, + "transport": TransportSettings, + "identity": IdentitySettings, + "messaging": MessagingSettings, + "attachments": AttachmentsSettings, + } + return { + name: kind.model_validate( + {key: value for key, value in base.items() if key in kind.model_fields} + ) + for name, kind in groups.items() + } + + +@asynccontextmanager +async def running_app(**overrides: Any) -> AsyncIterator[AsyncClient]: + app = create_app(**split_settings(**overrides), overrides=(MemoryStorageProvider(),)) + async with ( + app.router.lifespan_context(app), + AsyncClient(transport=ASGITransport(app=app), base_url="http://testserver") as http, + ): + APPS[id(http)] = app + try: + yield http + finally: + APPS.pop(id(http), None) + + +async def resolve[DependencyT](http: AsyncClient, dependency: type[DependencyT]) -> DependencyT: + return await APPS[id(http)].state.dishka_container.get(dependency) + + +@pytest.fixture +async def client() -> AsyncIterator[AsyncClient]: + async with running_app() as http: + yield http + + +async def scalar(sql: str, *args: Any) -> Any: + connection = await asyncpg.connect(DATABASE["url"]) + try: + return await connection.fetchval(sql, *args) + finally: + await connection.close() + + +def basic(username: str, password: str) -> dict[str, str]: + encoded = b64encode(f"{username}:{password}".encode()).decode() + return {"Authorization": f"Basic {encoded}"} + + +def bearer(token: str) -> dict[str, str]: + return {"Authorization": f"Bearer {token}"} + + +def new_user(label: str, password: str = PASSWORD) -> dict[str, str]: + return { + "username": f"u_{label}_{uuid4().hex[:12]}", + "display_name": f"Студент {label}", + "password": password, + } + + +async def login(http: AsyncClient, username: str, password: str = PASSWORD) -> dict[str, Any]: + response = await http.post("/api/v1/auth/sessions", headers=basic(username, password)) + assert response.status_code == 201, response.text + return response.json() + + +async def register(http: AsyncClient, label: str, password: str = PASSWORD) -> dict[str, Any]: + body = new_user(label, password) + response = await http.post("/api/v1/users", json=body) + assert response.status_code == 201, response.text + user = response.json() + TOKENS[user["id"]] = (await login(http, body["username"], password))["token"] + return user + + +async def make_chat( + http: AsyncClient, owner: dict[str, Any], *members: dict[str, Any] +) -> dict[str, Any]: + member_ids = [owner["id"], *[member["id"] for member in members]] + response = await http.post( + "/api/v1/chats", + json={"title": "Тестовый чат", "member_ids": member_ids}, + headers=actor(owner), + ) + assert response.status_code == 201, response.text + return response.json() + + +def actor(user: dict[str, Any]) -> dict[str, str]: + return bearer(TOKENS[user["id"]]) + + +def keyed(user: dict[str, Any], key: str | None = None) -> dict[str, str]: + return {**actor(user), "Idempotency-Key": key or uuid4().hex} diff --git a/student-tests/fakes.py b/student-tests/fakes.py new file mode 100644 index 00000000..2b0e23f8 --- /dev/null +++ b/student-tests/fakes.py @@ -0,0 +1,81 @@ +from collections.abc import AsyncIterator, Awaitable, Callable +from datetime import datetime, timedelta +from typing import Any + +from dishka import Provider, Scope, provide + +from messenger.attachments.application.errors import ObjectMissingError +from messenger.attachments.application.ports import ObjectContent, ObjectStorage +from messenger.shared.application.errors import Unavailable + + +class NullLogger: + __slots__ = () + + def info(self, event: str, /, **fields: Any) -> None: ... + + def warning(self, event: str, /, **fields: Any) -> None: ... + + def error(self, event: str, /, **fields: Any) -> None: ... + + +class FrozenClock: + __slots__ = ("_current",) + + def __init__(self, start: datetime) -> None: + self._current = start + + def now(self) -> datetime: + self._current += timedelta(microseconds=1) + return self._current + + +async def pieces(data: bytes, size: int = 65_536) -> AsyncIterator[bytes]: + for start in range(0, len(data), size): + yield data[start : start + size] + + +class MemoryObjectStorage: + __slots__ = ("after_put", "failing", "objects") + + def __init__(self) -> None: + self.objects: dict[str, tuple[bytes, str]] = {} + self.failing = False + self.after_put: Callable[[str], Awaitable[None]] | None = None + + async def put(self, key: str, data: bytes, content_type: str) -> None: + if self.failing: + raise Unavailable("S3 недоступно") + self.objects[key] = (bytes(data), content_type) + if self.after_put is not None: + await self.after_put(key) + + async def open(self, key: str) -> ObjectContent: + if self.failing: + raise Unavailable("S3 недоступно") + if key not in self.objects: + raise ObjectMissingError(key) + data = self.objects[key][0] + return ObjectContent(size=len(data), chunks=pieces(data)) + + async def read(self, key: str) -> bytes: + content = await self.open(key) + return b"".join([chunk async for chunk in content.chunks]) + + async def copy(self, source: str, target: str) -> None: + if self.failing: + raise Unavailable("S3 недоступно") + if source not in self.objects: + raise ObjectMissingError(source) + self.objects[target] = self.objects[source] + + async def delete(self, key: str) -> None: + if self.failing: + raise Unavailable("S3 недоступно") + self.objects.pop(key, None) + + +class MemoryStorageProvider(Provider): + @provide(scope=Scope.APP) + def storage(self) -> ObjectStorage: + return MemoryObjectStorage() diff --git a/student-tests/pictures.py b/student-tests/pictures.py new file mode 100644 index 00000000..f53a5296 --- /dev/null +++ b/student-tests/pictures.py @@ -0,0 +1,35 @@ +import struct +import zlib +from io import BytesIO +from pathlib import Path + +from PIL import Image + +SAMPLE = (Path(__file__).resolve().parents[1] / "tests" / "fixtures" / "sample.png").read_bytes() +SIGNATURE = b"\x89PNG\r\n\x1a\n" + + +def chunk(kind: bytes, data: bytes) -> bytes: + crc = zlib.crc32(kind + data) + return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", crc) + + +def png(width: int, height: int, *, size: int | None = None, rows: bool = True) -> bytes: + header = struct.pack(">IIBBBBB", width, height, 1, 0, 0, 0, 0) + raw = (b"\x00" * (1 + (width + 7) // 8)) * height if rows else b"\x00" + head = SIGNATURE + chunk(b"IHDR", header) + chunk(b"IDAT", zlib.compress(raw, 9)) + tail = chunk(b"IEND", b"") + if size is not None: + head += chunk(b"prVt", b"\x00" * (size - len(head) - len(tail) - 12)) + return head + tail + + +def encoded(kind: str, width: int = 640, height: int = 480) -> bytes: + buffer = BytesIO() + Image.effect_noise((width, height), 64).convert("RGB").save(buffer, kind) + return buffer.getvalue() + + +JPEG = encoded("JPEG") +GIF = encoded("GIF") +WEBP = encoded("WEBP") diff --git a/student-tests/pipeline.py b/student-tests/pipeline.py new file mode 100644 index 00000000..bd91fde1 --- /dev/null +++ b/student-tests/pipeline.py @@ -0,0 +1,164 @@ +import asyncio +from datetime import timedelta +from uuid import UUID + +from httpx import AsyncClient + +from conftest import resolve +from fakes import NullLogger +from messenger.attachments.application.commands import ( + CleanStaging, + ProcessJob, + PublishOutbox, + RecoverExpiredLeases, + ReplayAttachment, +) +from messenger.attachments.application.pipeline import ( + CleanStagingHandler, + ProcessJobHandler, + PublishOutboxHandler, + RecoverExpiredLeasesHandler, + ReplayAttachmentHandler, +) +from messenger.attachments.application.ports import ( + Faults, + Jitter, + ObjectStorage, + Rendition, + VariantRenderer, +) +from messenger.attachments.domain.jobs import Job, Outcome, ReplayResult, RetryPolicy +from messenger.attachments.entrypoints.schemas import JobMessage +from messenger.attachments.infrastructure.faults import EnvironmentFaults +from messenger.attachments.infrastructure.images import RenderLimits, render_variants +from messenger.attachments.infrastructure.postgres import PostgresAttachmentsUnitOfWork +from messenger.shared.application.errors import Unavailable +from messenger.shared.infrastructure.ids import Uuid7Generator +from messenger.shared.infrastructure.postgres import Database + +IDLE_SECONDS = 0.1 +HALF = 0.5 +LIMITS = RenderLimits(10_485_760, 40_000_000) +FAST = RetryPolicy(max_attempts=5, base_seconds=0.001, cap_seconds=0.001) + + +class FixedJitter: + __slots__ = ("_spread",) + + def __init__(self, spread: float) -> None: + self._spread = spread + + def spread(self) -> float: + return self._spread + + +class InlineRenderer: + __slots__ = () + + async def render(self, data: bytes, pipeline_version: int) -> list[Rendition]: + return render_variants(data, LIMITS) + + +class CapturingPublisher: + __slots__ = ("bodies", "failing", "job_ids") + + def __init__(self) -> None: + self.bodies: list[bytes] = [] + self.job_ids: list[UUID] = [] + self.failing = False + + async def publish(self, job_id: UUID, body: bytes) -> None: + if self.failing: + raise Unavailable("брокер недоступен") + self.job_ids.append(job_id) + self.bodies.append(body) + + +class InlinePipeline: + __slots__ = ( + "_database", + "faults", + "jitter", + "lease", + "policy", + "publisher", + "renderer", + "storage", + ) + + def __init__( + self, + database: Database, + storage: ObjectStorage, + *, + policy: RetryPolicy = FAST, + lease: timedelta = timedelta(seconds=60), + ) -> None: + self._database = database + self.storage = storage + self.policy = policy + self.lease = lease + self.publisher = CapturingPublisher() + self.renderer: VariantRenderer = InlineRenderer() + self.faults: Faults = EnvironmentFaults("", NullLogger()) + self.jitter: Jitter = FixedJitter(HALF) + + def uow(self) -> PostgresAttachmentsUnitOfWork: + return PostgresAttachmentsUnitOfWork(self._database) + + async def publish(self) -> list[Job]: + start = len(self.publisher.bodies) + await PublishOutboxHandler(self.uow(), self.publisher, self.faults)(PublishOutbox(100)) + return [ + JobMessage.model_validate_json(body).job() for body in self.publisher.bodies[start:] + ] + + async def process(self, job: Job) -> Outcome: + handler = ProcessJobHandler( + self.uow(), + self.storage, + self.renderer, + self.faults, + ids=Uuid7Generator(), + jitter=self.jitter, + policy=self.policy, + lease=self.lease, + ) + return await handler(ProcessJob(job)) + + async def drain(self, rounds: int = 20, idle: int = 3) -> list[Outcome]: + outcomes: list[Outcome] = [] + quiet = 0 + for _ in range(rounds): + jobs = await self.publish() + quiet = 0 if jobs else quiet + 1 + if quiet == idle: + return outcomes + if not jobs: + await asyncio.sleep(IDLE_SECONDS) + outcomes.extend([await self.process(job) for job in jobs]) + return outcomes + + async def recover(self) -> int: + handler = RecoverExpiredLeasesHandler(self.uow(), self.policy) + return await handler(RecoverExpiredLeases(100)) + + async def clean(self, retention: timedelta = timedelta(days=1)) -> int: + command = CleanStaging(100, retention) + return await CleanStagingHandler(self.uow(), self.storage)(command) + + async def replay(self, attachment_id: UUID, operator: str = "test") -> ReplayResult: + command = ReplayAttachment(attachment_id, operator) + return await ReplayAttachmentHandler(self.uow())(command) + + +async def pipeline_of( + http: AsyncClient, *, policy: RetryPolicy = FAST, lease: timedelta = timedelta(seconds=60) +) -> InlinePipeline: + database = await resolve(http, Database) + storage = await resolve(http, ObjectStorage) + return InlinePipeline(database, storage, policy=policy, lease=lease) + + +async def settle(http: AsyncClient) -> list[Outcome]: + return await (await pipeline_of(http)).drain() diff --git a/student-tests/stack/test_attachment_faults.py b/student-tests/stack/test_attachment_faults.py new file mode 100644 index 00000000..be9af4b7 --- /dev/null +++ b/student-tests/stack/test_attachment_faults.py @@ -0,0 +1,472 @@ +import asyncio +import json +import socket +import subprocess +import time +from collections.abc import AsyncIterator, Iterator +from datetime import UTC, datetime, timedelta +from hashlib import sha256 +from http.client import HTTPResponse +from typing import Any +from urllib.parse import quote, urlsplit +from uuid import uuid4 + +import httpx +import pytest +from httpx import AsyncClient, Response + +from pictures import GIF, JPEG, SAMPLE, WEBP, png +from stand import BASE_URL, bearer, compose, enroll, lb_up, psql, waited + +pytestmark = pytest.mark.stack +LIMIT = 10_485_760 +BUCKET = "messenger-attachments" +REQUEST_TIMEOUT = 8 +SWEEP_WAIT = 90 +CODES = { + 400: "invalid_request", + 404: "not_found", + 413: "payload_too_large", + 415: "unsupported_media_type", +} +HEAD = ( + 'curl -s -o /dev/null -w "%{http_code}" -I --aws-sigv4 "aws:amz:${RUSTFS_REGION:-us-east-1}:s3"' + ' --user "$(cat /run/secrets/s3_root_user):$(cat /run/secrets/s3_root_password)"' + ' "http://127.0.0.1:9000/$BUCKET/$KEY"' +) +FAULTS = """ +CREATE TABLE lab4_fault_seen (id uuid PRIMARY KEY, object_key text, filename text); +CREATE FUNCTION lab4_fault_seen() RETURNS trigger LANGUAGE plpgsql AS $$ +BEGIN + INSERT INTO lab4_fault_seen VALUES (NEW.id, NEW.staging_key, NEW.filename); + RETURN NEW; +END $$; +CREATE FUNCTION lab4_fault_commit() RETURNS trigger LANGUAGE plpgsql AS $$ +BEGIN + IF NEW.filename LIKE 'fault-fail-%' THEN + RAISE EXCEPTION 'injected commit failure' USING ERRCODE = '55P03'; + END IF; + PERFORM pg_sleep(1.8); + RETURN NEW; +END $$; +CREATE FUNCTION lab4_fault_sleeper() RETURNS text LANGUAGE plpgsql AS $$ +DECLARE found text; +BEGIN + FOR i IN 1..500 LOOP + PERFORM pg_stat_clear_snapshot(); + SELECT host(client_addr) INTO found FROM pg_stat_activity + WHERE wait_event = 'PgSleep' AND query LIKE 'UPDATE attachments%' LIMIT 1; + IF found IS NOT NULL THEN RETURN found; END IF; + PERFORM pg_sleep(0.02); + END LOOP; + RETURN NULL; +END $$; +CREATE TRIGGER lab4_fault_seen AFTER INSERT ON attachments FOR EACH ROW + WHEN (NEW.filename LIKE 'fault-%') EXECUTE FUNCTION lab4_fault_seen(); +CREATE TRIGGER lab4_fault_commit BEFORE UPDATE ON attachments FOR EACH ROW + WHEN (NEW.filename LIKE 'fault-%' AND OLD.status = 'pending' AND NEW.status = 'queued') + EXECUTE FUNCTION lab4_fault_commit(); +""" +CLEANUP = """ +DROP TRIGGER IF EXISTS lab4_fault_commit ON attachments; +DROP TRIGGER IF EXISTS lab4_fault_seen ON attachments; +DROP FUNCTION IF EXISTS lab4_fault_commit(), lab4_fault_seen(), lab4_fault_sleeper(); +DROP TABLE IF EXISTS lab4_fault_seen; +""" + + +def sql(text: str, **variables: str) -> str: + return psql(text, ON_ERROR_STOP="1", **variables) + + +def object_status(key: str) -> int: + found = compose( + "exec", "-T", "-e", f"KEY={key}", "-e", f"BUCKET={BUCKET}", "s3", "sh", "-c", HEAD + ) + return int(found) + + +def bucket_keys() -> set[str]: + listing = compose( + "run", + "--rm", + "--no-deps", + "-T", + "s3-cli", + "object", + "list", + "--recursive", + "--json", + f"root/{BUCKET}/staging/", + ) + return {item["key"] for item in json.loads(listing)["items"]} + + +def seen(name: str) -> tuple[str, str] | None: + row = sql("SELECT id, object_key FROM lab4_fault_seen WHERE filename = :'name'", name=name) + if not row: + return None + attachment_id, key = row.split("|") + return attachment_id, key + + +def status_of(attachment_id: str) -> str: + return sql("SELECT status FROM attachments WHERE id = :'id'", id=attachment_id) + + +def logs(since: str) -> list[dict[str, Any]]: + out = compose("logs", "api", "--since", since, "--no-log-prefix") + return [json.loads(line) for line in out.splitlines() if line.startswith("{")] + + +def moment() -> str: + return (datetime.now(UTC) - timedelta(seconds=1)).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def kill(container: str) -> None: + subprocess.run(["docker", "kill", "-s", "KILL", container], check=True, capture_output=True) + + +def replica_at(address: str) -> str: + ids = compose("ps", "-q", "api").split() + template = "{{.Id}} {{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}" + found = subprocess.run( + ["docker", "inspect", "-f", template, *ids], check=True, capture_output=True, text=True + ).stdout.split("\n") + return next(line.split()[0] for line in found if line.endswith(f" {address}")) + + +def raw( + path: str, headers: dict[str, str], body: bytes, *, hang_up: bool +) -> tuple[int, dict[str, Any], float]: + parts = urlsplit(BASE_URL) + head = f"POST {path} HTTP/1.1\r\nHost: {parts.netloc}\r\nConnection: close\r\n" + head += "".join(f"{name}: {value}\r\n" for name, value in headers.items()) + "\r\n" + started = time.perf_counter() + with socket.create_connection((parts.hostname, parts.port or 80), timeout=30) as sock: + sock.sendall(head.encode("latin-1") + body) + if hang_up: + return 0, {}, 0.0 + response = HTTPResponse(sock) + response.begin() + return response.status, json.loads(response.read()), time.perf_counter() - started + + +@pytest.fixture +async def http() -> AsyncIterator[AsyncClient]: + async with AsyncClient(base_url=BASE_URL, timeout=30, trust_env=False) as client: + yield client + + +@pytest.fixture(scope="module") +def faults() -> Iterator[None]: + sql(CLEANUP + FAULTS) + try: + yield + finally: + sql(CLEANUP) + + +async def scene(http: AsyncClient) -> dict[str, Any]: + (alice, alice_token), (bob, bob_token), (eve, eve_token) = [ + await enroll(http, label) for label in ("alice", "bob", "eve") + ] + chats = [] + for owner, token, members in ( + (alice, alice_token, [bob]), + (alice, alice_token, []), + (eve, eve_token, []), + ): + created = await http.post( + "/api/v1/chats", + json={"title": "Стенд", "member_ids": [owner["id"], *(m["id"] for m in members)]}, + headers=bearer(token), + ) + assert created.status_code == 201 + chats.append(created.json()["id"]) + return {"alice": alice_token, "bob": bob_token, "eve": eve_token, "chats": chats} + + +async def upload( + http: AsyncClient, + token: str, + chat: str, + body: bytes | AsyncIterator[bytes] = SAMPLE, + *, + filename: str | None = "sample.png", + kind: str | None = "image/png", +) -> Response: + query = "" if filename is None else f"?filename={quote(filename, safe='')}" + headers = {**bearer(token), **({"Content-Type": kind} if kind else {})} + return await http.post( + f"/api/v1/chats/{chat}/attachments{query}", content=body, headers=headers + ) + + +async def message(http: AsyncClient, token: str, chat: str, body: dict[str, Any]) -> Response: + headers = {**bearer(token), "Idempotency-Key": uuid4().hex} + return await http.post(f"/api/v1/chats/{chat}/messages", json=body, headers=headers) + + +def expect(label: str, response: Response, status: int) -> None: + print(f"{label}: {response.status_code}") + assert response.status_code == status, f"{label}: {response.status_code} {response.text}" + assert response.headers["X-Instance-Id"] + error = response.json()["error"] + assert error["code"] == CODES[status] + assert error["request_id"] == response.headers["X-Request-Id"] + + +async def test_user_errors_never_become_500(http: AsyncClient) -> None: + since = moment() + s = await scene(http) + alice, eve, (chat, solo, other) = s["alice"], s["eve"], s["chats"] + created = 0 + attachment = await upload(http, alice, chat) + exact = await upload(http, alice, chat, png(16, 16, size=LIMIT)) + assert (attachment.status_code, exact.status_code) == (202, 202) + assert exact.json()["size_bytes"] == LIMIT + created += 2 + mine = attachment.json()["id"] + foreign = (await upload(http, alice, solo)).json()["id"] + created_elsewhere = 1 + oversize = png(16, 16, size=LIMIT + 1) + + async def chunked() -> AsyncIterator[bytes]: + for start in range(0, len(oversize), 1 << 20): + yield oversize[start : start + (1 << 20)] + + path = f"/api/v1/chats/{chat}/attachments?filename=big.png" + status, body, _ = await asyncio.to_thread( + raw, + path, + {**bearer(alice), "Content-Type": "image/png", "Content-Length": str(LIMIT + 1)}, + b"", + hang_up=False, + ) + print(f"10 MiB + 1 с Content-Length, тело не отправлено: {status}") + assert (status, body["error"]["code"]) == (413, "payload_too_large") + expect("10 MiB + 1 chunked", await upload(http, alice, chat, chunked()), 413) + expect( + "PNG 100000×100000 (бомба)", + await upload(http, alice, chat, png(100_000, 100_000, rows=False)), + 413, + ) + expect("PNG 8001×5000", await upload(http, alice, chat, png(8001, 5000, rows=False)), 413) + for label, content, kind in ( + ("обрезанный PNG", SAMPLE[: len(SAMPLE) // 2], "image/png"), + ("обрезанный JPEG", JPEG[: len(JPEG) // 2], "image/jpeg"), + ("JPEG как image/png", JPEG, "image/png"), + ("GIF как image/png", GIF, "image/png"), + ("WebP как image/png", WEBP, "image/png"), + ("текст как image/png", b"plain text, not an image", "image/png"), + ("PNG как image/jpeg", SAMPLE, "image/jpeg"), + ("пустое тело", b"", "image/png"), + ("application/octet-stream", SAMPLE, "application/octet-stream"), + ("без Content-Type", SAMPLE, None), + ): + expect(label, await upload(http, alice, chat, content, kind=kind), 415) + for name in ( + "../a.png", + "a/b.png", + "a\\b.png", + "a\x00.png", + "a\x01.png", + "a\x7f.png", + "", + " ", + "x" * 256, + None, + ): + expect(f"filename={name!r}"[:40], await upload(http, alice, chat, filename=name), 400) + expect("посторонний: upload в чужой чат", await upload(http, eve, chat), 404) + expect( + "посторонний: метаданные", + await http.get(f"/api/v1/attachments/{mine}", headers=bearer(eve)), + 404, + ) + expect( + "посторонний: /content", + await http.get(f"/api/v1/attachments/{mine}/content", headers=bearer(eve)), + 404, + ) + expect( + "посторонний: ссылка на чужое вложение", + await message(http, eve, other, {"text": "x", "attachment_ids": [mine]}), + 404, + ) + expect( + "вложение другого чата", + await message(http, alice, chat, {"text": "x", "attachment_ids": [foreign]}), + 404, + ) + expect( + "неизвестное вложение", + await message(http, alice, chat, {"text": "x", "attachment_ids": [str(uuid4())]}), + 404, + ) + for label, ids in ( + ("11 вложений", [mine] + [str(uuid4()) for _ in range(10)]), + ("повтор вложения", [mine, mine]), + ("невалидный UUID", ["not-a-uuid"]), + ): + expect(label, await message(http, alice, chat, {"text": "x", "attachment_ids": ids}), 400) + expect( + "пустой текст без вложений", + await message(http, alice, chat, {"text": "", "attachment_ids": []}), + 400, + ) + expect( + "невалидный id вложения", + await http.get("/api/v1/attachments/x/content", headers=bearer(alice)), + 400, + ) + expect( + "variant=preview", + await http.get( + f"/api/v1/attachments/{mine}/content?variant=preview", headers=bearer(alice) + ), + 400, + ) + rows = sql("SELECT count(*) FROM attachments WHERE chat_id = :'chat'", chat=chat) + assert int(rows) == created + assert sql("SELECT count(*) FROM attachments WHERE chat_id = :'chat'", chat=solo) == str( + created_elsewhere + ) + statuses = [entry["status"] for entry in logs(since) if entry.get("event") == "request.handled"] + print( + f"строк вложений в чате: {rows} (= ответов 202); ответов в журнале реплик: " + f"{len(statuses)}, из них 5xx: {sum(code >= 500 for code in statuses)}" + ) + assert not [status for status in statuses if status >= 500] + + +async def test_commit_failure_is_compensated_at_once(http: AsyncClient, faults: None) -> None: + since = moment() + s = await scene(http) + name = f"fault-fail-{uuid4().hex}.png" + response = await upload(http, s["alice"], s["chats"][0], filename=name) + print( + "отказ COMMIT после записи объекта:", + response.status_code, + response.json()["error"]["code"], + ) + assert response.status_code == 503 + found = seen(name) + assert found is not None + attachment_id, key = found + print( + f"сразу после ответа: строка {status_of(attachment_id) or 'нет'},", + f"объект HEAD {object_status(key)}", + ) + assert status_of(attachment_id) == "" + assert object_status(key) == 404 + + def journaled() -> bool: + return any( + entry.get("event") == "audit.attachment_discarded" + and entry.get("attachment_id") == attachment_id + and entry.get("reason") == "compensation" + for entry in logs(since) + ) + + await waited(journaled, 10) + print("журнал: audit.attachment_discarded reason=compensation") + + +async def crash(http: AsyncClient, token: str, chat: str) -> tuple[str, str]: + name = f"fault-crash-{uuid4().hex}.png" + sent = asyncio.create_task(upload(http, token, chat, filename=name)) + address = await asyncio.to_thread(sql, "SELECT lab4_fault_sleeper()") + assert address, "UPDATE queued не дошёл до триггера" + victim = replica_at(address) + kill(victim) + try: + outcome = str((await sent).status_code) + except httpx.HTTPError as error: + outcome = type(error).__name__ + compose("up", "-d", "--wait", "--no-deps", "--no-recreate", "api") + await waited(lambda: lb_up() == 2, 60) + found = seen(name) + assert found is not None + print(f"реплика {victim[:12]} убита между PUT и COMMIT, клиент получил {outcome}") + return found + + +async def test_crash_between_put_and_commit_is_swept_after_grace( + http: AsyncClient, faults: None +) -> None: + s = await scene(http) + alice, chat = s["alice"], s["chats"][0] + kept = (await upload(http, alice, chat, filename="kept.png")).json() + await waited(lambda: status_of(kept["id"]) == "ready", 60) + assert ( + await message(http, alice, chat, {"text": "", "attachment_ids": [kept["id"]]}) + ).status_code == 201 + old, fresh = await crash(http, alice, chat), await crash(http, alice, chat) + for attachment_id, key in (old, fresh): + state = (status_of(attachment_id), object_status(key)) + print(f"остаток после падения: строка {state[0]}, объект HEAD {state[1]}") + assert state == ("pending", 200) + aged = "UPDATE attachments SET created_at = created_at - interval '2 hours' WHERE id = :'id'" + sql(aged, id=old[0]) + sql(aged, id=kept["id"]) + swept = await waited(lambda: status_of(old[0]) == "", SWEEP_WAIT) + print(f"старше grace: строка убрана через {swept:.1f} с, объект HEAD {object_status(old[1])}") + assert object_status(old[1]) == 404 + print(f"моложе grace: строка {status_of(fresh[0])}, объект HEAD {object_status(fresh[1])}") + assert (status_of(fresh[0]), object_status(fresh[1])) == ("pending", 200) + sql(aged, id=fresh[0]) + swept = await waited(lambda: status_of(fresh[0]) == "", SWEEP_WAIT) + print(f"тот же после сдвига: убран через {swept:.1f} с, объект HEAD {object_status(fresh[1])}") + assert object_status(fresh[1]) == 404 + content = await http.get(f"/api/v1/attachments/{kept['id']}/content", headers=bearer(s["bob"])) + assert content.status_code == 200 + assert sha256(content.content).hexdigest() == kept["sha256"] + print("готовое вложение в сообщении со сдвинутым created_at скачивается, SHA-256 совпадает") + + +async def test_aborted_upload_leaves_nothing(http: AsyncClient, faults: None) -> None: + since = moment() + s = await scene(http) + name, request_id = f"fault-abort-{uuid4().hex}.png", f"abort-{uuid4().hex}" + before = bucket_keys() + headers = { + **bearer(s["alice"]), + "Content-Type": "image/png", + "Content-Length": str(5 << 20), + "X-Request-Id": request_id, + } + path = f"/api/v1/chats/{s['chats'][0]}/attachments?filename={name}" + await asyncio.to_thread(raw, path, headers, SAMPLE[:8] + bytes(256 << 10), hang_up=True) + + def handled() -> list[int]: + return [ + e["status"] + for e in logs(since) + if e.get("request_id") == request_id and e.get("event") == "request.handled" + ] + + await waited(lambda: bool(handled()), 15) + print( + f"обрыв после 256 KiB из 5 MiB: статус в журнале реплики {handled()},", + f"намерение {seen(name)}", + ) + assert handled() == [400] + assert seen(name) is None + assert bucket_keys() == before + + +async def test_stalled_upload_is_cut_and_leaves_nothing(http: AsyncClient, faults: None) -> None: + s = await scene(http) + name = f"fault-stall-{uuid4().hex}.png" + headers = {**bearer(s["alice"]), "Content-Type": "image/png", "Content-Length": str(1 << 20)} + path = f"/api/v1/chats/{s['chats'][0]}/attachments?filename={name}" + status, body, elapsed = await asyncio.to_thread( + raw, path, headers, SAMPLE[:1024], hang_up=False + ) + print(f"тело зависло после 1 KiB: {status} {body['error']['code']} за {elapsed:.2f} с") + assert (status, body["error"]["code"]) == (503, "unavailable") + assert elapsed <= REQUEST_TIMEOUT + 2 + assert seen(name) is None + assert (await upload(http, s["alice"], s["chats"][0])).status_code == 202 diff --git a/student-tests/stack/test_faults.py b/student-tests/stack/test_faults.py new file mode 100644 index 00000000..6e78de49 --- /dev/null +++ b/student-tests/stack/test_faults.py @@ -0,0 +1,229 @@ +import asyncio +from collections.abc import AsyncIterator +from uuid import uuid4 + +import pytest +from httpx import AsyncClient + +from stand import ( + BASE_URL, + BASIC, + BEARER, + basic, + bearer, + compose, + enroll, + inside_replica, + lb_up, + new_user, + psql, + report, + started_at, + timed, + valkey, + wait_ready, + waited, +) + +pytestmark = pytest.mark.stack +RACERS = 50 + + +@pytest.fixture +async def http() -> AsyncIterator[AsyncClient]: + async with AsyncClient(base_url=BASE_URL, timeout=10, trust_env=False) as client: + yield client + + +async def test_registration_race_yields_exactly_one_user(http: AsyncClient) -> None: + body = new_user("race") + responses = await asyncio.gather( + *(http.post("/api/v1/users", json=body) for _ in range(RACERS)) + ) + codes = sorted(response.status_code for response in responses) + print(f"гонка {RACERS} регистраций: 201×{codes.count(201)}, 409×{codes.count(409)}") + assert codes.count(201) == 1 + assert codes.count(409) == RACERS - 1 + assert psql("SELECT count(*) FROM users WHERE username = :'name'", name=body["username"]) == "1" + + +async def test_chat_creation_rolls_back_on_unknown_member(http: AsyncClient) -> None: + alice, token = await enroll(http, "alice") + title = f"Откат {uuid4().hex}" + response = await http.post( + "/api/v1/chats", + json={"title": title, "member_ids": [alice["id"], str(uuid4())]}, + headers=bearer(token), + ) + print(f"чат с несуществующим участником: {response.status_code}") + assert response.status_code == 400 + assert response.json()["error"]["code"] == "invalid_request" + assert psql("SELECT count(*) FROM chats WHERE title = :'title'", title=title) == "0" + members = "SELECT count(*) FROM chat_members WHERE user_id = :'id'::uuid" + assert psql(members, id=alice["id"]) == "0" + + +async def test_wrong_password_and_unknown_user_get_same_basic_challenge(http: AsyncClient) -> None: + alice, _ = await enroll(http, "alice") + wrong, wrong_time = await timed( + http.post("/api/v1/auth/sessions", headers=basic(alice["username"], "x" * 20)) + ) + unknown, unknown_time = await timed( + http.post("/api/v1/auth/sessions", headers=basic(f"u_ghost_{uuid4().hex[:8]}")) + ) + report("неверный пароль", wrong, wrong_time) + report("неизвестный пользователь", unknown, unknown_time) + for response in (wrong, unknown): + assert response.status_code == 401 + assert response.headers["WWW-Authenticate"] == BASIC + assert wrong.json()["error"]["message"] == unknown.json()["error"]["message"] + + +async def test_logout_then_token_is_dead(http: AsyncClient) -> None: + _, token = await enroll(http, "alice") + assert (await http.get("/api/v1/users/me", headers=bearer(token))).status_code == 200 + logout = await http.delete("/api/v1/auth/sessions/current", headers=bearer(token)) + after = await http.get("/api/v1/users/me", headers=bearer(token)) + again = await http.delete("/api/v1/auth/sessions/current", headers=bearer(token)) + print(f"logout: {logout.status_code}, затем {after.status_code}, повтор {again.status_code}") + assert (logout.status_code, logout.content) == (204, b"") + assert after.status_code == again.status_code == 401 + assert again.headers["WWW-Authenticate"] == BEARER + + +async def test_valkey_serves_repeated_checks(http: AsyncClient) -> None: + _, token = await enroll(http, "cache") + await http.get("/api/v1/users/me", headers=bearer(token)) + before = int(valkey("info stats").split("keyspace_hits:")[1].split()[0]) + for _ in range(5): + assert (await http.get("/api/v1/users/me", headers=bearer(token))).status_code == 200 + after = int(valkey("info stats").split("keyspace_hits:")[1].split()[0]) + print(f"5 проверок токена: keyspace_hits {before} → {after}") + assert after - before >= 5 + + +async def test_valkey_stopped_is_bypassed_but_logout_waits(http: AsyncClient) -> None: + user, token = await enroll(http, "nocache") + try: + compose("stop", "valkey") + me, me_time = await timed(http.get("/api/v1/users/me", headers=bearer(token))) + stranger, stranger_time = await timed( + http.get("/api/v1/users/me", headers=bearer("x" * 43)) + ) + ready = await http.get("/health/ready") + login = await http.post("/api/v1/auth/sessions", headers=basic(user["username"])) + logout = await http.delete("/api/v1/auth/sessions/current", headers=bearer(token)) + report("valkey остановлен, действующий токен", me, me_time) + report("valkey остановлен, неизвестный токен", stranger, stranger_time) + codes = (ready.status_code, login.status_code, logout.status_code) + print("valkey остановлен: ready {}, вход {}, выход {}".format(*codes)) + assert me.json() == user and me_time < 1 + assert stranger.status_code == 401 and stranger.headers["WWW-Authenticate"] == BEARER + assert ready.status_code == 200 + assert login.status_code == 201 + assert logout.status_code == 503 and logout.json()["error"]["code"] == "unavailable" + finally: + compose("start", "valkey") + compose("up", "-d", "--wait", "valkey") + await asyncio.sleep(1.2) + retried = await http.delete("/api/v1/auth/sessions/current", headers=bearer(token)) + print(f"valkey снова запущен: повторный выход {retried.status_code}") + assert retried.status_code == 204 + assert (await http.get("/api/v1/users/me", headers=bearer(token))).status_code == 401 + + +async def test_valkey_restart_keeps_cache_and_revocations(http: AsyncClient) -> None: + _, alive = await enroll(http, "alive") + _, revoked = await enroll(http, "revoked") + assert (await http.get("/api/v1/users/me", headers=bearer(alive))).status_code == 200 + assert (await http.get("/api/v1/users/me", headers=bearer(revoked))).status_code == 200 + assert ( + await http.delete("/api/v1/auth/sessions/current", headers=bearer(revoked)) + ).status_code == 204 + keys_before = valkey("dbsize") + compose("restart", "valkey") + compose("up", "-d", "--wait", "valkey") + keys_after = valkey("dbsize") + await asyncio.sleep(1.2) + still = await http.get("/api/v1/users/me", headers=bearer(alive)) + dead = await http.get("/api/v1/users/me", headers=bearer(revoked)) + print(f"рестарт valkey: ключей {keys_before} → {keys_after}") + print(f"рестарт valkey: живой {still.status_code}, отозванный {dead.status_code}") + assert keys_after == keys_before + assert still.status_code == 200 + assert dead.status_code == 401 + + +async def test_valkey_flushall_loses_nothing(http: AsyncClient) -> None: + _, alive = await enroll(http, "alive") + _, revoked = await enroll(http, "revoked") + assert ( + await http.delete("/api/v1/auth/sessions/current", headers=bearer(revoked)) + ).status_code == 204 + valkey("flushall") + still = await http.get("/api/v1/users/me", headers=bearer(alive)) + dead = await http.get("/api/v1/users/me", headers=bearer(revoked)) + print(f"FLUSHALL: живой {still.status_code}, отозванный {dead.status_code}") + assert still.status_code == 200 + assert dead.status_code == 401 + + +async def test_postgres_stopped_gives_bounded_503_and_recovers(http: AsyncClient) -> None: + user, token = await enroll(http, "nodb") + assert (await http.get("/api/v1/users/me", headers=bearer(token))).status_code == 200 + api_started = started_at("api") + try: + compose("stop", "postgres") + checks = { + "ready": await timed(http.get("/health/ready")), + "me": await timed(http.get("/api/v1/users/me", headers=bearer(token))), + "stranger": await timed(http.get("/api/v1/users/me", headers=bearer("x" * 43))), + "login": await timed( + http.post("/api/v1/auth/sessions", headers=basic(user["username"])) + ), + "register": await timed(http.post("/api/v1/users", json=new_user("nodb"))), + } + live_inside = inside_replica("/health/live") + excluded = await waited(lambda: lb_up() == 0, 5) + through_lb = await http.get("/health/live") + for name, (response, elapsed) in checks.items(): + report(f"postgres остановлен, {name}", response, elapsed) + assert elapsed < 2 + assert response.status_code == 503 + assert response.json()["error"]["code"] == "unavailable" + print( + f"live внутри реплики {live_inside}; LB исключил все реплики ещё через {excluded:.2f} с" + ) + report("live через LB без реплик", through_lb, 0) + assert live_inside == 200 + assert lb_up() == 0 + assert excluded < 2 + assert through_lb.status_code == 503 + assert through_lb.json()["error"]["request_id"] == through_lb.headers["X-Request-Id"] + assert "X-Instance-Id" not in through_lb.headers + finally: + compose("start", "postgres") + recovered = await wait_ready(http) + await waited(lambda: lb_up() == len(compose("ps", "-q", "api").split())) + me = await http.get("/api/v1/users/me", headers=bearer(token)) + print(f"postgres запущен: готовность через {recovered:.1f} с, прежний токен {me.status_code}") + assert me.json() == user + assert started_at("api") == api_started + + +async def test_api_restart_keeps_session_without_extending_it(http: AsyncClient) -> None: + body = new_user("restart") + assert (await http.post("/api/v1/users", json=body)).status_code == 201 + session = (await http.post("/api/v1/auth/sessions", headers=basic(body["username"]))).json() + compose("restart", "api") + recovered = await wait_ready(http) + me = await http.get("/api/v1/users/me", headers=bearer(session["token"])) + stored = psql( + "SELECT to_char(expires_at AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS.US\"Z\"') " + "FROM sessions s JOIN users u ON u.id = s.user_id WHERE u.username = :'name'", + name=body["username"], + ) + print(f"рестарт api: готовность через {recovered:.1f} с, токен {me.status_code}") + print(f"expires_at при входе {session['expires_at']}, в БД после рестарта {stored}") + assert me.status_code == 200 + assert stored.rstrip("Z").rstrip("0") == session["expires_at"].rstrip("Z").rstrip("0") diff --git a/student-tests/stand.py b/student-tests/stand.py new file mode 100644 index 00000000..7264b1d2 --- /dev/null +++ b/student-tests/stand.py @@ -0,0 +1,132 @@ +import asyncio +import csv +import os +import subprocess +import time +from base64 import b64encode +from collections.abc import Awaitable, Callable +from pathlib import Path +from typing import Any +from uuid import uuid4 + +import httpx +from httpx import AsyncClient, Response, TransportError + +ROOT = Path(__file__).resolve().parents[1] +BASE_URL = os.environ.get("BASE_URL", "http://localhost:8080") +PASSWORD = "Пароль-для-стенда-01" +BASIC = 'Basic realm="messenger", charset="UTF-8"' +BEARER = 'Bearer realm="messenger"' +VALKEY_AUTH = 'REDISCLI_AUTH="$(sed -n "s/^requirepass //p" /run/secrets/valkey.conf)"' + + +def compose(*args: str, stdin: str | None = None) -> str: + done = subprocess.run( + ["docker", "compose", *args], + cwd=ROOT, + check=True, + capture_output=True, + text=True, + input=stdin, + ) + return done.stdout.strip() + + +def psql(sql: str, **variables: str) -> str: + assigned = [item for name, value in variables.items() for item in ("-v", f"{name}={value}")] + command = ["exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-tA"] + return compose(*command, *assigned, stdin=sql) + + +def valkey(command: str) -> str: + return compose("exec", "-T", "valkey", "sh", "-c", f"{VALKEY_AUTH} valkey-cli {command}") + + +def started_at(service: str) -> str: + return subprocess.run( + [ + "docker", + "inspect", + "--format", + "{{.State.StartedAt}}", + *compose("ps", "-q", service).split(), + ], + check=True, + capture_output=True, + text=True, + ).stdout.strip() + + +def inside_replica(path: str) -> int: + replica = compose("ps", "-q", "api").split()[0] + probe = f"import urllib.request as u;print(u.urlopen('http://127.0.0.1:8080{path}').status)" + done = subprocess.run( + ["docker", "exec", replica, "python", "-c", probe], + capture_output=True, + text=True, + check=False, + ) + return int(done.stdout.strip() or 0) + + +def lb_up() -> int: + stats = httpx.get(f"http://{compose('port', 'lb', '8404')}/stats;csv", timeout=2).text + rows = csv.DictReader(stats.lstrip("# ").splitlines()) + servers = (r for r in rows if r["pxname"] == "api" and r["svname"] != "BACKEND") + return sum(1 for r in servers if r["status"].startswith("UP")) + + +def basic(username: str, password: str = PASSWORD) -> dict[str, str]: + return {"Authorization": "Basic " + b64encode(f"{username}:{password}".encode()).decode()} + + +def bearer(token: str) -> dict[str, str]: + return {"Authorization": f"Bearer {token}"} + + +def new_user(label: str) -> dict[str, str]: + return { + "username": f"u_{label}_{uuid4().hex[:12]}", + "display_name": f"Стенд {label}", + "password": PASSWORD, + } + + +async def enroll(http: AsyncClient, label: str) -> tuple[dict[str, Any], str]: + body = new_user(label) + created = await http.post("/api/v1/users", json=body) + assert created.status_code == 201, created.text + session = await http.post("/api/v1/auth/sessions", headers=basic(body["username"])) + assert session.status_code == 201, session.text + return created.json(), session.json()["token"] + + +async def timed(call: Awaitable[Response]) -> tuple[Response, float]: + started = time.perf_counter() + response = await call + return response, time.perf_counter() - started + + +async def wait_ready(http: AsyncClient, seconds: float = 60) -> float: + started = time.perf_counter() + while time.perf_counter() - started < seconds: + try: + if (await http.get("/health/ready")).status_code == 200: + return time.perf_counter() - started + except TransportError: + pass + await asyncio.sleep(0.25) + raise AssertionError("стенд не вернулся в готовность") + + +async def waited(condition: Callable[[], bool], seconds: float = 30) -> float: + started = time.perf_counter() + while time.perf_counter() - started < seconds: + if condition(): + return time.perf_counter() - started + await asyncio.sleep(0.05) + raise AssertionError("условие не выполнилось за отведённое время") + + +def report(event: str, response: Response, elapsed: float) -> None: + print(f"{event}: {response.status_code} за {elapsed:.3f} с") diff --git a/student-tests/unit/test_attachments_domain.py b/student-tests/unit/test_attachments_domain.py new file mode 100644 index 00000000..db91c483 --- /dev/null +++ b/student-tests/unit/test_attachments_domain.py @@ -0,0 +1,96 @@ +from datetime import UTC, datetime, timedelta +from uuid import uuid4 + +import pytest + +from messenger.attachments.domain.errors import InvalidFilename, UnsupportedImage +from messenger.attachments.domain.events import AttachmentDiscarded, AttachmentQueued +from messenger.attachments.domain.models import Attachment, AttachmentStatus +from messenger.attachments.domain.values import Filename, ImageInfo, ImageKind + +MOMENT = datetime(2026, 1, 1, tzinfo=UTC) + + +@pytest.mark.parametrize( + "raw", + [ + *("", " ", "\t\n", "../a.png", "a/b.png", "a\\b.png"), + *("a\x00.png", "a\x01.png", "a\x7f.png", "a\n.png", "\x9f.png", "x" * 256), + ], +) +def test_filename_rejects_dangerous_and_empty(raw: str) -> None: + with pytest.raises(InvalidFilename): + Filename(raw) + + +@pytest.mark.parametrize("raw", ["sample.png", "фото 1.png", "я" * 255, "👋.jpg", "a..b.png"]) +def test_filename_keeps_display_names(raw: str) -> None: + assert Filename(raw).value == raw + + +@pytest.mark.parametrize( + ("header", "kind"), + [ + ("image/png", ImageKind.PNG), + ("IMAGE/JPEG", ImageKind.JPEG), + ("image/png; charset=binary", ImageKind.PNG), + ], +) +def test_declared_type_ignores_case_and_parameters(header: str, kind: ImageKind) -> None: + assert ImageKind.declared(header) is kind + + +@pytest.mark.parametrize("header", [None, "", "image/gif", "application/octet-stream", "image/*"]) +def test_declared_type_accepts_only_png_and_jpeg(header: str | None) -> None: + with pytest.raises(UnsupportedImage): + ImageKind.declared(header) + + +def test_attachment_is_pending_until_queued() -> None: + attachment_id = uuid4() + attachment = Attachment.stage( + attachment_id=attachment_id, + chat_id=uuid4(), + owner_id=uuid4(), + filename=Filename("a.png"), + image=ImageInfo(ImageKind.PNG, 2, 3), + size_bytes=10, + sha256="0" * 64, + now=MOMENT, + ) + assert attachment.status is AttachmentStatus.PENDING + assert attachment.staging_key == f"staging/{attachment_id}" + assert (attachment.original_key, attachment.published_version, attachment.variants) == ( + None, + 0, + (), + ) + assert attachment.collect_events() == [] + attachment.enqueue(MOMENT - timedelta(seconds=1)) + assert attachment.status is AttachmentStatus.QUEUED + assert not attachment.ready + assert attachment.updated_at == MOMENT + events = attachment.collect_events() + assert [type(event) for event in events] == [AttachmentQueued] + assert attachment.collect_events() == [] + + +def test_discarded_intent_records_its_reason() -> None: + attachment = Attachment.stage( + attachment_id=uuid4(), + chat_id=uuid4(), + owner_id=uuid4(), + filename=Filename("a.png"), + image=ImageInfo(ImageKind.PNG, 1, 1), + size_bytes=1, + sha256="0" * 64, + now=MOMENT, + ) + attachment.discard("abandoned") + (event,) = attachment.collect_events() + assert isinstance(event, AttachmentDiscarded) + assert (event.attachment_id, event.chat_id, event.reason) == ( + attachment.id, + attachment.chat_id, + "abandoned", + ) diff --git a/student-tests/unit/test_authentication.py b/student-tests/unit/test_authentication.py new file mode 100644 index 00000000..6837b434 --- /dev/null +++ b/student-tests/unit/test_authentication.py @@ -0,0 +1,64 @@ +from base64 import b64encode + +import pytest + +from messenger.identity.entrypoints.authentication import ( + basic_credentials, + bearer_token, + challenge_for, +) + + +def encoded(raw: bytes) -> str: + return "Basic " + b64encode(raw).decode() + + +def test_basic_splits_on_first_colon() -> None: + assert basic_credentials(encoded(b"alice:pa:ss:word")) == ("alice", "pa:ss:word") + + +def test_basic_decodes_utf8_password() -> None: + assert basic_credentials(encoded("alice:пароль👋".encode())) == ("alice", "пароль👋") + + +def test_basic_scheme_is_case_insensitive() -> None: + assert basic_credentials("bAsIc " + b64encode(b"alice:secret").decode()) == ("alice", "secret") + + +@pytest.mark.parametrize( + "raw", + [ + "", + "Basic", + "Basic ", + "Basic !!!not-base64!!!", + "Basic YWxpY2U", + encoded(b"no-colon-here"), + encoded("алиса:пароль".encode()), + encoded(b"alice:\xff\xfe"), + "Bearer " + b64encode(b"alice:secret").decode(), + ], +) +def test_basic_rejects_malformed(raw: str) -> None: + assert basic_credentials(raw) is None + + +def test_bearer_accepts_token68() -> None: + assert bearer_token("Bearer abc-DEF_123.~+/==") == "abc-DEF_123.~+/==" + assert bearer_token("bearer token") == "token" + + +@pytest.mark.parametrize("raw", ["", "Bearer", "Bearer ", "Bearer a b", "Basic abc", "x" * 40]) +def test_bearer_rejects_malformed(raw: str) -> None: + assert bearer_token(raw) is None + + +def test_bearer_rejects_oversized_token() -> None: + assert bearer_token("Bearer " + "x" * 513) is None + + +def test_challenge_depends_only_on_path() -> None: + assert challenge_for("/api/v1/auth/sessions") == 'Basic realm="messenger", charset="UTF-8"' + assert challenge_for("/api/v1/auth/sessions/") == 'Basic realm="messenger", charset="UTF-8"' + assert challenge_for("/api/v1/auth/sessions/current") == 'Bearer realm="messenger"' + assert challenge_for("/api/v1/users/me") == 'Bearer realm="messenger"' diff --git a/student-tests/unit/test_cursor.py b/student-tests/unit/test_cursor.py new file mode 100644 index 00000000..5458b9ee --- /dev/null +++ b/student-tests/unit/test_cursor.py @@ -0,0 +1,56 @@ +from datetime import UTC, datetime +from uuid import uuid4 + +import pytest + +from messenger.messaging.domain.errors import InvalidPageRequest +from messenger.messaging.infrastructure.cursor import SignedCursorCodec + +KEY = (datetime(2026, 5, 17, 12, 30, 45, 123456, tzinfo=UTC), uuid4().int) + + +def codec(secret: bytes = b"secret-for-tests") -> SignedCursorCodec: + return SignedCursorCodec(secret) + + +def test_roundtrip_preserves_key() -> None: + subject = codec() + assert subject.decode("messages:a", subject.encode("messages:a", KEY)) == KEY + + +def test_cursor_is_opaque() -> None: + encoded = codec().encode("messages:a", KEY) + assert "2026" not in encoded + assert str(KEY[1]) not in encoded + + +def test_cursor_from_another_scope_is_rejected() -> None: + subject = codec() + encoded = subject.encode("messages:chat-one", KEY) + with pytest.raises(InvalidPageRequest): + subject.decode("messages:chat-two", encoded) + + +def test_forged_cursor_is_rejected() -> None: + encoded = codec(b"one-secret").encode("messages:a", KEY) + with pytest.raises(InvalidPageRequest): + codec(b"another-secret").decode("messages:a", encoded) + + +@pytest.mark.parametrize( + "raw", + ["", "garbage", "1.2.3", "2.1.2.3", "1.notanumber.ff.AAAA", "1.1.zz.AAAA"], +) +def test_broken_cursor_is_rejected(raw: str) -> None: + with pytest.raises(InvalidPageRequest): + codec().decode("messages:a", raw) + + +def test_tampered_payload_is_rejected() -> None: + subject = codec() + encoded = subject.encode("messages:a", KEY) + head, _, signature = encoded.rpartition(".") + version, micros, identifier = head.split(".") + tampered = f"{version}.{int(micros) + 1}.{identifier}.{signature}" + with pytest.raises(InvalidPageRequest): + subject.decode("messages:a", tampered) diff --git a/student-tests/unit/test_domain.py b/student-tests/unit/test_domain.py new file mode 100644 index 00000000..95462776 --- /dev/null +++ b/student-tests/unit/test_domain.py @@ -0,0 +1,239 @@ +from dataclasses import fields +from datetime import UTC, datetime, timedelta +from itertools import pairwise +from uuid import UUID, uuid4 + +import pytest + +from messenger.identity.domain.errors import InvalidUserData +from messenger.identity.domain.models import Session, User, token_digest +from messenger.identity.domain.values import DisplayName, Password, Username +from messenger.messaging.domain.errors import ( + ChatNotFound, + IdempotencyConflict, + InvalidAttachmentList, + InvalidChatData, + InvalidMessageText, +) +from messenger.messaging.domain.models import Chat, Receipt +from messenger.messaging.domain.values import MessageText +from messenger.shared.infrastructure.clock import MonotonicUtcClock + +MOMENT = datetime(2026, 1, 1, tzinfo=UTC) +UUID_TWICE = uuid4() + + +@pytest.mark.parametrize( + "raw", ["ab", "a" * 33, "Alice", "alice-01", "алиса", "", "a b", "alice\n", "alice\r\n"] +) +def test_username_rejects_invalid(raw: str) -> None: + with pytest.raises(InvalidUserData): + Username(raw) + + +@pytest.mark.parametrize("raw", ["abc", "a_b_9", "a" * 32]) +def test_username_accepts_valid(raw: str) -> None: + assert Username(raw).value == raw + + +@pytest.mark.parametrize("raw", ["", " ", "\t\n", "x" * 101]) +def test_display_name_rejects_invalid(raw: str) -> None: + with pytest.raises(InvalidUserData): + DisplayName(raw) + + +@pytest.mark.parametrize("raw", ["", " " * 20, "x" * 11, "x" * 129, "\t" * 12]) +def test_password_rejects_invalid(raw: str) -> None: + with pytest.raises(InvalidUserData): + Password(raw) + + +@pytest.mark.parametrize("raw", ["x" * 12, "п" * 128, "пароль:с:двоеточием", "👋" * 12]) +def test_password_counts_code_points(raw: str) -> None: + assert Password(raw).value == raw + + +def test_password_is_not_printed() -> None: + assert "секретный-пароль" not in repr(Password("секретный-пароль")) + + +def test_session_token_is_opaque_and_stored_as_digest() -> None: + user_id = uuid4() + token, session = Session.issue(user_id=user_id, issued_at=MOMENT, ttl=timedelta(hours=1)) + assert len(token) >= 43 + assert str(user_id) not in token + assert session.token_hash == token_digest(token) + assert len(session.token_hash) == 32 + assert session.expires_at == MOMENT + timedelta(hours=1) + assert token not in repr(session) + + +def test_session_tokens_are_unique() -> None: + tokens = { + Session.issue(user_id=uuid4(), issued_at=MOMENT, ttl=timedelta(seconds=1))[0] + for _ in range(1000) + } + assert len(tokens) == 1000 + + +def test_message_text_counts_code_points_not_bytes() -> None: + emoji = "👋" * 4000 + assert len(MessageText(emoji).value) == 4000 + with pytest.raises(InvalidMessageText): + MessageText("👋" * 4001) + + +def test_message_text_rejects_whitespace() -> None: + for raw in (" ", "\n\t "): + with pytest.raises(InvalidMessageText): + MessageText(raw) + assert MessageText("").value == "" + + +def make_chat(*members: object) -> Chat: + ids = [uuid4() for _ in members] if members else [uuid4()] + return Chat.create( + chat_id=uuid4(), + title="Чат", + member_ids=ids, + creator_id=ids[0], + created_at=MOMENT, + ) + + +def test_chat_requires_creator_in_members() -> None: + members = [uuid4(), uuid4()] + with pytest.raises(InvalidChatData): + Chat.create( + chat_id=uuid4(), + title="Чат", + member_ids=members, + creator_id=uuid4(), + created_at=MOMENT, + ) + + +def test_chat_rejects_duplicate_members() -> None: + same = uuid4() + with pytest.raises(InvalidChatData): + Chat.create( + chat_id=uuid4(), + title="Чат", + member_ids=[same, same], + creator_id=same, + created_at=MOMENT, + ) + + +def test_chat_rejects_more_than_hundred_members() -> None: + members = [uuid4() for _ in range(101)] + with pytest.raises(InvalidChatData): + Chat.create( + chat_id=uuid4(), + title="Чат", + member_ids=members, + creator_id=members[0], + created_at=MOMENT, + ) + + +def test_outsider_cannot_post() -> None: + chat = make_chat(1) + with pytest.raises(ChatNotFound): + chat.post(message_id=uuid4(), sender_id=uuid4(), text="привет", now=MOMENT) + + +def test_post_never_moves_history_back() -> None: + chat = make_chat(1) + sender = chat.member_ids[0] + first = chat.post( + message_id=uuid4(), sender_id=sender, text="первое", now=MOMENT + timedelta(seconds=1) + ) + second = chat.post(message_id=uuid4(), sender_id=sender, text="второе", now=MOMENT) + assert second.created_at == first.created_at + timedelta(microseconds=1) + assert chat.last_message_at == second.created_at + + +def test_post_keeps_clock_when_it_is_ahead() -> None: + chat = make_chat(1) + sender = chat.member_ids[0] + first = chat.post(message_id=uuid4(), sender_id=sender, text="первое", now=MOMENT) + later = MOMENT + timedelta(seconds=5) + second = chat.post(message_id=uuid4(), sender_id=sender, text="второе", now=later) + assert first.created_at == MOMENT + assert second.created_at == later + + +def test_post_validates_membership_before_text() -> None: + chat = make_chat(1) + with pytest.raises(ChatNotFound): + chat.post(message_id=uuid4(), sender_id=uuid4(), text="", now=MOMENT) + + +def test_empty_text_needs_an_attachment() -> None: + chat = make_chat(1) + sender = chat.member_ids[0] + with pytest.raises(InvalidMessageText): + chat.post(message_id=uuid4(), sender_id=sender, text="", now=MOMENT) + ids = (uuid4(), uuid4()) + message = chat.post( + message_id=uuid4(), sender_id=sender, text="", now=MOMENT, attachment_ids=ids + ) + assert message.attachment_ids == ids + with pytest.raises(InvalidMessageText): + chat.post(message_id=uuid4(), sender_id=sender, text=" ", now=MOMENT, attachment_ids=ids) + + +@pytest.mark.parametrize("ids", [[uuid4() for _ in range(11)], [UUID_TWICE, UUID_TWICE]]) +def test_attachment_list_is_bounded_and_unique(ids: list[UUID]) -> None: + chat = make_chat(1) + with pytest.raises(InvalidAttachmentList): + chat.post( + message_id=uuid4(), + sender_id=chat.member_ids[0], + text="x", + now=MOMENT, + attachment_ids=ids, + ) + + +def test_membership_is_checked_before_attachments() -> None: + chat = make_chat(1) + with pytest.raises(ChatNotFound): + chat.post( + message_id=uuid4(), + sender_id=uuid4(), + text="", + now=MOMENT, + attachment_ids=[uuid4() for _ in range(11)], + ) + + +def test_receipt_replays_only_the_same_request() -> None: + chat = make_chat(1) + message = chat.post(message_id=uuid4(), sender_id=chat.member_ids[0], text="раз", now=MOMENT) + receipt = Receipt("key-1", b"a" * 32, message) + assert receipt.replay(b"a" * 32) is message + with pytest.raises(IdempotencyConflict): + receipt.replay(b"b" * 32) + + +def test_user_registration_emits_event() -> None: + user, event = User.register( + user_id=uuid4(), + username=Username("alice_01"), + display_name=DisplayName("Алиса"), + password_hash="$argon2id$v=19$m=19456,t=2,p=1$c2FsdA$aGFzaA", + registered_at=MOMENT, + ) + assert event.name == "user.registered" + assert {field.name for field in fields(event)} == {"name", "user_id", "registered_at"} + assert user.created_at == MOMENT + assert "argon2id" not in repr(user) + + +def test_clock_is_strictly_increasing() -> None: + clock = MonotonicUtcClock() + stamps = [clock.now() for _ in range(1000)] + assert all(later > earlier for earlier, later in pairwise(stamps)) + assert all(stamp.tzinfo is UTC for stamp in stamps) diff --git a/student-tests/unit/test_pipeline_domain.py b/student-tests/unit/test_pipeline_domain.py new file mode 100644 index 00000000..60935840 --- /dev/null +++ b/student-tests/unit/test_pipeline_domain.py @@ -0,0 +1,93 @@ +import json +from datetime import UTC, datetime, timedelta +from pathlib import Path +from uuid import uuid4 + +import pytest + +from messenger.attachments.domain.jobs import Job, RetryPolicy +from messenger.attachments.domain.variants import fit, variant_key +from messenger.attachments.entrypoints.schemas import JobMessage +from messenger.attachments.infrastructure.jitter import RandomJitter + +SCHEMA = json.loads( + (Path(__file__).resolve().parents[2] / "contracts" / "jobs.schema.json").read_text() +) + + +@pytest.mark.parametrize( + ("size", "fitted"), + [ + ((320, 200), (256, 160)), + ((300, 200), (256, 170)), + ((801, 601), (256, 192)), + ((17, 1000), (4, 256)), + ((3000, 2), (256, 1)), + ((2, 3000), (1, 256)), + ((64, 64), (64, 64)), + ((256, 100), (256, 100)), + ((257, 257), (256, 256)), + ], +) +def test_thumbnail_fits_the_box_rounding_down( + size: tuple[int, int], fitted: tuple[int, int] +) -> None: + assert fit(*size, 256) == fitted + + +@pytest.mark.parametrize(("spread", "share"), [(0.0, 0.5), (0.5, 0.75), (0.999, 0.9995)]) +def test_retry_delay_has_equal_jitter_up_to_the_cap(spread: float, share: float) -> None: + policy = RetryPolicy(max_attempts=5, base_seconds=2, cap_seconds=10) + delays = [policy.delay(attempt, spread) for attempt in range(1, 6)] + ceilings = [2, 4, 8, 10] + assert delays[:4] == [timedelta(seconds=ceiling * share) for ceiling in ceilings] + assert delays[4] is None + assert policy.exhausted(5) + assert not policy.exhausted(4) + + +def test_jitter_spreads_retries_between_half_and_full_delay() -> None: + policy, jitter = RetryPolicy(5, 2, 60), RandomJitter() + delays = {policy.delay(3, jitter.spread()) for _ in range(200)} + assert all(timedelta(seconds=4) <= delay < timedelta(seconds=8) for delay in delays if delay) + assert len(delays) > 150 + + +def test_job_message_follows_the_contract_schema() -> None: + job = Job(uuid4(), uuid4(), 1, 3, datetime(2026, 9, 1, 12, 0, tzinfo=UTC)) + payload = job.payload() + assert set(SCHEMA["required"]) <= set(payload) <= set(SCHEMA["properties"]) + assert payload["schema_version"] == SCHEMA["properties"]["schema_version"]["const"] + assert payload["type"] == SCHEMA["properties"]["type"]["const"] + assert payload["created_at"] == "2026-09-01T12:00:00.000000Z" + assert JobMessage.model_validate_json(json.dumps(payload)).job() == job + assert job.retried(4).payload()["job_id"] == payload["job_id"] + + +@pytest.mark.parametrize("example", SCHEMA["examples"]) +def test_contract_examples_are_accepted(example: dict[str, object]) -> None: + assert JobMessage.model_validate_json(json.dumps(example)).attempt == example["attempt"] + + +@pytest.mark.parametrize( + "change", + [ + {"schema_version": 2}, + {"type": "attachment.process.v2"}, + {"attempt": 0}, + {"pipeline_version": 0}, + {"filename": "a.png"}, + {"traceparent": "01-xyz"}, + {"created_at": "2026-09-01T12:00:00"}, + ], +) +def test_messages_off_the_schema_are_rejected(change: dict[str, object]) -> None: + message = {**SCHEMA["examples"][0], **change} + with pytest.raises(ValueError, match="validation error"): + JobMessage.model_validate_json(json.dumps(message)) + + +def test_variant_keys_carry_version_and_content_hash() -> None: + attachment = uuid4() + key = variant_key(attachment, 2, "thumbnail", "ab" * 32, "jpg") + assert key == f"variants/{attachment}/2/thumbnail-{'ab' * 8}.jpg" diff --git a/student-tests/unit/test_render_pool.py b/student-tests/unit/test_render_pool.py new file mode 100644 index 00000000..a22ec6d2 --- /dev/null +++ b/student-tests/unit/test_render_pool.py @@ -0,0 +1,91 @@ +import asyncio +import os +import sys +import time +from collections.abc import AsyncIterator + +import pytest + +from messenger.attachments.application.ports import Rendition +from messenger.attachments.domain.errors import ImageTooDemanding +from messenger.attachments.infrastructure.images import ( + ProcessVariantRenderer, + RenderLimits, + render_variants, +) +from messenger.shared.application.errors import Unavailable +from pictures import SAMPLE + +LIMITS = RenderLimits(max_bytes=10_485_760, max_pixels=40_000_000) +HOG_BYTES = 512 << 20 + + +def misbehaving(data: bytes, limits: RenderLimits) -> list[Rendition]: + if data == b"slow": + time.sleep(30) + if data == b"crash": + os._exit(1) + if data == b"oom": + raise MemoryError + if data == b"hog": + bytearray(HOG_BYTES) + return render_variants(data, limits) + + +@pytest.fixture +async def renderer() -> AsyncIterator[ProcessVariantRenderer]: + pool = ProcessVariantRenderer( + LIMITS, + concurrency=1, + timeout=3.0, + memory_bytes=256 << 20, + tasks_per_child=10, + render=misbehaving, + ) + yield pool + pool.close() + + +async def test_renders_in_a_separate_process(renderer: ProcessVariantRenderer) -> None: + assert await renderer.render(SAMPLE, 1) == render_variants(SAMPLE, LIMITS) + + +async def test_timeout_kills_the_render_and_frees_the_slot( + renderer: ProcessVariantRenderer, +) -> None: + started = time.monotonic() + with pytest.raises(ImageTooDemanding): + await renderer.render(b"slow", 1) + assert time.monotonic() - started < 10 + assert len(await renderer.render(SAMPLE, 1)) == 3 + + +async def test_crashed_render_process_is_a_transient_error( + renderer: ProcessVariantRenderer, +) -> None: + with pytest.raises(Unavailable): + await renderer.render(b"crash", 1) + assert len(await renderer.render(SAMPLE, 1)) == 3 + + +async def test_memory_error_is_a_permanent_error(renderer: ProcessVariantRenderer) -> None: + with pytest.raises(ImageTooDemanding): + await renderer.render(b"oom", 1) + assert len(await renderer.render(SAMPLE, 1)) == 3 + + +async def test_cancelled_render_does_not_block_the_slot(renderer: ProcessVariantRenderer) -> None: + slow = asyncio.create_task(renderer.render(b"slow", 1)) + await asyncio.sleep(1) + slow.cancel() + with pytest.raises(asyncio.CancelledError): + await slow + assert len(await asyncio.wait_for(renderer.render(SAMPLE, 1), 10)) == 3 + + +@pytest.mark.skipif(sys.platform != "linux", reason="RLIMIT_AS применяется только в Linux") +async def test_address_space_limit_applies_to_the_render_process( + renderer: ProcessVariantRenderer, +) -> None: + with pytest.raises(ImageTooDemanding): + await renderer.render(b"hog", 1) diff --git a/student-tests/unit/test_renderer.py b/student-tests/unit/test_renderer.py new file mode 100644 index 00000000..c7367ae0 --- /dev/null +++ b/student-tests/unit/test_renderer.py @@ -0,0 +1,126 @@ +from io import BytesIO + +import pytest +from PIL import Image + +from messenger.attachments.application.ports import Rendition +from messenger.attachments.domain.errors import ImageTooDemanding, UnreadableImage +from messenger.attachments.infrastructure.images import RenderLimits, render_variants +from pictures import SAMPLE + +LIMITS = RenderLimits(max_bytes=10_485_760, max_pixels=40_000_000) +FORMATS = { + "thumbnail": ("image/jpeg", "jpg", "JPEG"), + "crop": ("image/jpeg", "jpg", "JPEG"), + "optimized": ("image/webp", "webp", "WEBP"), +} + + +def encoded(image: Image.Image, kind: str = "PNG", **options: object) -> bytes: + buffer = BytesIO() + image.save(buffer, kind, **options) + return buffer.getvalue() + + +def rendered(data: bytes) -> dict[str, Image.Image]: + renditions: list[Rendition] = render_variants(data, LIMITS) + assert [rendition.name for rendition in renditions] == ["thumbnail", "crop", "optimized"] + images = {} + for rendition in renditions: + content_type, extension, kind = FORMATS[rendition.name] + assert (rendition.content_type, rendition.extension) == (content_type, extension) + image = Image.open(BytesIO(rendition.data)) + assert image.format == kind + assert image.size == (rendition.width, rendition.height) + assert not image.getexif() + assert not {"icc_profile", "exif", "xmp"} & set(image.info) + images[rendition.name] = image + return images + + +def sizes(data: bytes) -> tuple[tuple[int, int], ...]: + images = rendered(data) + return tuple(images[name].size for name in ("thumbnail", "crop", "optimized")) + + +def test_fixture_gives_three_variants() -> None: + assert sizes(SAMPLE) == ((256, 160), (128, 128), (320, 200)) + + +@pytest.mark.parametrize( + ("mode", "size", "thumbnail", "optimized"), + [ + ("L", (17, 1000), (4, 256), (17, 1000)), + ("RGB", (64, 64), (64, 64), (64, 64)), + ("RGB", (3000, 2), (256, 1), (1280, 1)), + ("I;16", (300, 200), (256, 170), (300, 200)), + ("CMYK", (801, 601), (256, 192), (801, 601)), + ("RGB", (4000, 3000), (256, 192), (1280, 960)), + ], +) +def test_sizes_round_down_never_upscale_and_crop_is_square( + mode: str, size: tuple[int, int], thumbnail: tuple[int, int], optimized: tuple[int, int] +) -> None: + kind = "JPEG" if mode == "CMYK" or size == (4000, 3000) else "PNG" + assert sizes(encoded(Image.new(mode, size), kind)) == (thumbnail, (128, 128), optimized) + + +def test_crop_takes_the_centre() -> None: + image = Image.new("RGB", (300, 100), (255, 0, 0)) + image.paste((0, 0, 255), (100, 0, 200, 100)) + crop = rendered(encoded(image))["crop"].convert("RGB") + pixels = [crop.getpixel((x, 64)) for x in (2, 64, 125)] + assert all(isinstance(p, tuple) and p[0] < 60 and p[2] > 200 for p in pixels) + + +@pytest.mark.parametrize("mode", ["RGBA", "LA"]) +def test_transparency_is_composed_on_white(mode: str) -> None: + transparent = Image.new(mode, (300, 300), (0,) * len(mode)) + for image in rendered(encoded(transparent)).values(): + assert image.convert("RGB").getpixel((5, 5)) == (255, 255, 255) + + +def test_palette_transparency_is_composed_on_white() -> None: + palette = Image.new("P", (200, 100)) + image = rendered(encoded(palette, transparency=0))["thumbnail"] + assert image.size == (200, 100) + assert image.convert("RGB").getpixel((5, 5)) == (255, 255, 255) + + +@pytest.mark.parametrize( + ("orientation", "thumbnail", "optimized"), + [(6, (128, 256), (200, 400)), (3, (256, 128), (400, 200))], +) +def test_exif_orientation_is_applied_before_sizing( + orientation: int, thumbnail: tuple[int, int], optimized: tuple[int, int] +) -> None: + exif = Image.Exif() + exif[0x0112] = orientation + data = encoded( + Image.new("RGB", (400, 200), (10, 200, 10)), "JPEG", exif=exif, icc_profile=b"x" * 64 + ) + assert sizes(data) == (thumbnail, (128, 128), optimized) + + +def test_same_bytes_give_same_variants() -> None: + assert render_variants(SAMPLE, LIMITS) == render_variants(SAMPLE, LIMITS) + + +@pytest.mark.parametrize( + "data", + [b"", b"garbage", SAMPLE[: len(SAMPLE) // 2], b"GIF89a" + bytes(20)], + ids=["empty", "garbage", "truncated", "gif"], +) +def test_undecodable_data_is_a_permanent_error(data: bytes) -> None: + with pytest.raises(UnreadableImage): + render_variants(data, LIMITS) + + +def test_pixel_limit_is_checked_before_decoding() -> None: + with pytest.raises(UnreadableImage): + render_variants(SAMPLE, RenderLimits(LIMITS.max_bytes, 320 * 200 - 1)) + + +def test_byte_limit_is_checked_before_decoding() -> None: + with pytest.raises(ImageTooDemanding): + render_variants(SAMPLE, RenderLimits(len(SAMPLE) - 1, LIMITS.max_pixels)) diff --git a/student-tests/unit/test_timeouts.py b/student-tests/unit/test_timeouts.py new file mode 100644 index 00000000..4050372a --- /dev/null +++ b/student-tests/unit/test_timeouts.py @@ -0,0 +1,60 @@ +import re +from pathlib import Path +from typing import Any + +import yaml + +from messenger.shared.infrastructure.postgres import PING_TIMEOUT_SECONDS + +ROOT = Path(__file__).resolve().parents[2] +DEFAULT = re.compile(r"\$\{\w+:?-([^}]*)\}") +UNITS = {"ms": 0.001, "s": 1.0, "": 1.0} + + +def seconds(value: object) -> float: + found = re.fullmatch(r"([\d.]+)(ms|s|)", str(value)) + assert found is not None, value + return float(found[1]) * UNITS[found[2]] + + +def services() -> dict[str, Any]: + return yaml.safe_load(DEFAULT.sub(r"\1", (ROOT / "compose.yaml").read_text()))["services"] + + +def balancer() -> dict[str, float]: + text = (ROOT / "infra/haproxy/haproxy.cfg").read_text() + found = { + f"timeout {name}": seconds(value) + for name, value in re.findall(r"timeout ([\w-]+) (\S+)\n", text) + } + found["hard-stop-after"] = seconds(re.findall(r"hard-stop-after (\S+)", text)[0]) + check = re.findall(r"default-server .*", text)[0] + for key in ("inter", "fastinter", "fall", "rise"): + found[key] = seconds(re.findall(rf"\b{key} (\S+)", check)[0]) + return found + + +def test_timeouts_shrink_inwards_and_shutdown_fits_the_stop_budget() -> None: + lb, stand = balancer(), services() + api = {k: seconds(v) for k, v in stand["api"]["environment"].items() if k.endswith("SECONDS")} + hook = seconds( + re.findall(r"STOP_TIMEOUT:-(\d+)", (ROOT / "scripts/stop-one.sh").read_text())[0] + ) + grace = seconds(stand["api"]["stop_grace_period"]) + detection = lb["inter"] + (lb["fall"] - 1) * lb["fastinter"] + hung = detection + lb["fall"] * lb["timeout check"] + database = api["DB_ACQUIRE_TIMEOUT_SECONDS"] + api["DB_COMMAND_TIMEOUT_SECONDS"] + drain, shutdown = api["DRAIN_SECONDS"], api["SHUTDOWN_TIMEOUT_SECONDS"] + print( + f"обнаружение {detection} с (зависание {hung} с), окно {drain} с, остановка {shutdown} с," + f" хук {hook} с, stop_grace_period {grace} с; LB server {lb['timeout server']} с >" + f" запрос {api['REQUEST_TIMEOUT_SECONDS']} с > БД {database} с;" + f" readiness {2 * PING_TIMEOUT_SECONDS} с < check {lb['timeout check']} с" + ) + assert lb["timeout server"] > api["REQUEST_TIMEOUT_SECONDS"] > database + assert lb["timeout check"] > 2 * PING_TIMEOUT_SECONDS + assert drain > detection + assert shutdown - drain >= database + assert shutdown < hook < grace < 10 + assert lb["hard-stop-after"] < seconds(stand["lb"]["stop_grace_period"]) + assert (detection, hung) == (1.5, 3.5) diff --git a/uv.lock b/uv.lock new file mode 100644 index 00000000..5372a9c3 --- /dev/null +++ b/uv.lock @@ -0,0 +1,1269 @@ +version = 1 +revision = 5 +requires-python = "==3.14.*" + +[[package]] +name = "aio-pika" +version = "10.2.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiormq" }, + { name = "yarl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5d/31/70f0d7a9fe1affc3241d86d24c24714a4a2ac648961efd3d835ef734a6a1/aio_pika-10.2.1.tar.gz", hash = "sha256:e4f77afd9179a3cca5a904c3b0a1190256a36d9f90650db908b1b2245eb27487", size = 101040, upload-time = "2026-10-25T11:47:24.577Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b4/ae/f69eb62f240f743d5d3aff659a6ed76077f73b722a0d8f9f9559ab825836/aio_pika-10.2.1-py3-none-any.whl", hash = "sha256:66c85d311e30f1dd3e28109d686c9bf1d9dd1a74582004c7cb7bed284d8fe3b1", size = 61568, upload-time = "2026-10-25T11:47:23.119Z" }, +] + +[[package]] +name = "aiobotocore" +version = "3.10.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiohttp" }, + { name = "aioitertools" }, + { name = "botocore" }, + { name = "jmespath" }, + { name = "multidict" }, + { name = "python-dateutil" }, + { name = "wrapt" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/16/40/659f2ac3915d329e3e8e5bf674d35ce1774f3117c6f60339d5e05e505c36/aiobotocore-3.10.0.tar.gz", hash = "sha256:35480ef11c51e23087273b85d97f80da2f356a46ce5f3fb88e24483cd343ffdb", size = 530781, upload-time = "2026-10-23T10:44:50.843Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7f/bc/3bb8965204ba51bbf4d021bc89d1357cd2179a17ef71b5fc429e21b511a2/aiobotocore-3.10.0-py3-none-any.whl", hash = "sha256:de0ccbcb82b6b0009f1b6667f48442e9d0432f9b8bfe6172fa51777cbea89d03", size = 102759, upload-time = "2026-10-23T10:44:49.109Z" }, +] + +[[package]] +name = "aiohappyeyeballs" +version = "2.7.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ce/f4/eec0465c2f67b2664688d0240b3212d5196fd89e741df67ddb81f8d35658/aiohappyeyeballs-2.7.1.tar.gz", hash = "sha256:065665c041c42a5938ed220bdcd7230f22527fbec085e1853d2402c8a3615d9d", size = 24757, upload-time = "2026-07-01T17:11:55.501Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/43/1947f06babed6b3f1d7f38b0c767f52df66bfb2bc10b468c4a7de9eceff2/aiohappyeyeballs-2.7.1-py3-none-any.whl", hash = "sha256:9243213661e29250eb41368e5daa826fc017156c3b8a11440826b2e3ed376472", size = 15038, upload-time = "2026-07-01T17:11:54.055Z" }, +] + +[[package]] +name = "aiohttp" +version = "3.14.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiohappyeyeballs" }, + { name = "aiosignal" }, + { name = "attrs" }, + { name = "frozenlist" }, + { name = "multidict" }, + { name = "propcache" }, + { name = "yarl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6c/4c/bdccd81e9ee225b69c60e7766c9a5b05364f118f4d383713b89a682d772d/aiohttp-3.14.5.tar.gz", hash = "sha256:5558a7f5a05af9ecf744af91e5baefc436f93c9333e656c27ec253f9a6bbe178", size = 8078052, upload-time = "2026-10-11T01:05:12.408Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/43/be/3184a1d34a8be665569eadb7e9e764b4629e4f3413e241cb2e4d6fecf3b3/aiohttp-3.14.5-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:939042d5cda21d41a6f512e7cc8b8e33a2aebff863352251da495fbd91b673b5", size = 540673, upload-time = "2026-10-11T01:01:41.354Z" }, + { url = "https://files.pythonhosted.org/packages/60/2a/d35f3ba4cf157b072e3b674bf9983047ca5ea5173c995d32d877e1191d36/aiohttp-3.14.5-cp314-cp314-android_24_x86_64.whl", hash = "sha256:6da32b5ff3fd78d244e37300463434c7145162bfd2b6e9e915ab164da37f7343", size = 554087, upload-time = "2026-10-11T01:01:43.719Z" }, + { url = "https://files.pythonhosted.org/packages/fc/d2/61a33880ca4eaca95a9c60ca3f6beed15555af1028652dfaac601627787b/aiohttp-3.14.5-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:1b438b73c38111818d0c9d6a5c2bfed8584c8e503a49ef085d70e874ec846738", size = 522613, upload-time = "2026-10-11T01:01:46.154Z" }, + { url = "https://files.pythonhosted.org/packages/31/1d/de579b299d2225dc2c6fd99d579d91f16c02a913fb5af9a3cf2fe9bd88ba/aiohttp-3.14.5-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:755933b107ea7a6a9ac916f635a70595a5b1a32fac10a8ff0b9f2ab88555550c", size = 529682, upload-time = "2026-10-11T01:01:48.477Z" }, + { url = "https://files.pythonhosted.org/packages/f4/4a/ddb923564e15e053b6e060b0036e1694dcadcb13aa476c5a87dcad20e336/aiohttp-3.14.5-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:b3cc509327c7b27f6f4727a8830f4004f6df7766e179f2f4b8e54e65c0bec5d3", size = 540120, upload-time = "2026-10-11T01:01:50.474Z" }, + { url = "https://files.pythonhosted.org/packages/3d/36/a640fbecaa53727a5900b892bdbe17b5f3e8cc88903e22864fe41b654def/aiohttp-3.14.5-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:7bd8ac754ebd6733a3e2a0dd1674c4d8ab086196803fd8dcd776f07b4e2607d9", size = 818795, upload-time = "2026-10-11T01:01:52.665Z" }, + { url = "https://files.pythonhosted.org/packages/ef/b6/d52ca608859e271b5fa7944074802dc45f60e52c318a4ddc34edbf73586e/aiohttp-3.14.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:e724a7b6091f0b1ac064f9d1b15ff9ec52e6033a86cdae649e5f086e32a3c0db", size = 549667, upload-time = "2026-10-11T01:01:54.65Z" }, + { url = "https://files.pythonhosted.org/packages/ce/b5/05b8ac39a76ff4bca89f42a4c2471560c71f894ff6e4bc16158c951874e3/aiohttp-3.14.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c32e26310cc10e547f53cd13d39a369034f69dcb7d749d5cb0e5f67bc196b6ba", size = 548922, upload-time = "2026-10-11T01:01:56.547Z" }, + { url = "https://files.pythonhosted.org/packages/19/b0/5aa186d56ce2334dabe29b70bd99dc8ae926ee44184c0de64a53d415a4f3/aiohttp-3.14.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1eb8167961ec4dfcc8cb9dd50bd0ee72519f7ef496be95203e49e27b01618382", size = 1882422, upload-time = "2026-10-11T01:01:59.258Z" }, + { url = "https://files.pythonhosted.org/packages/db/f7/7d5c91bb9620db300c8ddb05337a9014301acb626223faff3abcb8ea47d7/aiohttp-3.14.5-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c1d60eafd9c7e8e74abd03a5b00df44e7febfe6d9b89b559c0a6551eef0699d4", size = 1843621, upload-time = "2026-10-11T01:02:01.417Z" }, + { url = "https://files.pythonhosted.org/packages/30/0a/b208953b96d8f24b75f6da704f508e6c5cf3022f52b60c61933df082e89c/aiohttp-3.14.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:137351bf20bbed9a65e839f4a4452ac377389bdb2f2857d2acffef38f5e9f2d1", size = 1927915, upload-time = "2026-10-11T01:02:03.697Z" }, + { url = "https://files.pythonhosted.org/packages/f6/79/90ebcccb55e2d1e11a1fed581d83bb966e38fb35fb4b2577fdc980f8707a/aiohttp-3.14.5-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fba47bc2c3d7303c3d027c6cf4d07626c37b1314ac81f5820c31032e0ca1f677", size = 2036339, upload-time = "2026-10-11T01:02:06.046Z" }, + { url = "https://files.pythonhosted.org/packages/a6/66/55a8904b3a129fafdf94f9cc0a2e4ca09a3c914650be52355db7ad0bbdb6/aiohttp-3.14.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:94684b879ac1d71e4238850c99b62dc1b28d9086b156a2555f082010b85a865c", size = 1888041, upload-time = "2026-10-11T01:02:08.384Z" }, + { url = "https://files.pythonhosted.org/packages/0b/b8/96b25da7329a52e42c812b1e8b076386039ec4fc312afa043d173d8147fc/aiohttp-3.14.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:56572c42e3ecd636de8d2c3dd54cf5fc939cb5c32eb56297f176a0d366fac622", size = 1703849, upload-time = "2026-10-11T01:02:10.903Z" }, + { url = "https://files.pythonhosted.org/packages/1b/43/fbf976e3ae4c038d6f5c84945ab2150298c2d71201674d4e53d158063e75/aiohttp-3.14.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a95529a92a446db351675f4aab518feaf5e99842f63f5dd17160c2b74f382db3", size = 1849703, upload-time = "2026-10-11T01:02:13.15Z" }, + { url = "https://files.pythonhosted.org/packages/8b/7d/218e912f4c1d89bde7ac551409be57ad2f6121638e56942d395a7cb1fa58/aiohttp-3.14.5-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:3edbece0379b8b4aaa67619b8aa2399bb66fce372cd5911098a434ea77220aa0", size = 1852695, upload-time = "2026-10-11T01:02:15.295Z" }, + { url = "https://files.pythonhosted.org/packages/5a/42/252a1b9287e3b6e393a1c3bd1776f36af30f5f25f071bbf2b0cb7eba9116/aiohttp-3.14.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:56d9828f204331a5ca8850fcfe2bcce95a149f1f223f60cc7216e5524978e480", size = 1893493, upload-time = "2026-10-11T01:02:17.93Z" }, + { url = "https://files.pythonhosted.org/packages/78/97/71cae83d5100556fad1521684f7cd1e3e578432644f850245ed3bd969310/aiohttp-3.14.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:20064a177a070d789ee64a50b01a9161d3468e989baacfc6c714aa685c4b332f", size = 1693468, upload-time = "2026-10-11T01:02:20.666Z" }, + { url = "https://files.pythonhosted.org/packages/1f/69/73d88e97a8b5f0ca7a946d0011c0de99fb188b1687c7948ecd0553dc5bf0/aiohttp-3.14.5-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:81c2b3dfd56c62bee6108e4852d5970b4cf9086390b6983f52b666e878c1f115", size = 1915398, upload-time = "2026-10-11T01:02:23.011Z" }, + { url = "https://files.pythonhosted.org/packages/05/f0/881644bcb15d4b258daea9b720a0af9dc4330496cc8d6ade9090cdd0cffc/aiohttp-3.14.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:09ec102b4b8c9a920275733bbc11fdbb615efe6f9231a06007c0218d336fb77a", size = 1871470, upload-time = "2026-10-11T01:02:25.278Z" }, + { url = "https://files.pythonhosted.org/packages/7f/de/19d9ebbcce5aedaa3242d8a99ff8816a60bdb7629fb0084bf4a45bfd1f62/aiohttp-3.14.5-cp314-cp314-win32.whl", hash = "sha256:9c428eb2bd8817588d16a0ab898aa4eb5d141f896aa2b394cc79a4cf61d9a8e2", size = 485517, upload-time = "2026-10-11T01:02:27.579Z" }, + { url = "https://files.pythonhosted.org/packages/a9/74/8cdaf0e58c2588371670d5a9a8215bbb971d36940b6e5d051967dd05c07d/aiohttp-3.14.5-cp314-cp314-win_amd64.whl", hash = "sha256:6f967dde489ca6a8c02d093ab245d2cbf50ccb5c36adf0188b17b0ca39d24b67", size = 511755, upload-time = "2026-10-11T01:02:29.685Z" }, + { url = "https://files.pythonhosted.org/packages/1a/6b/e0100e25502430a531c7cf1482a378d0b65bf728ab60c01ee270e56bc469/aiohttp-3.14.5-cp314-cp314-win_arm64.whl", hash = "sha256:1d2d981b53dd09a319e3570ef8cc3bbc3ef86f5a7abef0f6b2bff3867db3a9e7", size = 495151, upload-time = "2026-10-11T01:02:32.163Z" }, + { url = "https://files.pythonhosted.org/packages/9d/c2/ca2ead7b655688c53c03aeeb6e96e6851c9ff08d6be7b13802f53a6ae8fd/aiohttp-3.14.5-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:9ce66feae6ac65327379460380549bf1b8df8e17c4e25df2a2bcf168272e3bed", size = 855396, upload-time = "2026-10-11T01:02:34.443Z" }, + { url = "https://files.pythonhosted.org/packages/a7/70/22206fea409255a240c926ce11de48de354ae2bb90ca44f709c05497585d/aiohttp-3.14.5-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:27c2322e03f66101acb09869ce1cf1efc04994ee95e1735b69827bf8c8b9d781", size = 567734, upload-time = "2026-10-11T01:02:36.644Z" }, + { url = "https://files.pythonhosted.org/packages/ce/e5/79a36c118308b56f8667d67e05d2fb6dc638ab45985704cdb199631bedaa/aiohttp-3.14.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ff75a7537413a86e7cafe98e0e1d6e3dc4b15c6349896e7d5c6b881bfdb6d550", size = 567379, upload-time = "2026-10-11T01:02:38.757Z" }, + { url = "https://files.pythonhosted.org/packages/63/a3/2ebec7dece3b1f02c30d2e484647f6f7b13952b1bb40a4cb285b849e8432/aiohttp-3.14.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9c061aa954daaf57d2a4b8374f9fca621ef0e1b603584431c220c22458c59b6d", size = 2050686, upload-time = "2026-10-11T01:02:41.169Z" }, + { url = "https://files.pythonhosted.org/packages/3a/d2/7e4d093db2f4450482652e7ef19a9e19919028f5135aa52bc4078c3beb80/aiohttp-3.14.5-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:1612fa5857b37bf32e5c1eaeefb96e3b01e9c70679eec81f0934e8a600080863", size = 1915077, upload-time = "2026-10-11T01:02:43.563Z" }, + { url = "https://files.pythonhosted.org/packages/a6/88/bd40d09958442a0a1df67da81de496361d6e2afc04f9db2950d835da750b/aiohttp-3.14.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:adbeee7d6fd4cf5fe0aece2fb3edc4243615d3180430ba8149d01a90670cac99", size = 2027204, upload-time = "2026-10-11T01:02:46.185Z" }, + { url = "https://files.pythonhosted.org/packages/63/eb/3a601c1f8d3103c1a60ea981f20924855da9f575d2007fc38f8898b792fb/aiohttp-3.14.5-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b2966998927d7bed9db12c0a4647b0c7b179755878fc9c357fe1ffd3e3b0c1a5", size = 2149520, upload-time = "2026-10-11T01:02:48.812Z" }, + { url = "https://files.pythonhosted.org/packages/22/d0/4e41bfe1b1ce1cb6f6d2e59fa7a88ef5cf92c402b2d07e9018778ba9edbc/aiohttp-3.14.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8e317e0fb6b16212c881d2205a7d87414c29acd69320b3aa6dce9d9c7b86fe4f", size = 2002039, upload-time = "2026-10-11T01:02:51.293Z" }, + { url = "https://files.pythonhosted.org/packages/6f/5e/72067019545c502b881b031153c437752ecef48d7d213bc0218ccebb4bfb/aiohttp-3.14.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:50343c1757b4b6f6708eeaf24534b32f19dfb99fb1b762c00420867a62fc81e0", size = 1786164, upload-time = "2026-10-11T01:02:53.533Z" }, + { url = "https://files.pythonhosted.org/packages/d9/fe/7741efd6119bfb7a00827fe6f7b84b4409de58d888ae21adc5a9a6824992/aiohttp-3.14.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:083673c7a94c3ea035caaa5ca04288bdb44887abfe1f5ba23294e6a4b03efd2d", size = 1970689, upload-time = "2026-10-11T01:02:55.888Z" }, + { url = "https://files.pythonhosted.org/packages/43/e7/342a13bf67f34d269bf2f7e870ecd72b99c832cc6f0a271a9210c0ebfb84/aiohttp-3.14.5-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:2528cb4c6b92008c76ac9ac6298624069bb2db91ff4929905512d1d84485f658", size = 1951967, upload-time = "2026-10-11T01:02:58.467Z" }, + { url = "https://files.pythonhosted.org/packages/e7/d4/fdb3b27340617e5e64df18a70fa89097778652ea5c7c7e2f79def76999c3/aiohttp-3.14.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:b1b8ece1e71132d2afba4dbc0c3d62c766e25165990b25db1196c04969eb3d84", size = 1968511, upload-time = "2026-10-11T01:03:00.883Z" }, + { url = "https://files.pythonhosted.org/packages/83/b2/e8f88298de78d1a951f36f9f966d38ec6ed1d4721303ba02064a545e8aa6/aiohttp-3.14.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:fce9523df31cea6284f3e2c479876750d7687cf671d7b25d32b19effc0e86441", size = 1778497, upload-time = "2026-10-11T01:03:03.206Z" }, + { url = "https://files.pythonhosted.org/packages/22/68/9ccdb93d664345c546be7f34480b921774d8c0d98f47e70d7e03b115d475/aiohttp-3.14.5-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:09e0eb18c7e0c8777e2f9149de63799195b9b3ca1b5c81ba6f32f2c6b8628210", size = 1999676, upload-time = "2026-10-11T01:03:05.829Z" }, + { url = "https://files.pythonhosted.org/packages/57/4a/a33cfa6dcb00e94194ae4fe710432ca4ed111e016356b4ba4d2f4c3a124c/aiohttp-3.14.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6774814fd5c338e72ee0da5cbb9432816df450e69c019f72b5d29bdec2a1792d", size = 1963437, upload-time = "2026-10-11T01:03:08.196Z" }, + { url = "https://files.pythonhosted.org/packages/f4/20/eacbecfea3b5c3dcbfc9b023e3a5460f43e5dda16d06a2877ebe7184c3f3/aiohttp-3.14.5-cp314-cp314t-win32.whl", hash = "sha256:33f706574e32c6e694f352a856e05caf18f7f2c871b3e87b41c55ea452b409ab", size = 503973, upload-time = "2026-10-11T01:03:10.481Z" }, + { url = "https://files.pythonhosted.org/packages/ff/78/18eec294f6c8c5dc845dcf6d730a0147d8d0f17e86138a7bdb85e43a30fa/aiohttp-3.14.5-cp314-cp314t-win_amd64.whl", hash = "sha256:5ba14a839fbe87cf7c12a6b5661c05f324a296eb8363141edb3944ba63d4c9d3", size = 531190, upload-time = "2026-10-11T01:03:12.716Z" }, + { url = "https://files.pythonhosted.org/packages/9d/39/e53f8169acc85271ebd12b5b32ad7f1541b35639ccbe0f49034c64785d10/aiohttp-3.14.5-cp314-cp314t-win_arm64.whl", hash = "sha256:1061b364556e8172e8d46b0b183adeeb73e8c42d30ebc745591e1bd89acad52e", size = 510360, upload-time = "2026-10-11T01:03:15.08Z" }, + { url = "https://files.pythonhosted.org/packages/68/30/173960c42b05a6c59f7558e4b12a4b0d9ba376cf6aa9bde7f9e08a30ca8d/aiohttp-3.14.5-py3-none-any.whl", hash = "sha256:efc21a454892828368b11c2c780de0ff8bc991f73f6b99c6b66e56205470929b", size = 279517, upload-time = "2026-10-11T01:05:08.523Z" }, +] + +[[package]] +name = "aioitertools" +version = "0.13.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fd/3c/53c4a17a05fb9ea2313ee1777ff53f5e001aefd5cc85aa2f4c2d982e1e38/aioitertools-0.13.0.tar.gz", hash = "sha256:620bd241acc0bbb9ec819f1ab215866871b4bbd1f73836a55f799200ee86950c", size = 19322, upload-time = "2025-11-06T22:17:07.609Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/10/a1/510b0a7fadc6f43a6ce50152e69dbd86415240835868bb0bd9b5b88b1e06/aioitertools-0.13.0-py3-none-any.whl", hash = "sha256:0be0292b856f08dfac90e31f4739432f4cb6d7520ab9eb73e143f4f2fa5259be", size = 24182, upload-time = "2025-11-06T22:17:06.502Z" }, +] + +[[package]] +name = "aiormq" +version = "7.2.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pamqp" }, + { name = "yarl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/71/11/8b93eac7630c5087c34df3d34ce0bdf50d6ea2fb83061f17e8c73877141d/aiormq-7.2.4.tar.gz", hash = "sha256:5fd20bb9583d73de85572198496b65cf028f27527c1733c9840d458e0b40e8fa", size = 81569, upload-time = "2026-10-24T12:36:22.703Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/29/a7/346b74a5f19d0acbd9d428f5f7b070537a8a079e45a74fba58c7831e058d/aiormq-7.2.4-py3-none-any.whl", hash = "sha256:2b9d6dade171318185ab8f507717b630f20b8b2303a075f9293d3f6b0d166f1d", size = 40694, upload-time = "2026-10-24T12:36:21.054Z" }, +] + +[[package]] +name = "aiosignal" +version = "1.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "frozenlist" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/61/62/06741b579156360248d1ec624842ad0edf697050bbaf7c3e46394e106ad1/aiosignal-1.4.0.tar.gz", hash = "sha256:f47eecd9468083c2029cc99945502cb7708b082c232f9aca65da147157b251c7", size = 25007, upload-time = "2025-07-03T22:54:43.528Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fb/76/641ae371508676492379f16e2fa48f4e2c11741bd63c48be4b12a6b09cba/aiosignal-1.4.0-py3-none-any.whl", hash = "sha256:053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e", size = 7490, upload-time = "2025-07-03T22:54:42.156Z" }, +] + +[[package]] +name = "annotated-doc" +version = "0.0.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5a/8e/38aa427ed5402449e226975b649c5dc73ccadfefeb95e6aecb8f8ea4b6b6/annotated_doc-0.0.5.tar.gz", hash = "sha256:c7e58ce09192557605d8bbd92836d7e1d520ac9580096042c0bfd197efacf1bb", size = 10758, upload-time = "2026-07-28T13:50:58.129Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3e/30/e900b21425a860e195f32e37657aa1f7c7f2b1bfb26f03ca209b90933c06/annotated_doc-0.0.5-py3-none-any.whl", hash = "sha256:117bac03a25ede5df5440e855b32d556049ca169ead221505badf432fed4b101", size = 5302, upload-time = "2026-07-28T13:50:57.239Z" }, +] + +[[package]] +name = "annotated-types" +version = "0.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5f/56/a8120250d128bed162cd73c76d45f6ef9991f3e068f62a8ee060afa3104a/annotated_types-0.8.0.tar.gz", hash = "sha256:13b2beaad985e05e2d6407ee4c4f35590b11f8d693a258a561055cac8f64cab7", size = 15893, upload-time = "2026-07-23T20:16:13.995Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/99/91/8acff4f5e50511b911bbccb72b8628a49c68ce14148cd9f6431094859a90/annotated_types-0.8.0-py3-none-any.whl", hash = "sha256:f072f4d804ea359e4eaf198b1af7a8b0943881a87f31bb764f8bf219bb9419e0", size = 13427, upload-time = "2026-07-23T20:16:12.938Z" }, +] + +[[package]] +name = "anyio" +version = "4.15.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "idna" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a9/d2/f4d173e22df740bc37b1db102b386ba719b66e95b0f0d751f556b387e6d2/anyio-4.15.1.tar.gz", hash = "sha256:9f28306018cbd6d329e64a36d58256edff76dd996fe423bc957326e578b82a94", size = 276966, upload-time = "2026-09-05T10:42:39.44Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/12/b8/4bd346e22b28902df4d651910f5242c28d84e4a5c2435ca5c3f797ed7e2e/anyio-4.15.1-py3-none-any.whl", hash = "sha256:6152fdbbf9a77fdec97731721bebf7c4c44f7c29b424b0065826173efc7ed101", size = 132079, upload-time = "2026-09-05T10:42:37.923Z" }, +] + +[[package]] +name = "argon2-cffi" +version = "25.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "argon2-cffi-bindings" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0e/89/ce5af8a7d472a67cc819d5d998aa8c82c5d860608c4db9f46f1162d7dab9/argon2_cffi-25.1.0.tar.gz", hash = "sha256:694ae5cc8a42f4c4e2bf2ca0e64e51e23a040c6a517a85074683d3959e1346c1", size = 45706, upload-time = "2025-06-03T06:55:32.073Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4f/d3/a8b22fa575b297cd6e3e3b0155c7e25db170edf1c74783d6a31a2490b8d9/argon2_cffi-25.1.0-py3-none-any.whl", hash = "sha256:fdc8b074db390fccb6eb4a3604ae7231f219aa669a2652e0f20e16ba513d5741", size = 14657, upload-time = "2025-06-03T06:55:30.804Z" }, +] + +[[package]] +name = "argon2-cffi-bindings" +version = "26.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0b/43/bb8b6e8708d49a5ab36781333af092d9f483b198a2710d01281204640055/argon2_cffi_bindings-26.1.0.tar.gz", hash = "sha256:63505c71542a44b68b1e38060450fb006404170da375feb31af153e7f9c6205d", size = 1790807, upload-time = "2026-08-20T07:44:22.492Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e7/d2/0ae991f1b2181e5be49007c574710a800ad36c2978683addb3e67c474e55/argon2_cffi_bindings-26.1.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:21ca0396fe5ec995dd54431c32698189666f9224810acfa752e50d2bd94d9df2", size = 25521, upload-time = "2026-08-20T07:32:43.019Z" }, + { url = "https://files.pythonhosted.org/packages/7e/e4/ad91d8297638aa2258aad4501c306aca99480dfe76ccd638173fa3702db9/argon2_cffi_bindings-26.1.0-cp310-abi3-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:78de2d65e0b9ea7ce9d1b1c3e87297b2d7305a02c266ee2a2d6910daddd7ee69", size = 27177, upload-time = "2026-08-20T07:32:44.158Z" }, + { url = "https://files.pythonhosted.org/packages/6f/86/5363df11b86d02cf3662208e7406496327649cc90eb365bf6f4e8a54a41f/argon2_cffi_bindings-26.1.0-cp310-abi3-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:27f1821903e2ceadcb88ec2b45ef190897b7682449c772f4d9b53e42c520cf29", size = 26597, upload-time = "2026-08-20T07:32:45.172Z" }, + { url = "https://files.pythonhosted.org/packages/f4/b5/a14dcc592652347dad23ee93b278a4da5d2a25c9ed3ebd10d68eea823a4f/argon2_cffi_bindings-26.1.0-cp310-abi3-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d88e5f7e60f28ae0b0cc6b2f16c43e87cd642a196a86f85e0d8bb6fe016fc16d", size = 27403, upload-time = "2026-08-20T07:32:46.13Z" }, + { url = "https://files.pythonhosted.org/packages/b3/81/b4a20d4902af7f796390bf9245ff83c5217dfa7367efa1d14986956c482b/argon2_cffi_bindings-26.1.0-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:34b7d9c24a4165a2c61cc8ae11d44d48c9ce2830fb536cb7914e11fdd9962728", size = 27132, upload-time = "2026-08-20T07:32:47.13Z" }, + { url = "https://files.pythonhosted.org/packages/7e/1b/c8de358af07b1c490e0fcb863ef98e46ddb486e45567aca5a60bd68d9daa/argon2_cffi_bindings-26.1.0-cp310-abi3-musllinux_1_2_riscv64.whl", hash = "sha256:224865cbbcb7a2bd1356741dff12b0134df726b6d44bb7b500df8e303cbd9e81", size = 27588, upload-time = "2026-08-20T07:32:48.087Z" }, + { url = "https://files.pythonhosted.org/packages/48/2f/7ee62a6e79f9309f9d9982d301b22a00010adb580c05c8109b94d7b33de0/argon2_cffi_bindings-26.1.0-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:ffff613aaa9ce6236766e2fc6dc560bb5abde7a2e2416e3db1f9ae395a2b4dd4", size = 26785, upload-time = "2026-08-20T07:32:48.977Z" }, + { url = "https://files.pythonhosted.org/packages/e9/10/960d0ee93d4897741bcaf4799c697dae2d81499f66fd1ed042a7dd54c1f4/argon2_cffi_bindings-26.1.0-cp310-abi3-win32.whl", hash = "sha256:a86c069c91a747a2c4e5c51473590aeb48172fff9b2130d23729a42d98665ecb", size = 23898, upload-time = "2026-08-20T07:32:50.114Z" }, + { url = "https://files.pythonhosted.org/packages/6d/3a/0cc14a05810e6add9bce5e87693334baa2222de5f647fa31781885b6573f/argon2_cffi_bindings-26.1.0-cp310-abi3-win_amd64.whl", hash = "sha256:2c36ff87b5dfaa477d0bd51e9d7f6abdae7c8955d2983c97419085d842154b3e", size = 25730, upload-time = "2026-08-20T07:32:51.091Z" }, + { url = "https://files.pythonhosted.org/packages/4e/db/d83cf2af140547f0b9cdaece05b2dc2dcbf991be4667331d073eff771435/argon2_cffi_bindings-26.1.0-cp310-abi3-win_arm64.whl", hash = "sha256:f9c4420a7a864fe1b86ce35befc95b8e39fb852493b81cf798671ddc265de638", size = 24478, upload-time = "2026-08-20T07:32:52.111Z" }, + { url = "https://files.pythonhosted.org/packages/76/38/de696045960f5b846d428c0fb6c130ed3da87aac2af209b05c193815404c/argon2_cffi_bindings-26.1.0-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:db0fcd827ca61622a01b220aadfbece01939acf53888f2cb98cd93e9b1e2c97e", size = 15449, upload-time = "2026-08-20T07:32:54.075Z" }, + { url = "https://files.pythonhosted.org/packages/91/0a/c25af768f6b75a5a71e31207f87c540656b2808c015260444a22763221ad/argon2_cffi_bindings-26.1.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:28524438cd3e723f25412f63d4fd516ff5bae9ae5aa56acbe2a1404398a0cf31", size = 25683, upload-time = "2026-08-20T07:32:55.05Z" }, + { url = "https://files.pythonhosted.org/packages/a8/7e/be212c751ab0bcea7f646615f933bf262e8e50b3f7bef32f861d0a2d066b/argon2_cffi_bindings-26.1.0-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ac82fc756a446b6ccd7139ce70efa9d8bbe541e7ad579a12dcb52764b7175c5f", size = 27311, upload-time = "2026-08-20T07:32:56.166Z" }, + { url = "https://files.pythonhosted.org/packages/a6/ee/f84b28e4afd13d3cac36c1d8fa8c239d2dc2c51cd978d02ee5d5ad98d9bb/argon2_cffi_bindings-26.1.0-cp314-cp314t-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6a4e68eed961a8de6928d1c17ff3dc2a547e0e923c17f8f1cd79fb7bc9502f98", size = 26771, upload-time = "2026-08-20T07:32:57.206Z" }, + { url = "https://files.pythonhosted.org/packages/21/c3/95c07a023691ecd529da9cb6a8f0779e13ebc1bdfaa86d145fdc1c6e7e79/argon2_cffi_bindings-26.1.0-cp314-cp314t-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:151dfaad9de753f4af2a7854e707e4784f2acc434340ade64239c5b104b2d605", size = 27568, upload-time = "2026-08-20T07:32:58.361Z" }, + { url = "https://files.pythonhosted.org/packages/e6/31/3a18e31406d8694b4d6a31573c3e572fff6bed318bb744453eb653766d22/argon2_cffi_bindings-26.1.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:061a6919145bbf282ebf1f9c59d3135d4833c25313c8595c0d68cf7712ddfce2", size = 27280, upload-time = "2026-08-20T07:32:59.343Z" }, + { url = "https://files.pythonhosted.org/packages/0b/39/d4be4577e178b2397aa5b5575c8a309bf0da2afe05fe0c72c8f398662d63/argon2_cffi_bindings-26.1.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:62ff20cd130c956c7c9144d5fe35228f98b51c579b2439e988b27ef93e16c02a", size = 27776, upload-time = "2026-08-20T07:33:00.325Z" }, + { url = "https://files.pythonhosted.org/packages/71/47/78f4dd96f7411339f723b96fe24039c1bd5835102b8a5ba71ac4ec712ac7/argon2_cffi_bindings-26.1.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:19423e5d7ac1cc354baab59eaabf18db2ec04ef6593b5abe5a34f323c4a8f87a", size = 26932, upload-time = "2026-08-20T07:33:01.272Z" }, + { url = "https://files.pythonhosted.org/packages/3b/cd/96bfd37434cc0a848a9066c291d84b28846c4c9ea289ed9866b1164d622b/argon2_cffi_bindings-26.1.0-cp314-cp314t-win32.whl", hash = "sha256:4f84cdd868978d7b7350a566c254042d44216d9e37f241f3a6d3b1dfebeede35", size = 24878, upload-time = "2026-08-20T07:33:02.189Z" }, + { url = "https://files.pythonhosted.org/packages/f1/42/d8b6810abd9b1bd2f47ebbccf460da59c9f32e94888bea4f7b137d998797/argon2_cffi_bindings-26.1.0-cp314-cp314t-win_amd64.whl", hash = "sha256:2b741888c93147444fdfc851abd81cc207f37f7f7da42062a00deb3888e57da8", size = 26656, upload-time = "2026-08-20T07:33:03.222Z" }, + { url = "https://files.pythonhosted.org/packages/a9/d1/095d95eaf2ed1d9f77268cf3291bde148c6cd56121f8db2c74c1ba618a0e/argon2_cffi_bindings-26.1.0-cp314-cp314t-win_arm64.whl", hash = "sha256:6ab674f668d5962a3a4136ae0812519b0f1586874263723a32181d60d64137e1", size = 25378, upload-time = "2026-08-20T07:33:04.332Z" }, +] + +[[package]] +name = "asyncpg" +version = "0.32.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/80/4e/59dc964f962f09e3ed472e5d2d3ba670a41a2be25080dc62ab3db507ff5e/asyncpg-0.32.0.tar.gz", hash = "sha256:45e64e56714d888330b884aad1dfb363d0bf43fb343e3d1a8968525f3bade478", size = 1075156, upload-time = "2026-10-06T20:32:40.251Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/25/25/a30ca6417f9142c6a63a7caf5f33717902b2d0ca8a8ff8fc72c6cc2fa77d/asyncpg-0.32.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:5ac18d9ee7a8ca70aed276f79b249d9f37e4d55e3525db1002b5f0b62ddec4f5", size = 691699, upload-time = "2026-10-06T20:31:24.168Z" }, + { url = "https://files.pythonhosted.org/packages/c1/b5/59f10f2381a073c199cd868fce0d8f7aa448b08412de4dc4dbe4118bcee9/asyncpg-0.32.0-cp314-cp314-macosx_11_0_x86_64.whl", hash = "sha256:e1120ef2ae3a5e514c9ea9fce83519ba692710ea5f38434eadbbf12789073dfe", size = 715194, upload-time = "2026-10-06T20:31:25.969Z" }, + { url = "https://files.pythonhosted.org/packages/54/59/79a5aebd58250bedefa6dcd43b22b037d9cf0054ceb4c718c53ebf04e63f/asyncpg-0.32.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4fa68acb42f22436597016e5d7feef7b0b5c49b4c56aece3fdb3ba0da2326cb2", size = 3729978, upload-time = "2026-10-06T20:31:27.541Z" }, + { url = "https://files.pythonhosted.org/packages/68/db/fc91b503b3ec66cf242d83c799388285ea5f0ee238435d53dd9c1a8648a9/asyncpg-0.32.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:63417b8f7369c54f6754c1fbd5a2968fbe632ff55bfbedd56a0177b6a96bd251", size = 3794539, upload-time = "2026-10-06T20:31:29.617Z" }, + { url = "https://files.pythonhosted.org/packages/40/bd/7359320499fdb2733206191b8fd15b7ec602656cbc1444bff7a8c66a365c/asyncpg-0.32.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:2c6366841a792d0a4d16991de240a8053b7c4772a18a5f27fa6fad09c0e359fb", size = 3632884, upload-time = "2026-10-06T20:31:31.298Z" }, + { url = "https://files.pythonhosted.org/packages/18/75/dd3c3dd99f1db55b9736d23a44da29501f07f852bf4df91507f37b156fb1/asyncpg-0.32.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:c3ef1dfd11919280e011ffd1c873323c5088a94fd2c3f77946a5250cf306e2eb", size = 3764931, upload-time = "2026-10-06T20:31:32.916Z" }, + { url = "https://files.pythonhosted.org/packages/38/4f/161b275759725a774d170a383c1208996865ebad50d6891e60d35461a3e6/asyncpg-0.32.0-cp314-cp314-win32.whl", hash = "sha256:77cf9d7023f063ae6f9e443077b55af0dc1807dd9afff1ae656b93ee0cddedc9", size = 557690, upload-time = "2026-10-06T20:31:34.856Z" }, + { url = "https://files.pythonhosted.org/packages/b5/03/880d0db1faedf8b740a57a7ba50e115651a0f05c5905140195813879b086/asyncpg-0.32.0-cp314-cp314-win_amd64.whl", hash = "sha256:2f87452025b47ce80dcc3a0be2b5d1f8aab5deec2516d266f1643d4e53cc40d5", size = 634859, upload-time = "2026-10-06T20:31:36.512Z" }, + { url = "https://files.pythonhosted.org/packages/79/bb/2e86b462a2a2a795eaa7838266db019876b8e7a12c465b903517a4e87fd0/asyncpg-0.32.0-cp314-cp314-win_arm64.whl", hash = "sha256:d0e4508a3d62b0f42d7a99c030c364050b11e75f61c9dd4861e5fdda7cb60636", size = 594013, upload-time = "2026-10-06T20:31:37.91Z" }, + { url = "https://files.pythonhosted.org/packages/20/1d/5369c4438496e654121cbda75be2e8043d1fcae3552b856d44011a19b723/asyncpg-0.32.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:afec11e0b9c001e69966becacd2f948cc8949b4916ec4c0f4dc9b52e47de4528", size = 743832, upload-time = "2026-10-06T20:31:39.261Z" }, + { url = "https://files.pythonhosted.org/packages/60/b0/4b92582c2339a164275a6418ccaeeb0453b72f2e0d7003702379cb50e852/asyncpg-0.32.0-cp314-cp314t-macosx_11_0_x86_64.whl", hash = "sha256:418d266a553e932bf961bb43bfd610ee6c5425fb1b9a599a5828fd12bae8f5c4", size = 769568, upload-time = "2026-10-06T20:31:40.691Z" }, + { url = "https://files.pythonhosted.org/packages/3d/88/919d9ff7ca3c3b96aa404b88b6a53e142b4422623c5ee5a69c4b733240ce/asyncpg-0.32.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b1666e1b747ebbc75c87cb31972704ae8a3ca15b950f94456e97d26781c67d10", size = 3948962, upload-time = "2026-10-06T20:31:42.456Z" }, + { url = "https://files.pythonhosted.org/packages/27/8b/e9f412ae9a3e3f0eb23415249e8d5933e7aeb01068b4083fc86714043d1f/asyncpg-0.32.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:83510bb25d38f0415e155aa3a7af78621369891f5ecd8730d012d9cb26143ffc", size = 3874815, upload-time = "2026-10-06T20:31:44.094Z" }, + { url = "https://files.pythonhosted.org/packages/08/71/24364e9ff7bb9860548452513f295306b12f5b24e8fb0b78f1605c443946/asyncpg-0.32.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:87957755d11639cf248c6aaa094eee9d150f07065866d1710c9427e02dfc0790", size = 3762465, upload-time = "2026-10-06T20:31:45.908Z" }, + { url = "https://files.pythonhosted.org/packages/2e/e1/33cb7e805ec6806b196473e2c7a2ba9d5af3ad2928930aa06359c8eeef87/asyncpg-0.32.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:764227423bf30a3001d3da6df90e82d30a2a097d762e4ee5fa074236eda262f4", size = 3797285, upload-time = "2026-10-06T20:31:47.53Z" }, + { url = "https://files.pythonhosted.org/packages/be/e7/85eb86d6040725f5c191fd6af9f10769c60ed971634b47f4b4bcab293d44/asyncpg-0.32.0-cp314-cp314t-win32.whl", hash = "sha256:f2342b1f3e87b2096320a77edcbb830fbd23b1d4d4842c57567764430b95e4fc", size = 594006, upload-time = "2026-10-06T20:31:49.197Z" }, + { url = "https://files.pythonhosted.org/packages/f9/aa/ea75defe55718457bcf41cde42248db5bbee65fce8c6f0a0e43d9eca1723/asyncpg-0.32.0-cp314-cp314t-win_amd64.whl", hash = "sha256:5c3a48908cb0a02393e5bdab7fa92aefd700f2a93212bf91f04aa9657b4f554d", size = 674647, upload-time = "2026-10-06T20:31:50.547Z" }, + { url = "https://files.pythonhosted.org/packages/0d/0b/078d362872c6c72dd5d11c214dde8dac65b1c87ece96fd2fc2f786a8f66c/asyncpg-0.32.0-cp314-cp314t-win_arm64.whl", hash = "sha256:f8eadd207c26850a2e15f3c2a1096b5d051ea6758a26f2f3e65ce16f84297ed8", size = 624589, upload-time = "2026-10-06T20:31:52.291Z" }, +] + +[[package]] +name = "attrs" +version = "26.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9a/8e/82a0fe20a541c03148528be8cac2408564a6c9a0cc7e9171802bc1d26985/attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32", size = 952055, upload-time = "2026-03-19T14:22:25.026Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548, upload-time = "2026-03-19T14:22:23.645Z" }, +] + +[[package]] +name = "botocore" +version = "1.43.117" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jmespath" }, + { name = "python-dateutil" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6e/a0/d0a5c44ad6a01f195681ddcb4ce203f06c712811daf1d15f8f8785e8a9f8/botocore-1.43.117.tar.gz", hash = "sha256:71993702832cc37c2495048b9945f2929d306fccb3fb767e9964e437e637fa6c", size = 16366888, upload-time = "2026-10-21T19:29:59.274Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/10/94/050e35c9febf6dd2fce4a76be587ae4a323964865700467d9d7b082f7133/botocore-1.43.117-py3-none-any.whl", hash = "sha256:5920ef9b3002e1361846549078549d9154a6e0004c22853c01ce7b279d05d495", size = 16061765, upload-time = "2026-10-21T19:29:56.001Z" }, +] + +[[package]] +name = "certifi" +version = "2026.7.22" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a3/c2/24167ea9858356b47a87a50d39908bfdb72ceeefe0041586e704e5376b3a/certifi-2026.7.22.tar.gz", hash = "sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55", size = 138112, upload-time = "2026-07-22T03:35:12.644Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0b/a7/71ac2cff56fec219ed242bb11b8efb69fcc4bec75db06fb7bfe35de520e6/certifi-2026.7.22-py3-none-any.whl", hash = "sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775", size = 136983, upload-time = "2026-07-22T03:35:11.276Z" }, +] + +[[package]] +name = "cffi" +version = "2.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pycparser", marker = "implementation_name != 'PyPy'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9e/ef/008a1939e372c06329a3fce4279c02f328488f3526744906eeec3da7ad5f/cffi-2.1.1.tar.gz", hash = "sha256:dd31f52ea1086513bb9df30f8fcee9b8918323ae067a3d5b78bc826a000712be", size = 530807, upload-time = "2026-08-03T21:21:18.939Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d3/7b/d6bbf82b8b96e7391438898c42f5bd96dd02030fd5b64937d248220003e2/cffi-2.1.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:7dbb61fe3a7699468030f71bbe5f8a0e326a151daa91beb11a6fc1f980c55e1c", size = 194064, upload-time = "2026-08-03T21:20:17.148Z" }, + { url = "https://files.pythonhosted.org/packages/94/e6/bcc91b283be94735e268487a054004f0aa19947b6348fa367db53230abc8/cffi-2.1.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:f24fb43132a4c6b4cb4eb029492919b2db645be6808d738f244fd146c03c32cb", size = 196720, upload-time = "2026-08-03T21:20:18.268Z" }, + { url = "https://files.pythonhosted.org/packages/d9/99/c4b0c17cacdc9c3b8f280026286a9826d6a208c0f047591a3c3ce99b91fd/cffi-2.1.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:d28630f5854ab07ab1fd4aba756de52326c82e6be15d414b12793f1975048b54", size = 184964, upload-time = "2026-08-03T21:20:19.708Z" }, + { url = "https://files.pythonhosted.org/packages/b3/a9/9db617d05d7367c1ad0ab00b3aa6e6f9281edd689b4ee9ea0e5a84e89c97/cffi-2.1.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:661c298b4821edebead0c91edd2b00374d67ad7c5a1f7a91d4442633b79d6a72", size = 184962, upload-time = "2026-08-03T21:20:20.833Z" }, + { url = "https://files.pythonhosted.org/packages/67/b8/b42132ca113dc567d37684437b46ca1dafc885902b02a110a02d5b511857/cffi-2.1.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:58acb8ab8e295e6c5ea12f888cbb13cf21511ef2a3303a23f4325c29d17fe5c1", size = 222328, upload-time = "2026-08-03T21:20:22.118Z" }, + { url = "https://files.pythonhosted.org/packages/80/10/c5c0cbf0a657aecf59ef511409734230bf556f05a0d6c9eed7aa5c0a0166/cffi-2.1.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:456a61fa52d579ebf9df2e9552ead5129855dbaff6c1e5a9b1bc408809bdc062", size = 209985, upload-time = "2026-08-03T21:20:23.401Z" }, + { url = "https://files.pythonhosted.org/packages/d5/6c/bfa0b87b03b9238148beca990292843c9396ba069b54496596594173de7b/cffi-2.1.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:a4f00aa42f75d6e4595e8866e748cc1705adc0cddfeb2ca86d0d03993d63ba03", size = 208530, upload-time = "2026-08-03T21:20:24.628Z" }, + { url = "https://files.pythonhosted.org/packages/e9/02/4e7d553a7ac4b4238b38b3c1b80d486e9d4436f8d2acbf87a0997fe3f402/cffi-2.1.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b0431303acaea1089ad4b3e9ce4e6518193def1118d4073ca848635ee4ea2e96", size = 221525, upload-time = "2026-08-03T21:20:25.758Z" }, + { url = "https://files.pythonhosted.org/packages/82/1d/a4aaf9babd75acb4d5f223bff71533bee748dd770a382619a798960ee9ba/cffi-2.1.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:64faea20f4e2613363a1a9b9c7dd73058f3ecd00133a511e72ad7c511658f527", size = 225053, upload-time = "2026-08-03T21:20:26.985Z" }, + { url = "https://files.pythonhosted.org/packages/81/10/5dc0e7bdd18e22107054288283380fc97a06ae3f1656a106908d666a3c88/cffi-2.1.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5c58fe613dc5e5336357eff555824a314d8e43282600435c8d1cb6a7a2fedd13", size = 223213, upload-time = "2026-08-03T21:20:28.277Z" }, + { url = "https://files.pythonhosted.org/packages/0b/e9/d0061c364cde06ee43168a0d076ac1da512cbc380d44767b844ba34fe2b6/cffi-2.1.1-cp314-cp314-win32.whl", hash = "sha256:1a18a57b58cfb21fc28d72e876acf10eaed67a1ed96226f92af4df681d571c4c", size = 177682, upload-time = "2026-08-03T21:20:44.288Z" }, + { url = "https://files.pythonhosted.org/packages/a7/06/1c3e01e3ba14c39f6d10bfbac52753b7e22259e38088e5cfe1d704918690/cffi-2.1.1-cp314-cp314-win_amd64.whl", hash = "sha256:3222ba5d678f80a030e6afbcc33dc1ae5cb45facabb61cee2c7016b8432fde48", size = 187949, upload-time = "2026-08-03T21:20:45.623Z" }, + { url = "https://files.pythonhosted.org/packages/87/5b/da4e39efe18eeb89cf580ea9cfc66b6a7c3eadb808fc0cc1d3a295cb5a5d/cffi-2.1.1-cp314-cp314-win_arm64.whl", hash = "sha256:ab36d55f9ed2d067327667c2fea18dda018eb628dd6347aa01dda6cf1f5d3836", size = 182947, upload-time = "2026-08-03T21:20:46.955Z" }, + { url = "https://files.pythonhosted.org/packages/23/59/40338bf421c5accea1d45158170c87006ef1cd371b05c077e76476949728/cffi-2.1.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:7750c6449dff7864bb9bb27ddfb0267756189201a3afc911d82b3caacd70dfc3", size = 188504, upload-time = "2026-08-03T21:20:29.495Z" }, + { url = "https://files.pythonhosted.org/packages/7d/47/5ecf1023850036e674c77ec4de86182d309ae344e39e7cba984b7df5d647/cffi-2.1.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:0beceaabe56af686895136a2de78db54ecd8e4046b236b8fd6d6cb61389e9bf2", size = 188259, upload-time = "2026-08-03T21:20:31.291Z" }, + { url = "https://files.pythonhosted.org/packages/2a/9c/92934c3bea9f785b23eba304538c0b4d37a2a96d2431eb3a1bc87a11aa19/cffi-2.1.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:49cbc70e6542d4ccccb936558d1064a8012541e78f821f955cff24e357776c94", size = 223864, upload-time = "2026-08-03T21:20:32.571Z" }, + { url = "https://files.pythonhosted.org/packages/4d/45/ba4c93527bc38616a8bd36488acb69a2212d60486794f0c1f318949bbb76/cffi-2.1.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:e2d65b31f36619cda3999b78b2aa9632e76b78448e7a56fc4240824200e7c4fc", size = 211538, upload-time = "2026-08-03T21:20:33.808Z" }, + { url = "https://files.pythonhosted.org/packages/80/e9/b6ef565e452acb932fb0cb5443f44a78efbd1233e566f02b5a83855e9115/cffi-2.1.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:28907ab9bfb6aa13184cfc17c6b8e1023c5ab6fd7076d8c20a35e59fe04f8f29", size = 210688, upload-time = "2026-08-03T21:20:34.974Z" }, + { url = "https://files.pythonhosted.org/packages/9a/95/eff5f0cee78d2eabc7eebffec40d3fc1876b5f3c95582e018bb4b99601f2/cffi-2.1.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:51b31d1c98274844cfd7838ce00bfc27c7423a4dc00fc0772fc3331c2cc90676", size = 223803, upload-time = "2026-08-03T21:20:36.564Z" }, + { url = "https://files.pythonhosted.org/packages/fa/01/579d39fb8bef00a335a23d83757b44feb24cd6345a2c451b64cb67b9c362/cffi-2.1.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:5e7cecbaadb83884793e05828cee59b210b24583b9c7425d0ba6a754fe22eb4e", size = 226763, upload-time = "2026-08-03T21:20:37.816Z" }, + { url = "https://files.pythonhosted.org/packages/8d/b0/0b44f47c60b01b57b6e2bbd92343f13a85a1d93bc46ccf6e47e244acd99c/cffi-2.1.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:25792eac27877609e7bb06d42ff88278a6624fff2ba9bbb523c09616b117e80f", size = 225688, upload-time = "2026-08-03T21:20:38.959Z" }, + { url = "https://files.pythonhosted.org/packages/eb/d2/3b7176cb570a1d3e27faf67b72f591af508036e0d8b2be2ef9af9e8c84bb/cffi-2.1.1-cp314-cp314t-win32.whl", hash = "sha256:8ef53b2de9bcb9197d31854256575d59dbac0cba72ac627bb291ef5eceb74be4", size = 182868, upload-time = "2026-08-03T21:20:40.388Z" }, + { url = "https://files.pythonhosted.org/packages/56/78/31f00c1bcd97c9bbf55f1bfdf5bc809a5de8887473e90bb9960dca825e80/cffi-2.1.1-cp314-cp314t-win_amd64.whl", hash = "sha256:616f097f2fe415bc92a247f02e11f634e1f9e9a83d327e3c915c15089c87869e", size = 194104, upload-time = "2026-08-03T21:20:41.725Z" }, + { url = "https://files.pythonhosted.org/packages/7b/1b/58496f2ed0a35de575250c02a43ab3cc2c04d494a88fed31c1cabc0fd176/cffi-2.1.1-cp314-cp314t-win_arm64.whl", hash = "sha256:ad2c86c495b899d862ea0f4b42891b8713a3bd45dd4105c7fd51c2a72f39f3a5", size = 186402, upload-time = "2026-08-03T21:20:43.042Z" }, +] + +[[package]] +name = "click" +version = "8.5.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c7/0e/7fa0ef50764b67090eca4114772a2abf8b6148198475e54c660b97caeee6/click-8.5.0.tar.gz", hash = "sha256:ba0d2089de75ea0310e2dde03160e6ca10009947fb95a182f9b54021bb272e34", size = 382235, upload-time = "2026-08-26T13:33:14.56Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/58/50/6c0d534c5f134586a8e1ba4e330569e32f057e33372ae556463212fb4cd3/click-8.5.0-py3-none-any.whl", hash = "sha256:255bc9599cf7748b4b1a446ccc735421bd08a2ae529a8b88597d3de5664ee360", size = 125251, upload-time = "2026-08-26T13:33:12.928Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "dishka" +version = "1.10.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/97/da/b75b5fe665d16725e1d46e291c8dd4b0ed1731c7964f41087fda92f03d40/dishka-1.10.1.tar.gz", hash = "sha256:51fb624b9acc948b90f57128460061d1f3a1780121fe1c7d14c565e8bbff2083", size = 282008, upload-time = "2026-04-25T12:07:57.587Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2c/08/dd3259a319ba45c48d975caf237c830c7873a203288cb4ab186cc5bef585/dishka-1.10.1-py3-none-any.whl", hash = "sha256:84f40c1115eb391ce3fb558fd7946c5bcaf6361c35b77b7a185b7697a5720ec7", size = 120062, upload-time = "2026-04-25T12:07:59.153Z" }, +] + +[[package]] +name = "dishka-faststream" +version = "0.7.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "dishka" }, + { name = "faststream" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1c/32/c8df3791072a5ea11abaa8e142536fe8cd7745f3fe25e4be4fdf3ac112f5/dishka_faststream-0.7.0.tar.gz", hash = "sha256:b8c6e95bf30f32fc67b3289697cee51c3047778afe195aa9a1062d21e64bfb7b", size = 118092, upload-time = "2026-06-04T17:34:30.856Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0e/9f/b1918e3ee6814f1b94489ff367b9178e2730a3d23370c24f2f2bce5886e5/dishka_faststream-0.7.0-py3-none-any.whl", hash = "sha256:6f68ce1520b9f15478c8e8407e0d36cdcf1261d237a81e0090fb34ab4775e67e", size = 10551, upload-time = "2026-06-04T17:34:29.953Z" }, +] + +[[package]] +name = "fast-depends" +version = "3.0.10" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3a/76/1e67c34a18a75aff77f25de2d56de8ad3d38c05915ae513ba35293fd920b/fast_depends-3.0.10.tar.gz", hash = "sha256:234c1dcfcdbc1cdb0ebe543fbf8e8234c069ba39518fbdf1f71dad6e3e6111ad", size = 19807, upload-time = "2026-10-28T07:36:21.82Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/af/af933c80bd2c60070dec7ea324350ce698d579bada6ab6511800f749ab57/fast_depends-3.0.10-py3-none-any.whl", hash = "sha256:e49306f54b691a59076e6fe1fdae9418bff4af80d1c9d9fb82a6d517b4d4c04c", size = 25535, upload-time = "2026-10-28T07:36:20.374Z" }, +] + +[package.optional-dependencies] +pydantic = [ + { name = "pydantic" }, +] + +[[package]] +name = "fastapi" +version = "0.141.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-doc" }, + { name = "pydantic" }, + { name = "starlette" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8a/02/91e3416a8fdd715abb903a952a6bec7cdd8d14eed55d415fc8595524c319/fastapi-0.141.1.tar.gz", hash = "sha256:e8822fc40db1e1858054d7a949a888695bc9bdce70139178e33bd2871a453ca1", size = 425799, upload-time = "2026-07-29T17:18:05.568Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/03/10388a42375ee7e4ac9b94eb2c5c569c8b5795e377e701c9ac3ad63de890/fastapi-0.141.1-py3-none-any.whl", hash = "sha256:bfb91aa2d334c61cb35ba9a116fc123b3d3df31640b801cf57a7a78ec3f603b3", size = 131954, upload-time = "2026-07-29T17:18:04.364Z" }, +] + +[[package]] +name = "faststream" +version = "0.7.8" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "fast-depends", extra = ["pydantic"] }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/87/b6/d15b1c88f1adc3bf11e4e23f8a057e489e2e60a47f7d4b821a3ae264f4c2/faststream-0.7.8.tar.gz", hash = "sha256:9e367d9ba8b59eeafad19f46641b11f20d23a7ed75ea27b2503856a8ae0c075a", size = 392145, upload-time = "2026-10-22T20:50:29.387Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a5/79/daae6f842665756f5fa45683d4762333348c4ad4dba2e9d9f37644717d74/faststream-0.7.8-py3-none-any.whl", hash = "sha256:ffe412278cf116216078dcf29dcec93274994a98d036be2bc388071dff7e259e", size = 618852, upload-time = "2026-10-22T20:50:27.388Z" }, +] + +[package.optional-dependencies] +rabbit = [ + { name = "aio-pika" }, +] + +[[package]] +name = "frozenlist" +version = "1.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2d/f5/c831fac6cc817d26fd54c7eaccd04ef7e0288806943f7cc5bbf69f3ac1f0/frozenlist-1.8.0.tar.gz", hash = "sha256:3ede829ed8d842f6cd48fc7081d7a41001a56f1f38603f9d49bf3020d59a31ad", size = 45875, upload-time = "2025-10-06T05:38:17.865Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f1/c8/85da824b7e7b9b6e7f7705b2ecaf9591ba6f79c1177f324c2735e41d36a2/frozenlist-1.8.0-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:cee686f1f4cadeb2136007ddedd0aaf928ab95216e7691c63e50a8ec066336d0", size = 86127, upload-time = "2025-10-06T05:37:08.438Z" }, + { url = "https://files.pythonhosted.org/packages/8e/e8/a1185e236ec66c20afd72399522f142c3724c785789255202d27ae992818/frozenlist-1.8.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:119fb2a1bd47307e899c2fac7f28e85b9a543864df47aa7ec9d3c1b4545f096f", size = 49698, upload-time = "2025-10-06T05:37:09.48Z" }, + { url = "https://files.pythonhosted.org/packages/a1/93/72b1736d68f03fda5fdf0f2180fb6caaae3894f1b854d006ac61ecc727ee/frozenlist-1.8.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:4970ece02dbc8c3a92fcc5228e36a3e933a01a999f7094ff7c23fbd2beeaa67c", size = 49749, upload-time = "2025-10-06T05:37:10.569Z" }, + { url = "https://files.pythonhosted.org/packages/a7/b2/fabede9fafd976b991e9f1b9c8c873ed86f202889b864756f240ce6dd855/frozenlist-1.8.0-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:cba69cb73723c3f329622e34bdbf5ce1f80c21c290ff04256cff1cd3c2036ed2", size = 231298, upload-time = "2025-10-06T05:37:11.993Z" }, + { url = "https://files.pythonhosted.org/packages/3a/3b/d9b1e0b0eed36e70477ffb8360c49c85c8ca8ef9700a4e6711f39a6e8b45/frozenlist-1.8.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:778a11b15673f6f1df23d9586f83c4846c471a8af693a22e066508b77d201ec8", size = 232015, upload-time = "2025-10-06T05:37:13.194Z" }, + { url = "https://files.pythonhosted.org/packages/dc/94/be719d2766c1138148564a3960fc2c06eb688da592bdc25adcf856101be7/frozenlist-1.8.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:0325024fe97f94c41c08872db482cf8ac4800d80e79222c6b0b7b162d5b13686", size = 225038, upload-time = "2025-10-06T05:37:14.577Z" }, + { url = "https://files.pythonhosted.org/packages/e4/09/6712b6c5465f083f52f50cf74167b92d4ea2f50e46a9eea0523d658454ae/frozenlist-1.8.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:97260ff46b207a82a7567b581ab4190bd4dfa09f4db8a8b49d1a958f6aa4940e", size = 240130, upload-time = "2025-10-06T05:37:15.781Z" }, + { url = "https://files.pythonhosted.org/packages/f8/d4/cd065cdcf21550b54f3ce6a22e143ac9e4836ca42a0de1022da8498eac89/frozenlist-1.8.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:54b2077180eb7f83dd52c40b2750d0a9f175e06a42e3213ce047219de902717a", size = 242845, upload-time = "2025-10-06T05:37:17.037Z" }, + { url = "https://files.pythonhosted.org/packages/62/c3/f57a5c8c70cd1ead3d5d5f776f89d33110b1addae0ab010ad774d9a44fb9/frozenlist-1.8.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:2f05983daecab868a31e1da44462873306d3cbfd76d1f0b5b69c473d21dbb128", size = 229131, upload-time = "2025-10-06T05:37:18.221Z" }, + { url = "https://files.pythonhosted.org/packages/6c/52/232476fe9cb64f0742f3fde2b7d26c1dac18b6d62071c74d4ded55e0ef94/frozenlist-1.8.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:33f48f51a446114bc5d251fb2954ab0164d5be02ad3382abcbfe07e2531d650f", size = 240542, upload-time = "2025-10-06T05:37:19.771Z" }, + { url = "https://files.pythonhosted.org/packages/5f/85/07bf3f5d0fb5414aee5f47d33c6f5c77bfe49aac680bfece33d4fdf6a246/frozenlist-1.8.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:154e55ec0655291b5dd1b8731c637ecdb50975a2ae70c606d100750a540082f7", size = 237308, upload-time = "2025-10-06T05:37:20.969Z" }, + { url = "https://files.pythonhosted.org/packages/11/99/ae3a33d5befd41ac0ca2cc7fd3aa707c9c324de2e89db0e0f45db9a64c26/frozenlist-1.8.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:4314debad13beb564b708b4a496020e5306c7333fa9a3ab90374169a20ffab30", size = 238210, upload-time = "2025-10-06T05:37:22.252Z" }, + { url = "https://files.pythonhosted.org/packages/b2/60/b1d2da22f4970e7a155f0adde9b1435712ece01b3cd45ba63702aea33938/frozenlist-1.8.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:073f8bf8becba60aa931eb3bc420b217bb7d5b8f4750e6f8b3be7f3da85d38b7", size = 231972, upload-time = "2025-10-06T05:37:23.5Z" }, + { url = "https://files.pythonhosted.org/packages/3f/ab/945b2f32de889993b9c9133216c068b7fcf257d8595a0ac420ac8677cab0/frozenlist-1.8.0-cp314-cp314-win32.whl", hash = "sha256:bac9c42ba2ac65ddc115d930c78d24ab8d4f465fd3fc473cdedfccadb9429806", size = 40536, upload-time = "2025-10-06T05:37:25.581Z" }, + { url = "https://files.pythonhosted.org/packages/59/ad/9caa9b9c836d9ad6f067157a531ac48b7d36499f5036d4141ce78c230b1b/frozenlist-1.8.0-cp314-cp314-win_amd64.whl", hash = "sha256:3e0761f4d1a44f1d1a47996511752cf3dcec5bbdd9cc2b4fe595caf97754b7a0", size = 44330, upload-time = "2025-10-06T05:37:26.928Z" }, + { url = "https://files.pythonhosted.org/packages/82/13/e6950121764f2676f43534c555249f57030150260aee9dcf7d64efda11dd/frozenlist-1.8.0-cp314-cp314-win_arm64.whl", hash = "sha256:d1eaff1d00c7751b7c6662e9c5ba6eb2c17a2306ba5e2a37f24ddf3cc953402b", size = 40627, upload-time = "2025-10-06T05:37:28.075Z" }, + { url = "https://files.pythonhosted.org/packages/c0/c7/43200656ecc4e02d3f8bc248df68256cd9572b3f0017f0a0c4e93440ae23/frozenlist-1.8.0-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:d3bb933317c52d7ea5004a1c442eef86f426886fba134ef8cf4226ea6ee1821d", size = 89238, upload-time = "2025-10-06T05:37:29.373Z" }, + { url = "https://files.pythonhosted.org/packages/d1/29/55c5f0689b9c0fb765055629f472c0de484dcaf0acee2f7707266ae3583c/frozenlist-1.8.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:8009897cdef112072f93a0efdce29cd819e717fd2f649ee3016efd3cd885a7ed", size = 50738, upload-time = "2025-10-06T05:37:30.792Z" }, + { url = "https://files.pythonhosted.org/packages/ba/7d/b7282a445956506fa11da8c2db7d276adcbf2b17d8bb8407a47685263f90/frozenlist-1.8.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:2c5dcbbc55383e5883246d11fd179782a9d07a986c40f49abe89ddf865913930", size = 51739, upload-time = "2025-10-06T05:37:32.127Z" }, + { url = "https://files.pythonhosted.org/packages/62/1c/3d8622e60d0b767a5510d1d3cf21065b9db874696a51ea6d7a43180a259c/frozenlist-1.8.0-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:39ecbc32f1390387d2aa4f5a995e465e9e2f79ba3adcac92d68e3e0afae6657c", size = 284186, upload-time = "2025-10-06T05:37:33.21Z" }, + { url = "https://files.pythonhosted.org/packages/2d/14/aa36d5f85a89679a85a1d44cd7a6657e0b1c75f61e7cad987b203d2daca8/frozenlist-1.8.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:92db2bf818d5cc8d9c1f1fc56b897662e24ea5adb36ad1f1d82875bd64e03c24", size = 292196, upload-time = "2025-10-06T05:37:36.107Z" }, + { url = "https://files.pythonhosted.org/packages/05/23/6bde59eb55abd407d34f77d39a5126fb7b4f109a3f611d3929f14b700c66/frozenlist-1.8.0-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:2dc43a022e555de94c3b68a4ef0b11c4f747d12c024a520c7101709a2144fb37", size = 273830, upload-time = "2025-10-06T05:37:37.663Z" }, + { url = "https://files.pythonhosted.org/packages/d2/3f/22cff331bfad7a8afa616289000ba793347fcd7bc275f3b28ecea2a27909/frozenlist-1.8.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cb89a7f2de3602cfed448095bab3f178399646ab7c61454315089787df07733a", size = 294289, upload-time = "2025-10-06T05:37:39.261Z" }, + { url = "https://files.pythonhosted.org/packages/a4/89/5b057c799de4838b6c69aa82b79705f2027615e01be996d2486a69ca99c4/frozenlist-1.8.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:33139dc858c580ea50e7e60a1b0ea003efa1fd42e6ec7fdbad78fff65fad2fd2", size = 300318, upload-time = "2025-10-06T05:37:43.213Z" }, + { url = "https://files.pythonhosted.org/packages/30/de/2c22ab3eb2a8af6d69dc799e48455813bab3690c760de58e1bf43b36da3e/frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:168c0969a329b416119507ba30b9ea13688fafffac1b7822802537569a1cb0ef", size = 282814, upload-time = "2025-10-06T05:37:45.337Z" }, + { url = "https://files.pythonhosted.org/packages/59/f7/970141a6a8dbd7f556d94977858cfb36fa9b66e0892c6dd780d2219d8cd8/frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:28bd570e8e189d7f7b001966435f9dac6718324b5be2990ac496cf1ea9ddb7fe", size = 291762, upload-time = "2025-10-06T05:37:46.657Z" }, + { url = "https://files.pythonhosted.org/packages/c1/15/ca1adae83a719f82df9116d66f5bb28bb95557b3951903d39135620ef157/frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:b2a095d45c5d46e5e79ba1e5b9cb787f541a8dee0433836cea4b96a2c439dcd8", size = 289470, upload-time = "2025-10-06T05:37:47.946Z" }, + { url = "https://files.pythonhosted.org/packages/ac/83/dca6dc53bf657d371fbc88ddeb21b79891e747189c5de990b9dfff2ccba1/frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:eab8145831a0d56ec9c4139b6c3e594c7a83c2c8be25d5bcf2d86136a532287a", size = 289042, upload-time = "2025-10-06T05:37:49.499Z" }, + { url = "https://files.pythonhosted.org/packages/96/52/abddd34ca99be142f354398700536c5bd315880ed0a213812bc491cff5e4/frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:974b28cf63cc99dfb2188d8d222bc6843656188164848c4f679e63dae4b0708e", size = 283148, upload-time = "2025-10-06T05:37:50.745Z" }, + { url = "https://files.pythonhosted.org/packages/af/d3/76bd4ed4317e7119c2b7f57c3f6934aba26d277acc6309f873341640e21f/frozenlist-1.8.0-cp314-cp314t-win32.whl", hash = "sha256:342c97bf697ac5480c0a7ec73cd700ecfa5a8a40ac923bd035484616efecc2df", size = 44676, upload-time = "2025-10-06T05:37:52.222Z" }, + { url = "https://files.pythonhosted.org/packages/89/76/c615883b7b521ead2944bb3480398cbb07e12b7b4e4d073d3752eb721558/frozenlist-1.8.0-cp314-cp314t-win_amd64.whl", hash = "sha256:06be8f67f39c8b1dc671f5d83aaefd3358ae5cdcf8314552c57e7ed3e6475bdd", size = 49451, upload-time = "2025-10-06T05:37:53.425Z" }, + { url = "https://files.pythonhosted.org/packages/e0/a3/5982da14e113d07b325230f95060e2169f5311b1017ea8af2a29b374c289/frozenlist-1.8.0-cp314-cp314t-win_arm64.whl", hash = "sha256:102e6314ca4da683dca92e3b1355490fed5f313b768500084fbe6371fddfdb79", size = 42507, upload-time = "2025-10-06T05:37:54.513Z" }, + { url = "https://files.pythonhosted.org/packages/9a/9a/e35b4a917281c0b8419d4207f4334c8e8c5dbf4f3f5f9ada73958d937dcc/frozenlist-1.8.0-py3-none-any.whl", hash = "sha256:0c18a16eab41e82c295618a77502e17b195883241c563b00f0aa5106fc4eaa0d", size = 13409, upload-time = "2025-10-06T05:38:16.721Z" }, +] + +[[package]] +name = "gunicorn" +version = "26.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d9/8a/e4ef6ee11701b6cd64702848415ffb69eeff85cb388a3c6c7fe86f22f3f8/gunicorn-26.2.0.tar.gz", hash = "sha256:62b864895d9ebff0b2f9867ba04fe811c93121596540830c9c916d0769668447", size = 787921, upload-time = "2026-08-24T15:05:59.3Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fe/85/7522a52e5e2f42faf1a129113ab63e548c42e103e9af395b7bfe65e403e2/gunicorn-26.2.0-py3-none-any.whl", hash = "sha256:bd249d0b3f7972f7432f0a6b6ff3b3ee2d129f70cd1ff6c09a9dd9e29a2b88e3", size = 228389, upload-time = "2026-08-24T15:05:57.67Z" }, +] + +[[package]] +name = "h11" +version = "0.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/ee/02a2c011bdab74c6fb3c75474d40b3052059d95df7e73351460c8588d963/h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1", size = 101250, upload-time = "2025-04-24T03:35:25.427Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, +] + +[[package]] +name = "httpcore" +version = "1.0.9" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "h11" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/94/82699a10bca87a5556c9c59b5963f2d039dbd239f25bc2a63907a05a14cb/httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8", size = 85484, upload-time = "2025-04-24T22:06:22.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" }, +] + +[[package]] +name = "httptools" +version = "0.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/43/e5/d471fcb0e14523fe1c3f4ba58ca52480e7bd70ad7109a3846bc75892f7fb/httptools-0.8.0.tar.gz", hash = "sha256:6b2a32f18d97e16e90827d7a819ffa8dbd8cc245fc4e1fa9d1095b54ef4bd999", size = 271342, upload-time = "2026-05-25T22:17:48.841Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1a/12/fa3fbf5f9517b273edea2dc982aa82a8c634091e67c590792b729017bc6f/httptools-0.8.0-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:de242a49b5d18e0a8776e654e9f6bf6d89f3875a5c35b425a0e7ce940feb3fd6", size = 206183, upload-time = "2026-05-25T22:17:24.004Z" }, + { url = "https://files.pythonhosted.org/packages/30/fc/5e7c4cb443370f2090a3aba0453a07384d29ff66b7435bb90e77e1037599/httptools-0.8.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:159e9ab5f701ccd42e555a12f1ad8ff69702910fc1c996cf2bb66e5fcb7a231b", size = 112079, upload-time = "2026-05-25T22:17:25.216Z" }, + { url = "https://files.pythonhosted.org/packages/ba/53/771bd891eb0f236f32145d6a1775777ec85745f3cc983a1f23d1a3b8ddfe/httptools-0.8.0-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:c4a9f1707e4823d54dfec6c33fa3697d302aed536ed352a7ebb5a061ddb869d0", size = 481596, upload-time = "2026-05-25T22:17:26.186Z" }, + { url = "https://files.pythonhosted.org/packages/62/42/94e15bc68ce3d423243c45d7f1b0c7561f13844f97dc52ae23182fb65628/httptools-0.8.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d76ad7b951387e3632c8716a9bb03ac5b45c5f16119aa409db0459520887944e", size = 480865, upload-time = "2026-05-25T22:17:27.542Z" }, + { url = "https://files.pythonhosted.org/packages/1c/7c/fe2980fc03723272e30f135b62360b075f513dfe7cc73aef36c7f04012bd/httptools-0.8.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a3b7387147361c3fd47a0bde763c5c91b5b4cd4dc9989b8ece84ff436c99843b", size = 463189, upload-time = "2026-05-25T22:17:28.546Z" }, + { url = "https://files.pythonhosted.org/packages/15/1b/47fc5fff68acd1bfa20b4734059c9a06cadb88119dcd5258b5b0d21d91c8/httptools-0.8.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f256d6ce930c52ca1cb2a960b7da03548c454e7d28b06059ad41bfe789036ce0", size = 466610, upload-time = "2026-05-25T22:17:29.816Z" }, + { url = "https://files.pythonhosted.org/packages/60/bd/07b13c93ffd9bec9546e0d43f8e19378dd696dbd278511406bc07371ef1f/httptools-0.8.0-cp314-cp314-win_amd64.whl", hash = "sha256:19d1ee275bb59ba2643ba9a3a1e51cc0c788caf2b8df506368e03f56fdd08527", size = 92705, upload-time = "2026-05-25T22:17:31.133Z" }, + { url = "https://files.pythonhosted.org/packages/fd/c4/121648f68ce066d7bd762d6b6d97e620847642d38d54f3d90ff11d947629/httptools-0.8.0-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:de1ed58a974e75d56560acc7e7fed01a454994429456f65209789992e41f2568", size = 215023, upload-time = "2026-05-25T22:17:32.401Z" }, + { url = "https://files.pythonhosted.org/packages/b9/b0/312a062ae741ae3e8baa8c8bf20be81b2e67337b259ab4349bebc7b6142e/httptools-0.8.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:e93c227b595c6926c1acee96891dd9da4be338cfbe82e5cd3bb9d8dd7dc4ac0b", size = 117405, upload-time = "2026-05-25T22:17:33.742Z" }, + { url = "https://files.pythonhosted.org/packages/fc/37/fccd705f795386bb05bf413012fecff2a33e5aa8c2f069096de3e9fd8702/httptools-0.8.0-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:2a021c3a8e65cc125390d72f59b968afca3bdcaff25bd67965e0a055a14946ca", size = 558497, upload-time = "2026-05-25T22:17:34.732Z" }, + { url = "https://files.pythonhosted.org/packages/bd/39/f172e8003576de35f5ba77ff417cf0e34429d35dc014deef15afa337a72c/httptools-0.8.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:48774d39cbb70e2b1f71f88852a3087ae1d3a1eb80482bb48c13067ab080c14f", size = 571585, upload-time = "2026-05-25T22:17:35.813Z" }, + { url = "https://files.pythonhosted.org/packages/3e/b9/f5564760af99f3dbbf3f9104dc00e5da27e96cf433c6bdcf77617f70bf3f/httptools-0.8.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:88eead8ec8680a9f146c655bc88445a325bd7921cfd8194c7337e9467282427d", size = 543297, upload-time = "2026-05-25T22:17:37.08Z" }, + { url = "https://files.pythonhosted.org/packages/99/67/8d9f2c313618e161b82f3873188e7196126da1d6e29688df40eb3997c77a/httptools-0.8.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:2c032fa028f46871ec7e1fc59fc15e8023eab3e6bbe6ece786a1611719a5d081", size = 539535, upload-time = "2026-05-25T22:17:38.032Z" }, + { url = "https://files.pythonhosted.org/packages/48/63/b906c01e53f50d432c0defe43ce52764a111dc1bdd028bafbeb54dcfd008/httptools-0.8.0-cp314-cp314t-win_amd64.whl", hash = "sha256:384c17174464c8e873398b7af24f0b1f44d992c820328413951a625323155d77", size = 108209, upload-time = "2026-05-25T22:17:39.473Z" }, +] + +[[package]] +name = "httpx" +version = "0.28.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "certifi" }, + { name = "httpcore" }, + { name = "idna" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size = 141406, upload-time = "2024-12-06T15:37:23.222Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, +] + +[[package]] +name = "idna" +version = "3.20" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f5/08/8eea9d4b8302028f3abb2c0813953f7aec26d33b7a8960ed760e65ff29fa/idna-3.20.tar.gz", hash = "sha256:a7db850025b95ded1eae8a46181a1a6c56c92c96f0e2b005d9ff8dc0210cab44", size = 216463, upload-time = "2026-09-17T14:11:04.752Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/58/a2/bb081bab032533a855d44de1d56f8e8426114ff1ba5d1f07a438a0a654f8/idna-3.20-py3-none-any.whl", hash = "sha256:ab7ae7122974553370f0bdb919e1a960b2cd1bc1ef0276416d896db81c14582c", size = 69583, upload-time = "2026-09-17T14:11:03.168Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, +] + +[[package]] +name = "jmespath" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d3/59/322338183ecda247fb5d1763a6cbe46eff7222eaeebafd9fa65d4bf5cb11/jmespath-1.1.0.tar.gz", hash = "sha256:472c87d80f36026ae83c6ddd0f1d05d4e510134ed462851fd5f754c8c3cbb88d", size = 27377, upload-time = "2026-01-22T16:35:26.279Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/14/2f/967ba146e6d58cf6a652da73885f52fc68001525b4197effc174321d70b4/jmespath-1.1.0-py3-none-any.whl", hash = "sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64", size = 20419, upload-time = "2026-01-22T16:35:24.919Z" }, +] + +[[package]] +name = "mtusi-messenger" +version = "1.6.0" +source = { editable = "." } +dependencies = [ + { name = "aiobotocore" }, + { name = "argon2-cffi" }, + { name = "asyncpg" }, + { name = "dishka" }, + { name = "dishka-faststream" }, + { name = "fastapi" }, + { name = "faststream", extra = ["rabbit"] }, + { name = "gunicorn" }, + { name = "orjson" }, + { name = "pillow" }, + { name = "pydantic" }, + { name = "pydantic-settings" }, + { name = "redis" }, + { name = "structlog" }, + { name = "uvicorn", extra = ["standard"] }, + { name = "uvicorn-worker" }, +] + +[package.dev-dependencies] +dev = [ + { name = "httpx" }, + { name = "pytest" }, + { name = "pytest-asyncio" }, + { name = "ruff" }, + { name = "ty" }, +] + +[package.metadata] +requires-dist = [ + { name = "aiobotocore", specifier = ">=3.10.0" }, + { name = "argon2-cffi", specifier = ">=25.1.0" }, + { name = "asyncpg", specifier = ">=0.32.0" }, + { name = "dishka", specifier = ">=1.7.0" }, + { name = "dishka-faststream", specifier = ">=0.1" }, + { name = "fastapi", specifier = ">=0.118.0" }, + { name = "faststream", extras = ["rabbit"], specifier = ">=0.7.2,<0.8" }, + { name = "gunicorn", specifier = ">=26.2.0" }, + { name = "orjson", specifier = ">=3.12.0" }, + { name = "pillow", specifier = ">=12.0.0" }, + { name = "pydantic", specifier = ">=2.11.0" }, + { name = "pydantic-settings", specifier = ">=2.11.0" }, + { name = "redis", specifier = ">=8.1.0" }, + { name = "structlog", specifier = ">=26.1.0" }, + { name = "uvicorn", extras = ["standard"], specifier = ">=0.37.0" }, + { name = "uvicorn-worker", specifier = ">=0.4.0" }, +] + +[package.metadata.requires-dev] +dev = [ + { name = "httpx", specifier = ">=0.28.0" }, + { name = "pytest", specifier = ">=8.4.0" }, + { name = "pytest-asyncio", specifier = ">=1.2.0" }, + { name = "ruff", specifier = ">=0.14.0" }, + { name = "ty", specifier = ">=0.0.1a21" }, +] + +[[package]] +name = "multidict" +version = "7.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7b/6b/29ef8e752f5854dcc267ded0652c475bec99b86f255957a0d4fcad657b18/multidict-7.2.1.tar.gz", hash = "sha256:e8af3e6f1bdbc31fd762b094133b478b89988eaebc63cbe82b1d9e3bbd15cede", size = 369994, upload-time = "2026-10-24T20:41:35.457Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/de/0c/2eff3c4f36f45eddf11fb10b35069d9ebcef751b16483a585527061354ac/multidict-7.2.1-cp314-cp314-android_24_x86_64.whl", hash = "sha256:3d22ebb716ac79bf32c2a1d15250f017faef719114af4376e8dd814d6633cf0a", size = 102695, upload-time = "2026-10-24T20:41:04.31Z" }, + { url = "https://files.pythonhosted.org/packages/e2/46/4b8667914db791922e826a847f80324cb1f0acba9b05343fcb53c5a237f9/multidict-7.2.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:1d369ed297e79b417766327ece16bb7e89306c588df63c364a85d90d4aeac843", size = 101581, upload-time = "2026-10-24T20:40:27.297Z" }, + { url = "https://files.pythonhosted.org/packages/5a/40/80c50aa45b607086d43ff3d467c25450b85c7d9bb7e4bbcd521c3b030be9/multidict-7.2.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:a9cc121b5ee2c8b4bd4b6b526d5a26bb782f4d620463f73707691b2004b8c349", size = 103677, upload-time = "2026-10-24T20:40:28.633Z" }, + { url = "https://files.pythonhosted.org/packages/f3/e0/c822cf8d92f33f0d6fc0f733b6f5b287956133024765f31b7151b68f6198/multidict-7.2.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:02009b402129a73e1f3da68f48eb999afbacd29f25aef0d5052b84ff5c2786a2", size = 172007, upload-time = "2026-10-24T20:40:29.243Z" }, + { url = "https://files.pythonhosted.org/packages/0b/d8/b36e8bbc49a59f38e6411bdf96e07c29462460d77e59527309d72469b1f8/multidict-7.2.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:cb7fc8820b206932ec6d8f8985aeba6edfd2932e13b6c4bf2434d6bcbbeef3e1", size = 97224, upload-time = "2026-10-24T20:41:05.627Z" }, + { url = "https://files.pythonhosted.org/packages/8e/3f/c014da364e2f9707945f921864a6b5c615e88a0ec0518c7999b3b2092405/multidict-7.2.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:cadd02ca804d6ebe799a7af19197d9730094fe2cc74b70eb4f374f4b8c1886c1", size = 103824, upload-time = "2026-10-24T20:40:28.61Z" }, + { url = "https://files.pythonhosted.org/packages/c9/73/bcd1fff09ba644499fcd864e6f86a596e51e90f5504c326354bdba3bb8a0/multidict-7.2.1-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:846b17193f1c373d6f05ada894fadf56bce2473067caea0aab51828849894b31", size = 486848, upload-time = "2026-10-24T20:40:29.976Z" }, + { url = "https://files.pythonhosted.org/packages/52/12/d6731ea19a185f76a1dc6b76f239fa3567732c8575188b47f7a715742c21/multidict-7.2.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6efdeed2723431999cf3bcbc84c72dbe0f65d16bb1f84e15a657f5b60ff18ca6", size = 500914, upload-time = "2026-10-24T20:40:30.626Z" }, + { url = "https://files.pythonhosted.org/packages/71/db/332a5b62d4c2c26950aede813a0da97e42cbc16b656ac936eb2c6e01f58f/multidict-7.2.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:f9c118e866164ce05498d9106a4bd01e745321403cd6f0b77a0b2587768f8c06", size = 462069, upload-time = "2026-10-24T20:41:06.892Z" }, + { url = "https://files.pythonhosted.org/packages/64/8d/c8c6826dc3e9b742bd21f3e84e1d8c46daffb584f45fd93429541e5e7f11/multidict-7.2.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:dc6f3dcb08c4b41e64f2d6cf272b5686d9144db585b2e138a8b1664376ab3921", size = 515356, upload-time = "2026-10-24T20:40:29.998Z" }, + { url = "https://files.pythonhosted.org/packages/92/c8/c13ffe0d352c4caaa17da0a74e9203c718606c62a1d62f98439feb1783d5/multidict-7.2.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:27cc258247b13008742f0e239933ad01dec5ed4e6dfc63849e30e04e1aa5bc82", size = 539793, upload-time = "2026-10-24T20:40:31.55Z" }, + { url = "https://files.pythonhosted.org/packages/3b/8f/bff861e3bd5a7cc13bd1f040d22aee8aa32b4f2755bba29b0c31a1d2ae25/multidict-7.2.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:822a7fcd33f6922b650e1b26594f09d53c5d011fe655fea7ded27b2728ce2a1b", size = 506944, upload-time = "2026-10-24T20:40:32.258Z" }, + { url = "https://files.pythonhosted.org/packages/a4/2f/f40fc4a7070798e31630ae0be583e0857fcab71a1bf80c37481ded49c553/multidict-7.2.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0f30d6eaf3cb309b6c21096c6c2b4cf3800f195a78b1babae85f481ce851bdc2", size = 443550, upload-time = "2026-10-24T20:41:08.334Z" }, + { url = "https://files.pythonhosted.org/packages/ba/7e/068b48b20c1df58dddb7885f5e1befd6def91c7b6048f2644ccd81f57fa9/multidict-7.2.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:edffdb3950d3a8b769ddaec3a0962fb7b76b3c96f317f6b88500455a537dded2", size = 486514, upload-time = "2026-10-24T20:40:31.371Z" }, + { url = "https://files.pythonhosted.org/packages/fe/0d/a2f8b1fdd7cc2f497bb2fb0cc59dc7e2ced1fa4b0cd9b2912973e384ea77/multidict-7.2.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:8c3ff4f5c376df7126695a52af61e6084b69c9e52dcb9f4713e71ce9819e1bfd", size = 459258, upload-time = "2026-10-24T20:40:32.999Z" }, + { url = "https://files.pythonhosted.org/packages/36/aa/74652a34732f16a951fce9ca118e659f1a9dbe102ee52d9bcea4a8001dad/multidict-7.2.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:d5738cd929f2af8651ebb284467acb58b1ec0f2f4d42489cd0597172bc82e15b", size = 487418, upload-time = "2026-10-24T20:40:33.649Z" }, + { url = "https://files.pythonhosted.org/packages/dd/91/ae90df6402911d310cee3f44a73914ea555d5219b1976a8f2842dc247eac/multidict-7.2.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:81a5f84a76401c4330e6db4e170dddd58860b30c4e8a44a181e5c2809c7f9c9d", size = 502308, upload-time = "2026-10-24T20:41:09.776Z" }, + { url = "https://files.pythonhosted.org/packages/3c/f3/5a1ab9f594e652bece00f1243bf565063790838250054ef9310999c856f7/multidict-7.2.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:efe695cc2081633b02fcb95bc1b13070f6647c9ad52d42a8beeb3c2a7ea062dc", size = 439169, upload-time = "2026-10-24T20:40:32.896Z" }, + { url = "https://files.pythonhosted.org/packages/38/a4/8c4fd4ecbea568de3b4605dc7323488f65a204475e1aac98db9d36fdbfd3/multidict-7.2.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:0c52fdd8cb8194fcb6969a417149aa1d09d6e48b69fb1b34ee1b1e4cb51ec183", size = 519617, upload-time = "2026-10-24T20:40:34.488Z" }, + { url = "https://files.pythonhosted.org/packages/1a/b5/4964b2d8f8b4ff03df7f24e8dc7d0de534fad0430e553ff0693aa2bcfb75/multidict-7.2.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:02d9c04d395a36eeebe89e45f2269614a4675b8e83fddcba15bba2be903971e5", size = 499402, upload-time = "2026-10-24T20:40:35.035Z" }, + { url = "https://files.pythonhosted.org/packages/ee/f2/c7d5c514e066fc6f9b9a2a9583f46a3d8178c49b7e91cadcc59241f22c12/multidict-7.2.1-cp314-cp314-win32.whl", hash = "sha256:b97bffa5772b7e853303de094829fcd3a9bd49fddd70534651ee4933691f6587", size = 83635, upload-time = "2026-10-24T20:41:11.337Z" }, + { url = "https://files.pythonhosted.org/packages/08/8f/2ec03cddf5f283f979813f8c5d38f5df798e21cb07a124dfc311ed28d8e9/multidict-7.2.1-cp314-cp314-win_amd64.whl", hash = "sha256:6e22756c6e2d4f57ead0d2e6be179d8a16d661bee424b81de0c6c0c1305ec39f", size = 94544, upload-time = "2026-10-24T20:40:34.202Z" }, + { url = "https://files.pythonhosted.org/packages/2e/75/3e9803944df2233cd43924c04ef057160a75c35e87f4fc083de7c855ab74/multidict-7.2.1-cp314-cp314-win_arm64.whl", hash = "sha256:b63acdf4df5a78a15223bbb510bb3e0f8234dfa807610ccb8796d0842db2c801", size = 92767, upload-time = "2026-10-24T20:40:35.787Z" }, + { url = "https://files.pythonhosted.org/packages/96/6d/e3a4b4de34cde67f98887ac0c134e2436e1eafdcef62e81f9f8b7ac8e189/multidict-7.2.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:15da10cf11fd8d8716720c95b93180e94c8317405d89d08524c579ef360128ac", size = 183485, upload-time = "2026-10-24T20:40:36.397Z" }, + { url = "https://files.pythonhosted.org/packages/e6/b7/c5aa68302159285e544c7f1b687a48d2b000217b9a90aff880052d6cb867/multidict-7.2.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:c753b820738cddc097892fee95e7cd716fbdfc9640414c5e9ce9463ad125321b", size = 103109, upload-time = "2026-10-24T20:41:12.496Z" }, + { url = "https://files.pythonhosted.org/packages/54/b5/7574cad64ee957cb65a32ac4f842663fbe948c1749f688a93eef0677644c/multidict-7.2.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:69befda4b1ef57027ea95dc6ed733568642f330fb3203a3e5c5ef107d41e0a1f", size = 109785, upload-time = "2026-10-24T20:40:35.496Z" }, + { url = "https://files.pythonhosted.org/packages/38/7b/ced76f7d87cce5ed15ae404213c7f9fdc594d0caaedbbacc6b8cdfac81e6/multidict-7.2.1-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:8648dd5dc4d5db32b5936dc5e85e9a4d79db1135513d40d1f54ca15503750324", size = 511246, upload-time = "2026-10-24T20:40:37.078Z" }, + { url = "https://files.pythonhosted.org/packages/51/91/b598741d494865ed002df52f37f9e92bf9ab5a0524098a3f7667c1bd6ac6/multidict-7.2.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5efddcdbcb73f5485dca2b6624edc8ee063604b77bd3a3eb772826063965819f", size = 545199, upload-time = "2026-10-24T20:40:37.773Z" }, + { url = "https://files.pythonhosted.org/packages/e4/1d/2e3c15a07da380a7845e7d0969fde25cd6fca078a09bf30cf6deb06a3480/multidict-7.2.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:9c44bd978adddf39df5834c6a1efd7d075c84769a7a084168d035a2a276d9008", size = 489912, upload-time = "2026-10-24T20:41:13.782Z" }, + { url = "https://files.pythonhosted.org/packages/d6/5b/f6e2668e393430240da34a13503c190493dac8bef352ce7e26e831dc7949/multidict-7.2.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cff7657e79224dea46f2f2e58eb99186656ca63ebe356212210c5c00e9beb814", size = 553445, upload-time = "2026-10-24T20:40:36.916Z" }, + { url = "https://files.pythonhosted.org/packages/05/ca/d7e0605545eebc8053167683fc6aa0d49114df2e75a2c474f2b65f8f083a/multidict-7.2.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7fb3bc8d58e1b7c1d9fa1d995e30a4ce9dbe49f4c63c755a1ffdecdfbd026ccb", size = 573757, upload-time = "2026-10-24T20:40:38.554Z" }, + { url = "https://files.pythonhosted.org/packages/1b/de/9a74903ffdbe3712ba392166935c91785ccdff82dc4b6a15b28a4df52a14/multidict-7.2.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:66af55b07e4593790157e85f897f06983ded8a8692cb2fb048f3049c1d3fa172", size = 543116, upload-time = "2026-10-24T20:40:39.052Z" }, + { url = "https://files.pythonhosted.org/packages/8c/9d/8eb3ee4465610cd599b08b3320262a3448dcd03299565f4c725ee1504bbc/multidict-7.2.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ed6c107642a9acaa7ede32dcd428a6c87e302c5d555ecd3ec132ffa8c6155b4b", size = 478536, upload-time = "2026-10-24T20:41:15.442Z" }, + { url = "https://files.pythonhosted.org/packages/49/fa/346308a741f2c06b23da186022fd72a75b9923f4fe7c88705605e20506d2/multidict-7.2.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:bc9decb36abd00ed5d123db9df70eb9a0da45d6376d86d64f4a94bfd5a1b83fe", size = 526429, upload-time = "2026-10-24T20:40:38.392Z" }, + { url = "https://files.pythonhosted.org/packages/eb/10/96986b2404bba1bdb54e364e682a2a9fc97a93c9f6598a2c35c065cbe28f/multidict-7.2.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:5a7df927c7999d1d3196176a8d4139d3a0e1765936f5de8c768097275b2d617a", size = 488886, upload-time = "2026-10-24T20:40:39.949Z" }, + { url = "https://files.pythonhosted.org/packages/8c/5b/59b11e1e2f93d39b34c554bfe3beb35777aa976c35af078f23eda41fa809/multidict-7.2.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:4eb2650aa240b62ab0c6d87238bee5d14aa4d411c0d69ad61be53e026e0785ba", size = 510875, upload-time = "2026-10-24T20:40:40.484Z" }, + { url = "https://files.pythonhosted.org/packages/83/af/79e0036cc9ecc160235f3bcbd337524cb44ebd266cfc77e7532d9bd06850/multidict-7.2.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:83fbd7cfaca614975efdbdcd9e2df45f3e987e819216468fb73e1db6f0365b98", size = 539190, upload-time = "2026-10-24T20:41:17.519Z" }, + { url = "https://files.pythonhosted.org/packages/a0/b5/342db062f9ae8816c1b940360c251dff21afeb88a9c4b1694c2b3348f2db/multidict-7.2.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:a878fef2797a7c082aa47559af68299372edebc7367bd5128dff2d36e6d820e1", size = 474879, upload-time = "2026-10-24T20:40:39.953Z" }, + { url = "https://files.pythonhosted.org/packages/8a/60/1420f7daae59b331f5506e5b32a79f60fa475e636f53bf4e6cacad5782ec/multidict-7.2.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:f332b16911ffc668baec6aaf8f08a09e1af34a8ee87ccac26bd722589e264645", size = 549769, upload-time = "2026-10-24T20:40:41.241Z" }, + { url = "https://files.pythonhosted.org/packages/ce/af/c4b548057bee365137be7b1ed12cb6339b6911ad95d99a83d4b26e9bcb2a/multidict-7.2.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:81c6fc88ac9bf6920a0cdf22d3c383964167f12d35eb630463fc6d5ac89e460d", size = 535840, upload-time = "2026-10-24T20:40:41.927Z" }, + { url = "https://files.pythonhosted.org/packages/ba/df/70710e4bc50b29ced1ed69ec3a99124c56ac34747c6a64b05611f7b5856a/multidict-7.2.1-cp314-cp314t-win32.whl", hash = "sha256:0c7b4d8cc9e6ae7070b8129bdb840215552d4e2689ca765d37eb208a24919209", size = 90136, upload-time = "2026-10-24T20:41:18.854Z" }, + { url = "https://files.pythonhosted.org/packages/ac/cb/a330b1474608eef92e6a0a5861d61b8f2d3152d1a71b65dad8b56edea150/multidict-7.2.1-cp314-cp314t-win_amd64.whl", hash = "sha256:b5fb6b048931ea925b484f0c7c01b007981415a00a6c62ee74bbf8d229d39c08", size = 101163, upload-time = "2026-10-24T20:40:41.847Z" }, + { url = "https://files.pythonhosted.org/packages/66/fe/1563cd1336111755f79a60d06ffecd11dfbce5eb8c32ece9eae888d5fbef/multidict-7.2.1-cp314-cp314t-win_arm64.whl", hash = "sha256:e64b86d16158954825d12799271979c6944c1fd1d03575b2f22e869c3f7d00dc", size = 98630, upload-time = "2026-10-24T20:40:42.671Z" }, + { url = "https://files.pythonhosted.org/packages/9d/96/1aaad0e5df33f1f5af21775c411349c0234a6edd581ea403bc81919a6eab/multidict-7.2.1-py3-none-any.whl", hash = "sha256:08ea94663d9e7c25d14d78222f7aee906c472d4837766bec62477c38f6e419dc", size = 28145, upload-time = "2026-10-24T20:40:59.159Z" }, +] + +[[package]] +name = "orjson" +version = "3.12.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0f/f3/742fb1f62b825f2c010697eaf4e828004bc2a81e7e806666989c132c7c42/orjson-3.12.0.tar.gz", hash = "sha256:d14203fb1aae2ad9b3d52f8a0e82aeb10197ef1c9bc61da7f358bd70b00123d5", size = 4142915, upload-time = "2026-08-14T16:13:30.607Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/12/9d/3931253e6f3148abf2cbe14830367042a4806b362ea520df2303db188fb9/orjson-3.12.0-cp314-cp314-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:9e6fee342a48760e854d743e7a81534d8e2925a6f46e09f750cf56b50fd1de5d", size = 223391, upload-time = "2026-08-14T16:12:59.184Z" }, + { url = "https://files.pythonhosted.org/packages/8a/0e/b4a4f1e305367245877b967a0bad70fcf001d77c54ac4339a120b66fdae4/orjson-3.12.0-cp314-cp314-macosx_15_0_arm64.whl", hash = "sha256:8c3bb86dd10f39b3fbf434b7d5dc7cac77d6fc8ac572ae30a10731ede2c4b647", size = 123659, upload-time = "2026-08-14T16:13:00.548Z" }, + { url = "https://files.pythonhosted.org/packages/96/f3/6782c6fa85e2702bc66be183c3b421486167dcf266ee4dc1403fe3824870/orjson-3.12.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:2bb3ce43203936072dd8b4917b01d3aecfc02329bfb42510cb7cfb24708adc9c", size = 113337, upload-time = "2026-08-14T16:13:02.009Z" }, + { url = "https://files.pythonhosted.org/packages/bf/79/b32ab64bacda9d0fa4942ef483bd03cabf0eaf2be819ca9fb7ff610c559d/orjson-3.12.0-cp314-cp314-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:6a2a79c89984dc719817d388c8709e0efc2a2795a934eaa746b4882eb6045adc", size = 130112, upload-time = "2026-08-14T16:13:03.404Z" }, + { url = "https://files.pythonhosted.org/packages/ee/49/6e6142999ca01509219be5e5a9c338a3e5ea011f63e91ff473fbbf3734ed/orjson-3.12.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f06dd838d1e07d9b1de0932ec0485ec92c4d5f5d1ad4817a656268c3e88be1e1", size = 130520, upload-time = "2026-08-14T16:13:04.798Z" }, + { url = "https://files.pythonhosted.org/packages/49/d0/3745af0a4cc9867784f29722929cec4d10bd1c877cd754b01ba6d96eb21a/orjson-3.12.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c6b11be792c3d2c6a4be2af4ebf97a68d0bf5f580aca6e86a418a354f6cc846a", size = 131053, upload-time = "2026-08-14T16:13:06.14Z" }, + { url = "https://files.pythonhosted.org/packages/c3/f4/6fe5a22fa478fffb190e65c338c84df5c311ef597b363150a17cc57063c0/orjson-3.12.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:477ecaf6b9f88f873341b91fcc736119ca81b5e002a9f7f308ff5b4f2ce2a70e", size = 135321, upload-time = "2026-08-14T16:13:07.544Z" }, + { url = "https://files.pythonhosted.org/packages/ff/41/b1b0ec30289646a81a76e2dbaae2686b96fcccb7cb0323dc1dd78cbc7875/orjson-3.12.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f3c0683136acdc29afdf88a5bc2f7d3d0e34087788d1d63c0144b805a87a196f", size = 127485, upload-time = "2026-08-14T16:13:08.88Z" }, + { url = "https://files.pythonhosted.org/packages/bf/2b/277404bdcc21c93b112b963655b76443ebfe828f8a3ff1de7d90f8850eb3/orjson-3.12.0-cp314-cp314-win32.whl", hash = "sha256:d39f3f5c3927e2dc0913fe5bbc1a2f6b1b9d1bba1de6358340d0ad0d0c00ca92", size = 128048, upload-time = "2026-08-14T16:13:10.305Z" }, + { url = "https://files.pythonhosted.org/packages/41/2b/395b36fa2b4ce7af70b651d715e88f80d884b2c2b14a6b53e84d554fb5f0/orjson-3.12.0-cp314-cp314-win_amd64.whl", hash = "sha256:0b1ac5bf6609b2716c7954011c5fef6254922df029f45d032ee4ebf5d363cbed", size = 121858, upload-time = "2026-08-14T16:13:11.634Z" }, + { url = "https://files.pythonhosted.org/packages/ea/a3/833e895ff452859eebe75093d26691fe9108f1a7a6a08435d7a5780ea652/orjson-3.12.0-cp314-cp314-win_arm64.whl", hash = "sha256:50fae885cb073eac7556353ff3df93312b0d5137b0a5056b2bb63f97ed9a93c7", size = 126749, upload-time = "2026-08-14T16:13:13.117Z" }, +] + +[[package]] +name = "packaging" +version = "26.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/fa/3944b40b07da9ce895c0e6303a5ab7d53da063554f534556b134a54d6093/packaging-26.3.tar.gz", hash = "sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79", size = 313412, upload-time = "2026-08-04T18:15:28.737Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl", hash = "sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c", size = 129956, upload-time = "2026-08-04T18:15:27.159Z" }, +] + +[[package]] +name = "pamqp" +version = "4.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/31/4c/33a0ddaaac7bc42f9a542dbaaee8b580ceca3f89bf5da7c498d1fa97ff9a/pamqp-4.0.1.tar.gz", hash = "sha256:9dd13b828e346622793981f14a5df817fce5de998c746209d6c0154eb8403970", size = 137192, upload-time = "2026-07-06T16:37:51.732Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/14/1dfc08b743ba995a38dee0ea09beb46a05c7fe8ac53d729095905f7bf11d/pamqp-4.0.1-py3-none-any.whl", hash = "sha256:a547f45128b06e42ce8d7a739b0cfcc40f2c724770622eaaff4a3f587b1cf7d0", size = 32773, upload-time = "2026-07-06T16:37:50.623Z" }, +] + +[[package]] +name = "pillow" +version = "13.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ce/a4/9cc30690a394a8305e8be4c9036aa16d0dfddadd09079860b582f562496d/pillow-13.0.0.tar.gz", hash = "sha256:cdcd1ba258178f7356d74170c9a81ba4eab6a5d55748bc402a5a2a6fe6be3510", size = 47058631, upload-time = "2026-10-16T22:18:09.718Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8e/45/fbad9c7f5e2f1745c373a434d46072bc84eddfefaa28b852425ac957fda6/pillow-13.0.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:44629d4498c9cb6b2051806155f22141df0d8a929c56134b166655232dfb2f8c", size = 4190721, upload-time = "2026-10-16T22:16:27.524Z" }, + { url = "https://files.pythonhosted.org/packages/a2/bf/2074902a259a90692c5dbf132885d21671644cbbd3f690047516243a55db/pillow-13.0.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:bf97ba1b5542050e9b0c74e632323b6d8136cc99d64e193abebee22fe6013c76", size = 4290236, upload-time = "2026-10-16T22:16:29.562Z" }, + { url = "https://files.pythonhosted.org/packages/ff/a5/3d1bb93b8ea38d15d21356c1b8e9f50971ac592aaa248f7fb71739094c0e/pillow-13.0.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:aa694c49787040943570d61c9b8fc01271bd2a2cfdd55f0d896f8aecdce1a981", size = 3723093, upload-time = "2026-10-16T22:16:31.249Z" }, + { url = "https://files.pythonhosted.org/packages/84/a1/be060135d85187b25fa8b2573786c0e45c62ae27527fff5257675eb97f4a/pillow-13.0.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:49c12af515073e9bf511e4ef736743bf70d531a643668722279314b937742ba4", size = 5572253, upload-time = "2026-10-16T22:16:33.103Z" }, + { url = "https://files.pythonhosted.org/packages/b5/f4/0bf6f1ac9ba7ee2ffe9914aeb410a1035a0c1a3c42f122388d7fc950bc4b/pillow-13.0.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:d9b0a2bee3f3e03e9f29057aedb807dbf1e1d96ee1b9d0f422d9db6c8583271d", size = 4852269, upload-time = "2026-10-16T22:16:35.102Z" }, + { url = "https://files.pythonhosted.org/packages/83/16/b603274617b3e6728da3a0133d4fba1ea8d7fbe0c4ea5fef9cbffa398b8b/pillow-13.0.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5933fcea5a5ba64c686f55c886a115cbfd92f2900460784a1e79909fdbafd7b6", size = 6516058, upload-time = "2026-10-16T22:16:37.228Z" }, + { url = "https://files.pythonhosted.org/packages/1d/f2/1371707ce438ba9a43f9d24089828494ec6f9396e28cd7903aa704dd6306/pillow-13.0.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e91c2b1cbeae768c98752e67e717e283d68dae45717b41f2d91eaee648e37b7d", size = 7142574, upload-time = "2026-10-16T22:16:39.172Z" }, + { url = "https://files.pythonhosted.org/packages/b9/a9/62cf68587f910bc87c5951bcb6e9430c7643a65076adf5f18b4bf350e2bd/pillow-13.0.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7f6299e494e46253d40c34653cac369412f44ca53b5f85d848d3f95b1ed0546c", size = 6614295, upload-time = "2026-10-16T22:16:41.117Z" }, + { url = "https://files.pythonhosted.org/packages/6c/c9/9e9b48178649730299be0bafbf5f810a5b6977538f5a92d47e4ba5f5d608/pillow-13.0.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f9bd7d260d986f3553518c5d723544346fa48031f8aef3265d786aeb59845bc0", size = 7224841, upload-time = "2026-10-16T22:16:43.118Z" }, + { url = "https://files.pythonhosted.org/packages/8f/48/c44e91384b1b6c072a714a63d00608acc16a6a968735e478e4be4384e5ac/pillow-13.0.0-cp314-cp314-win32.whl", hash = "sha256:2e31c4e68cffd7b1f116127626813b82fcba19edf6b940a7b0c266d6484d0055", size = 6508694, upload-time = "2026-10-16T22:16:45.883Z" }, + { url = "https://files.pythonhosted.org/packages/50/e2/7ad17747517e4c813dca78257d6bb593466f8231baf1384bc98d92745edb/pillow-13.0.0-cp314-cp314-win_amd64.whl", hash = "sha256:f7cbb7a64e7ec664775195e4f81923bc8a93d9451b963f844e59118f63147a7a", size = 7281830, upload-time = "2026-10-16T22:16:47.725Z" }, + { url = "https://files.pythonhosted.org/packages/41/9f/e5b51f2ce8ef4563215334035403ce61a244f93a2e65c24a940131618f87/pillow-13.0.0-cp314-cp314-win_arm64.whl", hash = "sha256:a6aaa7871aebca081eb5c45f83e845bbd9712ae99fa9286dbfdcee835e63f4b4", size = 2695468, upload-time = "2026-10-16T22:16:49.515Z" }, + { url = "https://files.pythonhosted.org/packages/97/73/a53502f731f6c84805a4ce87a8881e86d07a65aee64728b900d112dc703e/pillow-13.0.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:7b999816b5850a35ada08443e6e03945f1f6488144a83afe2d37b240a552aafa", size = 5578196, upload-time = "2026-10-16T22:16:51.291Z" }, + { url = "https://files.pythonhosted.org/packages/21/50/7e949959a56b5559c78ef172b70fc6025706acb289f3f648a15329e4fa6f/pillow-13.0.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:67871a13cf47275201553b536c025da777f997fa562a19eb798af4ad4b97c18f", size = 4857276, upload-time = "2026-10-16T22:16:53.399Z" }, + { url = "https://files.pythonhosted.org/packages/c9/52/43ded0ac08e77e39d17bd6d894162572201b5e8fdc4a022b14c55a1960a4/pillow-13.0.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e8ca8f69b7c77fa96dad99f8638c1fc4f1c0bed4acaa1baf7a3c31bdb80a0ab1", size = 6607008, upload-time = "2026-10-16T22:16:55.525Z" }, + { url = "https://files.pythonhosted.org/packages/0d/2e/4a57ed216d5bf877387cba4d44c02aa87abe51ad2e06404aa6b597c4da39/pillow-13.0.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6bce6a29c3898191f4c9998e4d4e8f69790caeabde46e54da4bc235ce5bde683", size = 7224281, upload-time = "2026-10-16T22:16:57.738Z" }, + { url = "https://files.pythonhosted.org/packages/5c/3d/2929b248e135752d63aaeabf53f6de797e905b7b6263f9c1e2adade20aba/pillow-13.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:816752a5ef76a80484c8c441c85593b7ea9304f6231fc5776ad1fbff7e773058", size = 6695600, upload-time = "2026-10-16T22:16:59.842Z" }, + { url = "https://files.pythonhosted.org/packages/bc/33/fb0da992cf084ed1ee6ee05e96b6191e35c351eb707ca2df874feb24ce47/pillow-13.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5bdb38ba220b61b4ec31dd1046dea6b016d3c6850a23f43ed8b58332b2fc668e", size = 7300589, upload-time = "2026-10-16T22:17:01.686Z" }, + { url = "https://files.pythonhosted.org/packages/bb/af/c30dd9ef6a3dce84ecb1b2393cac629047a32e66279056c6cb46c3ffd65c/pillow-13.0.0-cp314-cp314t-win32.whl", hash = "sha256:06903aac074d00c988e77a1fc435286dc498506d4ca8d6f69df9babddf4ecc15", size = 6517204, upload-time = "2026-10-16T22:17:03.722Z" }, + { url = "https://files.pythonhosted.org/packages/85/d5/ef3a8ae7609c8ab26570abf18194882534b480d245a9148b895a2262c004/pillow-13.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:9a635068d9793c7f0d929f39f18b58f1bd86c236e8e7b64167b86af9232ebca0", size = 7286695, upload-time = "2026-10-16T22:17:05.543Z" }, + { url = "https://files.pythonhosted.org/packages/84/3c/0861db0c69a03689ebe3e7599d35668a414deb5aaafd8255215e4e62e3d3/pillow-13.0.0-cp314-cp314t-win_arm64.whl", hash = "sha256:0d1a9cf7bc6917538b20f8d06b9ab2147e542f68849e96fc58eb66bdf6acc692", size = 2698098, upload-time = "2026-10-16T22:17:07.674Z" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + +[[package]] +name = "propcache" +version = "0.5.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b3/9a/9fbf4e4ec0c2d7f1c32519fff782ef467859b8faa9fbc5331a96f6395d43/propcache-0.5.4.tar.gz", hash = "sha256:ff6b113f50bc066a698db5d944d2c6dc7507168dd3341e255a8892fd0715a558", size = 61545, upload-time = "2026-09-16T00:17:14.386Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/33/c9/07e227b930c8ae513b8ef1aae3793499be097bffcdf7aee4fb8b33db4cd1/propcache-0.5.4-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:e6720ba44ad7e72174314d0e1fb0172494cff5c73a3a8a2159c3d2402ff15565", size = 85933, upload-time = "2026-09-16T00:15:16.073Z" }, + { url = "https://files.pythonhosted.org/packages/e4/e1/6710bb44510c4e4a8e0f004bbaf3cecfd048141309c77bae56d4e5a6ebc1/propcache-0.5.4-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:4cfe0a92ae30151869e67a4b5f5e105e4e03ad30b3f38e5211b5bf77d0881993", size = 50179, upload-time = "2026-09-16T00:15:17.377Z" }, + { url = "https://files.pythonhosted.org/packages/e2/22/b533b493d7025456f44518b33e53e000021a20fe7c27b88cf3d341df7186/propcache-0.5.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1d759d05634f1b038fb625a66662a8c85e5a8fec912da381b5149ddac107482b", size = 51942, upload-time = "2026-09-16T00:15:18.589Z" }, + { url = "https://files.pythonhosted.org/packages/f1/74/70ac8430e28f21e442c7bcb964eb46c4363f6881ade4aa0e978bfd8d503a/propcache-0.5.4-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:251c63dd46a0659bb875cb254dc4c1e79ee91a847c737cd62373295afc2235dc", size = 232647, upload-time = "2026-09-16T00:15:19.905Z" }, + { url = "https://files.pythonhosted.org/packages/72/95/f222f13b6fe623310be0eb61a673bf26df439ce27e563ca8e422d0818777/propcache-0.5.4-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7a8d5ff04eb1f85698a78d20c62a14676e7b960dcafde09a388d60ad377d355d", size = 241541, upload-time = "2026-09-16T00:15:21.3Z" }, + { url = "https://files.pythonhosted.org/packages/a2/3e/763e370340db16115c5e63ad46e21ef0770a7f06928b3d3b62d8f8edfca4/propcache-0.5.4-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7b9100a93b372418d8688f3f2a3e5b45c64d70ca4d6176e121aca1e3bfc1e32f", size = 245332, upload-time = "2026-09-16T00:15:22.802Z" }, + { url = "https://files.pythonhosted.org/packages/96/d3/e97cd6f5de2176bd90ed4076c7a9b5e09d0f0b9687d00a576507988bb62c/propcache-0.5.4-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cc07876cfb079b6f6f36d21ce75784ad6c2c6b563eeac0ed26c2fa2669b85df9", size = 232757, upload-time = "2026-09-16T00:15:24.374Z" }, + { url = "https://files.pythonhosted.org/packages/f9/4c/6766e5f60bcda26d244333aa71d0a702c1c9b21b251d543c7af5953d1eee/propcache-0.5.4-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0951315a6b3142ee2167404d707743f0157c110091342b1aa0accac5cf0e4acf", size = 204389, upload-time = "2026-09-16T00:15:25.667Z" }, + { url = "https://files.pythonhosted.org/packages/b8/5e/ec4bb09a70b26ea99d76a8292c3383b960b296de2b347ac9986678f1761c/propcache-0.5.4-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:bee7d3aed13d56f54e681df38c3a23031bc9e3863f687d9d598825c9146acd7d", size = 228217, upload-time = "2026-09-16T00:15:27.11Z" }, + { url = "https://files.pythonhosted.org/packages/e1/7d/b53922ba7d9e5bf797324e63aa05906ec240871899f779628df068743e2d/propcache-0.5.4-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:4e985382be6d15da8d0c2710a6fa7b9070fc9ecdeefb7f580e88373984ec8be3", size = 216947, upload-time = "2026-09-16T00:15:28.532Z" }, + { url = "https://files.pythonhosted.org/packages/ff/39/b62eee45e5ea4de094a258cbb3b01c1e856ca51ddfd95b43135c5effd1eb/propcache-0.5.4-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:9e9ab13760aa8b6d0881ae7cb04fd891d8d490cd2554ea8e79bb278399169bcc", size = 233457, upload-time = "2026-09-16T00:15:29.977Z" }, + { url = "https://files.pythonhosted.org/packages/cc/a9/feec61ed296d993db9dd097e0f6723e3f576a647722367547495e4c5b05c/propcache-0.5.4-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:1b2f3bec4261a94019575481c726c29850f72e27907773c75b1de421e20e9f9d", size = 204131, upload-time = "2026-09-16T00:15:31.74Z" }, + { url = "https://files.pythonhosted.org/packages/92/4d/411ef380cddad28dc001f1c6d75ec72c76cd3817030f68ec1ccfba0ec6c1/propcache-0.5.4-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:720cf832eb2d0b0dfee129cb3335a26f6ce3cc45ee1187e8f0731758caa16792", size = 234820, upload-time = "2026-09-16T00:15:33.087Z" }, + { url = "https://files.pythonhosted.org/packages/15/37/c988229753629ef1cfd5198337a83e624780ea2b3787efe9e747c05aad2d/propcache-0.5.4-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:9fb0a5be8d9aa213150e8d8148a42aca4984b285bcad1e69587dc4298edd929b", size = 228350, upload-time = "2026-09-16T00:15:34.533Z" }, + { url = "https://files.pythonhosted.org/packages/12/49/5ef1c5cf98591da3c5b952b39e6a298084cc1ce353bc70f85e82397a5036/propcache-0.5.4-cp314-cp314-win32.whl", hash = "sha256:30cc1cebaf9aef49db06357a50398323ae04d70460c0491837d026ab7d6452ea", size = 43578, upload-time = "2026-09-16T00:15:35.957Z" }, + { url = "https://files.pythonhosted.org/packages/1e/9e/a0ac821a2229186af5e2e3c3635a78abb23cfddca57f38513ab5d70420f3/propcache-0.5.4-cp314-cp314-win_amd64.whl", hash = "sha256:0a095db8e15a6020db149ecbed6461939fe74f6acaa3ae8b702a1fe8c38cd983", size = 46304, upload-time = "2026-09-16T00:15:37.655Z" }, + { url = "https://files.pythonhosted.org/packages/a1/19/c8d0d36a9d16cba5dcee67d389c9333b988c8986a653a61c00a451817a46/propcache-0.5.4-cp314-cp314-win_arm64.whl", hash = "sha256:45488d1a5f9ab5bd90aaa1ca20f50fe1922b8ffad71a2009d2adf41355897aac", size = 43440, upload-time = "2026-09-16T00:15:39.091Z" }, + { url = "https://files.pythonhosted.org/packages/2c/e9/42f1da77cacfc184e6ec929557ef653b7961bbf6f1da460b9221273948b3/propcache-0.5.4-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:53eaa697c4d0422ff4cb714d00231b43352064d97b944033b30c1d57cc506ec0", size = 90672, upload-time = "2026-09-16T00:15:40.306Z" }, + { url = "https://files.pythonhosted.org/packages/cf/2f/4b79940908c6ab8c795097c102999d7bc1f7e0b8604dfd1c232f9d99d67a/propcache-0.5.4-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:886b59c4d28ca97dd23b025fdfc50a0356be934efbbbca89ad26230067f86fe5", size = 52586, upload-time = "2026-09-16T00:15:41.575Z" }, + { url = "https://files.pythonhosted.org/packages/eb/07/02196ae6320c110235bb343f90dbd34be41f8b8964a3ee30db84ec12579e/propcache-0.5.4-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:3fa15757fea1dfcd5b7745cad9f4638929605531bd4018ab2adff7955f1a403d", size = 54335, upload-time = "2026-09-16T00:15:43.027Z" }, + { url = "https://files.pythonhosted.org/packages/6f/44/f48b9a131985659924df5fa5093f68fe72c7ee375329802989ba3126efc6/propcache-0.5.4-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6f0093ac3e9daada202c2082439d414a625c57184727a46e112a3fb2a81cb788", size = 297567, upload-time = "2026-09-16T00:15:44.373Z" }, + { url = "https://files.pythonhosted.org/packages/04/a1/418d956d2735139f77fc35262179f1f52c23aa666de5a8ab3819c1ae7854/propcache-0.5.4-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:3cd3a7edb6b95b9b33998135ebfa18d709da82290fb8f27c858970b5a12c8b56", size = 297477, upload-time = "2026-09-16T00:15:46.048Z" }, + { url = "https://files.pythonhosted.org/packages/69/fd/ff811fdb6d3d3e67fd9bbfb75881675d34a42d0ef29a45d33e3e233dde07/propcache-0.5.4-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c174bfd1c48a1b51a3078e95586dde718374bac79719ab3541ec9e74aec40574", size = 302669, upload-time = "2026-09-16T00:15:47.458Z" }, + { url = "https://files.pythonhosted.org/packages/fc/57/527910c455b5ec62f6871bef45d4f79fea16cb8c966ba0d4a07f0339ddc4/propcache-0.5.4-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a219f0ac59817a9114dd2aa57c13180f993e819ba658c7ddab4b66ed1ee0d370", size = 287908, upload-time = "2026-09-16T00:15:48.99Z" }, + { url = "https://files.pythonhosted.org/packages/1d/86/f69ab82707534a0cb2057bdca04f9200a71214c7551800f9d34d6ac39e4f/propcache-0.5.4-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:17a7400cec0256f0a71ae71f9da398f9894c956ff6668a1c9d317b3367316320", size = 249804, upload-time = "2026-09-16T00:15:50.486Z" }, + { url = "https://files.pythonhosted.org/packages/27/19/60677af50d93be4256213de7cd487f056944c048b9c0b6f2e45b3a30f666/propcache-0.5.4-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:978f28401afbc76cdc3df9e1717b4229a06b626a1dcc75db4e1f2beb3884c3e9", size = 282344, upload-time = "2026-09-16T00:15:52.029Z" }, + { url = "https://files.pythonhosted.org/packages/7c/f7/a0057808a91fb3b6a5f3602b528f0cdcb3d53e0ff8315d73fabdfdf8fec4/propcache-0.5.4-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:4a1f4f5ffa55dce6307631f3cb2948e117e665966ea512e0d502b16c24f567e7", size = 270167, upload-time = "2026-09-16T00:15:53.466Z" }, + { url = "https://files.pythonhosted.org/packages/83/c8/f4a865490df0dc0c8531d4e59ac411cb6dc24bb255d2396a6f1c60a368f4/propcache-0.5.4-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:213bb68d9ced5cf2bf717b1071bf2b09b4b04c426256f9fe6d054c60318424c4", size = 286551, upload-time = "2026-09-16T00:15:54.995Z" }, + { url = "https://files.pythonhosted.org/packages/b0/67/b4faebde9da4e8173d0e5a30e8cd31335914af7ef350b988f27fec588cfd/propcache-0.5.4-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:286867fb156488c251a3721766e380ac4495e4fd6b51aaa1403d89ce7f4359d9", size = 249595, upload-time = "2026-09-16T00:15:56.505Z" }, + { url = "https://files.pythonhosted.org/packages/f6/40/52e1dd5636e9f5a27f6b5a4b4e2f33c322fd72afe956c397d82523ec4a80/propcache-0.5.4-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:445ee3bfb46e85838387fb3c536a73cc0b994dc192b004e40e170adc54aa2a7e", size = 286700, upload-time = "2026-09-16T00:15:57.985Z" }, + { url = "https://files.pythonhosted.org/packages/d5/0e/30b2b324b93ff31a0bab539c102aae59e84e444031b2742150a7646aa1bb/propcache-0.5.4-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:48cb48c5346a97de792254af77715aa2529c2a1ebc5f586aa0aae44a02f1fe57", size = 280500, upload-time = "2026-09-16T00:15:59.487Z" }, + { url = "https://files.pythonhosted.org/packages/64/36/721bb59f682ff060d0c8df64274fca8cd0521b1a54506c2eedaef795b7f5/propcache-0.5.4-cp314-cp314t-win32.whl", hash = "sha256:03b229037d25b801e7af53fd52b9fc49d9439b036fca1e087e02780631adfa97", size = 46121, upload-time = "2026-09-16T00:16:01.349Z" }, + { url = "https://files.pythonhosted.org/packages/c1/86/0b1b80fa1ac3a0aac44e2922a6964fbe9cd52af5eab8fa933bf9e90b030c/propcache-0.5.4-cp314-cp314t-win_amd64.whl", hash = "sha256:8a1fc236528c457cd739c88abe823da851b7ab645d72792f88658114cc340c12", size = 49154, upload-time = "2026-09-16T00:16:02.901Z" }, + { url = "https://files.pythonhosted.org/packages/69/4f/9fe6f05a47cb550c823155052116f710064b6be5c6e8ec4e9faae7e18115/propcache-0.5.4-cp314-cp314t-win_arm64.whl", hash = "sha256:135036c5cfc93864affb0f9af9a27e5d7a71cb7bd745e7b6dbfc2d56cc30e827", size = 46005, upload-time = "2026-09-16T00:16:04.266Z" }, + { url = "https://files.pythonhosted.org/packages/f5/cd/785c64ed382f3f04201870267b02783f63b4678c2acfddc177a3ebcc2727/propcache-0.5.4-py3-none-any.whl", hash = "sha256:62c60aec739ed00124573cce1178138fd690c7676352d67a37328c1cf51d7468", size = 16338, upload-time = "2026-09-16T00:17:13.106Z" }, +] + +[[package]] +name = "pycparser" +version = "3.12" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cb/27/eeb9f307bf07060f3d9f9aa0fb8c15b01bdfded0a941e4b2719c7f94cacd/pycparser-3.12.tar.gz", hash = "sha256:f03387877eab82ce01bbdfd0ba7252ee55a90b1c708d5129e2ce915475cf1db6", size = 158785, upload-time = "2026-10-22T13:40:03.56Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b6/ae/315bbeee126a769cbde43cf4908acb4b40408d8bf8bbf334dd7368d8dadf/pycparser-3.12-py3-none-any.whl", hash = "sha256:a076f3f489b39b43ed13563a1dafe42447445bec2e294154ec5c57fcdda4c85e", size = 62600, upload-time = "2026-10-22T13:40:02.021Z" }, +] + +[[package]] +name = "pydantic" +version = "2.13.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-types" }, + { name = "pydantic-core" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/53/ef/fc4f868f4e2cee79f863883abffceff107875f569b848507319842d2a681/pydantic-2.13.5.tar.gz", hash = "sha256:51a9c5f7b2f8e636f04c6cada605d9b6a3bf1348fdf945a3d8869b19bba0ee08", size = 845750, upload-time = "2026-08-28T14:04:00.916Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/eb/47/c95ffc2009878c7aac0c5e08528022dcb885933252a88b5f170058014464/pydantic-2.13.5-py3-none-any.whl", hash = "sha256:346a034f080da3755d8e9cb5e00e8b07de1d39e4f6e2c87d8ab7cafa0b269a73", size = 472589, upload-time = "2026-08-28T14:03:59.136Z" }, +] + +[[package]] +name = "pydantic-core" +version = "2.46.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/af/f9/8a06bea35ef8daf588f707784c973a7046e0034c8d8cfb08828eeffb8b75/pydantic_core-2.46.5.tar.gz", hash = "sha256:10416c15b8839ecc4ef4d0885da76da6fd0f67333a0eb8aff6d93c4b8f2910fc", size = 472262, upload-time = "2026-08-28T10:01:31.677Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8e/8a/14596f2a8367da50cf7cbac48169ee5d9c8e11d486a3b527082384630c72/pydantic_core-2.46.5-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:c1c43ad4339643d70ebb8124e1305a7dab423001eff58bb41a0f731adbc98355", size = 2074081, upload-time = "2026-08-28T09:59:16.141Z" }, + { url = "https://files.pythonhosted.org/packages/ae/d5/d8a4eb6d6c7f66b91dd37c576d76e9e60fba900caf5372c17bcf949febc2/pydantic_core-2.46.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1a353f84de772f423b5ffb11d7ae352fbbef0f446f3c0b0af0f8236d7233606e", size = 1920497, upload-time = "2026-08-28T09:59:18.065Z" }, + { url = "https://files.pythonhosted.org/packages/8e/26/092079428f86e927e030b2c0ced87df69dbb1c875cdeaa67bf42ea2be746/pydantic_core-2.46.5-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5086029a57366b8cf81b130a43908738095c270c21a8d7f0e8bdfdb89718e2f3", size = 1952130, upload-time = "2026-08-28T09:59:20.476Z" }, + { url = "https://files.pythonhosted.org/packages/08/c3/8ec0e290a9ebaebd64047bf5fda94be835c6b1551b02437e4b76778fbcd7/pydantic_core-2.46.5-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:46c25dda9d092a06c08db76ffe0a197107904d0dfac653f7d5306bbcd6d6119c", size = 2026371, upload-time = "2026-08-28T09:59:22.227Z" }, + { url = "https://files.pythonhosted.org/packages/01/72/4fd20ad520fb8da0157f95b27a7eb05a72790ef08138e7701ac972c342ea/pydantic_core-2.46.5-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:37ea7b83c935e5b0d68c9449b82651accf78a10828b2c02b2f2d9e9496446c21", size = 2202822, upload-time = "2026-08-28T09:59:24.277Z" }, + { url = "https://files.pythonhosted.org/packages/31/b0/d16e0771206b29314f0d52198b720be21e8a99ab2bf11e3bc0d7c9cebdff/pydantic_core-2.46.5-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e64e88d5585bea9ce95861079de72006c7fa6d3df4e3a3b65ba31eb979c15c9f", size = 2262756, upload-time = "2026-08-28T09:59:26.608Z" }, + { url = "https://files.pythonhosted.org/packages/2c/9b/59634b7ac631c63b2a37760eb6943af3e29573d6b59a4abc5e7f019d4cee/pydantic_core-2.46.5-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:54d510bac3ee52247af28ed4bb18a1e799f040ac60fd2bf5ccd4c92f1fbe786f", size = 2068352, upload-time = "2026-08-28T09:59:29.044Z" }, + { url = "https://files.pythonhosted.org/packages/08/7c/570abb1ad2155348dc754ea91be22e5aaa18eb6d69a6068f7c6f2679a6ed/pydantic_core-2.46.5-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:a2a5e1d0ff29adddc9f6d6821a66302e4493f8ca898b715b6b1182c2c201ea0a", size = 2104777, upload-time = "2026-08-28T09:59:30.95Z" }, + { url = "https://files.pythonhosted.org/packages/8e/25/5bf74adc65a1ac5b7be3f6cb0bcb5433615c1598a801c19d830d84c98ded/pydantic_core-2.46.5-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:03b9666e41e35d8909852ba191a0607520f81b74eaf12ccf8737005dbb313821", size = 2156312, upload-time = "2026-08-28T09:59:32.604Z" }, + { url = "https://files.pythonhosted.org/packages/90/6a/2ef38830675e050121040618135564ed56b860b45433b02d9b4ebece46f3/pydantic_core-2.46.5-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:a91c17edf6eea2402cb5457b4c89e99bc5ed1004aa34c4adf1d4258c1a5c22c2", size = 2150067, upload-time = "2026-08-28T09:59:34.453Z" }, + { url = "https://files.pythonhosted.org/packages/90/ef/a7dbb03a14a64c2a4621f989c615ed9a892535a6cad938fc27079f919d80/pydantic_core-2.46.5-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:b49924c73a235e969511bf2aabdff3beebf9820931f646c80274d5d780010c47", size = 2304516, upload-time = "2026-08-28T09:59:36.194Z" }, + { url = "https://files.pythonhosted.org/packages/68/f8/6bb4c4b80e8a6fde1904c64a51c62a1d04fcdfa3ea521a66b2ddefa1d885/pydantic_core-2.46.5-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:2cbd9a5eff05e51c447c34dfa4632145b26b09120cf04bd0c871e44c1a5e1c9a", size = 2335223, upload-time = "2026-08-28T09:59:37.931Z" }, + { url = "https://files.pythonhosted.org/packages/2a/80/f46b8c681195190b2c1f1c7c0a81abce60663e987613e09ef64d433dd96b/pydantic_core-2.46.5-cp314-cp314-win32.whl", hash = "sha256:2d5d76654becf5efd62c9e51c3756c67b49498b0c9a40884934c40807adbd074", size = 1934827, upload-time = "2026-08-28T09:59:39.836Z" }, + { url = "https://files.pythonhosted.org/packages/f7/3c/60674207246bc0a4009d2391b7c7251c7159f279c8d2ab8aae8ef46f3dee/pydantic_core-2.46.5-cp314-cp314-win_amd64.whl", hash = "sha256:fa10ef4112775900e7a0661068635eb67b2ab824fbde764de6e0e21982a93db0", size = 2042648, upload-time = "2026-08-28T09:59:41.792Z" }, + { url = "https://files.pythonhosted.org/packages/69/0c/117c562c7c1babdf44576b72a5e496906506c93690387ecfbca7c729ae2e/pydantic_core-2.46.5-cp314-cp314-win_arm64.whl", hash = "sha256:045ab3b6d308439e32b81cc173bba5b9018bc6ed896afd0c65b3b009b1699af5", size = 1989652, upload-time = "2026-08-28T09:59:43.702Z" }, + { url = "https://files.pythonhosted.org/packages/e8/66/9336ae58f9eb68c41d121894e52c4c89eccb07eb8f602a04ee9c3f37736a/pydantic_core-2.46.5-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:8816f3d218beb4b787de5c9759c259b8fa61f9dec42dc7811f320a33771778b7", size = 2065829, upload-time = "2026-08-28T09:59:45.364Z" }, + { url = "https://files.pythonhosted.org/packages/c5/02/bc19b47a96c2d3109760711acf22369e56bd7e405ca52f7ade164d2ead57/pydantic_core-2.46.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:bce57638e08ac148e5778cce7feb968307a727d66f8e2274a543d0cf0c9ad6a3", size = 1905716, upload-time = "2026-08-28T09:59:47.18Z" }, + { url = "https://files.pythonhosted.org/packages/52/a4/70b47c0509923dd98ccfed04fb3e32ea3849c82a0ff2205bb41009b43c00/pydantic_core-2.46.5-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:976e1128455aa595ea04c79ccfedff1aaeab96ee013fcc916bed120c4f0ad94f", size = 1934216, upload-time = "2026-08-28T09:59:49.241Z" }, + { url = "https://files.pythonhosted.org/packages/52/ab/aa03b65f7bb198585edf806b906c3223ecf1795543e39e23aec4cce27ad2/pydantic_core-2.46.5-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e7b891faeedeafba41b2983e5001a81b6a915b69544c7e7570d1989ce1c36ac7", size = 2010635, upload-time = "2026-08-28T09:59:51.692Z" }, + { url = "https://files.pythonhosted.org/packages/3c/8b/0da06343f30b84ec549aafd309c6456223d5dc8bd36af504c573faad561d/pydantic_core-2.46.5-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5f194189415698233dd1114a093a9b56e61e2c57e11b469be3b0506f46f0771c", size = 2209369, upload-time = "2026-08-28T09:59:53.582Z" }, + { url = "https://files.pythonhosted.org/packages/d6/5b/844c4defaa34a3df66eb9257087d121d70c201298b96abdf9f492fc2f1bf/pydantic_core-2.46.5-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:82a36973cf8a2ef5406f4fe2edbf8ed0c99629535d959e0b100c76a32535a111", size = 2253238, upload-time = "2026-08-28T09:59:55.484Z" }, + { url = "https://files.pythonhosted.org/packages/f4/64/a4e536cb16d7f61a7fd3120b46c577fc7fa7325992f69c4f52bc786d77d8/pydantic_core-2.46.5-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cdbb78909f52b981d3b2d56b97328d71eb0b974c36bd77c920123a7ebb192829", size = 2065740, upload-time = "2026-08-28T09:59:58.038Z" }, + { url = "https://files.pythonhosted.org/packages/5f/75/aaa38c6bc2d085f6605b34eabdc6a8a4e0b2e61fc9c8e6e52b28e97b3125/pydantic_core-2.46.5-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:52e24eacdb536cade636aa90fb851835222becff8484b7001fdc78cb0290f2aa", size = 2087425, upload-time = "2026-08-28T09:59:59.898Z" }, + { url = "https://files.pythonhosted.org/packages/55/ae/fcab4cfc39aba3689e1d20c8b5250ad280957022c09af2ed9cd585602a5e/pydantic_core-2.46.5-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:37ae34309d7bd8c0d61ab839668058f2a7962ea1fc51d105d2db228fe0618034", size = 2139306, upload-time = "2026-08-28T10:00:03.057Z" }, + { url = "https://files.pythonhosted.org/packages/2d/f4/f1d03a4bc9d9acbc62f4d742b8a319af52f71885079868b2ff8e48a651ee/pydantic_core-2.46.5-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:0cdbada856a1c69a7624a64d3d9aefe79300bd6ef827b43a4f265010b9b55184", size = 2144589, upload-time = "2026-08-28T10:00:05.645Z" }, + { url = "https://files.pythonhosted.org/packages/83/f3/7a53bb1356de514a4cd295f25b6ac39237895620c0462d2592b76c16e114/pydantic_core-2.46.5-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:545f26c504b27c3758439a5e6d9349931f0a04f855668d5fe323c89e82300a38", size = 2288882, upload-time = "2026-08-28T10:00:07.931Z" }, + { url = "https://files.pythonhosted.org/packages/cd/94/5a81583660c175c59d49ffb09f4b3a44debeaf86a19fca664ae1cdd9ee32/pydantic_core-2.46.5-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:ff218293c9c806138dca139765e3b067621be52bcd93cdc14c7711be7ddc90a9", size = 2335210, upload-time = "2026-08-28T10:00:10.177Z" }, + { url = "https://files.pythonhosted.org/packages/5a/9f/5d685c2693b972d1a59c998586e8823712b66603aeff47ee60a4bdaafd37/pydantic_core-2.46.5-cp314-cp314t-win32.whl", hash = "sha256:97cf3eb53a8cccacf9d46686a0926186c9bfb5574f2ed66d3639d5fe117cd3a9", size = 1921180, upload-time = "2026-08-28T10:00:12.35Z" }, + { url = "https://files.pythonhosted.org/packages/70/12/5c94ee16d65a37a15f9e869f5e6256df111154491173801a4c5e800ab548/pydantic_core-2.46.5-cp314-cp314t-win_amd64.whl", hash = "sha256:d2f9fc07a8042a8f95925b35c4f04f469707c981fc33245b6ca187cf5d2dd290", size = 2020515, upload-time = "2026-08-28T10:00:14.774Z" }, + { url = "https://files.pythonhosted.org/packages/63/19/67830dda664e6bdf9285ee2e40f355d0d7d6b92aa0c42e8d217bb8d33d36/pydantic_core-2.46.5-cp314-cp314t-win_arm64.whl", hash = "sha256:acf8a67ba51f4ca9ddbd0e6b3000a65ac51ab734661778b3e7ba64d99a710f2f", size = 1989276, upload-time = "2026-08-28T10:00:16.984Z" }, +] + +[[package]] +name = "pydantic-settings" +version = "2.15.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pydantic" }, + { name = "python-dotenv" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/68/ca/31c57507b13119d7d3cfa1576dad2911a4861e3be07b579395f4e9d393f9/pydantic_settings-2.15.0.tar.gz", hash = "sha256:694b793e84f766ba76a90ebdefc01d0a9a045dab0382bee70393da93712ad117", size = 261253, upload-time = "2026-08-07T09:24:57.419Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/30/a4/2bffa9f8e804325a09867f0e9d30795c80ea9f8d62560bd1b6ad6220eb2f/pydantic_settings-2.15.0-py3-none-any.whl", hash = "sha256:0ba092c291c94baceb5eff768aa0d56400a457585bc0175925a5a5510303da42", size = 69413, upload-time = "2026-08-07T09:24:55.839Z" }, +] + +[[package]] +name = "pygments" +version = "2.21.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/49/2e/ced460408999b33da6b31b0021b0f37d329e202d4169aeb164493778f25b/pygments-2.21.0.tar.gz", hash = "sha256:610ca751c9bc2492b38eb9a38a7fbc93edbbb2d7182edaf34e66ae493dee5c8c", size = 5005329, upload-time = "2026-08-17T08:02:48.824Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/46/17f022dd3e953bf20a04a028a21ec746d942f8d2af30fa0f124fa0e6a684/pygments-2.21.0-py3-none-any.whl", hash = "sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9", size = 1250147, upload-time = "2026-08-17T08:02:44.912Z" }, +] + +[[package]] +name = "pytest" +version = "9.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", size = 1636369, upload-time = "2026-06-19T10:58:32.857Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, +] + +[[package]] +name = "pytest-asyncio" +version = "1.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/43/7c/d36d04db312ecf4298932ef77e6e4a9e8ad017906e24e34f0b0c361a2473/pytest_asyncio-1.4.0.tar.gz", hash = "sha256:c6c0d2259945122819f171a32ecea2c349ead889ee28176caaf492143424be42", size = 58514, upload-time = "2026-05-26T09:56:04.083Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/03/e2/08a497ef684b88559c9cc5f4ad53a37e7b99e727094a86d6ea32536d5d3c/pytest_asyncio-1.4.0-py3-none-any.whl", hash = "sha256:933ca923a23075a87fb7070c0ec272a6848489824d887c85c812670932835aa1", size = 16930, upload-time = "2026-05-26T09:56:02.576Z" }, +] + +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, +] + +[[package]] +name = "python-dotenv" +version = "1.2.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6a/53/ed9d74092561d4b01a2ef1349d52cdbc135e526c245f366b089cfca6de49/python_dotenv-1.2.3.tar.gz", hash = "sha256:a20a594dabeaa385725aa239d5244871c143ecb356add8a20fcf23773a6c3a35", size = 58945, upload-time = "2026-08-16T16:54:54.067Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0d/17/c5c6b53ddc18f297992099b3d9ec16c855c0ccc83263a21fe4d1c625ec6c/python_dotenv-1.2.3-py3-none-any.whl", hash = "sha256:904552145e8bfed22162c09dab1c2b9b54fefa7b23ba780f4f26ca0316b0f0d9", size = 22780, upload-time = "2026-08-16T16:54:52.473Z" }, +] + +[[package]] +name = "pyyaml" +version = "6.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9d/8c/f4bd7f6465179953d3ac9bc44ac1a8a3e6122cf8ada906b4f96c60172d43/pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac", size = 181814, upload-time = "2025-09-25T21:32:35.712Z" }, + { url = "https://files.pythonhosted.org/packages/bd/9c/4d95bb87eb2063d20db7b60faa3840c1b18025517ae857371c4dd55a6b3a/pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310", size = 173809, upload-time = "2025-09-25T21:32:36.789Z" }, + { url = "https://files.pythonhosted.org/packages/92/b5/47e807c2623074914e29dabd16cbbdd4bf5e9b2db9f8090fa64411fc5382/pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", size = 766454, upload-time = "2025-09-25T21:32:37.966Z" }, + { url = "https://files.pythonhosted.org/packages/02/9e/e5e9b168be58564121efb3de6859c452fccde0ab093d8438905899a3a483/pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788", size = 836355, upload-time = "2025-09-25T21:32:39.178Z" }, + { url = "https://files.pythonhosted.org/packages/88/f9/16491d7ed2a919954993e48aa941b200f38040928474c9e85ea9e64222c3/pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5", size = 794175, upload-time = "2025-09-25T21:32:40.865Z" }, + { url = "https://files.pythonhosted.org/packages/dd/3f/5989debef34dc6397317802b527dbbafb2b4760878a53d4166579111411e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764", size = 755228, upload-time = "2025-09-25T21:32:42.084Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ce/af88a49043cd2e265be63d083fc75b27b6ed062f5f9fd6cdc223ad62f03e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35", size = 789194, upload-time = "2025-09-25T21:32:43.362Z" }, + { url = "https://files.pythonhosted.org/packages/23/20/bb6982b26a40bb43951265ba29d4c246ef0ff59c9fdcdf0ed04e0687de4d/pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac", size = 156429, upload-time = "2025-09-25T21:32:57.844Z" }, + { url = "https://files.pythonhosted.org/packages/f4/f4/a4541072bb9422c8a883ab55255f918fa378ecf083f5b85e87fc2b4eda1b/pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3", size = 143912, upload-time = "2025-09-25T21:32:59.247Z" }, + { url = "https://files.pythonhosted.org/packages/7c/f9/07dd09ae774e4616edf6cda684ee78f97777bdd15847253637a6f052a62f/pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3", size = 189108, upload-time = "2025-09-25T21:32:44.377Z" }, + { url = "https://files.pythonhosted.org/packages/4e/78/8d08c9fb7ce09ad8c38ad533c1191cf27f7ae1effe5bb9400a46d9437fcf/pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba", size = 183641, upload-time = "2025-09-25T21:32:45.407Z" }, + { url = "https://files.pythonhosted.org/packages/7b/5b/3babb19104a46945cf816d047db2788bcaf8c94527a805610b0289a01c6b/pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c", size = 831901, upload-time = "2025-09-25T21:32:48.83Z" }, + { url = "https://files.pythonhosted.org/packages/8b/cc/dff0684d8dc44da4d22a13f35f073d558c268780ce3c6ba1b87055bb0b87/pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702", size = 861132, upload-time = "2025-09-25T21:32:50.149Z" }, + { url = "https://files.pythonhosted.org/packages/b1/5e/f77dc6b9036943e285ba76b49e118d9ea929885becb0a29ba8a7c75e29fe/pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c", size = 839261, upload-time = "2025-09-25T21:32:51.808Z" }, + { url = "https://files.pythonhosted.org/packages/ce/88/a9db1376aa2a228197c58b37302f284b5617f56a5d959fd1763fb1675ce6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065", size = 805272, upload-time = "2025-09-25T21:32:52.941Z" }, + { url = "https://files.pythonhosted.org/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", size = 829923, upload-time = "2025-09-25T21:32:54.537Z" }, + { url = "https://files.pythonhosted.org/packages/f0/7a/1c7270340330e575b92f397352af856a8c06f230aa3e76f86b39d01b416a/pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", size = 174062, upload-time = "2025-09-25T21:32:55.767Z" }, + { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, +] + +[[package]] +name = "redis" +version = "8.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a8/99/604f0b666d4c616d891cf77ebb9db6bb21601344c051aebf1b72b9ff915f/redis-8.1.0.tar.gz", hash = "sha256:6e1a19beef9225c83efd689c7e6b7da2d5215b1f42cd13b7fc3714d0a09c7b25", size = 5254356, upload-time = "2026-07-30T08:51:00.269Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/66/9d/c5731f6e3608663d4d3656fd8d3aecee8b509c3082818f5a13eae925baea/redis-8.1.0-py3-none-any.whl", hash = "sha256:a4fe1aac3d3b3cc791d4b3d5931c5a956045dc951ee74d1c913ee3ac4d2ee9fb", size = 560618, upload-time = "2026-07-30T08:50:58.497Z" }, +] + +[[package]] +name = "ruff" +version = "0.16.9" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/96/bf/c935ca98e73fe8ce65b87ef08a280c0c1e85295d569228c15e87d8fdfaf1/ruff-0.16.9.tar.gz", hash = "sha256:12b625c6cfba78d285d9f48eda5f053374f1e53cb10ef17342a383750db99161", size = 4948764, upload-time = "2026-09-24T20:37:49.416Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0d/26/df51322b52ee1ada7eff2d071ea09d11d5c2d1dcc9f02594f5785c4d1635/ruff-0.16.9-py3-none-linux_armv6l.whl", hash = "sha256:95e6f022090368ab3b824c36276839c53b2adf1a3f4c09fefc33dfc400f6da96", size = 10082922, upload-time = "2026-09-24T20:37:13.045Z" }, + { url = "https://files.pythonhosted.org/packages/a5/27/7bf51f5a7aa375e9f339280a303aab44ca75dc1525f1cdc5991761685b0f/ruff-0.16.9-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:a5f27be168556594a86d2f415db0cf43f5291917849318f873c7e2791f7a8c67", size = 10236360, upload-time = "2026-09-24T20:37:16.049Z" }, + { url = "https://files.pythonhosted.org/packages/b6/63/09659283f92f02dff45809da194a70da2f688d87c55d8875c4fae3536072/ruff-0.16.9-py3-none-macosx_11_0_arm64.whl", hash = "sha256:1632eb1d6197f33bd00b1acbc5b71009e89a8895c158e2d2b03a834fac964ab6", size = 9892940, upload-time = "2026-09-24T20:37:17.957Z" }, + { url = "https://files.pythonhosted.org/packages/24/58/98de1b72ec172f5f8f1731236fe21585b3998bf7dfe9fcc44ae9ba626012/ruff-0.16.9-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b3f951b14d865d5952c89d40a5ca07e87abe24fa5453299878411e127748fb1c", size = 10032114, upload-time = "2026-09-24T20:37:19.942Z" }, + { url = "https://files.pythonhosted.org/packages/c8/7d/f1e17c54ab59d4bad1dce8ee3e22a7a1d0ef4745240decacdcf3832b5bb2/ruff-0.16.9-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:447fc07e1573afff7cb02803462b12b6c8ece7cf10e2cd78565fa6d7a1c0bf8d", size = 9910227, upload-time = "2026-09-24T20:37:21.872Z" }, + { url = "https://files.pythonhosted.org/packages/34/19/436f647a65075bbd3bab2668b3bdaa5120559b294694018cdcefabbbf30b/ruff-0.16.9-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8a3e039a6a40ed976c491722b60e0ae4a4aa1a86057f540ee7a37a5d19ae9120", size = 10547484, upload-time = "2026-09-24T20:37:24.229Z" }, + { url = "https://files.pythonhosted.org/packages/03/59/38430a6bc2f6d8095447ac39625cf8b6e9344a47e6c26225c8ba1bff3ffb/ruff-0.16.9-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4684dded7db60aa57cb118fa158630f5feade4af5782903b6053484bdf9bd129", size = 11412367, upload-time = "2026-09-24T20:37:26.307Z" }, + { url = "https://files.pythonhosted.org/packages/c8/bd/bbb6d7fc7f208c8b8c50dd5dc8206e4cfdb1e7a8fb852606a3adf370c880/ruff-0.16.9-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d29c934357e45642fda2f34c0b1f4025b4a6c01e15e4bf0016879d60078a142c", size = 10869787, upload-time = "2026-09-24T20:37:28.35Z" }, + { url = "https://files.pythonhosted.org/packages/bc/b8/9c543074918061abbefc3bd139bee22de35abedb00dde0f2d27288838962/ruff-0.16.9-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a21713e629d3e5bdb2f5c2def1cc7f04f47fa8e1a7eb0571b4a28e1da64bc728", size = 10406494, upload-time = "2026-09-24T20:37:30.624Z" }, + { url = "https://files.pythonhosted.org/packages/35/7a/5a8851bd146e7ccf8fd4b003f6c75c11f8fbdb0e60673b45097986b6bf41/ruff-0.16.9-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:7baa24ef5fc8e77aa93879e1d3f43754a01ae488e869f1ae30cf431afd4d2452", size = 10590083, upload-time = "2026-09-24T20:37:32.439Z" }, + { url = "https://files.pythonhosted.org/packages/87/f0/4c3467188f23f806960b46fa76575a7cd0514c9ba90562650b71efc96980/ruff-0.16.9-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:a41aac6230aadfaa133bdfa1614488531ffa3e0837567ae04c0da2058a9c0f9e", size = 10119151, upload-time = "2026-09-24T20:37:34.581Z" }, + { url = "https://files.pythonhosted.org/packages/15/34/5a4def5adea572ce6aea0bb64f21f928ee317b80e0db747d7979b01d7261/ruff-0.16.9-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:c2529fb5896d49115b0e9aa8f887490b34bbe76baf879ec2264ac59406869ce7", size = 9911544, upload-time = "2026-09-24T20:37:36.796Z" }, + { url = "https://files.pythonhosted.org/packages/62/5d/d15ebea7499eef9373318c0ee6ca127832927c6529731f6d48e18dce7ca9/ruff-0.16.9-py3-none-musllinux_1_2_i686.whl", hash = "sha256:41e3870277694177429b56406d65dfbdb2c2802c52b715edaf6a0b829c69d4ee", size = 10269884, upload-time = "2026-09-24T20:37:38.857Z" }, + { url = "https://files.pythonhosted.org/packages/d1/56/c5d3cd119ded7a3c7aba0e961b69cb3df701c91662c98ad694467d060ce1/ruff-0.16.9-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:8adbe4e58af167f767d7b2ba5e83c42e878350796cf78c2f5e14ab9903a92588", size = 10749366, upload-time = "2026-09-24T20:37:41.042Z" }, + { url = "https://files.pythonhosted.org/packages/ac/fe/734ec7527029ac757ecf821f143c9f3fcf69149c21f53a044a899b430f5a/ruff-0.16.9-py3-none-win32.whl", hash = "sha256:0e1dbc2073624dee6618d41d0098690a7244654af746704b64759e12b6b6b385", size = 10152355, upload-time = "2026-09-24T20:37:43.025Z" }, + { url = "https://files.pythonhosted.org/packages/14/21/26e4643629b3ebb44f0a06f9c9a53058d63d989415f63a9a3c28e2ee7f22/ruff-0.16.9-py3-none-win_amd64.whl", hash = "sha256:6bd40fec8cd4c8a3d4dd589bd8ad4e6320c13c29234159bfd959a40d529d597b", size = 10592965, upload-time = "2026-09-24T20:37:44.944Z" }, + { url = "https://files.pythonhosted.org/packages/51/60/5fb1a39dbb5ae314d5f59bc7348a63c1d5c20f3cd83914c4b5cb0be31d2d/ruff-0.16.9-py3-none-win_arm64.whl", hash = "sha256:ed1a252039200f57a59eebc063b54beabea67bfbaaca0eeaa7f54b5fbcda2284", size = 10458649, upload-time = "2026-09-24T20:37:46.882Z" }, +] + +[[package]] +name = "six" +version = "1.17.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, +] + +[[package]] +name = "starlette" +version = "1.7.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7b/2b/3850dc6bf7ef71b088962eba31dafc6cffd2f96e577ebb0bb316df96da3e/starlette-1.7.0.tar.gz", hash = "sha256:c79f74ea63cff761804fbbfb182f1e0b440c2d07b164d24700c5a1bab5d6ff5d", size = 2736246, upload-time = "2026-09-23T07:30:26.35Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4e/d6/1ec1b290f9e0fb067899b61e1d37a30c923068bad260b216dbe37a7d2967/starlette-1.7.0-py3-none-any.whl", hash = "sha256:67f8e99895493dd2911a03f11314af6ceebeae4e704bb9f43dfc6a9db151c93e", size = 78980, upload-time = "2026-09-23T07:30:24.567Z" }, +] + +[[package]] +name = "structlog" +version = "26.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5e/89/b4a0bcfdf4f71a3dea31379f095929613d7e4528a0996bca6aa964cd0dca/structlog-26.1.0.tar.gz", hash = "sha256:f63a716cbd1b1291cf7661de7794b455acfa4c43c5bcf1630e6ad5ddc1adb3b7", size = 1459881, upload-time = "2026-06-06T07:33:39.348Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a9/18/489c97b834dfff9cf2fc2507cede4bcd4b11e67f84bc462acd1992496f86/structlog-26.1.0-py3-none-any.whl", hash = "sha256:e081a26d6c373e6d201eca24eede26d8ffab07f88f477822e679183428d3d91e", size = 73764, upload-time = "2026-06-06T07:33:38.046Z" }, +] + +[[package]] +name = "ty" +version = "0.0.84" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e1/f6/34f8869c45ea87fe6a931ffa824b1fe4428167173543d92988d631add355/ty-0.0.84.tar.gz", hash = "sha256:0cefdb0cd5d399418dbe83c349ba605047b7dff815ae6f460c80e8522b40be67", size = 7409537, upload-time = "2026-09-24T13:16:24.367Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/aa/0c/600648778e1c79133ce6b6a5c4c2531ae16b90c9b30f5041b38b574f61b8/ty-0.0.84-py3-none-linux_armv6l.whl", hash = "sha256:868438e2b9abfc835dd5b5b07ed693db95d8348ac40e3228252511d63ae87695", size = 13935333, upload-time = "2026-09-24T13:15:31.167Z" }, + { url = "https://files.pythonhosted.org/packages/c5/3b/27f06f49945c36170765fad82076898039ae3873338e9865f876dd62f12c/ty-0.0.84-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:b00f39b521f0b559cde0f6abf8dd47213b513b04f33d58efeb9b98486a16bb17", size = 13511328, upload-time = "2026-09-24T13:15:34.446Z" }, + { url = "https://files.pythonhosted.org/packages/fc/b6/b12dd130e7f30141f0645614f454c925991cd389b1cc6c8284fd6bf32326/ty-0.0.84-py3-none-macosx_11_0_arm64.whl", hash = "sha256:8f78c2915567f2ee62ce0c24d86091bc76f81577677755e242d09951db588b9a", size = 13445445, upload-time = "2026-09-24T13:15:37.476Z" }, + { url = "https://files.pythonhosted.org/packages/44/96/f012de4b3c1d9a4773326621380d768b4d48c47e620dc012775c5e01c7d7/ty-0.0.84-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:61155cad0921e890d86d1d911a40b3a0e924ef4bdc3b4f3865a6c89c7e3012dc", size = 13471235, upload-time = "2026-09-24T13:15:40.64Z" }, + { url = "https://files.pythonhosted.org/packages/05/32/dca630f5f5353fa7bf67eebccf1549df99b16656d21071d0df6677c826a5/ty-0.0.84-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:417c3bf14947ed16b0b92976ab333c27915590354e96d4f581709068d3811355", size = 13680704, upload-time = "2026-09-24T13:15:43.817Z" }, + { url = "https://files.pythonhosted.org/packages/44/8d/33702c8f62f05ad45b33644fb914d483de6a0a7bc8e51884e9b8773e6d69/ty-0.0.84-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e06ef117759f654f5379efe9ae79bda8a186016c016f1023d05c34f8e15c1546", size = 14518923, upload-time = "2026-09-24T13:15:46.802Z" }, + { url = "https://files.pythonhosted.org/packages/c8/a1/da3246e6ef2ae0e842bca4058b25c54bee55ab058ffff4ee54e72517ec01/ty-0.0.84-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a05d1d53db8b64410ab38c6e32c5898510584869d1d5dd9c0605db81c9aef7df", size = 15027731, upload-time = "2026-09-24T13:15:49.755Z" }, + { url = "https://files.pythonhosted.org/packages/20/17/b0099098a560aeb4f9fde6c912f9a81622c3e44e6555e1fbe322b368ec31/ty-0.0.84-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b92400494ab855cfb1204ac1101e3f53b2d09a8e21d171758b2cf9acdab1809e", size = 14748260, upload-time = "2026-09-24T13:15:52.713Z" }, + { url = "https://files.pythonhosted.org/packages/60/d5/32cd67aedb271f006e716ad093806c96c5d7e2af86be8255469dbd3d76c8/ty-0.0.84-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:af17eb391ae3027fed9be1780bea555c2b8a25ba68e184c627b4981b2c9eaab7", size = 14205858, upload-time = "2026-09-24T13:15:55.677Z" }, + { url = "https://files.pythonhosted.org/packages/43/c4/638ac62ca2e8eca1b92e3e16273085744fada728b993500819b5c258ca15/ty-0.0.84-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:44cf06af0e7ec78ea6d8ff33a3450fc19627756a4fba681f7410d4e3ef63047a", size = 14573196, upload-time = "2026-09-24T13:15:59.165Z" }, + { url = "https://files.pythonhosted.org/packages/a6/f6/9ee81c5d8921e5fd5f10889ae563c95cd3272b31df2ada528919ca1cef6b/ty-0.0.84-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:c6e56b443ff9ba462e34bfbc1e582004c2719ed35f8f93a433744b0ba0afb2d9", size = 13480760, upload-time = "2026-09-24T13:16:02.313Z" }, + { url = "https://files.pythonhosted.org/packages/f6/83/fbe5d1177667ae16d84c3242846b143809c9cc09568b333e5a9b4d5908f6/ty-0.0.84-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:8b6562fcc48820c1030e2a4509ef2c14ef93043ca4860280074314ac3a825b95", size = 13696346, upload-time = "2026-09-24T13:16:05.391Z" }, + { url = "https://files.pythonhosted.org/packages/86/9a/1b4ac9ddc34cc0ab4624bc219825d66af52f1ad87f4778ad08ad03fa06d5/ty-0.0.84-py3-none-musllinux_1_2_i686.whl", hash = "sha256:dee7421451efcbd70f03e95a8f2c8d9e5ec3d35edcd27b4095ee00ee062abfc4", size = 13959410, upload-time = "2026-09-24T13:16:08.661Z" }, + { url = "https://files.pythonhosted.org/packages/c9/04/70e289437a67dfd686056ab4fd91880261a8233033a6a95d29393741fb97/ty-0.0.84-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:31e96307cf621babe969307dd657783d672e2d8bdbd06493710da635bf63fa07", size = 14350294, upload-time = "2026-09-24T13:16:11.865Z" }, + { url = "https://files.pythonhosted.org/packages/9c/f4/ae7dbd118425c850981d23efb850f0d9d7f6b8d2b852bde2767434f668b4/ty-0.0.84-py3-none-win32.whl", hash = "sha256:4bdc85f91cbaae35825f2b17bb4ac5de0563367875cb0f42b2bf07a1bf7ff2ba", size = 13176439, upload-time = "2026-09-24T13:16:15.049Z" }, + { url = "https://files.pythonhosted.org/packages/29/ec/994d87252f065dda5a500bc8afc726d18f6582d81d6f4ff1feb8a267c800/ty-0.0.84-py3-none-win_amd64.whl", hash = "sha256:71cc9aa7a7c89d4e12c6a814e9c0c3310ad5bee44efedbd6f5a42d91e9ac0f0d", size = 13962879, upload-time = "2026-09-24T13:16:18.191Z" }, + { url = "https://files.pythonhosted.org/packages/6d/1c/e6e91852b72566b1ab9aca504712857dfb89324c12092a5bd61d42d34427/ty-0.0.84-py3-none-win_arm64.whl", hash = "sha256:6c9d3718c4a0d318bc9c92af9b2aa85c3294e7dff54cf1d69ad79568f56516da", size = 13656453, upload-time = "2026-09-24T13:16:21.51Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f6/cc/6253133b5bb138fc3306cebfbda2c520f545d36b5be2c7255cc528bb45d6/typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5", size = 113555, upload-time = "2026-07-02T08:40:05.92Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", size = 45571, upload-time = "2026-07-02T08:40:04.659Z" }, +] + +[[package]] +name = "typing-inspection" +version = "0.4.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a3/26/b09b8010994eccc3c09092e6b34058f36a460eea2d4c3e8b910c695975a0/typing_inspection-0.4.4.tar.gz", hash = "sha256:547274fa6b0a561ccf549cc9524b999a578e737d015d8709d021f9d0d13bea47", size = 76928, upload-time = "2026-08-12T12:37:25.997Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/67/81/4add07e5172b7ac40d8ed5ff580409a7801a4fe26d529bdd915401dabfbe/typing_inspection-0.4.4-py3-none-any.whl", hash = "sha256:65b8397ba37ccbce054456aaccddfc91e6e3083c92824df348d96ca832f3f147", size = 14750, upload-time = "2026-08-12T12:37:24.648Z" }, +] + +[[package]] +name = "urllib3" +version = "2.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e3/05/b17359e1cefb4f909b5e40b1b90a496d987258916dbbf88e842c729f510e/urllib3-2.8.0.tar.gz", hash = "sha256:63bf2ead4c879426ebf22ef2a781eeb4aa3b4ae798a0435506f8687fd5bb9b63", size = 458972, upload-time = "2026-09-15T19:29:36.253Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/92/9d/c4e665119135114480843e7ab388fa94d8480650450e6f8e26b70d323a4c/urllib3-2.8.0-py3-none-any.whl", hash = "sha256:0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3", size = 135717, upload-time = "2026-09-15T19:29:34.577Z" }, +] + +[[package]] +name = "uvicorn" +version = "0.54.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "h11" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/da/34/30e9280707135d2cfc589dfff3cb796bd07a3aeb1a3e415ba09dd89d7bb4/uvicorn-0.54.0.tar.gz", hash = "sha256:a2e33cbfaa0306f8e6b0c13e0cb89d7d7a2da3e62b90c66e18c33d9807b28620", size = 112283, upload-time = "2026-09-25T06:52:37.601Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/38/0c/b54a4fdd7f90a3af8b02ebc9ce6712c2c208b7926a2f7bad95c33ebbe943/uvicorn-0.54.0-py3-none-any.whl", hash = "sha256:505bdb0f318731d45f1f712071fc781a8981f6847a31c902c9f5e652d4f67faf", size = 87427, upload-time = "2026-09-25T06:52:35.829Z" }, +] + +[package.optional-dependencies] +standard = [ + { name = "httptools" }, + { name = "python-dotenv" }, + { name = "pyyaml" }, + { name = "uvloop", marker = "platform_python_implementation != 'PyPy' and sys_platform != 'cygwin' and sys_platform != 'win32'" }, + { name = "watchfiles" }, + { name = "websockets" }, +] + +[[package]] +name = "uvicorn-worker" +version = "0.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "gunicorn" }, + { name = "uvicorn" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/80/59/9101b9c0680fd80e9d26c07deb822a5d18a324339fcf9cd017885ee808ad/uvicorn_worker-0.4.0.tar.gz", hash = "sha256:8ee5306070d8f38dce124adce488c3c0b50f20cf0c0222b12c66188da7214493", size = 9361, upload-time = "2025-09-20T10:47:01.218Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/90/25/09cd7a90c8bb7fb693be0d6704fccd5f9778d5513214b7a01cc4a94ff314/uvicorn_worker-0.4.0-py3-none-any.whl", hash = "sha256:e2ed952cef976f5e9e429d7269640bbcafbd36c80aa80f1003c8c77a6797abde", size = 5364, upload-time = "2025-09-20T10:46:59.776Z" }, +] + +[[package]] +name = "uvloop" +version = "0.22.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/06/f0/18d39dbd1971d6d62c4629cc7fa67f74821b0dc1f5a77af43719de7936a7/uvloop-0.22.1.tar.gz", hash = "sha256:6c84bae345b9147082b17371e3dd5d42775bddce91f885499017f4607fdaf39f", size = 2443250, upload-time = "2025-10-16T22:17:19.342Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/90/cd/b62bdeaa429758aee8de8b00ac0dd26593a9de93d302bff3d21439e9791d/uvloop-0.22.1-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:3879b88423ec7e97cd4eba2a443aa26ed4e59b45e6b76aabf13fe2f27023a142", size = 1362067, upload-time = "2025-10-16T22:16:44.503Z" }, + { url = "https://files.pythonhosted.org/packages/0d/f8/a132124dfda0777e489ca86732e85e69afcd1ff7686647000050ba670689/uvloop-0.22.1-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:4baa86acedf1d62115c1dc6ad1e17134476688f08c6efd8a2ab076e815665c74", size = 752423, upload-time = "2025-10-16T22:16:45.968Z" }, + { url = "https://files.pythonhosted.org/packages/a3/94/94af78c156f88da4b3a733773ad5ba0b164393e357cc4bd0ab2e2677a7d6/uvloop-0.22.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:297c27d8003520596236bdb2335e6b3f649480bd09e00d1e3a99144b691d2a35", size = 4272437, upload-time = "2025-10-16T22:16:47.451Z" }, + { url = "https://files.pythonhosted.org/packages/b5/35/60249e9fd07b32c665192cec7af29e06c7cd96fa1d08b84f012a56a0b38e/uvloop-0.22.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c1955d5a1dd43198244d47664a5858082a3239766a839b2102a269aaff7a4e25", size = 4292101, upload-time = "2025-10-16T22:16:49.318Z" }, + { url = "https://files.pythonhosted.org/packages/02/62/67d382dfcb25d0a98ce73c11ed1a6fba5037a1a1d533dcbb7cab033a2636/uvloop-0.22.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:b31dc2fccbd42adc73bc4e7cdbae4fc5086cf378979e53ca5d0301838c5682c6", size = 4114158, upload-time = "2025-10-16T22:16:50.517Z" }, + { url = "https://files.pythonhosted.org/packages/f0/7a/f1171b4a882a5d13c8b7576f348acfe6074d72eaf52cccef752f748d4a9f/uvloop-0.22.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:93f617675b2d03af4e72a5333ef89450dfaa5321303ede6e67ba9c9d26878079", size = 4177360, upload-time = "2025-10-16T22:16:52.646Z" }, + { url = "https://files.pythonhosted.org/packages/79/7b/b01414f31546caf0919da80ad57cbfe24c56b151d12af68cee1b04922ca8/uvloop-0.22.1-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:37554f70528f60cad66945b885eb01f1bb514f132d92b6eeed1c90fd54ed6289", size = 1454790, upload-time = "2025-10-16T22:16:54.355Z" }, + { url = "https://files.pythonhosted.org/packages/d4/31/0bb232318dd838cad3fa8fb0c68c8b40e1145b32025581975e18b11fab40/uvloop-0.22.1-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:b76324e2dc033a0b2f435f33eb88ff9913c156ef78e153fb210e03c13da746b3", size = 796783, upload-time = "2025-10-16T22:16:55.906Z" }, + { url = "https://files.pythonhosted.org/packages/42/38/c9b09f3271a7a723a5de69f8e237ab8e7803183131bc57c890db0b6bb872/uvloop-0.22.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:badb4d8e58ee08dad957002027830d5c3b06aea446a6a3744483c2b3b745345c", size = 4647548, upload-time = "2025-10-16T22:16:57.008Z" }, + { url = "https://files.pythonhosted.org/packages/c1/37/945b4ca0ac27e3dc4952642d4c900edd030b3da6c9634875af6e13ae80e5/uvloop-0.22.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b91328c72635f6f9e0282e4a57da7470c7350ab1c9f48546c0f2866205349d21", size = 4467065, upload-time = "2025-10-16T22:16:58.206Z" }, + { url = "https://files.pythonhosted.org/packages/97/cc/48d232f33d60e2e2e0b42f4e73455b146b76ebe216487e862700457fbf3c/uvloop-0.22.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:daf620c2995d193449393d6c62131b3fbd40a63bf7b307a1527856ace637fe88", size = 4328384, upload-time = "2025-10-16T22:16:59.36Z" }, + { url = "https://files.pythonhosted.org/packages/e4/16/c1fd27e9549f3c4baf1dc9c20c456cd2f822dbf8de9f463824b0c0357e06/uvloop-0.22.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6cde23eeda1a25c75b2e07d39970f3374105d5eafbaab2a4482be82f272d5a5e", size = 4296730, upload-time = "2025-10-16T22:17:00.744Z" }, +] + +[[package]] +name = "watchfiles" +version = "1.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b3/68/e6aa0b77d217b31f8f486ec0cdfe5e00e6e38dc0be657e7d85819b9faf0a/watchfiles-1.3.0.tar.gz", hash = "sha256:99aee4a07847c06820765fd7b1b49ceac4f3f711ccb7d104655a33231de1c207", size = 108119, upload-time = "2026-09-21T09:08:55.664Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/68/fa/c0b840d5d8bafe640925408ce11095948c8945bb61be1e67d0ab629b872a/watchfiles-1.3.0-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:000b9688fc8133037a8b075c8ebf98f32844ff8964dda61e85c1db547dafc441", size = 399703, upload-time = "2026-09-21T09:08:13.518Z" }, + { url = "https://files.pythonhosted.org/packages/c7/8a/894799b485fe9473ad10422a0d9668e53fb78e3a2b5cc6061159572844ad/watchfiles-1.3.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:bbc1198edfdc90fda0600f825aa94150f428dfcbf8138746f55998e0e660d64c", size = 397647, upload-time = "2026-09-21T09:08:15.16Z" }, + { url = "https://files.pythonhosted.org/packages/5a/61/f277cd5ead05f5d37b98b41fbdd0c943ee0c04d49738f62e902cb291cf43/watchfiles-1.3.0-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4af9c464d410e8c44b58ddf1cbec7c3a02c1cb2c6c80df9c74659ba2fbda0ab5", size = 454994, upload-time = "2026-09-21T09:08:16.587Z" }, + { url = "https://files.pythonhosted.org/packages/46/66/ea4c01382975e53b1ac1b0a0ea1b353416b8127fb6e04f5c1c313119aab6/watchfiles-1.3.0-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:eb94e40b9a0db19636b5e3fed0a9803452ecc7381df13b385272abb779ee83ca", size = 460105, upload-time = "2026-09-21T09:08:17.961Z" }, + { url = "https://files.pythonhosted.org/packages/31/c3/1a501b597817096c9831c255f6270c124ce0e2faeb74124be24dcc93b86d/watchfiles-1.3.0-cp310-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7398930a2b76b9dc67a4e6b8bfc7baf30946892a86f417a3920c07bdd6febc30", size = 493761, upload-time = "2026-09-21T09:08:19.327Z" }, + { url = "https://files.pythonhosted.org/packages/5a/16/77352d05e152c7786a016e9f1decdb4507e8aa5bcf3343c72142a0413d14/watchfiles-1.3.0-cp310-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:025b108f2d5cc2799cb941f79380df3e8b22bfb589c218e801767fe78d598c95", size = 577500, upload-time = "2026-09-21T09:08:20.723Z" }, + { url = "https://files.pythonhosted.org/packages/95/fe/81001c55b24466fe9911135e50269fd3a29f43e446627a92adfab54553b6/watchfiles-1.3.0-cp310-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:be9ef3cd403d756a304a0a08a112163c63da7ec72c8c9d083c98818291ff8f29", size = 469209, upload-time = "2026-09-21T09:08:22.346Z" }, + { url = "https://files.pythonhosted.org/packages/8a/cf/e3ad894ed5909a7c44b0707e89bc25d19a3d78b3a7aec846119443b6ae56/watchfiles-1.3.0-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b5768b49e426fd5b550b012c866db347cdf15c398ef98dc557b6e6b72fa74cd1", size = 458353, upload-time = "2026-09-21T09:08:23.7Z" }, + { url = "https://files.pythonhosted.org/packages/7f/f5/2f106829e6e2bc91ad74614730f6df484202b9a4d20d927dbf2f08d37d44/watchfiles-1.3.0-cp310-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:d36c72fcdc08f143d87316a4c4c3b064e340da2c7383d39803950740ed418935", size = 468707, upload-time = "2026-09-21T09:08:25.167Z" }, + { url = "https://files.pythonhosted.org/packages/15/37/2741ab776b978669c1f3bba1e573f1a184f727ab8b4d3a6d9c61090800ed/watchfiles-1.3.0-cp310-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:f0c9865240e065a2247f4b5248528ae3d01e0e8ae071462c250c884dc16e3ee3", size = 632510, upload-time = "2026-09-21T09:08:26.532Z" }, + { url = "https://files.pythonhosted.org/packages/48/64/db4a4684275fe9b696ee5118c194705c1423b3d42181c3f8ee70194d5c73/watchfiles-1.3.0-cp310-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:380f513d26cc2e598266b88d66456e07d67ef4be8f0f435a1154d9c44b43d509", size = 661723, upload-time = "2026-09-21T09:08:28.004Z" }, + { url = "https://files.pythonhosted.org/packages/52/80/95ddb24f6a2f595c031c24432ab7b8aa7d6d1ada0c6d63a552166e06e8d1/watchfiles-1.3.0-cp310-abi3-win32.whl", hash = "sha256:509d9f74d2bec5c1f4cd868ddcfe0c9601f7ea53066dfcb19d8ede742c4de661", size = 277761, upload-time = "2026-09-21T09:08:29.335Z" }, + { url = "https://files.pythonhosted.org/packages/48/a1/1c4a1b3c8030cd03206232612be8a6e8799093490c220b0fcb935d02b75a/watchfiles-1.3.0-cp310-abi3-win_amd64.whl", hash = "sha256:1acabde19b67e673274e89a04d613b7fb1f122e2c4b8ec9a581e59c149264b61", size = 290643, upload-time = "2026-09-21T09:08:30.45Z" }, + { url = "https://files.pythonhosted.org/packages/82/e0/d2db577c632d26d08078c4390dd870b8555c4e558df25d22270f0166c71f/watchfiles-1.3.0-cp310-abi3-win_arm64.whl", hash = "sha256:d978cc1dd7ba44f5590d7de74f7a9c8be7262d9e469c2c0efb8bbfc2574321dc", size = 285510, upload-time = "2026-09-21T09:08:31.561Z" }, + { url = "https://files.pythonhosted.org/packages/4f/55/a122743607939429ab6a7272672f49303aa02bbd34c3f06676765a3861de/watchfiles-1.3.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:e0204f90677b7fb8d3a824279a71a01166bea9f6e53923b8056592d87b33b0b6", size = 395978, upload-time = "2026-09-21T09:08:32.747Z" }, + { url = "https://files.pythonhosted.org/packages/3e/9d/0ee1b0ea42acfe72afa2484f0222e2386dd09992e9034a81e63cc3cdab0a/watchfiles-1.3.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7c1125e99f84934a92996ae343501285617373acd66f16b59fd5654b27ad8d80", size = 387997, upload-time = "2026-09-21T09:08:33.953Z" }, + { url = "https://files.pythonhosted.org/packages/b7/f2/4319c6b683e8bd28465dfc53c750617a2a07f072eb2e3ab0182ccf812229/watchfiles-1.3.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a85e243c87109d9f48a2e0d10d6abdee8768cc114848b32281ec83d7f81d86ef", size = 448067, upload-time = "2026-09-21T09:08:35.104Z" }, + { url = "https://files.pythonhosted.org/packages/b7/71/9f37ac192a51a24de3a2d7fe9aac1e99b2b2668300674ff7a9f85d95beb1/watchfiles-1.3.0-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:dd538c71766c59c732e2c5cedc39323c99c11897b99f93eac773438e54af206a", size = 452881, upload-time = "2026-09-21T09:08:36.238Z" }, + { url = "https://files.pythonhosted.org/packages/34/07/1224667c32a4a96876ddd642da69733cbfe6a15897db0865aec5a882cebc/watchfiles-1.3.0-cp314-cp314t-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:da213822ec9082b62cadf3008e07a8c690ec7a3c89f6dc5e55a8cf07b56a1b26", size = 484802, upload-time = "2026-09-21T09:08:37.717Z" }, + { url = "https://files.pythonhosted.org/packages/bd/4d/7f29cd87e1f826df86fc0418d0b8f370ca0311c29c618406dc492aad9951/watchfiles-1.3.0-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d4593392e87669836670f87d62fc754059eaeb0e6157af44060a46558cebe5a3", size = 567522, upload-time = "2026-09-21T09:08:38.94Z" }, + { url = "https://files.pythonhosted.org/packages/e3/ac/3aa1f84abd4dbbfbae99a817dd931f77b439261c911189cad4c30fdfd470/watchfiles-1.3.0-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:26b81bc515a0f03f66a69a6fea86eea54c8eca4173c46dd7222569f79dd2f977", size = 461997, upload-time = "2026-09-21T09:08:40.142Z" }, + { url = "https://files.pythonhosted.org/packages/82/1c/27c66764d5b0dbb70a36770a32eca217eba739c28b24e8583a035087a3d7/watchfiles-1.3.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ca39661934749df580d89f3dde266f2af7c6968f4c21c72ce2dc387d299e6aa7", size = 451856, upload-time = "2026-09-21T09:08:41.305Z" }, + { url = "https://files.pythonhosted.org/packages/1e/b3/a45827084d953c1d0dd3d75abf85dcac26a0f5d115ad2fe0b7e9e931573b/watchfiles-1.3.0-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:04c5500faa725d69a99b0f63750e80795e455bb951849e9c98a811b4324997c4", size = 459889, upload-time = "2026-09-21T09:08:42.58Z" }, + { url = "https://files.pythonhosted.org/packages/7a/3d/07740d23ce2bb23419bb24055c435a9abe5fe03efdaea1ba0720b831aab6/watchfiles-1.3.0-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:ec3cd4bb181a7b6329134204c2277c7144ab5f62cce9c5335d6f901ee7ce8d6a", size = 624482, upload-time = "2026-09-21T09:08:44.064Z" }, + { url = "https://files.pythonhosted.org/packages/91/46/9540f8123eaec86df989f26d7704cad0448d9fb7271763a4f8f9fe23617d/watchfiles-1.3.0-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:e5871d4a7f788a9f64fb06f5793389d29d171e5c32d37c3ebfd520b017cc7694", size = 653724, upload-time = "2026-09-21T09:08:45.257Z" }, + { url = "https://files.pythonhosted.org/packages/25/9d/b9deb52f1e4528ffe44338c5ebab95c32ed79203082da661de50288e130a/watchfiles-1.3.0-cp314-cp314t-win32.whl", hash = "sha256:81c989d7267bfb7b676cd32c83bec97914e396870983cbf493e29415037bff5e", size = 270863, upload-time = "2026-09-21T09:08:46.518Z" }, + { url = "https://files.pythonhosted.org/packages/59/59/8b1881afccbafda06248be4185e28034dd73c80c85e4da24f2cb0c1321eb/watchfiles-1.3.0-cp314-cp314t-win_amd64.whl", hash = "sha256:d754e049006e81a9be49dc78ba7802081cee426e968cbc63ee19e164fa2d0e39", size = 286879, upload-time = "2026-09-21T09:08:47.874Z" }, + { url = "https://files.pythonhosted.org/packages/46/dd/41c12ae9745d5d72f709333287679d71cbba68ad7678fed872207bba94f4/watchfiles-1.3.0-cp314-cp314t-win_arm64.whl", hash = "sha256:a5b631db08fd3032db57ec6a92d1777cd60f88803287e4e941a4858c53aab1e0", size = 276924, upload-time = "2026-09-21T09:08:48.976Z" }, +] + +[[package]] +name = "websockets" +version = "17.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/18/72/fba934cb3dff7a85d811820efffcd141ddd52b5a2a01637f64551373ff4d/websockets-17.1.tar.gz", hash = "sha256:acfea4c20bf54384883ea33b1240fc1db4f52e190823a4e2b334bc3e8bfca96a", size = 187520, upload-time = "2026-08-26T17:25:33.063Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/90/0d/500cf5daea09d4669dff3a7d67159094a0bd6c4ef130381404f6edd3eb5f/websockets-17.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:0c9982938980e086da59f70d05f9418cd143401a601a0faac10fa48f7bb1cd3e", size = 217048, upload-time = "2026-08-26T14:56:36.03Z" }, + { url = "https://files.pythonhosted.org/packages/97/12/5b12c6168aa269cffbfd24d177cd492b130120403a418c7e89462e27b4ac/websockets-17.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:57b39dc8541cf7ed3f639da82bf7451060483967f9e733da1f8173e4095f0642", size = 214737, upload-time = "2026-08-26T14:56:37.43Z" }, + { url = "https://files.pythonhosted.org/packages/0c/36/e453e5106e4e2416f008ac222837c2f1637a063b08008afcd1088889b631/websockets-17.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:96abdecbaae746851b87c3a36cb4a661df93ca3d92f114270f79228bf1d00de6", size = 214955, upload-time = "2026-08-26T14:56:38.71Z" }, + { url = "https://files.pythonhosted.org/packages/dd/30/0204bb86176db02cdfc678ce65ed808a66fab87d250ce61a8790800a60b0/websockets-17.1-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:d9fc873e239c5abeb150bc24dbd1a7af23a9254526383ce0a077f5e20adbeb19", size = 224331, upload-time = "2026-08-26T14:56:39.924Z" }, + { url = "https://files.pythonhosted.org/packages/46/c8/d8372256e00c4e3cab1115c45075d1eeedb642a3f2b42bd70c4deae03f06/websockets-17.1-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:6f42912fa9eb4cb7c7ec9fde9b3332ba339eb8a8811981043d4029599f3d950b", size = 224685, upload-time = "2026-08-26T14:56:41.169Z" }, + { url = "https://files.pythonhosted.org/packages/12/7d/650355b8f67f908ff99603351d4458d1a0b787d627950a47c38db7e25308/websockets-17.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f98bf378d7a5be047a044a1a27c987a8f355e10e3b5754617dbe756248cbc5ce", size = 225927, upload-time = "2026-08-26T14:56:42.359Z" }, + { url = "https://files.pythonhosted.org/packages/34/6c/a9ffa5b903579eed76017870f055d75ecc73988d9d0c9b65a92ba0bf2a27/websockets-17.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d334d11398086bb5559606cb42d51c013ea7c061c7db701521392373d3c087f5", size = 227300, upload-time = "2026-08-26T14:56:43.538Z" }, + { url = "https://files.pythonhosted.org/packages/9b/5d/4551c2269066af7481ee44605a0813770961615b5b5da3e87a8f5cb859ea/websockets-17.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5c27336b1a0ac56569493e858497870347854372395f50483725f8cdacc5a45c", size = 226533, upload-time = "2026-08-26T14:56:44.669Z" }, + { url = "https://files.pythonhosted.org/packages/3c/43/237a99233e5c445759a613831b3a92e91905afc064dc3bd0ad33c35fd1e2/websockets-17.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:67258b00302a5aaf0b267771c7014b13429abd7ea17eebc4c55bd935ff101555", size = 225280, upload-time = "2026-08-26T14:56:45.83Z" }, + { url = "https://files.pythonhosted.org/packages/d3/b5/e9407a91613d1d1cd932414143a1012096b26674a782fc55a0bd23217ee4/websockets-17.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:455ffeea0879d313205df1e745e5883e1feb7f31ecd26be882f5f0babd3db04f", size = 222540, upload-time = "2026-08-26T14:56:47.053Z" }, + { url = "https://files.pythonhosted.org/packages/db/d2/db76628db0577b783205d9779f64d8e373416b04c62d1546be4b75dc8540/websockets-17.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:f7233eaf441a345a5943a929fd4b5ea3278f11aed35a9ed0f3106b8cb3ca846a", size = 225354, upload-time = "2026-08-26T14:56:48.32Z" }, + { url = "https://files.pythonhosted.org/packages/a9/4c/2174181c067b89a74ae18e2650c2ac29959f4b796afe876ab3f4d30d642c/websockets-17.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:c65da239a5ad553619804c1f9d65c1a0b3005381c6158ee14da2c7444cbd0c78", size = 223867, upload-time = "2026-08-26T14:56:49.579Z" }, + { url = "https://files.pythonhosted.org/packages/df/75/274decb9a8253561b5be3261e02a6676fc8ecdf31e95b722e53d5bfb8fd2/websockets-17.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:9fa1ffa08c81a4f809cdab6129f8e55bee4650b9d6d3461019dda73aacd146b6", size = 224652, upload-time = "2026-08-26T14:56:50.885Z" }, + { url = "https://files.pythonhosted.org/packages/9f/e6/49824f1fb4db7656d2f7492b1d8be16147b759d909490e32f4776843ee64/websockets-17.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:406b8107943a43ef4649b1e0cb0cdc052bbf08fe1c8905a623c4af9586e5cebb", size = 225822, upload-time = "2026-08-26T14:56:52.356Z" }, + { url = "https://files.pythonhosted.org/packages/b8/6a/5dc43838c0b02a95f42c47a0de33c5ddd7767a9feeb4d0d8777ac1cfefe4/websockets-17.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:4e8ffcb486c8490a34a4cef5e4409d8da5a1cb1681e5bf7d786ce5e84aa8540d", size = 223379, upload-time = "2026-08-26T14:56:53.699Z" }, + { url = "https://files.pythonhosted.org/packages/c2/62/585637cf06d6b321232f79c55dc14d65518d12cf87c94c44f5864068810e/websockets-17.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:fb88076df585b69c5761c387c0081aa87d7b9eb1b205a6535ca4777e25650d81", size = 224330, upload-time = "2026-08-26T14:56:55.184Z" }, + { url = "https://files.pythonhosted.org/packages/de/68/c3b234a6a1366b6ab5bbfaa4434a1b946e1dc4e8ddd6824bfd93a8835b7f/websockets-17.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5d4724255fb8398acd9e583b97eb2279cec20e0bd0f9a94bf75f6056ef9f13da", size = 224622, upload-time = "2026-08-26T14:56:56.393Z" }, + { url = "https://files.pythonhosted.org/packages/6a/d4/84cf3d1376f5d8207f55f43c1c818babd6b89447f5dcd01f18a6d5526796/websockets-17.1-cp314-cp314-win32.whl", hash = "sha256:be3f0129c5654517b2abf07dcb75bb1d9479759a4ccfb569e8293579e9fc029a", size = 217036, upload-time = "2026-08-26T14:56:57.652Z" }, + { url = "https://files.pythonhosted.org/packages/d0/0f/9e7ac63c5d7cb642952200814f584318e65146df008b7d375d5d9c6b2c97/websockets-17.1-cp314-cp314-win_amd64.whl", hash = "sha256:2a4dc6ef83f4559e0d05f313a375cb38f63c986096a9da99fe94fdd779d313e5", size = 217382, upload-time = "2026-08-26T14:56:59.065Z" }, + { url = "https://files.pythonhosted.org/packages/54/bb/1ae6b91f7f3ac05f5c9f14a72dc2181c115ff370bcd8a7f10f02c174adfd/websockets-17.1-cp314-cp314-win_arm64.whl", hash = "sha256:46c0331c9eaaf73a559f3a9e388466be0df96eb83d40f06f1ca6ab6613b35c82", size = 217268, upload-time = "2026-08-26T14:57:00.654Z" }, + { url = "https://files.pythonhosted.org/packages/b3/f0/f65644d0e0b2b90918a8c41503841cc4072a58f2bf76c09bc36e751fc0dd/websockets-17.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:d411ea5ca18ac1b12c0c94be88b60c18ca641ac43bcdfdf1c9f79d46cdbe1603", size = 217379, upload-time = "2026-08-26T14:57:02.181Z" }, + { url = "https://files.pythonhosted.org/packages/ff/35/4c46d1f620ac1a30f92b6eae78ee40a772a93f568647ca7ccdc5ea283cf8/websockets-17.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:07fa3e7c30e2c577928d359b56bf872a3e0cbcc15553eaa0907c1ee86344b56f", size = 214911, upload-time = "2026-08-26T14:57:03.478Z" }, + { url = "https://files.pythonhosted.org/packages/04/6e/4587e8406d7c1188e97b9cf466c081e93399380d447f885bfce81626cd37/websockets-17.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6de9acef07e3a78e9567fcd26c29011a4da8f050b13004bbf880a0fd82a6eea5", size = 215115, upload-time = "2026-08-26T14:57:04.692Z" }, + { url = "https://files.pythonhosted.org/packages/ec/06/1381c8fff525041025909eb80ace32489194a00ba22a0a8d428030afcc84/websockets-17.1-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:ea0ed9373b880115911d9d39634bccc95b8ce590c9c42e8589f5cacc3ef3cee2", size = 224696, upload-time = "2026-08-26T14:57:05.899Z" }, + { url = "https://files.pythonhosted.org/packages/36/9d/9034e867dc85340be058619751742b895f722326e83100d110063461ca07/websockets-17.1-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:50903d335bfda026c2fa11dd9aed09d8cbee0c451e3a85122a9acb041b7dc69b", size = 224975, upload-time = "2026-08-26T14:57:07.262Z" }, + { url = "https://files.pythonhosted.org/packages/40/eb/ed03aa3cae748ebf6397e5d44028f433f746bad09dc568ff754fda3a3c9b/websockets-17.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5a74531ce81af587f906ab42f194032388fcff8fc7938402e5917c9147a39441", size = 226151, upload-time = "2026-08-26T14:57:08.524Z" }, + { url = "https://files.pythonhosted.org/packages/b1/c9/cc1964a096d16f3b73cb1ee5f14f277f5a3bcac07c6e8f9a1dcded99f4c8/websockets-17.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:8fbf28e639544503b7d1c96452a5e5e043e4108d89b1f3fa02910603622d19db", size = 228292, upload-time = "2026-08-26T14:57:09.846Z" }, + { url = "https://files.pythonhosted.org/packages/1a/26/46da6dd0363c2db2e4876fd59a40fd40c1943a82d7018d0a33afbce47d52/websockets-17.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f612dc57f00c07cf4aa2673f7cbceabd654ad2457b7e639f061b794d6e11f9fd", size = 226722, upload-time = "2026-08-26T14:57:11.118Z" }, + { url = "https://files.pythonhosted.org/packages/78/98/ecd8f5e1c5d0e54c08ebc5c66852271112166db68107cb0e17ca1bf25009/websockets-17.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1c7ac77401227212dc6e849182feee50d57cf456ec6329ffd6979c94bb136c5c", size = 225451, upload-time = "2026-08-26T14:57:12.601Z" }, + { url = "https://files.pythonhosted.org/packages/65/4d/da8d2760db53e17aae763738b6ba834b1fcf16813d3632f3edb6951e1ec8/websockets-17.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:32a2a68d989d6e5b74a9d5095415c51189ebae29fceb7cf2b64a1c0318a81256", size = 223003, upload-time = "2026-08-26T14:57:13.875Z" }, + { url = "https://files.pythonhosted.org/packages/a4/40/ea401c141a79c5b1d0021a0dab9d0df2051c108f1620fbb39a6e7c714c3b/websockets-17.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:aec00f018d34c67500ff0438dc314b40277be4a1b983cbacbf53ccf7db63e257", size = 225704, upload-time = "2026-08-26T14:57:15.091Z" }, + { url = "https://files.pythonhosted.org/packages/e1/8e/07ab3f44215d89840d5385fdcaaab1fed8caeffa67c6899e15062957c12c/websockets-17.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:0014eaff8ad5b3b43feda2279f9d34bf2eaae040720b9fbbb55944b10f40b14d", size = 224192, upload-time = "2026-08-26T14:57:16.3Z" }, + { url = "https://files.pythonhosted.org/packages/58/93/ccf1af0a23e5748d4e22292a377d78d15cf294d7e707bbb11a8990ae6bd5/websockets-17.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:db9d7ee47f3ba531e278be539af39e2c7c7d28fb94897b6cd1120d63b0ef5922", size = 225082, upload-time = "2026-08-26T14:57:17.531Z" }, + { url = "https://files.pythonhosted.org/packages/e2/db/e32200f99ce282e728d2929f2c429db353cf3282db7d0eba99eb32c9fec1/websockets-17.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:ff3e2ba7a9f0a110b0555452e9b5a03a34e11662544e01beea15f144b48ba7b7", size = 226101, upload-time = "2026-08-26T14:57:18.802Z" }, + { url = "https://files.pythonhosted.org/packages/28/3d/e7a6e9777b29433620167c98f3caaff0d6b08b1239a273ef7f7fd1393349/websockets-17.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:6da17fc94bd270f5987b10bee113461ac36a36a98b0481ddcc98056e5a90001a", size = 223794, upload-time = "2026-08-26T14:57:20.313Z" }, + { url = "https://files.pythonhosted.org/packages/48/05/ac569090726dedd6656f3ee28b0c02dfb1ba76e898dceaccc2987a237cef/websockets-17.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:e8dc3fa6d6b7ead3f9de57895f41b116a28787548e066365d9d90f7356bcaad2", size = 224567, upload-time = "2026-08-26T14:57:21.634Z" }, + { url = "https://files.pythonhosted.org/packages/14/50/4ef62941111db6b31193f4fabbb65f845a5177579040cb8fe0d774d25034/websockets-17.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:b65d5fe48219dc2d5e158de9e6514e75600f379cc7e37108d35f31764c155566", size = 224993, upload-time = "2026-08-26T14:57:22.86Z" }, + { url = "https://files.pythonhosted.org/packages/28/42/2b95ada4ea19bf3a2072b68669ce4f4afb212690b727d31640576287fd68/websockets-17.1-cp314-cp314t-win32.whl", hash = "sha256:2cce251f3e2469b99b6802b55435bcdd07123b41870f54c87b336183af9d7e68", size = 217168, upload-time = "2026-08-26T14:57:24.466Z" }, + { url = "https://files.pythonhosted.org/packages/32/0a/67d5ee08dd8060a37d612fd40a625b5376ad19ae48fe1c8ad428c278b817/websockets-17.1-cp314-cp314t-win_amd64.whl", hash = "sha256:8f6c38cdcaf98a911d7acc25577f2f9e710f3a2fc2bde1563556784320196b51", size = 217508, upload-time = "2026-08-26T14:57:25.983Z" }, + { url = "https://files.pythonhosted.org/packages/76/a3/822005d0c674451d2411027b878cdc128a2b7ea5a30d337d9e279da22eba/websockets-17.1-cp314-cp314t-win_arm64.whl", hash = "sha256:d1e2f5fa2b6d01f0d85b4f223fea7ed1d504be282a02a81bd2be4817ef7a2f03", size = 217425, upload-time = "2026-08-26T14:57:27.324Z" }, + { url = "https://files.pythonhosted.org/packages/41/63/23572870e01836a98346075b9e17a8bc24a6ddd9800a3204ceee58677f3c/websockets-17.1-py3-none-any.whl", hash = "sha256:f221081107b8c48184d99f7019604486376e7ef826037e70aad6b02540732c23", size = 211134, upload-time = "2026-08-26T17:25:31.397Z" }, +] + +[[package]] +name = "wrapt" +version = "2.5.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/04/22/89e2f3bdae5cb34e0cab0cd86d7172dbf418de4b46c9b17b9c7a560dfa44/wrapt-2.5.1.tar.gz", hash = "sha256:f595bb0185aab3e9dc31950c95d914f56ea8278810c3b928f3426e12ed6d27bc", size = 184455, upload-time = "2026-10-14T00:39:39.24Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/25/3d/ec1937283863bbe0d90528e09f2b27cfc0dd7e608fc2b65c804967dee369/wrapt-2.5.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:cf63fffcdcd8c60f223d3967bb92cc4fc2e8b46f09e75b67a6a75e6f47c0fc43", size = 106439, upload-time = "2026-10-14T00:37:53.853Z" }, + { url = "https://files.pythonhosted.org/packages/93/39/cca8afb80dbb9fce6103e59db507a9415291c4dbe97ea055875ff62901fb/wrapt-2.5.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:9f0750cbc2e29e4f3c9529d3587d4e7ed8f60638ceafb80b87a95833b0c5acd9", size = 106378, upload-time = "2026-10-14T00:37:55.386Z" }, + { url = "https://files.pythonhosted.org/packages/aa/a0/e784d7a9fd277a2ee395490ec4df96608b7fe218bb1ef7dced2a1caea490/wrapt-2.5.1-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:3cf273b7e8d2038abb7f0a8c6550aff4f617b9d486a9965c8e8acc96a3a04de9", size = 251505, upload-time = "2026-10-14T00:37:57.022Z" }, + { url = "https://files.pythonhosted.org/packages/81/56/01ebc86b88056f5782b9d50f962fb398c6b82aa11efc98f50c64896d94e3/wrapt-2.5.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:380f72610181883f66b41442cfc7c0f7552b42169efb2113def26e6380013d37", size = 250744, upload-time = "2026-10-14T00:37:58.652Z" }, + { url = "https://files.pythonhosted.org/packages/5f/5c/0e8eaaf31e2d6e7bf13c6eae2fd5b85eaa24e21e06466e6e7a0f35532689/wrapt-2.5.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cef2a8f006410b6134a0d273ec037fea8cc7a6a914f1bd7555ad9788ad788c6e", size = 231598, upload-time = "2026-10-14T00:38:00.505Z" }, + { url = "https://files.pythonhosted.org/packages/6a/6b/6a3e257e65de6cc0027e78b423942697c74451520ba3797fd86455ec4df8/wrapt-2.5.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:9bad4dbb4e61624fcce5f301e37f9e743ecae4f1259a3777b3207eb7eba3dccd", size = 247998, upload-time = "2026-10-14T00:38:02.63Z" }, + { url = "https://files.pythonhosted.org/packages/22/38/b2b8f3ee22b05f33f5aefb052844a36a0d7edd1eaff2ff4249f97792bea8/wrapt-2.5.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:9a34640eb6295f33ca23462977de275fe8f3a50ab339b8918b96d69a7451e2e1", size = 228795, upload-time = "2026-10-14T00:38:04.317Z" }, + { url = "https://files.pythonhosted.org/packages/29/39/e6c86552286ac27b855042fb9c229c580ed2d4c3ced5d0a1dad5f5ee8c11/wrapt-2.5.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:26313f38d18d40a9975123a4ebff9da125ec63ab9ece4f05320a3d8d37d2c1fe", size = 239090, upload-time = "2026-10-14T00:38:06.119Z" }, + { url = "https://files.pythonhosted.org/packages/e7/7a/aea209f64e894573935b17de26ecfa0efe139b63f170a60be9f13734e0f5/wrapt-2.5.1-cp314-cp314-win32.whl", hash = "sha256:0591e6eace0d186c9ef1ecd1244be5a04e98041424cfca425b684ffe4f0d8030", size = 100919, upload-time = "2026-10-14T00:38:07.874Z" }, + { url = "https://files.pythonhosted.org/packages/57/24/847096aa49d42990137ed3b940743c8a6da806d39f6c455317114e8ebfde/wrapt-2.5.1-cp314-cp314-win_amd64.whl", hash = "sha256:25ed8b1b39234140d5b5c6a273130c7595e0abece417c3ca3cb378fcea5cd0fe", size = 106243, upload-time = "2026-10-14T00:38:09.49Z" }, + { url = "https://files.pythonhosted.org/packages/9c/ff/1cdc742133b9fb8558cdf42b2a6c2699bd7c72f7d0606286ec2f9142e20a/wrapt-2.5.1-cp314-cp314-win_arm64.whl", hash = "sha256:6201c7e122f40060a9b50696d80deec8f93b1a235ec0443f51d7a8a42f7044a6", size = 103974, upload-time = "2026-10-14T00:38:11.354Z" }, + { url = "https://files.pythonhosted.org/packages/fc/6f/c32dc64900f1970a7f991ff5d06788cd636ca2f3ee2f99709d82577ca198/wrapt-2.5.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:da847332447db5505162759a4cd5ac374eb8b74841fe97a98ef3de14edd2586d", size = 109552, upload-time = "2026-10-14T00:38:12.965Z" }, + { url = "https://files.pythonhosted.org/packages/e9/73/a9c8cc82b166e3de42f5fbd88089d2ef9b72e87aac7d6cdddb070335c20b/wrapt-2.5.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:9f437dd704abc4ee1bd03bb2d796d362d0e75915e8f3113a7900b3b7ec5f8b47", size = 109853, upload-time = "2026-10-14T00:38:14.565Z" }, + { url = "https://files.pythonhosted.org/packages/f6/48/f341d82e69ae47df2755847af1c744ff0732543482dbc2373e5e57676621/wrapt-2.5.1-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:03aa7d2256309b57ddbf317bff2cae5f47e50ea9ae8d582780ebe0b554347b42", size = 284103, upload-time = "2026-10-14T00:38:16.23Z" }, + { url = "https://files.pythonhosted.org/packages/fd/50/b87c6374377b08ee0783b6c5c31cd41a5e103bc54e7e857e800a0a965550/wrapt-2.5.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fcccaa1484f7dd1091602970988ab741491f9f974013c844f70e45ac1196b80d", size = 293883, upload-time = "2026-10-14T00:38:17.986Z" }, + { url = "https://files.pythonhosted.org/packages/38/e0/6d0810ae73f7a5180ec366577588ab3ad55fc1bc7e3623e82d5f8528dd2a/wrapt-2.5.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:8078186f719a92693199f1e06c4ec72e1e6d374c2e459da18ed5c39d6966d727", size = 263097, upload-time = "2026-10-14T00:38:20.107Z" }, + { url = "https://files.pythonhosted.org/packages/d6/d3/c890a46f4d395a7935e5a7436f374ceefa362eb612fdd39376dd775e0283/wrapt-2.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:1425fcf0e70b27053bd610d57bae975856e7897e3f6ba1456d2b80b9d7fd15d1", size = 289045, upload-time = "2026-10-14T00:38:21.913Z" }, + { url = "https://files.pythonhosted.org/packages/e5/c6/042e30e0d851ca6ea743e6978902527f9166da42d736f074245d1cc54c8f/wrapt-2.5.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:b238e955ba34ef2b8897f358b7b868b41b9a02ffd338014b62985fa91898cc4a", size = 260378, upload-time = "2026-10-14T00:38:23.662Z" }, + { url = "https://files.pythonhosted.org/packages/31/a4/5e65f90bf414c2f1c7eefc3d26c33af01d87ba33c4b879db4e1d4ba7fc3b/wrapt-2.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:25eb4d928a9abeaf70ca786a35861b46d1ab37cc4ce49ea70a070dacdead4dfe", size = 278746, upload-time = "2026-10-14T00:38:25.523Z" }, + { url = "https://files.pythonhosted.org/packages/4b/86/17a85475e218e05225a4f6d65237b12b280596e04520df0e3d8841c4eae3/wrapt-2.5.1-cp314-cp314t-win32.whl", hash = "sha256:df6e3a36170cda0d313be50fe5065948e7f12f3a181b38cbc262e9f2ee4824e1", size = 103415, upload-time = "2026-10-14T00:38:27.409Z" }, + { url = "https://files.pythonhosted.org/packages/27/1c/495b3aebbbe5aebf52ae5f9e8ddd0e072a412b9f5a9bc68e5eba43fa26ba/wrapt-2.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:bc5c0203d383403043fb86c964bd0bab4fcbfb26004ff4bb9c6d02ebc1d608ae", size = 109940, upload-time = "2026-10-14T00:38:29.145Z" }, + { url = "https://files.pythonhosted.org/packages/28/4d/030ecd98da4d052c264290c4fb9f984706c9a19026154c833580e8624a05/wrapt-2.5.1-cp314-cp314t-win_arm64.whl", hash = "sha256:a424e8a9776c06aef6313af1d0e3fe6e0838af4241d0c09eb0a3b46f2c9a5ff3", size = 106214, upload-time = "2026-10-14T00:38:30.795Z" }, + { url = "https://files.pythonhosted.org/packages/bc/0c/7da7513ddcc8f1d831ec4bfbedc9f7f174ecb91042bc16916fc1e0d06b22/wrapt-2.5.1-py3-none-any.whl", hash = "sha256:c6e6c226b1ca5402d7ae5fb34a0d21f1b49124fe4200e5884d1e19e53c47ac1d", size = 81849, upload-time = "2026-10-14T00:39:37.441Z" }, +] + +[[package]] +name = "yarl" +version = "1.25.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "idna" }, + { name = "multidict" }, + { name = "propcache" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/75/16/e8be8e2fb175bbf41a0680381a319f1199fae256588241a2ac8677eafb49/yarl-1.25.1.tar.gz", hash = "sha256:03dd38de09bc213e9a8b29761eec33ee1d5318dac0e49d8af36e4d27830e23a7", size = 246245, upload-time = "2026-09-15T19:35:02.264Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/80/cf/54023edfab7aa773b860503db0c56e962ccab0922803ee97988c176ea090/yarl-1.25.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:a9ca696eb02e5c02a8afd872ada510eba9b7fe6e68b9572c2e9a9b1941e31e2e", size = 143975, upload-time = "2026-09-15T19:32:16.416Z" }, + { url = "https://files.pythonhosted.org/packages/d7/a8/e6c1be0e6761d0f2d10bbf33a3e1e02b99dc83874d92945d7b461a72481e/yarl-1.25.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:a5877f2255aab518ebe528289037699201d5dc5f045f2396cb30aa02db22f57f", size = 104018, upload-time = "2026-09-15T19:32:18.364Z" }, + { url = "https://files.pythonhosted.org/packages/6e/bb/dda344765ffd3430afe1a1c66c866a57fae67786537d4f14607df6505ac1/yarl-1.25.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:7a5c3115595995779ee21f2567035793911c3802a43c74f3fbb0314929ec67ac", size = 104156, upload-time = "2026-09-15T19:32:20.459Z" }, + { url = "https://files.pythonhosted.org/packages/e5/5f/ed1538bcd06009fe990d6d283dd7667f639e62a81e35c6d8c6ef6c08fb3c/yarl-1.25.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:77e5099b99b37f3cf79c246998ca9f7313a78054cd1809ec46bc1afad47e1c4c", size = 116025, upload-time = "2026-09-15T19:32:22.766Z" }, + { url = "https://files.pythonhosted.org/packages/a2/af/2185daf56b99830d3356ecfada46faaa49945de6626e842b7728088d4980/yarl-1.25.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6efaf45df6a849cef613a03a94c845647456662f85438c886bb67a9c027c8c2c", size = 106985, upload-time = "2026-09-15T19:32:24.749Z" }, + { url = "https://files.pythonhosted.org/packages/c1/65/bc1ae564fb4b04a30b6a8f250e787772581c57e4c3d5cf07ac3359de3103/yarl-1.25.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d5f90e44653c4e0f78501ed9bb7d3fce835a8d62b7c6ed0cb16557534087e743", size = 123030, upload-time = "2026-09-15T19:32:27.084Z" }, + { url = "https://files.pythonhosted.org/packages/6a/3e/e2afcde10d74e53b3fa889960991efb3019beda2b1682a01de720a302056/yarl-1.25.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:632da579b2d879f6bad20f2cfa35ded1efe2f4f77f8abb26a6234a5b236acd2f", size = 126765, upload-time = "2026-09-15T19:32:29.332Z" }, + { url = "https://files.pythonhosted.org/packages/a2/be/415b00c0fe5a0615b062a456b26623d7ec91c2bee20faea1a14045aa0469/yarl-1.25.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:30eec96e8a91bd588ce897c9543f6d5d8d34b28fbcba28a4dedf20ebeae9fe57", size = 117199, upload-time = "2026-09-15T19:32:31.49Z" }, + { url = "https://files.pythonhosted.org/packages/97/27/3d8c63ddd3e8bcfd033748ab93876678ce59bacd66e4cb1ed851c9c5b37e/yarl-1.25.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:12b6bc4906e11f5e1a1cdcb12296e7afbd366c783cc8073403cd2fb74334e453", size = 115187, upload-time = "2026-09-15T19:32:34.137Z" }, + { url = "https://files.pythonhosted.org/packages/39/b7/7a81d0be1a502a26a0d4326c6f2ecb736c824f570ea1c6529f2b0b227b50/yarl-1.25.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:9d6ed3d17bccce4c05343e1ca8da13bc5c02c812a4e7282ddd05e8769322d3fc", size = 116085, upload-time = "2026-09-15T19:32:36.438Z" }, + { url = "https://files.pythonhosted.org/packages/f0/69/39fff459916aa0fab42215dc47b759586fd80f94aa56dfc4a7c15ba6e0dc/yarl-1.25.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:f38a70074041d3b7e138e452799f5174198bae5bd5ab2000917badf403908c5f", size = 107996, upload-time = "2026-09-15T19:32:38.959Z" }, + { url = "https://files.pythonhosted.org/packages/c0/39/80b9a55a3335590451d9ecf3eb593a8c635351f4c905ef056d7e8a8fd9e7/yarl-1.25.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:4ca89e4e21854ed27ec753297dde84b16c9f8e53b14a4866fb44457d643c19f8", size = 122549, upload-time = "2026-09-15T19:32:41.151Z" }, + { url = "https://files.pythonhosted.org/packages/42/7d/a179c6757818bb59372a4adafd09f7f26a3b4a0f04c3ae404b544c0b0c82/yarl-1.25.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:1ab7618921a93767387a4b83776f751588f5b5ae9bb5bc96620e2e2e00bca868", size = 115107, upload-time = "2026-09-15T19:32:43.072Z" }, + { url = "https://files.pythonhosted.org/packages/32/2b/a773ac867e4ab53a98ed98e5cefe3bae31e6f550252ca9d1de266f1a40c5/yarl-1.25.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:0ae12ff2b805fa02c4dab838005caef735e39986322698c48588d3beacb65c62", size = 120666, upload-time = "2026-09-15T19:32:45.061Z" }, + { url = "https://files.pythonhosted.org/packages/bc/41/52be6505e85b0f76b4f85b01b5de7e06a0512201abc2c95e14e099549174/yarl-1.25.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:90c30ed53546da833c700115c0064c22120d1b1560f474699fd31f22dd668233", size = 117505, upload-time = "2026-09-15T19:32:47.177Z" }, + { url = "https://files.pythonhosted.org/packages/f5/01/349c0386caedbbe488d519f252df54efac8a1459282d466c474bdd84a620/yarl-1.25.1-cp314-cp314-win_amd64.whl", hash = "sha256:acfa7e22aa6c6e7a5996a41d275bfa01efa7ea56ab890590280e9063e2cf5c1b", size = 103446, upload-time = "2026-09-15T19:32:49.615Z" }, + { url = "https://files.pythonhosted.org/packages/5c/f0/8ec63180f77912f0dc4e5a42760cb8c08d20da1d5ace3578a01b84d1f3d8/yarl-1.25.1-cp314-cp314-win_arm64.whl", hash = "sha256:8e7d98cdbb6d71e726f7d525952867096053d1f290dd4e3c50d7d313a136f414", size = 99159, upload-time = "2026-09-15T19:32:51.686Z" }, + { url = "https://files.pythonhosted.org/packages/47/7d/92d2220d6886b70ab1ed8579533ac2af2dfac716d5d929001daff7986df9/yarl-1.25.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:d21f0fa80a02d05299207eeaafef345d812ace96d5306e4ef265e1d419a615fa", size = 150071, upload-time = "2026-09-15T19:32:53.911Z" }, + { url = "https://files.pythonhosted.org/packages/64/fc/b245e448124bcda9340df38e3553fa222b50260fca027a84095e9bd8642d/yarl-1.25.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:17c9877a89fb6e2bca6f9087eb24cd7fb434653946ef5075e470d23d49b52287", size = 106780, upload-time = "2026-09-15T19:32:56.443Z" }, + { url = "https://files.pythonhosted.org/packages/51/e2/9a6ce2e334ebf218a30335ae76fb1696459430d42f733b8cb0d7d65b84d3/yarl-1.25.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:29273edf1530e397bd07cb784db1fbe0d2590b77569f2e24679a9c0a2d763b94", size = 107361, upload-time = "2026-09-15T19:32:58.827Z" }, + { url = "https://files.pythonhosted.org/packages/ed/70/66e8c76b569b450d16e190f15071c916c3df70b0e33927e415ac497cf0c2/yarl-1.25.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b7abffdf37af1cec6a2ad69b827aa84320db5894791bc8ed932dc93fb274b7e9", size = 114396, upload-time = "2026-09-15T19:33:02.24Z" }, + { url = "https://files.pythonhosted.org/packages/73/23/0d82838a05c57fdc05bc8b66e8c92dcc0df15e27463a5f163142d521c682/yarl-1.25.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:2239a02249d9326655419e0168a28ca9008938eaab31dc29fc875c217927a6c0", size = 104882, upload-time = "2026-09-15T19:33:04.494Z" }, + { url = "https://files.pythonhosted.org/packages/86/d4/ea08615c4edaa6049a13a2f1128944d068d1893abda7d708d4d7ea01599a/yarl-1.25.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:664ec6a520b74a1df2810666eb67695fcb77fa663e6ea0a25aaf2e529cb24dfa", size = 119485, upload-time = "2026-09-15T19:33:06.583Z" }, + { url = "https://files.pythonhosted.org/packages/1a/82/0898bdce9b1ae403b308b9c733d0d24af4a3464270c2c081f457b16c3e0d/yarl-1.25.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4f1c91f5a5980a937ff8e238e98e6897e1ad74a4b1e2c0d68c73b5ffbb3f5c0b", size = 122490, upload-time = "2026-09-15T19:33:08.653Z" }, + { url = "https://files.pythonhosted.org/packages/d1/38/97d79b81c342b78246cfedb74809e68841f3198d21653e10d3232bd9c622/yarl-1.25.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7c88edaec8c349ad4c5ad4c486a3defcc4b80ceb2f074436ffa0a87caf5e76a6", size = 115336, upload-time = "2026-09-15T19:33:11.056Z" }, + { url = "https://files.pythonhosted.org/packages/8e/9d/2577896554cd310dc470adb6da0b7dd0b435cb63e2565204a7ac240e504c/yarl-1.25.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:35dcbea443fafb3eece757ad4e514560ddeb6c34cfae1582c620d7b293d7feee", size = 111825, upload-time = "2026-09-15T19:33:13.204Z" }, + { url = "https://files.pythonhosted.org/packages/29/6b/7ac49d8ba84a5c4bd73415a4c949d22c749cb3762579b3d50e48019a78aa/yarl-1.25.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:882569ff613758cac762a457a5d72d6e211b28d4bcfea89d1d71ea942b02eac0", size = 114655, upload-time = "2026-09-15T19:33:15.553Z" }, + { url = "https://files.pythonhosted.org/packages/e5/18/e5942a16723f5b72f9b1297fd5a85a54f6300cd15c0dcb5005b90cd89156/yarl-1.25.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:d0f1489233a254bb3643d2f05de7d59019254d81daeca6b9162fe9edef57e0c7", size = 106395, upload-time = "2026-09-15T19:33:17.599Z" }, + { url = "https://files.pythonhosted.org/packages/7b/2d/549fa46240781513ebc47ae7eb418df428a163a2a3d644cc9cbb3ecb7846/yarl-1.25.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:f41753a76f4f63927d03a0d8ba8f5ce0f2083bec29a8cfaccc55371b1564b96b", size = 119277, upload-time = "2026-09-15T19:33:19.973Z" }, + { url = "https://files.pythonhosted.org/packages/76/16/4763f78dcdc0b3b9fb3842b04afe72b9320857c6a69300c62a0eab03d119/yarl-1.25.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:8fb0eb4955adf0579001581f2f71a126e8781ba61bcd120f127b0401163c6c2d", size = 112504, upload-time = "2026-09-15T19:33:22.464Z" }, + { url = "https://files.pythonhosted.org/packages/ae/b4/974e3edfe0d188393ce1cb9de400111c63fe61f4eb3b772a500d84c970d1/yarl-1.25.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:a1e32763e641a1566507d90a8d3b19bfc3cc04a9d4e5ae3e32189874ed4b58a3", size = 116243, upload-time = "2026-09-15T19:33:24.788Z" }, + { url = "https://files.pythonhosted.org/packages/b0/aa/157b940428da80c104ca09666a740e51c94963df65d5b112e06b52e4d7a8/yarl-1.25.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:65b5b2066651b7432d389e9799d979c703bcc6ef44266bb8153ef54e91e4aab3", size = 115822, upload-time = "2026-09-15T19:33:26.886Z" }, + { url = "https://files.pythonhosted.org/packages/7e/af/19fbdce41412e1b96825544cc52cd7029d3724655d0988237972f078bd29/yarl-1.25.1-cp314-cp314t-win_amd64.whl", hash = "sha256:734f6e5400352ac4254456003d462866c684703570929cff7a7bde015d0cb371", size = 107386, upload-time = "2026-09-15T19:33:29.009Z" }, + { url = "https://files.pythonhosted.org/packages/2a/99/f6431c8968e89be608d74b28ae2d024521b2953f27dd44e0dece5e04f67a/yarl-1.25.1-cp314-cp314t-win_arm64.whl", hash = "sha256:287e99ff5aa4dc1c7630bfc683ded6f106d756c99dec432a2d7f197a784f51c6", size = 102094, upload-time = "2026-09-15T19:33:31.151Z" }, + { url = "https://files.pythonhosted.org/packages/54/22/318c7980066769c6bcd9221ed2248294f5698811da099013098c670565ed/yarl-1.25.1-py3-none-any.whl", hash = "sha256:681c758b0490f9e96b78e5fa8e8dc6e648e9185bb6eaebe73183c33ea0c445f3", size = 63617, upload-time = "2026-09-15T19:34:59.616Z" }, +]