From 8c253a0e5af4d93ffae202fd1279845180631d21 Mon Sep 17 00:00:00 2001 From: andrejkamenskij Date: Thu, 19 Nov 2026 16:36:40 +0300 Subject: [PATCH] Implement lab6 observability with OpenTelemetry and Grafana --- .env.example | 89 +- .gitattributes | 4 + .github/pull_request_template.md | 25 +- .github/workflows/contracts.yml | 26 +- .gitignore | 2 + Dockerfile | 25 + Dockerfile.example | 8 - LAB6_GUIDE.md | 83 + LAB6_TASK.md | 105 ++ MAINTAINERS.md | 2 +- Makefile | 52 +- README.md | 141 +- REPORT.md | 214 ++- RUNBOOK.md | 46 + alembic.ini | 3 + compose.example.yaml | 17 - compose.yaml | 367 +++++ course-tests/README.md | 57 + course-tests/client.py | 217 +++ course-tests/fixtures/README.md | 9 + course-tests/fixtures/sample.png | Bin 0 -> 111817 bytes course-tests/image_probe.py | 26 + course-tests/load.py | 58 + course-tests/resilience.py | 129 ++ course-tests/smoke.py | 82 + evidence/README.md | 13 +- evidence/lab6-checks.txt | 10 + evidence/lab6-correlated-logs.json | 56 + evidence/lab6-db-incident-logs.json | 227 +++ evidence/lab6-db-incident-trace.json | 255 +++ evidence/lab6-incidents.txt | 5 + evidence/lab6-observability-summary.json | 40 + evidence/lab6-retry-trace.json | 1421 +++++++++++++++++ evidence/lab6-signals.txt | 2 + evidence/lab6-summary.json | 71 + evidence/lab6-trace.json | 1003 ++++++++++++ evidence/lab6-worker-incident.json | 32 + infra/README.md | 11 +- infra/haproxy.cfg | 43 + infra/minio/Dockerfile | 20 + infra/observability/alerts.yaml | 23 + infra/observability/collector.yaml | 67 + .../grafana/dashboards/messenger.json | 519 ++++++ .../provisioning/dashboards/provider.yaml | 10 + .../provisioning/datasources/sources.yaml | 34 + infra/observability/loki.yaml | 30 + infra/observability/prometheus.yaml | 12 + infra/observability/tempo.yaml | 21 + infra/otel/README.md | 7 - infra/s3/api-policy.json | 29 + infra/s3/reconciler-policy.json | 28 + infra/s3/worker-policy.json | 30 + lab.json | 2 +- migrations/env.py | 34 + migrations/script.py.mako | 17 + migrations/versions/001_initial.py | 60 + migrations/versions/002_redis_sessions.py | 22 + migrations/versions/003_admin.py | 16 + migrations/versions/004_idempotency.py | 25 + migrations/versions/005_message_unicode.py | 55 + migrations/versions/006_unicode_names.py | 40 + migrations/versions/007_attachments.py | 65 + migrations/versions/008_reconciliation.py | 37 + migrations/versions/009_pipeline.py | 67 + migrations/versions/010_trace_context.py | 15 + pyproject.toml | 7 + requirements-dev.txt | 2 +- requirements.txt | 25 + scripts/replicas.py | 56 + scripts/restart.example.sh | 10 - scripts/restart.sh | 7 + scripts/setup-env.py | 28 + scripts/stop-one.example.sh | 10 - scripts/stop-one.py | 34 + scripts/stop-one.sh | 4 + scripts/test-persistence.py | 46 + scripts/verify-lab6.py | 20 + src/__init__.py | 0 src/__main__.py | 13 + src/api/__init__.py | 0 src/api/dependencies.py | 106 ++ src/api/router.py | 17 + src/api/v1/__init__.py | 0 src/api/v1/routers/__init__.py | 0 src/api/v1/routers/attachments.py | 134 ++ src/api/v1/routers/auth/__init__.py | 0 src/api/v1/routers/auth/callback.py | 31 + src/api/v1/routers/auth/router.py | 50 + src/api/v1/routers/auth/schemas.py | 13 + src/api/v1/routers/chats/__init__.py | 0 src/api/v1/routers/chats/router.py | 86 + src/api/v1/routers/chats/schemas.py | 42 + src/api/v1/routers/messages/__init__.py | 0 src/api/v1/routers/messages/router.py | 96 ++ src/api/v1/routers/messages/schemas.py | 43 + src/api/v1/routers/users/__init__.py | 0 src/api/v1/routers/users/router.py | 44 + src/api/v1/routers/users/schemas.py | 29 + src/api/v1/schemas.py | 21 + src/core/__init__.py | 0 src/core/cache.py | 68 + src/core/callback.py | 24 + src/core/db/__init__.py | 0 src/core/db/models.py | 207 +++ src/core/db/repo/__init__.py | 0 src/core/db/repo/auth.py | 16 + src/core/db/repo/chat.py | 59 + src/core/db/repo/message.py | 151 ++ src/core/db/repo/user.py | 120 ++ src/core/db/session.py | 105 ++ src/core/db/types.py | 15 + src/core/errors.py | 29 + src/core/pagination.py | 43 + src/core/readiness.py | 89 ++ src/core/request_deadline.py | 77 + src/core/request_limits.py | 132 ++ src/core/security.py | 42 + src/core/session_store.py | 55 + src/core/settings.py | 130 ++ src/core/storage.py | 96 ++ src/core/storage_bootstrap.py | 96 ++ src/core/upload_coordination.py | 21 + src/main.py | 156 ++ src/maintenance/__init__.py | 1 + src/maintenance/reconcile.py | 361 +++++ src/observability/__init__.py | 1 + src/observability/signals.py | 182 +++ src/pipeline/__init__.py | 1 + src/pipeline/admin.py | 113 ++ src/pipeline/common.py | 58 + src/pipeline/dispatcher.py | 197 +++ src/pipeline/images.py | 95 ++ src/pipeline/worker.py | 364 +++++ src/service/__init__.py | 0 src/service/attachment.py | 209 +++ src/service/auth.py | 74 + src/service/chat.py | 61 + src/service/dto.py | 50 + src/service/exceptions.py | 30 + src/service/message.py | 64 + src/service/user.py | 63 + tests/api_regressions.py | 199 +++ tests/attachment_failover.py | 45 + tests/attachments.py | 99 ++ tests/idempotency.py | 116 ++ tests/lab5.py | 291 ++++ tests/lab5_limits.py | 44 + tests/lab5_migration.py | 90 ++ tests/observability.py | 207 +++ 149 files changed, 11752 insertions(+), 229 deletions(-) create mode 100644 .gitattributes create mode 100644 Dockerfile delete mode 100644 Dockerfile.example create mode 100644 LAB6_GUIDE.md create mode 100644 LAB6_TASK.md create mode 100644 RUNBOOK.md create mode 100644 alembic.ini delete mode 100644 compose.example.yaml create mode 100644 compose.yaml create mode 100644 course-tests/README.md create mode 100644 course-tests/client.py create mode 100644 course-tests/fixtures/README.md create mode 100644 course-tests/fixtures/sample.png create mode 100644 course-tests/image_probe.py create mode 100644 course-tests/load.py create mode 100644 course-tests/resilience.py create mode 100644 course-tests/smoke.py create mode 100644 evidence/lab6-checks.txt create mode 100644 evidence/lab6-correlated-logs.json create mode 100644 evidence/lab6-db-incident-logs.json create mode 100644 evidence/lab6-db-incident-trace.json create mode 100644 evidence/lab6-incidents.txt create mode 100644 evidence/lab6-observability-summary.json create mode 100644 evidence/lab6-retry-trace.json create mode 100644 evidence/lab6-signals.txt create mode 100644 evidence/lab6-summary.json create mode 100644 evidence/lab6-trace.json create mode 100644 evidence/lab6-worker-incident.json create mode 100644 infra/haproxy.cfg create mode 100644 infra/minio/Dockerfile create mode 100644 infra/observability/alerts.yaml create mode 100644 infra/observability/collector.yaml create mode 100644 infra/observability/grafana/dashboards/messenger.json create mode 100644 infra/observability/grafana/provisioning/dashboards/provider.yaml create mode 100644 infra/observability/grafana/provisioning/datasources/sources.yaml create mode 100644 infra/observability/loki.yaml create mode 100644 infra/observability/prometheus.yaml create mode 100644 infra/observability/tempo.yaml delete mode 100644 infra/otel/README.md create mode 100644 infra/s3/api-policy.json create mode 100644 infra/s3/reconciler-policy.json create mode 100644 infra/s3/worker-policy.json create mode 100644 migrations/env.py create mode 100644 migrations/script.py.mako create mode 100644 migrations/versions/001_initial.py create mode 100644 migrations/versions/002_redis_sessions.py create mode 100644 migrations/versions/003_admin.py create mode 100644 migrations/versions/004_idempotency.py create mode 100644 migrations/versions/005_message_unicode.py create mode 100644 migrations/versions/006_unicode_names.py create mode 100644 migrations/versions/007_attachments.py create mode 100644 migrations/versions/008_reconciliation.py create mode 100644 migrations/versions/009_pipeline.py create mode 100644 migrations/versions/010_trace_context.py create mode 100644 pyproject.toml create mode 100644 requirements.txt create mode 100644 scripts/replicas.py delete mode 100755 scripts/restart.example.sh create mode 100755 scripts/restart.sh create mode 100644 scripts/setup-env.py delete mode 100755 scripts/stop-one.example.sh create mode 100644 scripts/stop-one.py create mode 100755 scripts/stop-one.sh create mode 100644 scripts/test-persistence.py create mode 100644 scripts/verify-lab6.py create mode 100644 src/__init__.py create mode 100644 src/__main__.py create mode 100644 src/api/__init__.py create mode 100644 src/api/dependencies.py create mode 100644 src/api/router.py create mode 100644 src/api/v1/__init__.py create mode 100644 src/api/v1/routers/__init__.py create mode 100644 src/api/v1/routers/attachments.py create mode 100644 src/api/v1/routers/auth/__init__.py create mode 100644 src/api/v1/routers/auth/callback.py create mode 100644 src/api/v1/routers/auth/router.py create mode 100644 src/api/v1/routers/auth/schemas.py create mode 100644 src/api/v1/routers/chats/__init__.py create mode 100644 src/api/v1/routers/chats/router.py create mode 100644 src/api/v1/routers/chats/schemas.py create mode 100644 src/api/v1/routers/messages/__init__.py create mode 100644 src/api/v1/routers/messages/router.py create mode 100644 src/api/v1/routers/messages/schemas.py create mode 100644 src/api/v1/routers/users/__init__.py create mode 100644 src/api/v1/routers/users/router.py create mode 100644 src/api/v1/routers/users/schemas.py create mode 100644 src/api/v1/schemas.py create mode 100644 src/core/__init__.py create mode 100644 src/core/cache.py create mode 100644 src/core/callback.py create mode 100644 src/core/db/__init__.py create mode 100644 src/core/db/models.py create mode 100644 src/core/db/repo/__init__.py create mode 100644 src/core/db/repo/auth.py create mode 100644 src/core/db/repo/chat.py create mode 100644 src/core/db/repo/message.py create mode 100644 src/core/db/repo/user.py create mode 100644 src/core/db/session.py create mode 100644 src/core/db/types.py create mode 100644 src/core/errors.py create mode 100644 src/core/pagination.py create mode 100644 src/core/readiness.py create mode 100644 src/core/request_deadline.py create mode 100644 src/core/request_limits.py create mode 100644 src/core/security.py create mode 100644 src/core/session_store.py create mode 100644 src/core/settings.py create mode 100644 src/core/storage.py create mode 100644 src/core/storage_bootstrap.py create mode 100644 src/core/upload_coordination.py create mode 100644 src/main.py create mode 100644 src/maintenance/__init__.py create mode 100644 src/maintenance/reconcile.py create mode 100644 src/observability/__init__.py create mode 100644 src/observability/signals.py create mode 100644 src/pipeline/__init__.py create mode 100644 src/pipeline/admin.py create mode 100644 src/pipeline/common.py create mode 100644 src/pipeline/dispatcher.py create mode 100644 src/pipeline/images.py create mode 100644 src/pipeline/worker.py create mode 100644 src/service/__init__.py create mode 100644 src/service/attachment.py create mode 100644 src/service/auth.py create mode 100644 src/service/chat.py create mode 100644 src/service/dto.py create mode 100644 src/service/exceptions.py create mode 100644 src/service/message.py create mode 100644 src/service/user.py create mode 100644 tests/api_regressions.py create mode 100644 tests/attachment_failover.py create mode 100644 tests/attachments.py create mode 100644 tests/idempotency.py create mode 100644 tests/lab5.py create mode 100644 tests/lab5_limits.py create mode 100644 tests/lab5_migration.py create mode 100644 tests/observability.py diff --git a/.env.example b/.env.example index 2b32655..dcde4c0 100644 --- a/.env.example +++ b/.env.example @@ -1,30 +1,73 @@ -# Copy to .env for your LOCAL synthetic test environment; never commit .env. -# These are logical interface suggestions; map actual framework names in REPORT.md. APP_HOST=0.0.0.0 -APP_PORT=8080 +APP_PORT=8086 APP_ENV=lab +POSTGRES_DB=messenger +POSTGRES_USER=messenger +# Задай свой локальный пароль перед запуском. Не добавляй .env в Git. +POSTGRES_PASSWORD= SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. -DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres -CACHE_URL_FILE=/run/secrets/cache_url +# REDIS_URL задаётся в compose.yaml; Redis хранит сессии с TTL и кэш профилей на 60 секунд. +LOG_LEVEL=INFO +# DATABASE_URL и REDIS_URL передаёт Compose. При ручном запуске задай их в окружении. +# Остальные параметры и значения по умолчанию: src/core/settings.py (AppSettings). + +# Стартовый администратор; повторный запуск не меняет существующий пароль. +ADMIN_USERNAME=admin +ADMIN_PASSWORD=admin-password-123 +ADMIN_DISPLAY_NAME=Администратор + +# APP_ENV=production запрещает учебный ADMIN_PASSWORD. Пароль существующего +# администратора должен совпадать с настройкой; автоматического сброса нет. +# HTTP_MAX_BODY_BYTES=65536 +# HTTP_BODY_TIMEOUT_SECONDS=5 +# HTTP_REQUEST_TIMEOUT_SECONDS=5 +# HTTP_CONCURRENCY_LIMIT=64 +# PASSWORD_HASH_CONCURRENCY=2 + +# Lab3: число реплик и лимиты каждой API-реплики. Наружу доступен только LB. API_REPLICAS=2 -IDEMPOTENCY_TTL_SECONDS=86400 -S3_ENDPOINT=http://s3:9000 -S3_REGION=us-east-1 +API_CPUS=0.5 +API_MEMORY=512m +DATABASE_POOL_SIZE=10 +POSTGRES_CPUS=1.0 + +# Lab4. Local synthetic credentials; MinIO is accessible only inside Compose. +MINIO_ROOT_USER=messenger +MINIO_ROOT_PASSWORD=local-messenger-s3-password +S3_ACCESS_KEY_ID=messenger-api +S3_SECRET_ACCESS_KEY=local-messenger-api-password S3_BUCKET=messenger-attachments -S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id -S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key -MAX_UPLOAD_BYTES=10485760 -MAX_IMAGE_PIXELS=40000000 -BROKER_URL_FILE=/run/secrets/broker_url -WORKER_CONCURRENCY=2 +UPLOAD_CONCURRENCY=1 +# Manual API startup additionally needs S3_ENDPOINT=http://host:9000. + +# Automatic reconciliation runs every 300 seconds; default grace is five minutes. +S3_RECONCILE_ACCESS_KEY_ID=messenger-reconciler +S3_RECONCILE_SECRET_ACCESS_KEY=local-messenger-reconciler-password +RECONCILE_GRACE_SECONDS=300 + +# Lab5: separate durable Redis Streams broker and isolated image worker. +BROKER_URL=redis://broker:6379/0 +S3_WORKER_ACCESS_KEY_ID=messenger-worker +S3_WORKER_SECRET_ACCESS_KEY=local-messenger-worker-password +WORKER_CONCURRENCY=1 +WORKER_MEMORY=1g +WORKER_MEMORY_MB=768 +WORKER_CPUS=1.0 JOB_MAX_ATTEMPTS=5 -JOB_TIMEOUT_SECONDS=60 -STAGING_RETENTION_SECONDS=86400 -OTEL_SERVICE_NAME=messenger-api -# Worker must set a distinct service.name. +JOB_TIMEOUT_SECONDS=30 +JOB_LEASE_SECONDS=120 +JOB_RETRY_BASE_SECONDS=2 +JOB_RETRY_CAP_SECONDS=60 +STAGING_RETENTION_SECONDS=604800 +# Only enable on an isolated fault-test stand, never normal operation. +PIPELINE_FAULT_INJECTION=false + +# Lab6: telemetry is asynchronous and is not an API readiness dependency. +OTEL_ENABLED=true OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318 -OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf -OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab +GRAFANA_ADMIN_PASSWORD=local-grafana-password-change-me +GRAFANA_PORT=3000 +PROMETHEUS_PORT=9090 +LOKI_PORT=3100 +TEMPO_PORT=3200 +COLLECTOR_METRICS_PORT=8889 diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..cafd4fa --- /dev/null +++ b/.gitattributes @@ -0,0 +1,4 @@ +# Matplotlib SVG paths contain generated trailing separators. +evidence/**/*.svg -whitespace +# Preserve generated measurements byte-for-byte for evidence checksums. +evidence/**/*.csv -text -whitespace diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 59c9f4e..2013364 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,18 +1,13 @@ -## Лабораторная 6 +## Лабораторная 6 — наблюдаемость -Автор / группа: TODO -Целевая оценка (3/4/5): TODO -Трек 5, если заявлен: TODO +Автор: **Каменский Андрей, БПИ2301**. Целевая оценка: **4**. -Что реализовано и что перенесено: TODO -Команды запуска из чистого clone: TODO -Результаты проверок и ссылка на REPORT.md: TODO -Ограничения / известные ошибки: TODO -Источники и использование ИИ: TODO +Сохранён бизнес-API lab5. Добавлены OTLP SDK API/worker, отдельный Collector, +Prometheus, Loki, Tempo, Grafana provisioning и dashboard. W3C context проходит +через durable outbox/Redis Streams и retry. Есть SLI/SLO, локальные alert rules +и RUNBOOK.md. SDK/Collector имеют ограниченные очереди и память. -- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. -- [ ] Контракт и публичные тесты не ослаблены. -- [ ] make check и make test выполнены; приложен реальный вывод. -- [ ] Сценарии отказов/безопасности своего уровня воспроизведены. -- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. -- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. +Быстрая проверка: `make test test-observability`. +Контролируемые инциденты: `python3 tests/observability.py --base-url http://localhost:8086 --incidents`. +Отчёт: REPORT.md; результаты: evidence/lab6-*. Публичные tests/контракты не изменены. +Трек на оценку 5 не заявлен. diff --git a/.github/workflows/contracts.yml b/.github/workflows/contracts.yml index 11cf289..2d78596 100644 --- a/.github/workflows/contracts.yml +++ b/.github/workflows/contracts.yml @@ -1,4 +1,4 @@ -name: Skeleton contracts +name: Lab6 contracts and application on: [push, pull_request] permissions: contents: read @@ -9,8 +9,24 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: - python-version: '3.12' + python-version: '3.13' - run: python -m pip install -r requirements-dev.txt - - run: make check validate PYTHON=python -# This job validates the skeleton, not a student solution. -# Add build, readiness and make test in a separate application job. + - run: make check validate lint PYTHON=python + application: + needs: contracts + runs-on: ubuntu-latest + timeout-minutes: 25 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.13' + - run: python -m pip install -r requirements.txt -r requirements-dev.txt + - run: make verify-lab6 PYTHON=python + - uses: actions/upload-artifact@v4 + if: always() + with: + name: lab6-evidence + path: | + evidence/lab6-*.json + evidence/lab6-*.txt diff --git a/.gitignore b/.gitignore index b5a7189..a085fb7 100644 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,5 @@ coverage/ .vscode/ # Keep small, sanitized evidence summaries in evidence/ under version control. # Inspect git diff --cached: ignore rules are not a secret scanner. + +.ruff_cache/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..cfd86cb --- /dev/null +++ b/Dockerfile @@ -0,0 +1,25 @@ +FROM python:3.13.0-slim-bookworm + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 + +WORKDIR /app + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt \ + && python -m pip uninstall --yes pip \ + && addgroup --system app \ + && adduser --system --ingroup app app + +COPY --chown=app:app src ./src +COPY --chown=app:app migrations ./migrations +COPY --chown=app:app infra/s3 ./infra/s3 +COPY --chown=app:app alembic.ini ./alembic.ini + +USER app +EXPOSE 8080 + +HEALTHCHECK --interval=10s --timeout=3s --start-period=5s --retries=3 \ + CMD python -c "import urllib.request; from src.core.settings import settings; urllib.request.urlopen(f'http://127.0.0.1:{settings.app_port}/health/ready', timeout=settings.healthcheck_timeout_seconds)" + +CMD ["python", "-m", "src"] diff --git a/Dockerfile.example b/Dockerfile.example deleted file mode 100644 index e333a12..0000000 --- a/Dockerfile.example +++ /dev/null @@ -1,8 +0,0 @@ -# Teaching outline, not a runnable Dockerfile. Implement your own Dockerfile. -# 1. Choose a runtime/toolchain version and pin the base image tag/digest. -# 2. Copy dependency manifests first; install from the language lockfile. -# 3. Copy source and build. For compiled languages use a separate runtime stage. -# 4. Run the HTTP process as PID 1; listen on 0.0.0.0:8080. -# 5. Add non-root, signals and healthcheck according to the current rubric. -# 6. Pass runtime config/secrets at startup, never COPY .env or secret files. -# This file deliberately has no FROM/CMD: selecting/implementing them is the task. diff --git a/LAB6_GUIDE.md b/LAB6_GUIDE.md new file mode 100644 index 0000000..128d008 --- /dev/null +++ b/LAB6_GUIDE.md @@ -0,0 +1,83 @@ +# Как работает lab6 и что объяснять на защите + +Автор: **Каменский Андрей, БПИ2301**. + +## Что осталось от lab5 + +Картинка полностью передаётся API, проходит базовые проверки и сохраняется в MinIO +в staging/. В одной транзакции PostgreSQL создаются вложение queued и job/outbox. +API отвечает 202. Dispatcher отправляет job в отдельный Redis Streams broker. +Worker получает её, читает исходник из MinIO, создаёт thumbnail, crop и WebP, +сохраняет результат и фиксирует ready. Оригинал не перекодируется. + +RabbitMQ не нужен, потому что Redis Streams даёт хранимый журнал, consumer groups, +pending deliveries и подтверждения. Redis очереди отделён от Redis сессий. +Данные очереди лежат в volume, включены AOF, appendfsync=always и noeviction. + +| Механизм | Что делает у нас | +| --- | --- | +| XADD | Dispatcher добавляет ID задания и служебные поля в Stream | +| XREADGROUP | Worker получает очередную доставку в consumer group | +| Pending Entries List | Хранит доставки, которые consumer ещё не подтвердил | +| XAUTOCLAIM | Позволяет забрать зависшую неподтверждённую доставку | +| XACK | Подтверждает результат после DB COMMIT | +| XDEL | Удаляет уже подтверждённую запись, чтобы Stream не рос бесконечно | +| Outbox в PostgreSQL | Сохраняет намерение отправки, если broker недоступен | +| Lease и DB row lock | Защищают захват и восстанавливают работу после смерти worker | + +Один consumer берёт по одной записи; WORKER_CONCURRENCY задаёт число consumers. +Повтор доставки нормален: используем стабильный job_id и детерминированные ключи +результатов. Ready-дубль подтверждается без нового resize. Временные ошибки +повторяются с backoff/jitter, максимум пять попыток на цикл; постоянные дают failed. +CLI позволяет явно повторить failed и пишет audit. + +## Что нового в шестой + +Теперь каждый HTTP-запрос получает trace. Внутри него есть spans — отдельные +участки работы: db.SELECT/INSERT/UPDATE, s3.PUT/GET, queue.publish, queue.consume, +image.transform. Span содержит начало, конец, родителя и результат. + +После HTTP-ответа worker работает в другом процессе. Чтобы связь не потерялась, +мы сохраняем traceparent/tracestate в job, dispatcher извлекает их, создаёт +producer span и записывает новый carrier в очередь. Worker создаёт consumer span +с этим родителем. Retry сохраняет context попытки перед повторной публикацией. + +Логи http.completed/job.started/job.completed/db.error — структурный JSON. +trace_id связывает их с трассой, span_id указывает конкретную операцию. +request_id связывает HTTP-ответ и соответствующее событие. В telemetry не передаются +Authorization, пароль, текст сообщения, изображение, SQL или filename. + +| Компонент | Роль | +| --- | --- | +| OpenTelemetry SDK | Создаёт spans, counters/histograms и структурные события в API/worker | +| Collector | Принимает OTLP, ограничивает память, группирует и отправляет данные | +| Prometheus | Хранит временные ряды и вычисляет alert rules | +| Loki | Хранит логи, позволяет найти события по trace_id | +| Tempo | Хранит трассы и причинные связи между процессами | +| Grafana | Показывает данные из трёх источников; не заменяет эти хранилища | + +## Почему метрики не содержат chat_id + +Каждое новое значение label создаёт новый временной ряд. UUID чатов или вложений +дали бы неограниченный рост. Поэтому route — шаблон /chats/{chat_id}/messages, +а labels — method, route, status_class, outcome, job_type или queue. +Технические trace/span IDs находятся в логах/трассах, не в labels метрик. + +## Что значит p95 + +95% измерений попали ниже этого времени. Мы собираем histogram buckets и +агрегируем buckets всех API-реплик, затем применяем histogram_quantile. +Усреднять p95 разных реплик или заменять p95 средней задержкой неправильно. + +## Что показать + +1. make test: весь бизнес-API работает с включённой телеметрией. +2. Dashboard в Grafana: свежие RPS/latency, очередь и worker. +3. tests/observability.py: один upload найден в Tempo вместе с producer/consumer, + DB/S3 и связанными логами API/worker. +4. Prometheus /alerts: правила и локальные firing/resolved. +5. RUNBOOK.md: как отличить остановленный worker от ожидания INSERT в БД. + +Недоступная telemetry не участвует в /health/ready. SDK экспортирует в фоне, +очереди ограничены; Collector при длительном отказе backend теряет telemetry, +но не бизнес-задания. Эти ограничения описаны в отчёте. diff --git a/LAB6_TASK.md b/LAB6_TASK.md new file mode 100644 index 0000000..a607238 --- /dev/null +++ b/LAB6_TASK.md @@ -0,0 +1,105 @@ +# Лабораторная 6. Наблюдаемость: OpenTelemetry и Grafana + +Научиться находить причину деградации через метрики, логи и трассы всей системы, включая фоновую обработку. + +Это **skeleton**, а не готовый сервер. Здесь есть задание, контракт и публичные проверки. Реализацию приложения, Dockerfile и требуемую инфраструктуру пишет студент. Язык и framework свободные; UI не обязателен. + +[Карта курса](COURSE.md) · [Процесс fork/PR](CONTRIBUTING.md) · [OpenAPI](contracts/openapi.json) · [Семантика API](contracts/README.md) · [Проверки](tests/README.md) · [Отчёт](REPORT.md) · [Примеры JSON](contracts/examples.json) · [Памятка преподавателю](MAINTAINERS.md) + +## Откуда начинаем + +Перенесите минимум lab5. Бизнес-API не меняется. Должны оставаться API-реплики, БД, S3, broker и worker. + +**Архитектура:** API/worker → OTLP → OTel Collector → хранилища metrics/logs/traces → Grafana; queue context связывает API и worker. + +## Что сделать + +1. Выберите хранилища: например Prometheus для метрик, Loki для логов, Tempo для трасс. Разрешена другая совместимая комбинация с Grafana; Grafana сама не хранит все три сигнала. +2. Добавьте SDK-инструментацию API и worker и отдельный Collector в общий Compose/Swarm; настройте receivers/processors/exporters и хранение данных. +3. Реализуйте набор сигналов contracts/observability.md и импортируемые dashboard/datasources. Не используйте user_id/chat_id/message_id как labels метрик. +4. Проверьте связь запроса, структурного лога и обработки задания. Секреты, тела сообщений и изображения не должны попадать в telemetry. +5. Создайте нагрузку, внедрите два ограниченных отказа и найдите их причину только с помощью сохранённых наблюдений; приложите доказательства. + +## Оценка «3»: работающий минимум + +- Весь минимум lab5 работает. Collector, хранилища трёх сигналов и Grafana включены в общий воспроизводимый стенд; конфиги и provisioning лежат в Git, persistent volumes сохраняют нужную историю. +- API и worker отдают OTLP в Collector. В Grafana видны: RPS/errors/duration API, успешные/ошибочные jobs и duration worker; доступны структурные логи и реальные трассы API и worker. +- Есть импортируемый dashboard, а не только screenshot. Сохраняются низкокардинальные атрибуты и идентификаторы trace/span в логах; содержимое сообщений, пароли/токены не экспортируются. +- На контролируемом запросе можно найти trace и связанный лог; на job — worker trace и лог. make test проходит при включённой телеметрии. Объяснено назначение каждого хранилища. + +## Оценка «4»: уровень стажёра/джуна + +Весь уровень 3 **и все** пункты ниже: + +- W3C trace context переносится API → broker → worker; видны producer/consumer spans с корректной связью parent или link. По одному upload найдена цепочка до S3/DB результата, включая retry. +- Dashboard показывает p50/p95/p99, долю ошибок, queue depth/возраст старейшей job, DB pool и saturation worker. Приведены запросы и единицы измерения; percentile вычисляется из histogram, не средних. +- Задан проверяемый SLI/SLO и окно; сохранены alert rules минимум для высокого error rate и растущего queue lag. Alerts проверены искусственным отказом; доставка наружу не нужна, достаточно локального состояния firing/resolved. +- Collector имеет memory limiter/batch и ограниченную буферизацию/поведение при недоступном backend. Остановка telemetry-хранилища не ломает сообщения и не вызывает бесконечный рост RAM. Отчёт содержит разбор двух инцидентов и шаги диагностики. + +## Оценка «5»: исследование и доказанный результат + +Весь уровень 4 **и один законченный трек на выбор**. Приведите гипотезу, повторяемую методику, результаты и ограничения. + +**Трек 1. Цена наблюдаемости.** Сравните telemetry off/on и ≥2 sampling/aggregation настройки при одинаковой нагрузке: latency, CPU/RAM, объём ingest, потери spans. Объясните, какие ошибки и редкие медленные запросы теряются; предложите и проверьте сбалансированную конфигурацию. + +**Трек 2. SLO и обнаружение инцидентов.** Обоснуйте SLO для API и времени до ready, реализуйте burn-rate alerts с несколькими окнами, воспроизведите краткий всплеск и длительную деградацию. Измерьте время обнаружения/восстановления, ложные срабатывания и покажите runbook, по которому другой студент находит причину. + +## Как начать и проверить + +```sh +# Работает прямо в skeleton, Python 3.9+; приложение не запускается: +make check + +# После реализации запустите свой сервер/стенд по REPORT.md. +# По умолчанию тест использует http://localhost:8080. +make test +``` + +`.env.example`, `compose.example.yaml` и `contracts/infrastructure.json` описывают настройки и роли компонентов. **compose.example.yaml — список ролей с пустым services, не запускаемый стенд.** Реализуйте свой `compose.yaml`/`stack.yaml`, закрепите версии образов, заполните локальный `.env` и опишите bootstrap/миграции. Все зависимости поднимаются из этого репозитория; ссылка «у меня БД уже установлена» не заменяет воспроизводимость. + +```sh +# Compose-вариант после реализации: +docker compose --env-file .env config --quiet +docker compose --env-file .env up -d --build +# Дождитесь /health/ready; затем make test. +# Для Swarm укажите эквивалентные build/push/deploy-команды в REPORT.md. +``` + + +Примеры запросов доступны в `examples/requests.http`, примеры JSON — в `contracts/examples.json`. Для полной проверки схемы (по желанию локально; CI делает её автоматически): + +```sh +python3 -m venv .venv +.venv/bin/python -m pip install -r requirements-dev.txt +make validate PYTHON=.venv/bin/python +``` + +Smoke не выставляет оценку автоматически. Проверку сохранности/отказов запускайте явно по [tests/README.md](tests/README.md), остальные доказательства приведите в отчёте. + +## Сценарий защиты + +1. С нуля поднять стенд, импортировать provisioning без ручного набора dashboard, запустить smoke и нагрузку. +2. Показать свежие метрики, одну трассу и соответствующий структурный лог API и worker; проверить, что это данные своего приложения. +3. Для 4: пройти от upload до worker в одном trace/связанных traces, остановить worker или ограничить DB, увидеть firing и затем resolved. +4. Остановить telemetry backend, проверить сохранение работы API и ограниченное поведение exporter. Для 5: повторить эксперимент. + +## Что должно быть в PR + +Collector configuration, конфиги backend, Grafana provisioning/dashboards, alert rules по уровню, схема сигналов и отчёт инцидентов. + +Заполните `REPORT.md` и [шаблон PR](.github/pull_request_template.md), укажите целевую оценку/трек. Не меняйте обязательный контракт и публичные tests ради зелёного результата. Сохраняйте возможности предыдущих лабораторных в пределах [объявленных переходов](COURSE.md). + +## Вопросы на защите + +- Чем лог, метрика и trace отвечают на разные вопросы об одном инциденте? +- Почему chat_id в labels может исчерпать хранилище? +- Как очередь переносит причинную связь между процессами? +- Почему среднее latency не заменяет p99 и почему нельзя усреднять p99 реплик? + +## Первичные материалы + +- [OTel Collector configuration](https://opentelemetry.io/docs/collector/configuration/) +- [OTel propagation](https://opentelemetry.io/docs/concepts/context-propagation/) +- [W3C Trace Context](https://www.w3.org/TR/trace-context/) +- [Prometheus histograms](https://prometheus.io/docs/practices/histograms/) +- [Grafana provisioning](https://grafana.com/docs/grafana/latest/administration/provisioning/) diff --git a/MAINTAINERS.md b/MAINTAINERS.md index 48b24e7..85601ae 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -1,4 +1,4 @@ -# Памятка преподавателю — лабораторная 6 +# Памятка преподавателю — лабораторная 5 ## Что подготовлено diff --git a/Makefile b/Makefile index 5d86d17..5e1e24c 100644 --- a/Makefile +++ b/Makefile @@ -1,30 +1,56 @@ PYTHON ?= python3 -BASE_URL ?= http://localhost:8080 +BASE_URL ?= http://localhost:8086 CA_FILE ?= ACTION_SCRIPT ?= TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) -.PHONY: help check validate test load test-persistence test-failover - +.PHONY: help up down migrate lint format check validate test load test-attachments test-idempotency test-api-regressions test-attachment-failover test-persistence test-failover test-lab5 test-lab5-faults verify-lab6 jobs-failed help: - @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" - + @echo "up: start; test: public lab6 smoke; test-observability: real signals; verify-lab6: quick acceptance" +up: + docker compose up -d --build --wait +down: + docker compose down +migrate: + docker compose run --rm migration +lint: + $(PYTHON) -m ruff check src + $(PYTHON) -m ruff format --check src +format: + $(PYTHON) -m ruff check --fix src + $(PYTHON) -m ruff format src check: $(PYTHON) scripts/check_contract.py - validate: $(PYTHON) scripts/validate_openapi.py - test: $(PYTHON) tests/smoke.py --base-url "$(BASE_URL)" $(TLS_ARGS) - load: $(PYTHON) tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS) - +test-attachments: + $(PYTHON) tests/attachments.py --base-url "$(BASE_URL)" $(TLS_ARGS) +test-idempotency: + $(PYTHON) tests/idempotency.py --base-url "$(BASE_URL)" +test-api-regressions: + $(PYTHON) tests/api_regressions.py --base-url "$(BASE_URL)" $(TLS_ARGS) +test-attachment-failover: + $(PYTHON) tests/attachment_failover.py --base-url "$(BASE_URL)" $(TLS_ARGS) test-persistence: - @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable restart hook"; exit 2) - $(PYTHON) tests/resilience.py --mode persistence --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) - + @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT=scripts/restart.sh"; exit 2) + $(PYTHON) scripts/test-persistence.py --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) test-failover: - @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2) + @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT=scripts/stop-one.sh"; exit 2) $(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) +test-lab5: + $(PYTHON) tests/lab5.py --base-url "$(BASE_URL)" +test-lab5-faults: + @test -n "$(COMPOSE_PROJECT_NAME)" || (echo "Set COMPOSE_PROJECT_NAME to your isolated test project"; exit 2) + $(PYTHON) tests/lab5.py --faults --project "$(COMPOSE_PROJECT_NAME)" --base-url "$(BASE_URL)" +verify-lab6: + $(PYTHON) scripts/verify-lab6.py +jobs-failed: + docker compose exec -T dispatcher python -m src.pipeline.admin failed + +.PHONY: test-observability +test-observability: + $(PYTHON) tests/observability.py --base-url "$(BASE_URL)" diff --git a/README.md b/README.md index a607238..7a28c65 100644 --- a/README.md +++ b/README.md @@ -1,105 +1,74 @@ -# Лабораторная 6. Наблюдаемость: OpenTelemetry и Grafana +# Лабораторная 6: OpenTelemetry и Grafana -Научиться находить причину деградации через метрики, логи и трассы всей системы, включая фоновую обработку. +Автор: **Каменский Андрей, БПИ2301**. Целевая оценка: **4**. -Это **skeleton**, а не готовый сервер. Здесь есть задание, контракт и публичные проверки. Реализацию приложения, Dockerfile и требуемую инфраструктуру пишет студент. Язык и framework свободные; UI не обязателен. +Мессенджер из lab5 с двумя API-репликами, PostgreSQL, Redis-сессиями, MinIO, +Redis Streams broker и worker. Добавлены реальные метрики, структурные логи, +сквозные трассы, готовый dashboard и локальные alerts. -[Карта курса](COURSE.md) · [Процесс fork/PR](CONTRIBUTING.md) · [OpenAPI](contracts/openapi.json) · [Семантика API](contracts/README.md) · [Проверки](tests/README.md) · [Отчёт](REPORT.md) · [Примеры JSON](contracts/examples.json) · [Памятка преподавателю](MAINTAINERS.md) +[Задание](LAB6_TASK.md) · [Отчёт](REPORT.md) · [Объяснение](LAB6_GUIDE.md) · +[Диагностика инцидентов](RUNBOOK.md) · [Контракт](contracts/observability.md) -## Откуда начинаем +## Запуск -Перенесите минимум lab5. Бизнес-API не меняется. Должны оставаться API-реплики, БД, S3, broker и worker. - -**Архитектура:** API/worker → OTLP → OTel Collector → хранилища metrics/logs/traces → Grafana; queue context связывает API и worker. - -## Что сделать - -1. Выберите хранилища: например Prometheus для метрик, Loki для логов, Tempo для трасс. Разрешена другая совместимая комбинация с Grafana; Grafana сама не хранит все три сигнала. -2. Добавьте SDK-инструментацию API и worker и отдельный Collector в общий Compose/Swarm; настройте receivers/processors/exporters и хранение данных. -3. Реализуйте набор сигналов contracts/observability.md и импортируемые dashboard/datasources. Не используйте user_id/chat_id/message_id как labels метрик. -4. Проверьте связь запроса, структурного лога и обработки задания. Секреты, тела сообщений и изображения не должны попадать в telemetry. -5. Создайте нагрузку, внедрите два ограниченных отказа и найдите их причину только с помощью сохранённых наблюдений; приложите доказательства. - -## Оценка «3»: работающий минимум - -- Весь минимум lab5 работает. Collector, хранилища трёх сигналов и Grafana включены в общий воспроизводимый стенд; конфиги и provisioning лежат в Git, persistent volumes сохраняют нужную историю. -- API и worker отдают OTLP в Collector. В Grafana видны: RPS/errors/duration API, успешные/ошибочные jobs и duration worker; доступны структурные логи и реальные трассы API и worker. -- Есть импортируемый dashboard, а не только screenshot. Сохраняются низкокардинальные атрибуты и идентификаторы trace/span в логах; содержимое сообщений, пароли/токены не экспортируются. -- На контролируемом запросе можно найти trace и связанный лог; на job — worker trace и лог. make test проходит при включённой телеметрии. Объяснено назначение каждого хранилища. - -## Оценка «4»: уровень стажёра/джуна - -Весь уровень 3 **и все** пункты ниже: - -- W3C trace context переносится API → broker → worker; видны producer/consumer spans с корректной связью parent или link. По одному upload найдена цепочка до S3/DB результата, включая retry. -- Dashboard показывает p50/p95/p99, долю ошибок, queue depth/возраст старейшей job, DB pool и saturation worker. Приведены запросы и единицы измерения; percentile вычисляется из histogram, не средних. -- Задан проверяемый SLI/SLO и окно; сохранены alert rules минимум для высокого error rate и растущего queue lag. Alerts проверены искусственным отказом; доставка наружу не нужна, достаточно локального состояния firing/resolved. -- Collector имеет memory limiter/batch и ограниченную буферизацию/поведение при недоступном backend. Остановка telemetry-хранилища не ломает сообщения и не вызывает бесконечный рост RAM. Отчёт содержит разбор двух инцидентов и шаги диагностики. - -## Оценка «5»: исследование и доказанный результат - -Весь уровень 4 **и один законченный трек на выбор**. Приведите гипотезу, повторяемую методику, результаты и ограничения. - -**Трек 1. Цена наблюдаемости.** Сравните telemetry off/on и ≥2 sampling/aggregation настройки при одинаковой нагрузке: latency, CPU/RAM, объём ingest, потери spans. Объясните, какие ошибки и редкие медленные запросы теряются; предложите и проверьте сбалансированную конфигурацию. - -**Трек 2. SLO и обнаружение инцидентов.** Обоснуйте SLO для API и времени до ready, реализуйте burn-rate alerts с несколькими окнами, воспроизведите краткий всплеск и длительную деградацию. Измерьте время обнаружения/восстановления, ложные срабатывания и покажите runbook, по которому другой студент находит причину. - -## Как начать и проверить +Нужен Docker с Compose v2. При первом запуске образы скачиваются; MinIO собирается +из закреплённого commit. Все компоненты входят в репозиторий. ```sh -# Работает прямо в skeleton, Python 3.9+; приложение не запускается: -make check - -# После реализации запустите свой сервер/стенд по REPORT.md. -# По умолчанию тест использует http://localhost:8080. -make test +python3 scripts/setup-env.py # если .env ещё нет +docker compose up -d --build --wait +python3 tests/smoke.py --base-url http://localhost:8086 +python3 tests/observability.py --base-url http://localhost:8086 ``` -`.env.example`, `compose.example.yaml` и `contracts/infrastructure.json` описывают настройки и роли компонентов. **compose.example.yaml — список ролей с пустым services, не запускаемый стенд.** Реализуйте свой `compose.yaml`/`stack.yaml`, закрепите версии образов, заполните локальный `.env` и опишите bootstrap/миграции. Все зависимости поднимаются из этого репозитория; ссылка «у меня БД уже установлена» не заменяет воспроизводимость. +- API: http://localhost:8086/docs +- Grafana: http://localhost:3000/d/messenger-lab6 +- Prometheus / локальные alerts: http://localhost:9090/alerts +- Tempo API: http://localhost:3200 +- Loki API: http://localhost:3100 -```sh -# Compose-вариант после реализации: -docker compose --env-file .env config --quiet -docker compose --env-file .env up -d --build -# Дождитесь /health/ready; затем make test. -# Для Swarm укажите эквивалентные build/push/deploy-команды в REPORT.md. -``` +Для просмотра Grafana вход не нужен: anonymous Viewer только на loopback. +Для редактирования: admin и GRAFANA_ADMIN_PASSWORD из локального .env. +Секреты setup-env.py создаёт случайно, не печатает и не перезаписывает существующий +.env. Конфиги datasource/dashboard автоматически загружаются из infra/observability/. +Остановить с сохранением данных: `docker compose down`. +Намеренно удалить все учебные данные текущего проекта: `docker compose down --volumes`. -Примеры запросов доступны в `examples/requests.http`, примеры JSON — в `contracts/examples.json`. Для полной проверки схемы (по желанию локально; CI делает её автоматически): +## Что добавлено + +API/worker → OTLP HTTP/protobuf → Collector → Prometheus / Loki / Tempo → Grafana. +Метрики отвечают «что и насколько ухудшилось», логи — «какое событие произошло», +трасса — «на каком участке потрачено время или возникла ошибка». + +W3C context сохраняется в PostgreSQL job/outbox, передаётся через Redis Stream и +восстанавливается worker. Upload, producer/consumer, DB и S3 находятся в одной +причинной цепочке. При retry следующий producer продолжает consumer предыдущей +попытки. X-Request-Id сохранён; дополнительно API возвращает X-Trace-Id. + +Dashboard показывает RPS, 5xx, p50/p95/p99 из histogram, jobs/outcomes, время до ready, +backlog/возраст, DB pool, worker saturation, CPU/RAM и связанные логи. Labels метрик +не содержат пользовательские ID, имена файлов, токены или сырые URL. + +## Проверки + +Быстрый тест запущенного стенда: `make test test-observability`. +Публичные tests skeleton lab6 и контракты не изменены. ```sh python3 -m venv .venv -.venv/bin/python -m pip install -r requirements-dev.txt -make validate PYTHON=.venv/bin/python +.venv/bin/python -m pip install -r requirements.txt -r requirements-dev.txt +make check validate lint PYTHON=.venv/bin/python +make verify-lab6 PYTHON=.venv/bin/python ``` -Smoke не выставляет оценку автоматически. Проверку сохранности/отказов запускайте явно по [tests/README.md](tests/README.md), остальные доказательства приведите в отчёте. +Для коротких контролируемых инцидентов уровня «4» на своём учебном стенде: -## Сценарий защиты +```sh +python3 tests/observability.py --base-url http://localhost:8086 --incidents +``` -1. С нуля поднять стенд, импортировать provisioning без ручного набора dashboard, запустить smoke и нагрузку. -2. Показать свежие метрики, одну трассу и соответствующий структурный лог API и worker; проверить, что это данные своего приложения. -3. Для 4: пройти от upload до worker в одном trace/связанных traces, остановить worker или ограничить DB, увидеть firing и затем resolved. -4. Остановить telemetry backend, проверить сохранение работы API и ограниченное поведение exporter. Для 5: повторить эксперимент. - -## Что должно быть в PR - -Collector configuration, конфиги backend, Grafana provisioning/dashboards, alert rules по уровню, схема сигналов и отчёт инцидентов. - -Заполните `REPORT.md` и [шаблон PR](.github/pull_request_template.md), укажите целевую оценку/трек. Не меняйте обязательный контракт и публичные tests ради зелёного результата. Сохраняйте возможности предыдущих лабораторных в пределах [объявленных переходов](COURSE.md). - -## Вопросы на защите - -- Чем лог, метрика и trace отвечают на разные вопросы об одном инциденте? -- Почему chat_id в labels может исчерпать хранилище? -- Как очередь переносит причинную связь между процессами? -- Почему среднее latency не заменяет p99 и почему нельзя усреднять p99 реплик? - -## Первичные материалы - -- [OTel Collector configuration](https://opentelemetry.io/docs/collector/configuration/) -- [OTel propagation](https://opentelemetry.io/docs/concepts/context-propagation/) -- [W3C Trace Context](https://www.w3.org/TR/trace-context/) -- [Prometheus histograms](https://prometheus.io/docs/practices/histograms/) -- [Grafana provisioning](https://grafana.com/docs/grafana/latest/administration/provisioning/) +Этот тест временно останавливает worker, блокирует INSERT в messages, проверяет +alerts firing/resolved и работу API при остановленном Tempo. Все воздействия +снимаются в finally. Результаты и реальные traces/logs находятся в evidence/. +По умолчанию verify-lab6 запускает только smoke и проверку сигналов. diff --git a/REPORT.md b/REPORT.md index b97ae42..bce2eec 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,203 @@ -# Отчёт — лабораторная 6 +# Лабораторная 6. Наблюдаемость: OpenTelemetry и Grafana -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +Автор: **Каменский Андрей, БПИ2301**. Целевая оценка: **4**. +Исследовательский трек на 5 не заявлен. Реализация продолжает собственную lab5. +Commit/PR в рамках этой работы не публиковались. -## Запуск из чистого clone +## Запуск и проверка -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Docker с Compose v2, Python 3.13 для полного набора инструментов. Все зависимости +поднимаются из этого репозитория. Предварительно установленный broker/БД/S3 не нужен. -## Архитектура и решения +```sh +python3 scripts/setup-env.py +docker compose config --quiet +docker compose up -d --build --wait +python3 tests/smoke.py --base-url http://localhost:8086 +python3 tests/observability.py --base-url http://localhost:8086 +``` -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +setup-env.py создаёт .env со случайными паролями и правами 0600, не выводит секреты +и не перезаписывает существующий файл. Первый запуск скачивает образы и компилирует +закреплённый MinIO. Миграции до 010 и bootstrap bucket/политик выполняются автоматически. +010 добавляет traceparent/tracestate к durable jobs, прежние данные сохраняются. -## Доказательства критериев +Grafana: http://localhost:3000/d/messenger-lab6. Просмотр без входа разрешён только +на локальном loopback, права Viewer. Редактор: admin / GRAFANA_ADMIN_PASSWORD из .env. +API: http://localhost:8086/docs. Prometheus и alerts: http://localhost:9090/alerts. -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +Обычная остановка `docker compose down` сохраняет данные. +Намеренное удаление всех учебных данных выбранного проекта: +`docker compose down --volumes`. Эта команда для обычного запуска не нужна. -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +Быстрая приёмка: `make verify-lab6 PYTHON=.venv/bin/python` после установки +requirements.txt и requirements-dev.txt. Опциональная проверка инцидентов: +`python3 tests/observability.py --base-url http://localhost:8086 --incidents`. +Тест снимает воздействия в finally. Долгие эксперименты и исследование цены +наблюдаемости не проводились по выбранному уровню и запросу на короткие проверки. -## Эксперименты и отказы +## Компоненты и версии -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +| Компонент | Версия / роль | +| --- | --- | +| Python / FastAPI / Pillow | 3.13.0 / 0.141.1 / 13.0.0; приложение и изображения | +| PostgreSQL / Redis | 16.6 / 7.4.2; данные, сессии и отдельный Streams broker | +| HAProxy / MinIO | 3.0.12 / закреплённый commit RELEASE.2025-10-15T17-29-55Z | +| OpenTelemetry Python | API/SDK/OTLP HTTP 1.31.1, semconv 0.52b1 | +| Collector contrib | 0.123.0; OTLP receiver, memory_limiter, batch, exporters | +| Prometheus | 3.2.1; временные ряды и вычисление alert rules | +| Loki | 3.4.2; структурные логи через нативный OTLP endpoint | +| Tempo | 2.7.2; реальные traces API/worker | +| Grafana | 11.6.0; импортируемые datasource/dashboard provisioning | -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +Схема: API/worker/dispatcher → OTLP HTTP/protobuf → Collector → Prometheus / Loki / +Tempo → Grafana. Collector не является долговременным хранилищем, Grafana — интерфейс. +Все backend имеют persistent volumes; Prometheus, Loki и Tempo настроены на 7 дней. +Конфигурация находится в infra/observability/. JSON dashboard содержит запросы и единицы. -## Ограничения и следующие шаги +Бизнес-API lab5 сохранён: пользователи, Bearer-сессии, два API, сообщения/пагинация, +идемпотентность, raw upload 202 queued, durable outbox, очередь, worker, оригинал и +thumbnail/crop/WebP, retry, failed и replay. Публичные tests/контракты skeleton lab6 +не изменены. Дополнительный X-Trace-Id облегчает поиск своего запроса. -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +## Инструментация и перенос контекста + +Каждый HTTP-запрос получает настоящий SERVER span. После маршрутизации его имя и +http.route используют шаблон, а не URL с UUID. SQLAlchemy events создают CLIENT spans +DB-операций без SQL/parameters. S3 transport создаёт CLIENT spans без object key/URL. +Worker создаёт CONSUMER span и дочерний image.transform; dispatcher — PRODUCER. + +При принятии файла текущий W3C context сохраняется вместе с job/outbox в PostgreSQL. +После перезапуска он не зависит от памяти API. Dispatcher извлекает его, создаёт +queue.publish и inject-ит traceparent/tracestate в envelope Redis Streams. +Worker extract-ит carrier и создаёт queue.consume с producer-родителем. +При retry текущий consumer context сохраняется в job; следующая публикация +продолжает ту же причинную цепь. Нет передачи тела картинки или произвольного URL. + +Resource attributes: service.name=messenger-api / messenger-worker / +messenger-dispatcher, service.version=1.6.0, service.instance.id=случайный UUID +жизни процесса, deployment.environment.name=lab. API, worker и dispatcher имеют +разные service.name; две API-реплики различаются instance ID. + +Отдельный безопасный event logger экспортирует JSON с timestamp, severity, +service_name, event, trace_id, span_id; HTTP содержит request_id. Не экспортируются +пароли, Authorization/session tokens, тела сообщений, изображения, filename, +SQL, parameters, S3 keys или signed URLs. В exceptions передаётся только класс +ошибки, без произвольного сообщения/stack trace. SDK exception auto-recording +для бизнес-spans отключён. Внутренние предупреждения SDK не пересылаются в Loki. + +## Сигналы и единицы + +| Контракт | Реальная Prometheus-метрика | Labels / единица | +| --- | --- | --- | +| HTTP requests | http_server_requests_total | method, route template, status_class; count | +| HTTP duration | http_server_duration_seconds_bucket / sum / count | те же labels; seconds | +| Jobs processed/failed | worker_jobs_total | job_type, outcome=ready/retry/failed; attempts count | +| Job processing duration | worker_job_duration_seconds_bucket | job_type/outcome; seconds | +| Acceptance → ready | attachment_ready_duration_seconds_bucket | job_type; seconds | +| Queue depth | queue_depth | queue; число queued + processing в PostgreSQL, включая outbox | +| Oldest job | queue_oldest_age_seconds | queue; seconds от принятия самой старой незавершённой job | +| DB pool | db_pool_used / db_pool_waiting | checked-out connections / ожидающие checkout | +| Worker saturation | worker_busy / worker_capacity | число занятых consumers / конфигурационная ёмкость | +| Process resources | process_memory_rss_bytes / process_cpu_time_seconds_total | текущий RSS bytes / CPU seconds | + +Collector также добавляет ограниченные resource labels. Пользовательские IDs, +request/trace IDs, raw URL и filename не используются в labels метрик. +job_id тоже не label. Технические trace/span IDs находятся только в логах/трассах. + +Histogram boundaries: 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, +30, 60, 120 секунд и +Inf. p95 API, например: + +```promql +histogram_quantile(0.95, + sum by (le) (rate(http_server_duration_seconds_bucket{route=~"/api/v1/.*"}[1m]))) +``` + +Для p50/p99 меняется quantile. Buckets агрегируются по всем репликам ДО quantile. +Средние и средние перцентили реплик не используются. HTTP duration измеряется +до готовности ответа/его заголовков, а не длительности полного download stream. +Backlog age не является histogram processing duration. + +## SLI, SLO и alerts + +SLI = 1 − число 5xx / число всех бизнес-запросов за скользящие **5 минут**. +SLO: **не менее 99%** запросов без server error в таком окне. Бизнес-запросы — +route под /api/v1/, достигшие API; health исключены, 4xx не считаются server error. +Ошибки самого HAProxy до попадания в API этот SLI не измеряет — ограничение стенда. +При отсутствии запросов SLI не следует трактовать как доказанное выполнение SLO. + +Prometheus rules в infra/observability/alerts.yaml, evaluation interval 2 секунды: + +- ApiHighErrorRate: 5xx/all >5% в окне 30 секунд, минимум 3 запроса, for=5 секунд. +- QueueLagGrowing: oldest accepted job >15 секунд и depth>0, for=5 секунд. + +Это короткие учебные диагностические пороги, а не multi-window burn-rate трека 5. +Отправка во внешние каналы отсутствует, проверяется локальное firing/resolved. +Правила содержат описание и ссылку на RUNBOOK.md. Prometheus ALERTS хранит историю +состояний; после восстановления правило становится inactive (resolved). + +## Два инцидента и диагностика + +**1. Worker остановлен.** API принимает файл и сообщение с queued-вложением. +queue_depth/oldest растут, QueueLagGrowing переходит в firing. По сохранённым +наблюдениям: публикация есть, свежего consumer/completion нет, HTTP не деградировал. +Причина локализуется в фоновой обработке. После запуска worker тот же ID доходит +до ready, backlog/age возвращаются в 0, alert resolved. + +Дополнительная краткая пауза S3 демонстрирует retry: consumer получает временный +отказ, следующая попытка сохраняет trace ID upload. После восстановления S3 +видны несколько consumer spans в одной трассе и готовый результат. + +**2. INSERT блокирован PostgreSQL.** Учебная SHARE-блокировка messages задерживает +запись, но совместима с SELECT readiness. Запросы получают ограниченный 503, +появляются ApiHighErrorRate firing, длинные ERROR db.INSERT spans и db.error logs. +Причина — ожидание DB-операции, а не image resize или просто медленный HTTP handler. +После снятия блокировки запись снова даёт 201; по выходу ошибок из окна alert resolved. + +Команды, подтверждения и trace/log evidence сохраняет tests/observability.py. +Сценарий не требует чтения секретов или ручного поиска по Docker stdout. + +## Ограничение буферов и отказ backend + +В SDK spans/logs передаются background processors: queue=256, batch=32, +schedule=500 ms, HTTP timeout=1 s. Метрики экспортируются раз в 2 s, +export timeout=1.5 s; агрегируются ограниченные наборы атрибутов. Нет sync export +в обработчике запроса. При переполнении SDK теряет telemetry, а не блокирует API. + +Collector: memory_limiter 192 MiB, spike 32 MiB при container limit 256 MiB; +batch до 256 записей, timeout 1 s. Для Loki/Tempo sending_queue=64 batch, +2 consumers, HTTP timeout 2 s, retry до 10 s с интервалом 1–3 s. Очереди in-memory: +при длительном отказе/restart возможны потери, они не называются durable telemetry. + +Метрики Collector фиксируют failures/queue. При остановке Tempo бизнес-сообщения +и новый attachment продолжают работать, а exporter показывает ошибки. Это +проверка ограниченного сценария и конфигурационных caps; длительного исследования +роста RAM под многодневной нагрузкой не проводилось. Бизнес-readiness не зависит +от Collector, Grafana, Prometheus, Loki или Tempo. + +## Доказательства и ограничения + +- evidence/lab6-checks.txt — offline/OpenAPI/lint и публичный smoke. +- evidence/lab6-signals.txt — короткая проверка трёх сигналов и dashboard queries. +- evidence/lab6-trace.json, lab6-correlated-logs.json — реальные связанные данные. +- evidence/lab6-incidents.txt, lab6-observability-summary.json — DB/backend проверки; + lab6-worker-incident.json — ранее проверенные остановка worker и retry. +- evidence/lab6-retry-trace.json — цепочка с повтором. +- evidence/lab6-db-incident-trace.json, lab6-db-incident-logs.json — диагностика DB. + +Стек однокопийный учебный, backend и host — точки отказа; нет HA/backups. +Sampling по умолчанию сохраняет все созданные traces; нагрузочных выводов о цене +такого режима нет. worker.capacity — конфигурация, не heartbeat; последнее значение +может временно сохраняться после остановки процесса. Loss telemetry при отказах +допускается и наблюдается, не обещается exactly-once доставка telemetry. + +## Источники и помощь + +Задание/контракты взяты из предоставленного skeleton lab6. + +- https://opentelemetry.io/docs/collector/configuration/ +- https://opentelemetry.io/docs/collector/internal-telemetry/ +- https://opentelemetry.io/docs/concepts/context-propagation/ +- https://redis.io/docs/latest/develop/data-types/streams/ +- https://grafana.com/docs/loki/latest/send-data/otel/ +- https://prometheus.io/docs/practices/histograms/ diff --git a/RUNBOOK.md b/RUNBOOK.md new file mode 100644 index 0000000..9b8a145 --- /dev/null +++ b/RUNBOOK.md @@ -0,0 +1,46 @@ +# Диагностика lab6 + +Автор: Каменский Андрей, БПИ2301. + +## QueueLagGrowing + +Условие: есть принятая незавершённая работа, возраст старейшей больше 15 секунд +не менее 5 секунд. Источник — PostgreSQL jobs, включая ещё не опубликованный outbox. + +1. Grafana: проверить queue depth и oldest age. Рост обоих при работающем API + означает накопление фоновой работы, а не медленный HTTP upload. +2. Loki: найти job.published, затем job.started/job.completed. Отсутствие свежих + consumer events при наличии публикаций указывает на worker/доставку. +3. Tempo: открыть trace upload. Если есть queue.publish, а consumer отсутствует, + обработка не началась. Если есть consumer со статусом ERROR и s3.GET — проверить S3. +4. Проверить процесс `docker compose ps worker`, восстановить `docker compose start worker`. +5. Убедиться, что прежний attachment_id стал ready, backlog/age упали до 0 и alert resolved. + +worker.capacity отражает конфигурацию, а не самостоятельный heartbeat: последнее +значение не доказывает, что процесс всё ещё жив. + +## ApiHighErrorRate + +Условие: 5xx / все бизнес-запросы > 5% за 30 секунд, минимум 3 запроса, +условие сохраняется 5 секунд. Health и 4xx в numerator не входят. + +1. Grafana: проверить 5xx ratio и p95/p99, DB pool used/waiting, CPU. +2. Loki: найти http.completed со status=503 и его trace_id; найти db.error. +3. Tempo: если длительный db.INSERT заканчивается ERROR, проблема в DB-операции. + Это отличается от длительного image.transform в worker или HTTP без DB-задержки. +4. Проверить блокировки/доступность PostgreSQL; снять учебную блокировку, восстановить БД. +5. Отправить сообщение, проверить 201. Дождаться ухода ошибок из окна 30 секунд + и отсутствия alertstate=firing. Prometheus хранит историю ALERTS для просмотра. + +## Backend telemetry недоступен + +Остановка Tempo не должна ломать сообщения или создание вариантов изображения. +Collector продолжает metrics/logs, в traces exporter растут failures/queue. +Максимум 64 batch в очереди exporter, retry ограничен 10 секундами. По исчерпании +данные отбрасываются; это допустимая потеря наблюдений, не потеря приложения. + +Проверить Collector self metrics на localhost:8889/metrics: +`otelcol_exporter_queue_size`, `otelcol_exporter_queue_capacity`, +`otelcol_exporter_send_failed_spans_total` и `otelcol_exporter_enqueue_failed_spans_total`. +Конкретное имя/суффикс зависит от закреплённого Collector; исходные строки сохраняет тест. +Восстановить `docker compose start tempo`, выполнить свежий запрос и найти новый trace. diff --git a/alembic.ini b/alembic.ini new file mode 100644 index 0000000..d3b7004 --- /dev/null +++ b/alembic.ini @@ -0,0 +1,3 @@ +[alembic] +script_location = %(here)s/migrations +prepend_sys_path = . diff --git a/compose.example.yaml b/compose.example.yaml deleted file mode 100644 index dfa1cfb..0000000 --- a/compose.example.yaml +++ /dev/null @@ -1,17 +0,0 @@ -# Topology worksheet, deliberately NOT a runnable stack (no application supplied). -# Implement compose.yaml or stack.yaml; do not submit this empty file as deployment. -# Service names below are logical roles, not imposed container names. -x-lab-number: 6 -x-required-roles: - - api - - postgres - - load_balancer - - s3 - - broker - - worker - - otel_collector - - telemetry_storage - - grafana -# TODO: implement real images/builds, network wiring, volumes, healthchecks, -# credentials and bootstrap. Details: contracts/infrastructure.json and README.md. -services: {} diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..2bb1c7c --- /dev/null +++ b/compose.yaml @@ -0,0 +1,367 @@ +x-logging: &logging + driver: json-file + options: + max-size: "10m" + max-file: "3" + +x-app: &app + build: . + image: ${APP_IMAGE:-mtusi-messenger:lab6} + env_file: ${ENV_FILE:-.env} + logging: *logging + +services: + minio: + build: ./infra/minio + image: ${MINIO_IMAGE:-mtusi-minio:RELEASE.2025-10-15T17-29-55Z} + logging: *logging + mem_limit: 512m + cpus: 1.0 + environment: + MINIO_ROOT_USER: ${MINIO_ROOT_USER:-messenger} + MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-local-messenger-s3-password} + MINIO_BROWSER: "off" + MINIO_API_STALE_UPLOADS_EXPIRY: "24h" + MINIO_API_STALE_UPLOADS_CLEANUP_INTERVAL: "1h" + volumes: + - minio_data:/data + healthcheck: + test: ["CMD", "wget", "-q", "--spider", "http://127.0.0.1:9000/minio/health/live"] + interval: 2s + timeout: 3s + retries: 60 + start_period: 10s + + s3-bootstrap: + <<: *app + image: ${APP_IMAGE:-mtusi-messenger:lab6}-s3-bootstrap + command: ["python", "-m", "src.core.storage_bootstrap"] + environment: + MINIO_ROOT_USER: ${MINIO_ROOT_USER:-messenger} + MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-local-messenger-s3-password} + S3_WORKER_ACCESS_KEY_ID: ${S3_WORKER_ACCESS_KEY_ID:-messenger-worker} + S3_WORKER_SECRET_ACCESS_KEY: ${S3_WORKER_SECRET_ACCESS_KEY:-local-messenger-worker-password} + S3_RECONCILE_ACCESS_KEY_ID: ${S3_RECONCILE_ACCESS_KEY_ID:-messenger-reconciler} + S3_RECONCILE_SECRET_ACCESS_KEY: ${S3_RECONCILE_SECRET_ACCESS_KEY:-local-messenger-reconciler-password} + DATABASE_URL: postgresql://unused:unused@postgres/unused + REDIS_URL: redis://redis:6379/0 + S3_ENDPOINT: http://minio:9000 + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-messenger-api} + S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-local-messenger-api-password} + depends_on: + minio: + condition: service_healthy + healthcheck: + disable: true + + postgres: + logging: *logging + image: postgres:16.6-bookworm + cpus: ${POSTGRES_CPUS:-1.0} + mem_limit: 512m + environment: + POSTGRES_DB: ${POSTGRES_DB:-messenger} + POSTGRES_USER: ${POSTGRES_USER:-messenger} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Задай POSTGRES_PASSWORD в .env} + volumes: + - postgres_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U $$POSTGRES_USER -d $$POSTGRES_DB"] + interval: 2s + timeout: 3s + retries: 30 + + redis: + logging: *logging + image: redis:7.4.2-bookworm + cpus: 0.25 + mem_limit: 128m + command: ["redis-server", "--appendonly", "yes", "--appendfsync", "always", "--maxmemory", "64mb", "--maxmemory-policy", "noeviction"] + volumes: + - redis_data:/data + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 2s + timeout: 3s + retries: 30 + + migration: + <<: *app + image: ${APP_IMAGE:-mtusi-messenger:lab6}-migration + command: ["alembic", "upgrade", "head"] + cpus: 0.5 + mem_limit: 256m + environment: + REDIS_URL: &redis_url redis://redis:6379/0 + DATABASE_URL: &database_url postgresql://${POSTGRES_USER:-messenger}:${POSTGRES_PASSWORD:?Задай POSTGRES_PASSWORD в .env}@postgres:5432/${POSTGRES_DB:-messenger} + depends_on: + postgres: + condition: service_healthy + healthcheck: + disable: true + + api: + <<: *app + cpus: ${API_CPUS:-0.5} + mem_limit: ${API_MEMORY:-512m} + stop_grace_period: 15s + deploy: + replicas: ${API_REPLICAS:-2} + read_only: true + tmpfs: + - /tmp:rw,noexec,nosuid,size=64m + cap_drop: [ALL] + security_opt: ["no-new-privileges:true"] + environment: + S3_ENDPOINT: http://minio:9000 + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-messenger-api} + S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-local-messenger-api-password} + S3_WORKER_ACCESS_KEY_ID: "" + S3_WORKER_SECRET_ACCESS_KEY: "" + # Do not inherit privileged S3 credentials from env_file into API containers. + MINIO_ROOT_USER: "" + MINIO_ROOT_PASSWORD: "" + S3_RECONCILE_ACCESS_KEY_ID: "" + S3_RECONCILE_SECRET_ACCESS_KEY: "" + OTEL_ENABLED: ${OTEL_ENABLED:-true} + OTEL_SERVICE_NAME: messenger-api + OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318 + UPLOAD_CONCURRENCY: ${UPLOAD_CONCURRENCY:-1} + APP_HOST: 0.0.0.0 + APP_PORT: 8080 + DATABASE_URL: *database_url + REDIS_URL: *redis_url + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + LOG_LEVEL: ${LOG_LEVEL:-INFO} + APP_WORKERS: 1 + DATABASE_POOL_SIZE: ${DATABASE_POOL_SIZE:-10} + DATABASE_POOL_TIMEOUT_SECONDS: 2 + DATABASE_CONNECT_TIMEOUT_SECONDS: 1 + DATABASE_COMMAND_TIMEOUT_SECONDS: 3 + DATABASE_STATEMENT_TIMEOUT_MS: 2500 + SHUTDOWN_TIMEOUT_SECONDS: 10 + HTTP_REQUEST_TIMEOUT_SECONDS: ${HTTP_REQUEST_TIMEOUT_SECONDS:-5} + expose: ["8080"] + depends_on: + s3-bootstrap: + condition: service_completed_successfully + migration: + condition: service_completed_successfully + redis: + condition: service_healthy + healthcheck: + interval: 2s + timeout: 3s + retries: 30 + start_period: 10s + + reconciler: + <<: *app + image: ${APP_IMAGE:-mtusi-messenger:lab6}-reconciler + command: ["python", "-m", "src.maintenance.reconcile", "--apply", "--interval", "300"] + mem_limit: 256m + cpus: 0.5 + read_only: true + cap_drop: [ALL] + security_opt: ["no-new-privileges:true"] + environment: + DATABASE_URL: *database_url + REDIS_URL: *redis_url + S3_ENDPOINT: http://minio:9000 + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_RECONCILE_ACCESS_KEY_ID: ${S3_RECONCILE_ACCESS_KEY_ID:-messenger-reconciler} + S3_RECONCILE_SECRET_ACCESS_KEY: ${S3_RECONCILE_SECRET_ACCESS_KEY:-local-messenger-reconciler-password} + RECONCILE_GRACE_SECONDS: ${RECONCILE_GRACE_SECONDS:-300} + MINIO_ROOT_USER: "" + MINIO_ROOT_PASSWORD: "" + S3_ACCESS_KEY_ID: "" + S3_SECRET_ACCESS_KEY: "" + depends_on: + migration: + condition: service_completed_successfully + s3-bootstrap: + condition: service_completed_successfully + healthcheck: + test: ["CMD", "python", "-c", "import os; os.kill(1, 0)"] + interval: 10s + timeout: 3s + retries: 3 + start_period: 2s + + broker: + image: redis:7.4.2-bookworm + logging: *logging + mem_limit: 256m + cpus: 0.25 + command: ["redis-server", "--appendonly", "yes", "--appendfsync", "always", "--maxmemory", "192mb", "--maxmemory-policy", "noeviction"] + volumes: + - broker_data:/data + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 2s + timeout: 3s + retries: 30 + + worker: + <<: *app + image: ${APP_IMAGE:-mtusi-messenger:lab6}-worker + command: ["python", "-m", "src.pipeline.worker"] + init: true + mem_limit: ${WORKER_MEMORY:-1g} + cpus: ${WORKER_CPUS:-1.0} + stop_grace_period: 45s + read_only: true + tmpfs: ["/tmp:rw,noexec,nosuid,size=128m"] + cap_drop: [ALL] + security_opt: ["no-new-privileges:true"] + environment: &pipeline_env + DATABASE_URL: *database_url + REDIS_URL: *redis_url + BROKER_URL: redis://broker:6379/0 + S3_ENDPOINT: http://minio:9000 + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_WORKER_ACCESS_KEY_ID: ${S3_WORKER_ACCESS_KEY_ID:-messenger-worker} + S3_WORKER_SECRET_ACCESS_KEY: ${S3_WORKER_SECRET_ACCESS_KEY:-local-messenger-worker-password} + MINIO_ROOT_USER: "" + MINIO_ROOT_PASSWORD: "" + S3_RECONCILE_ACCESS_KEY_ID: "" + S3_RECONCILE_SECRET_ACCESS_KEY: "" + S3_ACCESS_KEY_ID: "" + S3_SECRET_ACCESS_KEY: "" + WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-1} + WORKER_MEMORY_MB: ${WORKER_MEMORY_MB:-768} + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-30} + JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-120} + JOB_RETRY_BASE_SECONDS: ${JOB_RETRY_BASE_SECONDS:-2} + JOB_RETRY_CAP_SECONDS: ${JOB_RETRY_CAP_SECONDS:-60} + STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-604800} + OTEL_ENABLED: ${OTEL_ENABLED:-true} + OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318 + PIPELINE_FAULT_INJECTION: ${PIPELINE_FAULT_INJECTION:-false} + depends_on: + migration: + condition: service_completed_successfully + s3-bootstrap: + condition: service_completed_successfully + broker: + condition: service_healthy + healthcheck: + test: ["CMD", "python", "-c", "import os; os.kill(1, 0)"] + interval: 5s + timeout: 3s + retries: 3 + + dispatcher: + <<: *app + image: ${APP_IMAGE:-mtusi-messenger:lab6}-dispatcher + command: ["python", "-m", "src.pipeline.dispatcher"] + init: true + mem_limit: 256m + cpus: 0.25 + read_only: true + cap_drop: [ALL] + security_opt: ["no-new-privileges:true"] + environment: *pipeline_env + depends_on: + migration: + condition: service_completed_successfully + s3-bootstrap: + condition: service_completed_successfully + broker: + condition: service_healthy + healthcheck: + test: ["CMD", "python", "-c", "import os; os.kill(1, 0)"] + interval: 5s + timeout: 3s + retries: 3 + + otel-collector: + image: otel/opentelemetry-collector-contrib:0.123.0 + command: ["--config=/etc/otelcol/config.yaml"] + mem_limit: 256m + cpus: 0.5 + volumes: + - ./infra/observability/collector.yaml:/etc/otelcol/config.yaml:ro + ports: + - "127.0.0.1:${COLLECTOR_METRICS_PORT:-8889}:8888" + + prometheus: + image: prom/prometheus:v3.2.1 + mem_limit: 512m + cpus: 0.5 + command: ["--config.file=/etc/prometheus/prometheus.yaml", "--storage.tsdb.retention.time=7d"] + volumes: + - ./infra/observability/prometheus.yaml:/etc/prometheus/prometheus.yaml:ro + - ./infra/observability/alerts.yaml:/etc/prometheus/alerts.yaml:ro + - prometheus_data:/prometheus + ports: ["127.0.0.1:${PROMETHEUS_PORT:-9090}:9090"] + + loki: + image: grafana/loki:3.4.2 + mem_limit: 384m + cpus: 0.5 + command: ["-config.file=/etc/loki/config.yaml"] + volumes: + - ./infra/observability/loki.yaml:/etc/loki/config.yaml:ro + - loki_data:/loki + ports: ["127.0.0.1:${LOKI_PORT:-3100}:3100"] + + tempo: + image: grafana/tempo:2.7.2 + user: "0:0" + mem_limit: 384m + cpus: 0.5 + command: ["-config.file=/etc/tempo/config.yaml"] + volumes: + - ./infra/observability/tempo.yaml:/etc/tempo/config.yaml:ro + - tempo_data:/var/tempo + ports: ["127.0.0.1:${TEMPO_PORT:-3200}:3200"] + + grafana: + image: grafana/grafana:11.6.0 + mem_limit: 384m + cpus: 0.5 + environment: + GF_AUTH_ANONYMOUS_ENABLED: "true" + GF_AUTH_ANONYMOUS_ORG_ROLE: Viewer + GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:?Set GRAFANA_ADMIN_PASSWORD in .env} + GF_USERS_ALLOW_SIGN_UP: "false" + GF_ANALYTICS_REPORTING_ENABLED: "false" + GF_ANALYTICS_CHECK_FOR_UPDATES: "false" + GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH: /var/lib/grafana/dashboards/messenger.json + volumes: + - grafana_data:/var/lib/grafana + - ./infra/observability/grafana/provisioning:/etc/grafana/provisioning:ro + - ./infra/observability/grafana/dashboards:/var/lib/grafana/dashboards:ro + ports: ["127.0.0.1:${GRAFANA_PORT:-3000}:3000"] + + lb: + image: haproxy:3.0.12-alpine + logging: *logging + cpus: 0.5 + mem_limit: 128m + read_only: true + tmpfs: ["/tmp:rw,noexec,nosuid,size=4m"] + security_opt: ["no-new-privileges:true"] + ports: + - "127.0.0.1:${APP_PORT:-8080}:8080" + volumes: + - ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + depends_on: + api: + condition: service_healthy + stop_signal: SIGUSR1 + stop_grace_period: 15s + +volumes: + redis_data: + postgres_data: + minio_data: + broker_data: + prometheus_data: + loki_data: + tempo_data: + grafana_data: diff --git a/course-tests/README.md b/course-tests/README.md new file mode 100644 index 0000000..0dfa3f8 --- /dev/null +++ b/course-tests/README.md @@ -0,0 +1,57 @@ +# Публичные проверки + +Python **3.9+**, стандартная библиотека. Сервер уже должен быть запущен; проверки не создают готовое приложение и сами не управляют Docker. `make check` можно выполнить на исходном skeleton, `make test` — только после реализации API. Полная OpenAPI-валидация отдельно: `make validate` после установки `requirements-dev.txt` в venv. + +```sh +make check +make test BASE_URL=http://localhost:8080 +# без make: +python3 tests/smoke.py --base-url http://localhost:8080 +``` + +`smoke.py` выполняет **один сквозной сценарий**, автоматически выбирая профиль из `lab.json`: health → три пользователя → вход с lab2 → чат → сообщения → пагинация → ошибки доступа → идемпотентность с lab3 → файл с lab4 → thumbnail с lab5 → logout. Он проверяет JSON по используемому подмножеству схемы, коды/headers, Unicode, SHA-256 оригинала и фактические JPEG-размеры thumbnail. Это публичный минимум, не полный fuzz/security/load suite. + +Каждый запуск создаёт уникальных синтетических пользователей, чат и сообщения. Данные автоматически не удаляются: в API курса нет delete. Для чистого повтора используйте отдельный тестовый стенд/volume и осознанный сброс своего окружения. Session tokens хранятся только в памяти процесса и не печатаются/не сохраняются. + +## Проверка сохранности, lab2+ + +Скопируйте `scripts/restart.example.sh` в `scripts/restart.sh`, реализуйте перезапуск **API и всех используемых хранилищ** с сохранением volumes и задайте executable bit. Скрипт может опираться на ваш compose.yaml или stack.yaml. Он не должен удалять volumes, повторно seed-ить БД или менять тестовые данные. + +```sh +chmod +x scripts/restart.sh +make test-persistence ACTION_SCRIPT=scripts/restart.sh +``` + +Тест создаёт данные и сессию, запускает указанный файл без shell interpolation, ожидает готовности и проверяет те же данные **с прежним токеном**. В lab4+ также проверяется исходный файл. Hook может работать до 90 секунд, готовность затем ожидается до 90 секунд. TTL сессии для этого теста — минимум 300 секунд. Успех не доказывает, что hook действительно перезапустил БД: приложите команды/состояния контейнеров до и после. + +## Проверка отказа, lab3+ + +Реализуйте `scripts/stop-one.sh` на основе примера. `TARGET_INSTANCE_ID` в окружении содержит alias обслужившей тест API-реплики: сопоставьте его своему контейнеру/Swarm task и остановите именно его. Скрипт должен быстро вернуть 0 и **оставить эту реплику остановленной**; восстановление выполняйте отдельно. Swarm может создать новую task с другим ID — это допустимо. + +```sh +chmod +x scripts/stop-one.sh +make test-failover ACTION_SCRIPT=scripts/stop-one.sh +``` + +Перед отказом тест наблюдает ≥2 API-реплики; 15 секунд отправляет сообщения параллельно выполнению hook, повторяет неопределённый POST с тем же ключом, проверяет восстановление записи за ≤10 секунд, отсутствие остановленной реплики в последних ответах, сохранность подтверждённых сообщений и отсутствие дублей. Ошибки переходного периода учитываются. Затем восстановите реплику и повторите для другой. Скрипт — проверка одного отказа API, не гарантия HA БД/LB/host. Вывод hook скрыт, чтобы не утекли секреты; отлаживайте его отдельно с безопасным выводом. + +## Нагрузка + +```sh +python3 tests/load.py --duration 30 --concurrency 4 --output evidence/load.json +``` + +Это простой **closed-loop** генератор POST сообщений; он ограничен производительностью клиента и страдает coordinated omission. Он выдаёт successful RPS, ошибки и p50/p95/p99 **только успешных запросов**. Во время нагрузки нет прозрачных retry; timeout мог скрыть совершённую запись, поэтому это throughput HTTP-подтверждений, не точный счётчик COMMIT. Изменение размера истории входит в профиль. Для серьёзного исследования используйте k6/Locust/wrk либо свой обоснованный генератор, а не выводите максимальную пропускную способность из одного запуска этого скрипта. Для нагрузки pipeline изображений нужен отдельный сценарий студента. + +## HTTPS, lab7 + +```sh +make test BASE_URL=https://localhost:8443 CA_FILE=/absolute/path/to/ca.crt +make test-security BASE_URL=https://localhost:8443 CA_FILE=/absolute/path/to/ca.crt +``` + +Все скрипты принимают `--ca-file`/`CA_FILE` и проверяют серверный сертификат и hostname. `--insecure` отсутствует. TLS-тест сначала проверяет рабочее доверенное соединение, затем намеренно пустой trust store и именно ошибку проверки сертификата; network timeout не засчитывается как правильный отказ. **Внутреннее mTLS, identity/authorization, OpenBao policy и hardening проверяют отдельные тесты студента** по trust matrix. Клиентский сертификат внешнему учебному REST-клиенту не требуется: mTLS находится на внутренних связях. + +## Что ещё остаётся доказать + +Smoke не доказывает persistence, число реальных контейнеров, durability брокера, outbox, отсутствие утечек/уязвимостей, полноту telemetry или выполнение уровня 4/5. Конкурентные/нагрузочные/негативные проверки своего решения добавляйте отдельно. Список защиты текущей лабы находится в README.md. Проверки рассчитаны на localhost-стенд, не на production. diff --git a/course-tests/client.py b/course-tests/client.py new file mode 100644 index 0000000..f0499a2 --- /dev/null +++ b/course-tests/client.py @@ -0,0 +1,217 @@ +"""HTTP helpers for public black-box tests; Python 3.9+, standard library only.""" +import base64 +import datetime as dt +import hashlib +import json +import os +from pathlib import Path +import re +import ssl +import time +import urllib.error +import urllib.request +import uuid + +ROOT = Path(__file__).resolve().parents[1] +LAB = json.loads((ROOT / 'lab.json').read_text())['number'] +SPEC = json.loads((ROOT / 'contracts/openapi.json').read_text()) + +class Failure(AssertionError): + pass + +def check(condition, message): + if not condition: + raise Failure(message) + +def timestamp(value): + check(isinstance(value, str) and value.endswith('Z'), 'timestamp must be UTC ending in Z') + try: + return dt.datetime.fromisoformat(value[:-1] + '+00:00') + except ValueError as exc: + raise Failure('invalid RFC3339 timestamp') from exc + +def validate(value, schema, where='response'): + """Validate the JSON Schema subset used by the supplied contract, not arbitrary OpenAPI.""" + if '$ref' in schema: + target = SPEC + for part in schema['$ref'].split('/')[1:]: + target = target[part] + return validate(value, target, where) + kinds = schema.get('type', []) + if isinstance(kinds, str): + kinds = [kinds] + matches = {'null':value is None, 'boolean':isinstance(value,bool), 'integer':isinstance(value,int) and not isinstance(value,bool), 'number':isinstance(value,(int,float)) and not isinstance(value,bool), 'string':isinstance(value,str), 'array':isinstance(value,list), 'object':isinstance(value,dict)} + if kinds: + check(any(matches.get(k,False) for k in kinds), where + ': incorrect type') + if 'enum' in schema: + check(value in schema['enum'], where + ': invalid enum') + if 'const' in schema: + check(value == schema['const'], where + ': invalid const') + if isinstance(value,dict): + props = schema.get('properties',{}) + check(all(k in value for k in schema.get('required',[])), where + ': missing required fields') + if schema.get('additionalProperties') is False: + check(not set(value).difference(props), where + ': unexpected fields') + for key, sub in props.items(): + if key in value: + validate(value[key], sub, where + '.' + key) + if isinstance(value,list): + check(len(value) >= schema.get('minItems',0), where + ': too few items') + check(len(value) <= schema.get('maxItems',float('inf')), where + ': too many items') + if schema.get('uniqueItems'): + encoded = [json.dumps(x,sort_keys=True) for x in value] + check(len(encoded)==len(set(encoded)), where + ': duplicate items') + if 'items' in schema: + for item in value: + validate(item, schema['items'], where + '[]') + if isinstance(value,str): + check(len(value) >= schema.get('minLength',0), where + ': too short') + check(len(value) <= schema.get('maxLength',float('inf')), where + ': too long') + if 'pattern' in schema: + check(re.search(schema['pattern'], value) is not None, where + ': pattern mismatch') + if schema.get('format') == 'uuid': + try: + uuid.UUID(value) + except ValueError as exc: + raise Failure(where + ': invalid UUID') from exc + if schema.get('format') == 'date-time': + timestamp(value) + if isinstance(value,(int,float)) and not isinstance(value,bool): + check(value >= schema.get('minimum',-float('inf')), where + ': below minimum') + check(value <= schema.get('maximum',float('inf')), where + ': above maximum') + if 'anyOf' in schema: + for alternative in schema['anyOf']: + try: + validate(value, alternative, where) + break + except Failure: + pass + else: + raise Failure(where + ': no anyOf branch matches') + +def shape(value, name): + validate(value, SPEC['components']['schemas'][name]) + return value + +class NoRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, headers, newurl): + return None + +class Client: + def __init__(self, base_url=None, ca_file=None, timeout=10): + self.base = (base_url or os.environ.get('BASE_URL') or ('https://localhost:8443' if LAB==7 else 'http://localhost:8080')).rstrip('/') + check(self.base.startswith(('http://','https://')), 'BASE_URL must be HTTP(S)') + check(LAB != 7 or self.base.startswith('https://'), 'lab7 requires HTTPS') + self.timeout = timeout + context = ssl.create_default_context(cafile=ca_file or os.environ.get('CA_FILE') or None) + self.opener = urllib.request.build_opener(NoRedirect(), urllib.request.HTTPSHandler(context=context)) + + def call(self, method, path, expected=200, body=None, actor=None, headers=None, raw=None, schema=None): + h = {'Accept':'application/json'} + if actor: + h['X-User-Id' if LAB==1 else 'Authorization'] = actor['id'] if LAB==1 else 'Bearer ' + actor['token'] + if body is not None: + raw = json.dumps(body,ensure_ascii=False).encode('utf-8') + h['Content-Type'] = 'application/json' + h.update(headers or {}) + req = urllib.request.Request(self.base + path, data=raw, headers=h, method=method) + start = time.monotonic() + try: + response = self.opener.open(req, timeout=self.timeout) + except urllib.error.HTTPError as error: + response = error + except (urllib.error.URLError,TimeoutError,OSError) as exc: + # Do not print URL query, headers, request body, or credentials. + raise Failure(method + ' ' + path.split('?')[0] + ': connection/TLS failure (' + type(exc).__name__ + ')') from None + with response: + status = response.code + rh = response.headers + data = response.read(16*1024*1024 + 1) + check(len(data) <= 16*1024*1024, 'response exceeds test safety limit') + allowed = [expected] if isinstance(expected,int) else expected + check(status in allowed, method + ' ' + path.split('?')[0] + ': expected ' + str(allowed) + ', received ' + str(status)) + check(bool(rh.get('X-Request-Id')), 'missing X-Request-Id') + if LAB>=3: + check(bool(rh.get('X-Instance-Id')), 'missing X-Instance-Id') + payload = None + if schema or status>=400: + check(rh.get_content_type()=='application/json', 'expected application/json') + try: + payload = json.loads(data) + except (ValueError,UnicodeError): + raise Failure('invalid response JSON') from None + shape(payload, 'Error' if status>=400 else schema) + if status>=400: + check(payload['error']['request_id']==rh.get('X-Request-Id'),'error request_id differs from header') + if status==401 and LAB>=2: + scheme = 'Basic' if path=='/api/v1/auth/sessions' else 'Bearer' + check(rh.get('WWW-Authenticate','').lower().startswith(scheme.lower()), 'incorrect WWW-Authenticate challenge') + if status==204: + check(not data, '204 must not contain body') + return {'status':status,'headers':rh,'json':payload,'bytes':data,'elapsed':time.monotonic()-start} + + def user(self, label): + username = 'u_' + label + '_' + uuid.uuid4().hex[:12] + password = 'T3st-' + uuid.uuid4().hex + body = {'username':username,'display_name':'Студент ' + label} + if LAB>=2: + body['password'] = password + user = self.call('POST','/api/v1/users',201,body=body,schema='User')['json'] + check(user['username']==username,'username changed') + actor = {'id':user['id'],'user':user,'registration':body} + if LAB>=2: + credential = base64.b64encode((username+':'+password).encode()).decode() + session = self.call('POST','/api/v1/auth/sessions',201,headers={'Authorization':'Basic '+credential},schema='Session')['json'] + check(session['user']==user,'session user mismatch') + check(timestamp(session['expires_at']) > dt.datetime.now(dt.timezone.utc),'session already expired') + actor['token'] = session['token'] + return actor + + def bootstrap(self): + self.call('GET','/health/live',schema='Health') + self.call('GET','/health/ready',schema='Health') + alice, bob, eve = (self.user(label) for label in ('alice','bob','eve')) + chat = self.call('POST','/api/v1/chats',201,actor=alice,body={'title':'Контрактный тест','member_ids':[alice['id'],bob['id']]},schema='Chat')['json'] + check(set(chat['member_ids'])=={alice['id'],bob['id']},'chat member mismatch') + return alice,bob,eve,chat + + def message(self, actor, chat, text, key=None, attachments=None): + body = {'text':text} + if attachments is not None: + body['attachment_ids'] = attachments + headers = {'Idempotency-Key':key or uuid.uuid4().hex} if LAB>=3 else {} + return self.call('POST','/api/v1/chats/'+chat['id']+'/messages',201,actor=actor,body=body,headers=headers,schema='Message') + + def attachment(self, actor, chat): + content = (ROOT / 'tests/fixtures/sample.png').read_bytes() + att = self.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=sample.png',201 if LAB==4 else 202,actor=actor,raw=content,headers={'Content-Type':'image/png'},schema='Attachment')['json'] + check(att['status']==('ready' if LAB==4 else 'queued'),'incorrect upload status') + check(att['chat_id']==chat['id'] and att['owner_id']==actor['id'],'attachment ownership mismatch') + check(att['sha256']==hashlib.sha256(content).hexdigest() and att['size_bytes']==len(content),'original checksum/size mismatch') + return att,content + + def wait_ready(self, actor, att, wait_seconds=60): + deadline = time.monotonic()+wait_seconds + while time.monotonic()|3C0bDCg50hY&IhVa-}PAeB?jayAv^IER`~Iyjp-)LWvQLWRh=OeBYm zoGEFQvV&768SgUXUB9RA@1NcF=XKp~yIqgH?$_f%K51_)Dx@g1Wy==Pw!jr)^g?I<%@C1k??w5p=(q(7=5 z`O)3FZPp38c>W(UJfUrpsC1dH4|d6t6J5HOg!bOh-6L~LXiXxo=aMmpV`1f-lBN^-U^$ZC)OrursoPQL4nqDJjNOl@^j=pB-ybDDfPPX79G`*_N^9O*vMc%u6}|94&9p<2h~vcw#jcs+B+ z9RC-(0KXC<)T!ZAe6>F%@sZGTDL>tfqpVh;=h8E}8;4n#l!F#7o}7ks&}Vq+H1}nwTAdz zJ?%HQ6(uesSdI6XQ$vX(>|9kIG~FKTb)7_(v5z6FtAo*=Rl(KlV8kQK5JZ>@vZDME z){Ra?y{is3z*Q|)Z9`?t6}!Z^O;}mE6B7reT}$q*oF--rI=hxUScxH~4u+%JYw*rR z_g6ym7OL>>MU^Y@d2&@>)zB_+HwH^km&g=X-}0u`40E;<;jW>8|wQ>v%}NXE9kt7HAQ>0s*7r)G6t&&cLqP9&Q?*ar)${eh_xEh zp_B9^l!}v)^RN@QA;a1Qk5X|q@}eUVITUS|VOJvudM;ZB;ZQ?5N{`Po6?QSNSux8S zqsX~*R;_5}3Dec{3~J)7=4#lcc@)>qx)o9GgF#JH{=;~ijz>|q2QRvG>V9-?^rj0T zbm=HWjcZ3eo3HUGJq=~<#BJnR(koHs&fGTM5xOVhi%YGXl1r#d<8k^~?)?mX)RTuB zC*mKiDCNyO+^~aygr zUO(t_UcVt#(KYG8VYyfnc|}{%TMed@ubIJ-DV1d<0)7^|j)>k!{a`{EXvIQB+D#Ra|??I2ITFD2k7h?qZK|x)sGINL9fP zxrSLICMtDh^E*VX&tp$mL{A*Cuo1Nm3D)!s39ewti-yq~6|-NB+gn_kIBa1pIv&}v zc(1HE6|ML*L)xUIin}{Yo8oLz@{ntiRY@^6x$uhN!@14Y%L)p?TXWjj_^gGHjzfXs z#Iin$$zO%$b>m>OTFG3+h&QCP zzMX9Qtc&M5yf~Id$2q67oK5TzuT1RU7+UlmVEc-$oum5o@+B&&OfFwfy-m24nr4bG zXE|~UQr{}h2a~+Z8n;L34lhh)tJZ@6(TvM*&F z44!_#wdUsbm74x3=elzX`@Sh&eM7a_DC0V0T|KwqwNXK=?7M2Z^zwW{>|_Y3Z zcRb)Tu5(In-P@ zx`cxq`h@pdFvCF(eZu=K@Z%tdVtwy5SB^ps73VL0eN+KCWbLP%|7kOaI46|dKS2(C zp?dajQHC7C)x9=4)(<(9t%j?6Lxda(EXY=h-OQmyWOek-QOF@(=pDCE#+Bd-4w z_oIL;bICB@;$|K_lCW$3Dl~FF+0z8+?vGz5qG2Fog4) zSce>1AQabs+03D6^5}gW&GcW4k;_ehQ4fs915glKf*x{ec3p=(29c``m%9&0aFY)6!WKcRjwFv zC?@>PdiyBkQ1R8z8#W7&L&f`J|I|e1`wEBNJ!*PaVcbfYFLmpyYnq{av7%T3Zho?s5tTh+dr1x579uLBJBs*Wf8(Syr7E+Yt{Vc`d&j+7498z~O$d0Brv%=~ zWBU$y5o;4TA}z+_zA=X56I(=(1~`Sq(7Tj`I`;M?ZAJV5bATPZJge@e+IbgyOeMBml8p%3ly{5tp1C*ayL`NR zN05J80z$wo!&Sgdi8RwCNXSYM7mYv74CIm%97N;KGNZXE36BM)6g&(t@{C~r+=LOf z4WE}ts3|Xx_l~(w;JGQUnD>z>F3_s5;ka>nIVkD5{D$ks*~s?p8PO$)tMVJKn0^9( zW&g@gpIi1zYR~lamo|O4b0iu4xxmo2Kua&6uqvd-h$S>;c|f|8d?;Bdw<}7<(a0=fdh6J2~oSHGfg)J`pU@z&+L{0qx>362mZ5D@n1Paygra* zr$SgcIWajKpjCJqKW#Z8sg-R)yh)xfPa)sZ%EnBbXu(YM(Tud-WdzzGgUXA^Kg*NI z*;@W=(!_vekhG6&3_`mAKQVh;%R#oX0RMXSoR))JP{G%;P3;55^ix`6n7HhLXnLI1 z*uJ=g0TuRAnknHf zImAw6FR@rYl#ovTVmG}j^^W{m_EG8Hzx|S0)_0I+RohV=9Hpc81tr>}kJKlt->Va0;d#(K2@%^*y>;uw6cD06$PPKa*-_d?+ znM;J&#s{-Cq*W;nw(;lL#?qw}1KV2D#!=e;8oOynr3-CCW&Q3j2!xt}dfFSU=SJ&~ z2ENnQw5}Sqjm+k-wQ~;^ZqR4z+52+e+x-cTO5^p;mTLXoyWYy{pS92ZduRI0)Hbgw zNBv;g7{~s^fbJImo$rtM2xs%VlVm@iTGrpL-Xd{|F>*vtO1L`ZgZ^pt){`E3`b%mk z$`AdebBzhX!9)W}mL5jdPH!oUs7m>r@t@vO1a}KXFe6D%C6p^cNy~U3E8ff)X0v3^ zXE#LNOR3N^k<-^}X^Ct}aVvkEf|1Q`9Ct&8u&~14^jhSU^iSB4Ksgo|O_fta7v*QK9f%1$- zmo;c)yg#B!tWN2aMK?#bQG0swWb2y79sX^Ne4MgXwoZ~y-#aAWdde4?zp#a#cLYno zXtKvpc%P^IpM%*A*q7`*lzrqv{f`8$8>NCSPYKk|ZYCAVM{qAtE_xL0T52KX)7$m! zWpefGpTMQL7>%%FgpKv4HI^PnmZtpH$L|Ve9TpZpRiJbB6p7KJA*+JOIaLrAke$*` zmzJIPEC^y9=iW@I)W-(~bkgte%a%Uo710;y!m@rXynpB^gd6mH{Z0bcm3xOym-TDq zrIKE;r6@M~otht?P>0$3WoHoIPkDp~q@?5#@={*Qer#P%@CamC2&40Y5%N|)_Fa0Z;olkgK^a%Yc+MTVuBzlDjo{JWDd|Mk4?NBIcX;FCTZRqLd#$ZBp zMB#77wENef=&OZ-4E#yt=~e&2GzQ+qmPQ!zOhsF8K*jy6im(=` zM`10E#LTQGY!9XWx4}Ub&ldeFDnuDr-CEglo4l4;<^(K~#tcLmeH>Xj@=dCc8-hI_XKZzrce8l`pw+Z|AG4XcR z5FLT3Ya!-kT`7N_RTC!CK+KX4=6EV4g?eiXyq70%{-?Ary$qB3mO9b5{Fj(Kasy-i zw&VD8=&EdC>pP@=9|IW@T{QBZMk;K5-_bk2ome^29{MpXy1d|Ybkm3qrZVFkrt*D3 z07r#bpY=O*I)bzHu#)&Y(u+P@qoKZDj7lT7iZt98suUfHN0&4hz3Ia%Ib>9Z;=?!! zN~>X0dJXHfBMYJC2EGmJtr23BcqOYGVtyKDsH{7?t7FG%gt8H%d4Q*(g z=+javq~~LvFc{}ob-9m6B0{f(tnTMN8&M4nZ(gt94a|>Wt~RdM@J8k@6h41Ht@fv3 zy>jGv=pX$*jcu>ypJOh*ecUB>-bL^ZpU))GA8V-W(R1PCn*>f5q9o~mV)23rJCrTO zE<{UEXoo-hUlhFkLRm)4Chb!CV8G?HOSsToin!^a0FAUPE9taL`=|R^orHsFzxU(2 zSPAT1X@dLlG*(%8b#E^c6HTHf-O)fIKQZ?Rj#vo`>MJ9^-OUUBb~h_25ZQw0$6-eg z?-%@LB`Bz@+||u3&rKQ?dx~8|*+-JfdvlQ`diF%p!`@fO67*;!VCxV87^yY)#!NN;pqWaRy%8sxvy|0#Qa z^S_o9LOUe*1}8tu#2GQVB?RB>rydrGH5~5F5=;G?qk+th!49(Q*k{vPu^q2j!A5;7 zk2DVQqgLPkjt?wlG41sI${**J!_%bcyVG)&?TJ`d+i2{EZV|z?^h?Uq3(PBm;yZAp z@oo#Tvl*wvR4y#1Ct1@2#m??8;4q2YIEL9*N+|{*1CV=151pX)_YL-&)jkZH>bB)>~Ab} zsG9$RW5v^PZ=e?8-jq*3EtIEdq+H#y_hdAyeTU_udQ@W}#bYIt*cvciG2#ev?`# zx<1}>KrP?d=jHgkfm)tYr|7%nfLhS0RO-h~YM}(6!(%q7g%x#g$Po?H(%wOzdxQpR zLH9pJE&#QBoco6U9tqU)X$75su^p(Tt>|l1#t=};&?od+Dp1Q8wz=F|JL%9kJqfK6 z1=JG7ZJ=0t1GPjOy{9APfLgqVV~xf(sYTAAozzE<&o>qJHvh6>mJih8-MP4;nGe*G zZ_pl3-=vm&icja#CbbM{qVt~uwS0@ZJ#^6&-)c z-aswUZ6D^oqk&qy|9o08$OmfiUW!?-AMy#;Z%9*ooiq>Bf|pmZ6~EPRVDbl(6q&M6 zRw6h9)FOT(?E{{s&b|IW#s{Dl@s$3TKrJgCGn^6?pca+26~+so78>WH${H7_WsLJj zMSK~kN=^wyZe~tpe17FInX7&H-w{mwe`$#wzodYELn_r*g*PfLfSbs~m$kpq9A!w;Wv+pq4neo7fznmIG)mb#jwha)$7o zoK0#;n^SQk0ky<=d}Cb}2Wn9<|9AOD8c>Uh`H$t>X+SM1Q~&c=0BT_dRHWId0JR(l zHns3bq%P6#dXNJ8VA(EAy)QZJ+QQL zKH>6YmAQ%P(<)Bya=@>K|Zk9d*k<>_*>>c zEn~mT*Zu`+`L`{9?5H_Vi}~8mwwy7b7W1{WxNBoTE#~4s-hHzX68UG>-d=kesiFf%f7SCm196HrTM(C-+)>st^HK< z*MVACoD-_<>p(5Psqgx?r~G57*LCO z1UIf`45($S(Y${hsAVm#eQbX`P|L55p@k3TKrQClzl+#FE%VV&a}>;hT7FT-`u{Um zk@%N;HzzF~s3ji%l_RMN)Dl1Z)2M%wT2!Mi53d5X4EDb=FX0-^0JW5k{fZymq?S^H zU-1J#EiV>^%>B4PEh7tr()!;(EuW&vWB1L0T7G-auPT=UwZ!YKHTD3t4ECQ_{r(%M zh086=$%zMQ;c}~U#^QlmmK&$~ajHNqs!EqbSAbeR1yXVznFF=_-soFsH3w?>z0tjZ zDFteY|MPWKt`w*xK74JxeGI6j^y;sTp-pNj-GBMdm*{*yVdUMTqIdbnt%|oC-C}%A z^XryaKI~R+7Jrpom$*c_;UO9S)-8?4w%yx$LHs%ye@L=H1gK@(lsGl0OQ1ZU>)AGp z5Kzlb>cy_h+kjexsaL!BwgI&Wks`Y+wgI&W;Pw2QucZlAnW=Z(*qSW_)UsoIZ`Z_D zpq8!gwqvz`T0*aJwzUWYwd@!-+OiF(g?|xw*XCLsJNcTn0A2^E<>s|Y0lYp?OXjr? z+b)O$wdgY6ZXMeR)MCI~-)gV}s6~Fc{MrFNpccW7-TvRV0JUuWB|y~!YRTB*EV=p*tNr3fm--Pc1B*lrbHXx(z|V1ev?|%nJ=zYYy)Z$BiVF$@d34L z%NE6=fm-6b-t&Ew1Zpv6`fU9!2Gk;iHDlhq=1gz7_KI(K2T;qtuFG4O#Fwx6UlZiR z%K)_?n5->1+kskS{V!ZI+X~dOy+Fy|_Zm>k))KL0+iUAvLu9wdW0=uffm*i58!>IR z0<~=Q6XTue0&3YRCm4SKsAc4u%{H_sP|JR1)3xv0fm-DJP>a>I$6KfPfm*gR?qO1GV_Q_|HN`0;uKWizEvbDWDdU*#Nb|6g(cNMJ-!|=#oBPZk2vZEn5tz zMQlQgW~2txVu6$^_euX*Zjqj?CIQrb3uMa@qLs6}W! z>V^9epcdga?Ccjcpq5l`si<==fLhe@Q{uIOTAoP^S=8h&`zr+0qCGo${cq|tVrrXpm7_+mWQ?PBqGWfb|IYnKKJ2jH z?oN{YY_+VhT|HC6ilKi*@1Gs#l0RsiR?j(Usi(1|_Avd2#*zY1i#)MDJxd*^MSV$` zSeg#hqQ0ccy_OEtqOOAEUQY*Vkrbye^w=!P^V#W-O6K=F_PJ2 zpcW+-b;mb#pceI*%xgPYHYHkLk3WbCc9eFfdbT`L1<0?paIm9LApST zRtIXKkgm{#)qz@$cI+XHr31A{8ju$yCf1xpVSg3oe0=1a*H0;=Sbdy@rv3uE7=|C+SAEmfc=@s;X^gxYl3Q&tG_h?j7P*KrL1zY>$Q{P>WT8 za)5m@P>baJ(E>T37MJ8o4ZL)~71|ws$vMWB`o!)J_U z$UrRyeJk?11fUi|Z8rBxra0}6ylNkB4^T_4d}$v~52%GA9}-wg0cuHaH0paG57c7x zgvL+=YKa|=AY4%bYPmeDN(iR_wdl-`8C@jTYD_S-L{ zWp)lFUrI*>{Fi)53#dhFS~s8~Im;?H`I7duHc-p+lNMD@*SJYH>~+-T8Dk zP|GnAp(j@gsO326e9tQ>pq66{{eZP(pcbi;;^m@bpcbi;(&fZtpq8BmCm6;+EyNxh zsh{>hEqCrlBo0XdwH%8)lUO4K)FSO&?0+aRgx0nFjh6gPpcZWR_3dx8fm-Yg^}4fm zZc>X>wk;L~)H0vky1T;^s6~s#NajcZwP<%7U@7keYLWh!x4e`rO~03%D{X%qsKo|* zpgVIrP>b|*KJ(9Z@f}(u!)}Y6KrK5}?3a%v0=4Wst6h)_)MA;~wsT$+s3nj2Y5Umj zg3^YwhkJsg&GUg;j`YY$ciseQxw$--T)!Krt!-fi!|DXXWs+V zvh$X<#|@yCJpb6lHEE!h-8{o?>zzO?yLl$vK|6t3c6${1`y~D#SR`ic#ApJw*fwZ( zPbCAjNMFfaexD4~BK_xjq*o76%g(FzJewY%mYuEI8wY?|ta}bh|H%Vtap)=D`PYuB zTg`9pSn*)oX|dd1NF(KTkJN>Xr#eM4b<0kZHmu4WmIdl38oi*^qVrOwz4C=+N%^r; z4}e;9zEn;d1GN~=`RXX$2Wq)Dy?@SJ2dL%#^uf8UIzTP=@z^;v9iSFxm0L}x278?+ zkvG^cWUTK4wcHE7&0dswWcfuV%n6wb)PkWCb>7_vYPq*quuTW3#VN*qLKCRv$e^@y z$xWb^6NApqCAWZDjt++Fv{wSPIAe}JO@ANoV^Fs zQpoONWG5~6+mT{;gY0)=snhHCa z7p}D31Zr{WELgdJQwQKtqkIsvuVG~($Q zGP?AKGBwT}r9drO^fVpw6F@Dy>6JR>Hb5=v^mj5}oPb)KLTwv$fLhLE=<7Ur0Mt^v zQh0Ob0Z>cnO4`l$PCzXu)}sggoPb(v+78Tp*8ysA`g41w{w7e1(~|9a#GubP{m4{B z)1+HKEqmn^hDB~Q7*CEcPe-QYm-z}DwAj@n;+XnjFHL>m`dP*S3#}CqtG>NJErT8h zIVB1}EedOcj6FatyEz{f)&_uD3^;!j#9sil?BR4Phz|g@+~GE4j_g$_DVt9hu|A7M z0kx=G42f6=0ks4L=dub!fLau?hsM!BEov71A}@eiZk9!-q7{Hz_LdZI?_~nD>@B&& zZOH^`*?VD#aR8{LHZv#)uMO1Fk+~4mu?wgrppUZm=P*!95%-_WXU0G+XBP`uO(H-o zd%bV79*X>D`B%g}Nd7udi(&6$1%Wd_Eu)6Jdw(bZwHS*GMFwuRTGBH$jI#%UTFf~5 z3K1hfEdy8#hnfl0q7X4gvW8a6Xy(N;9s7Y=bU3q_&U=Ac_S%mbY5=u-6j?h%JqXmI zpfbFC5?U>8scFXee}Gy%Qr{}fp9X5V5fGjlrvTJ)hJ@=iR+tZ}J$0S?HgoI@P|I~L zJ=5R}P|KNkGmfqTP|F#(n;)4#Eym~pstHibx6Glvow-0QzNvExZl{4-&Uh5F{u2Re zQ7|7~cHC^WD435dpW1A-C`_I8$Ome<9uSynrvTJq96Hco3e@r?bLR902B_tC=E2j` zw}4ul`$~=f`~%eD)%Q)|>KIVV4enUx)iXdXxx}EptHw)%=M$nQtIS%EPpfi!Uj)oI zBo|kNi)p9nb|8O5@yxbK&ZkFxy`>?0Jlmvy$!zb9g*{QX%+@4aM}L{E4Fk0dx4DlV zH3MofTN`b2*le|!t&PPU+HAF$iI2t|gI3F{qO`ftTdT`RQ|npm-hM|jpcXUFc`Od7 z1!rA|?4Jc{86kRSUoZn|F~iLtf>uiL+4yfgy)ybmDIiQvs&0eER zWmZ;%TBW<7-GBP@PcnGKk)&I(@WWeZP&`4s@*nB)1s3oPyU_M?4sAbQ>keS~A zP>cQoyQqEvsO5I_@zMKcKrIX2w^oabfLfyT#v0XuT2TF;k>3}9S_ZfQ**Q@_Ed$(h z*<(>aEiW1|0MK9PsqnWebP%ZJcA!o6BQv0ug$=ERRx_ZMg$=cZCq+OlQGbe7^NWC5 zqQb}4FF~s%r|9bZhAvP`Y0>`ZKZVimzQW#jkLuq=0JUs8y0saoWm~K;mfxF&sB-Tz zkVtPhqJKL}kg0BWgWzU3d&0cxpduJar0 z0c!DBE>Aol4AdgmanJwzHlP;%U$WHNO={WZE=zp~)bhORHNSuiP>WtUt~NiF5f7l{=@KrLvJO}CdYP>WEuKDKU?TDsp0e{=+DX=eKHe@6qg zps;Psn~6XziLZo*_W-rDc3tLQG5~5x5ERC{0JXR>S=(}CfLh%AFC>`p1GUH$c=`Ko zwp#d0(95<7>--_IGVxEC(fmLyGVzT}8-Ab`em^wt#Ad67UrsLm*(S9l+6bZbfm+&` zO^M%SfLh%B;}Q`3KrQ@ka-`>*)RG`B9PhG8EeQ_7@ot;clJJ;+3IWt2!|?EThE~ha zHZP%2eV~@7%zgaN^?_PCnd1Dd2%r|1<%PuOGC(bEk(at>gn?RQHkz4!{6H--)2Eha z65BI8nNp_bcRm7YxoLQ!KvORuzsjpehb2^R`Aq5xv|2RFrP}k-t(?^L8!thtMKj}{ zGtg>rZMrgWzy_$rX34e55n3(B2cFq1Id4+Sz<)L>CpW2OAjw9>38eh5=v(=(GhK{q}Y_({P8N?liR!h%6a(Q1*0ku$PH8n5g1GVT+ zw9?LK0=3+vHql%)fm$@{&_;JRsf8S3BZ2~I@g^K2f3X2-xh3z$?vvW27ESA0NY_nj z8Spys(G#e}IVyC3s0q|!^U-%D`yZed&F`qVi_mIu8W_;5yb08jSL91Ly4h;cT+$_8 zg;t9V-h*8x1=J!Xo=>{BNi9;%Hp4nVE#QZ>Okpo2?ej{Ji+eO=^)EvZ>tz)Pib!O8cz| z)N&%;gIys7)N&%;lie%@)N-O$zp)OerHs}mRd@oZGC>2&hHqT*?RC)9UpnUG#K;S~7m?l;L==-9NYB^pblRjAIvwFx57g4k z{wKpm7pSFU1ml~AcGP>W+gSV}Ac zs0D#>W4HjdH1*sP^3nxrspHwvBXogU>UkFQFa%JG>qZ;f00GqEI_=9{%K&OY{H^7= zZ?;+x7u_~m*>w+PspjV=e~o{(kc<7?TP5CXt{3pe`opit&%FrW%ab+Y&*3G}l(oy> zgV~>YucjyD@bRQ4_P+aFr}UwCbA>YfLHzifhN#uC-psFGl|`G?P1mI#jF@x~3U9-Tf!FHzmY9HMne6l4#(+x%V$$s z_jI^UKZq>q;RU>yYc+12tA249giyi8;NuRIq|;Gl6!-#p8N8(L_tKYgf61islgM%KID-juc60U zrVjB%rp}#um<8>;lpNjoBdnv1$62{TrB1vS(ot5kP^mMogLIg6_Wy<_9komZzn4rX z;=N4AL#8SJd%75Zy)Mb3yO6EVAE8UK?zV5V>24Rgpi6S-sAux=tD<F+Y`J?m4iPze5Rpw94 zq=1`aO?JjJm#_MtA2O`zov6>`Uns?@;HvT)3fJH9r8SCa}QFU z7yPI^Yw!u`J^3dJk@s*=?1*!9jfu;h@`@8S_aD8K6s(SM{U7SGWYyPudtDUB9%ss% zng?C19@$#evHcM#dt6;m=UtHMVy?u33^Yni&c_v5R$iE~RGyZh;d+SfSYDY?;rg%B zgx9~0Z{6pNP`}B4P&uw~MT1+}q&)2^yyM&<;y;6jQGzu{_0)&>Q*$~f2SjBx-e*n$ zl}0vpEx8{>&1j~Zp+;SaB^j%9eiYip=m_14O+e&VQC;X~*ii`gDykFR6(%Ncs$gCc zOSqTu#dZ2FVe%^0b51@l(Dmb z**=JN*ADHE?%ae7ao70W=5AbahJ))BEpzwvltBYmct*LcXAMTU!ZXT!J!w$Y_0R4r zZtIjmKi5B6SKQar1`nYA%3Q79@R~b_`s*6P7gFnL&GA)SlZ`c9sv!Dh{jj{8C2E41 zWE}fiNxYgR7>qw@GR3eAbj}*FjLy{*isF)K=kd0I za#?AVI>nrEhF##r2@B2*#WL=m31`lP;-zz6^?wD!C`A7MwjllIvG%=Nh#jfYCPh^N zcd^HNcZnvMibz_R7S*z3L@P61nH1HrL_~XOw-rZU8hUfyvXNN}=kX4lb@sL_x%0>} zt~#-?&&I^}A$Kpatxwm)x0;J0KJE)K@vY(FjGQ<>6eGs5pAX7E3dl@tR1BBdcaD0h zS6?w4(Rxn4GN2=MC!sv`y{YZX$TX^R?`ctI6I;Z^a|I6qo~9}g9;BX93<~M+nyw&b z_gyj_MNDEHIPqD+w9ATe=e_R{8$HU2)qTHB(b7Yv=6AV=vvg@EvT97~rg+y=Jwr_J zwJb@|5IRoui)rZiauvz8m!nuR-ryMea`|3rr3qfYWMa8Cb-@I0;5*)Ms+XeZD`gN8 zZ^e<#@}gt1>d(j9aV|7kaK0$cjPo9lY&h=~XC|o5UtjX-`aUSmNPiF6IK)X)JSe3a zvSG=oR5X_^4dET>%~$M{@(bY|?QK=;l%5IU9q!FhydrflglE-TuXsiJc?i#VU!pw4txdGmkN&T`b5sDQ^f`3*&sS~FNBLq@_9%V{ zEpM(KJzL&33MJdmp_-mwLMtjBWnUZ&a~3E!`qk);+_w8z-KGV>zLD{hYcsJ}E}X1(wXw;=Sns4}{9 zl%H*0EF6n|+2)})(dM%7A1n0XaXmvpkwI`iM|C>K@zq zAiAAVwi;U;u^8{%aepd^j)kJku?Oveug3&g-A#W}{rU!g%1%ihITw{`5A!8X2Pkz@gNNu^^VF_`O$ zwRsA|qAH;c68F0Y{j<6)Xk`N8#tb@>RbJDrA#%Z(;q$K{>E2;^{E2@}NG;vI0+%S} zFbaXu$>tWMQ6Ssc@DMVBxt&j2QGSp8Uc(n%{(Kg&1<_W*(z~kJCVZ7j*xoKf;)8@{ z5s`hQC)j>wo4}Ycsgv5pOcAh_?>Ocix-6BHuh?Ph9lk7-ryw8UpI8vgyug<$GOe~`V)(X8hR>Pq zBk(|>AlyH-;GF-h?zaNpW!L130+)T0qWMKfjj=P#ZN$><6ahH}Z@pXc-shLEBn{D` zlEmfX$C*NnFPW&M8d3Apcc&Ej#Fa?2F3-l%uK)2ViGUazFH;k=MM7l?MeygCI(!b4 zAQ5~B(~J+CSR@p)T%M3m8xrs{qSBdejU&t&0Y77^&sRg<)9!sDojM=)@}6{G6j_pK z7wHU%Z0%m7{T4WA$Sdp4q4f(KG~!h!jL{YZIt`caC&0l(7p@&F>L7XDrFn+T~*@scRfEE=AZlcbHVh`?8oxDfy=f_Ax$M!5lsmL zsai_p|7@2cR<`5{X7p(l7KEN}x*Z(elrnHj%Zogv^}ZlDq`Wvd;)qe%oyA=y(oc8x zChOqtG}tn9)E~k2knX=Nzbktm zF_A27CnBSLTmCemfviTTBInu_5!f!oJLFe(MFE5Y>?_(St<*xiS5e@KckaS%yq~xK z%8^_RTi@MiOdNT@#?Ds@y)Q0hK-bP!vknuNH4tLwt9|rN$H!SwE$jaTE9^3jjHYN= zY%S@1g;YB25gQ}TvCZ6PblBE}*iJsKwIp%zPC+OkgIvu;hoD6W2>Em3RXc2PuB0(ETmNG`<|Eo)4OEl37P{tNcwpJ?=>xgeNp;q0V9y@ zz|AK=x7!G0`!!=4=$^59OJ|4?j&IoJvTKh$941>CT?7kC7VV1cS}W-qozE(eVyB0B z%&;|SW%LkiDA2*Bgz2?B9=C64z3XvOKStsX{|8v4$SCW3$c74MV_zhObT>08T}QNq z%TkWfWTk|Q_3z1soRN*D2zu#Ro@;FK2;@o-tLVLY7aB>_#q-=9ln;6rM*^lG1G4swkq@yW zJ)dMp8w_oW&aw1_d-Y57M;l3X^i{nQ19Staj4ml_ASLJ3L1W*SHIUBt>gZwT$<|A` zdv)}&AIsKDYv}t%a0Mt8f08C-`#j`BSVxJuDTVse1eP1IFoh`F=T)#v%ai({XSeXh z^!@s_PYmfjF2wwlbFvS-3e?Ve7MvkIp|{fy$S!zvykUDErmX6Xwo(sG2NBEYRBN@IgtZF9rN9sywhV?BI0Yp1Xh zT}#-(lQh`#imf0#=1CgufkvKip?*g&OOx9~ch?JVq`HPS^B&UQ=!MJe*Xs=Dic%sn zTJ<`^xXP6MbTs9==X4nN0JoDau6HGvtHEugJLp|G&obt=r5NbNpJTyVASFUCK7<8p zffQA}jbN5Gw>ia6Z{s{`AGbZ_px)QUjkXYPWw9`0vI`9SoB24* zn4BD=V|;!`p&XeK=1W+0D$FMjh4}`oo+xxD*M#{7u6h=}A>Y8%x5g!nIR5PpRtgNw zd`zsM4)l#+!W)S-)R8`{5+}@7LON_ZJn)~a{j_2XJXr~W52-_aCQ7fuh9!){hMUaZ z&wpU66h?=B42iD0I}(Rkc<;Syskv=@UcHb)mnyUg`xp^@+ar8+YoYi%Z#B1f-e;m8 zkEDf8dIoX=lyVpzx>IjGA_9}L;4L>srInkUl|{(S(hkF);hf`UW(9@egE%lF$Qr@; zws8L;y`9&^v@)hqPu|AAo)5vaQj{@%t?Q{H`{-VnnJ4SnBgKULk&9tVPu5FFGxLI& zgDvYfc#L^%%)!Up+q~&MIZS5@7bcE<`!Jo4xiE3;6USU>;pXz*_8DNVJmwbiKJ=+# zelmE@8)rBXN`D!=H+7<48{CxBE}2bxO~z7ldM_(aN&isBL^8$$G#X2L|5H|BE+3j; z1W2dZDPyEZl%GU0UIst~H$X`+Ou0q+oAMqp7eVFZ`9l|BkYZ&mc#H3Sf`(WW6Ss`y zsYi|v9A4Zd)tE#(93)uX>v_9(n$;~P?1-yHPGRwu$j{4$X+B65^iv$ZCtzP1I7^x} zF1Z(+CCz&O*SUt)MGj0t0{?Fbg8ah73WBqU*+)i}CxNpFCLu*G7+Q+NMm8lmBa2Y> z(UDJ+jFBSJK{)JCcVrqk3$o0Zs66?cJ9j9lRX9ZiaRM+Ww|Dq(Wm{vMjEc~M3KsPvxV6L() z(a?2z&i_u*C$Y2Xx5dty6@6OZWb$vUWV;s_gO&$9O-z+*JVVS7cw029@+VM*)$EDC9Qscd@l=}B`soqd_T*+ zafsC}c5#2aL&x`JzBEr^v566SC08>eD2FHW308>!#AdYu@ z<1156@HviWx3R<&7rcbr(D@U!+?sS1x$$LE;KHez?4f7gOZuOmPWR7sqQ8)Tp_Qni zi%mBeV{?Dd;*oK8gTFSBhaR8{$@6G0YUrmX*bOyhQJ$dRk$<9PV_Eq2nc~i?ix6l4OPzyS$k)dFIjZnWYu~{FFUDZs-ti6?L)LxPl=x zT3pV@M}^f?I0Vizq~X&u2hK87;qx=w{#t}J)*J=`+=^2P9+=r6wMK+L*OjvvDmnhA#fJ7L5x|?95_pUKDA?V z4xA<5o!aqo4xAa~^#3t+ zC;m{c@BhHF@4K-MLTKzmX=oZnQrRkm>?DfHpc*w~HyB1C%AhDz3TYW9$ueV*$Z1nK znZ;JAW@s7Z)bEidA8nSMrRhfTELO*7t zXo9n7c7%)`PrXND&LAtl|3$T!PpW;iY^_Q98DgE%IXDy5PM_IR`Tc%~>ACyFo|$ax z08)i@KxfFuMMKRyFKC1bhOzaF!eiDi?oY5d;4DSVP%JyDVPcxIL(`{|un)`S1Zj?S z5+Fqo)lsAw+O;O^_;TSRE9-lOhpoepEnj#9&hqGGXg7NioaNEI&~9#0)5OeTXyyIT zZnL*1mNOqIg{H5;9bYbZ+~rk~*F7j3fsrGC0o@P^@R{p(n~*r^tOCO1qUoMqD|E}heJS896f5lAFW@ZSCI(!CmB3k)3Zn}b`D$3|JUGix|7xY|P2en>s0(a$C2$rj^%GlO z37kcLbkTNF=q!}Elh`q#vrsfoBH2P`VR~T0-V2?DX@L!UCv+BOJT~kDILqJMulh%M zLTC9HNzQNA2WLs8z3R8p2WLsunv3kp2WQEDv&nN2oaJf%ekETlI7@2PY~)}0;4Jw; z`mv|LS-AKT=BZ7ih%cBiE|CJxqOW-}WEz}>gTKoBxe1(Q6OP5-LIG!?T*u!0CMeC{ z7mvj*uvL`6S(H9(O5lOBjP!H$YkA--i@eMEO;~UiKCdNz4hzo0YkSpSqz}$=ipbz< zP{3IzttW|(!CCsbAM~eq;4F{0Z}g}5;4F)Ro6LXo!C6kN^lQT@&=9b^ZkpFKY>|rAqKTd;o>-_W$Co7M&Hf|<`;~=23ahf|B0OdW~r+f z)2RbyNzjH1sdE=B%mTBpm$ujOfLY!x#nn9^0<#dme8rO~z$}#LG~^sG%Uh-l z&ie~6%XD-pr56Xxg7f}{FQfpo;Jm-%^C-Y92C>UppM=aZmYh~QCuEi}xwH}t+DVlGe>hE*b-784F13o&{!MFCD8pDgb7gSvp+zSpdwk zOnfC zfLYR3*wGXUFbnS2H$H`O`rp?*BJs~GqDbs1shS6VI>C4B6e5ii32bTE<<&*0x*ln7l@xmN5n`)YP)EPVyR?Hg~W)MKC(?YrpG+Os|T1xL`iPK1ehhh$5`~Z959PCVgF`v zU>12oN>7*wFbf1iAaEkgfwYU1$N{q$uyR(JNdmLT+OF+kt#XANh?x||1eoRaswOFn zF)+)4j1EzGC14gaR;S3A95BmjR<4M!2gfm!5j9ec`G0keoSN>u3s zvuH=`?Rg~v%py)zr-6K#Q6>{!<_ zu8ZN+FTS|2N=b^a2AIXV=f^5tQD7E19wEa_6qrR6tIfK)3YbL{tIO(K1*hm!X*evskgNi-gMovq%e0_G}OZW|0*f?>Qj~%p%&Yn|PuJm_@`{S`gU-%p&3} zBRH4=%p%e)&5OwZW)bO@;T_HZW)Vr3=0#@!vxub2@ZvLoSwwzGx0wO6w5~cWB{1uG zxaBab^v?U7Nef_>BZFx!pLBs)3~p^1%rgXLartDhm{AJMVpyG%paaaJf6Hy~oFOoa z;hUT|U0@ai7nw_W^Q&(wpOiKRX31-FnJ`4+cS&!^(d^UQLh?53Q$9m-U1R8!zuoo4 zu*7RbZC&(<+AdbAefMuy$))fLZdePWxL+AGp;ir@124 zfLRPU6Uxk7U=}633rWotm?c+BJt~o87!^kRv+N^1b)~8Rv)I#v4D<4VS=1IM zOXqTdS(F$1N{wBBSrk)u40^Z%vnZvy4O+MYvnZx|4aU0yv*dN*rWFi@%;HLxK|27m zG%9b%rhQKVj#EsJ%4S`t^w4!hjWgBekVxNm%;*yWm}LV8 zr}DiRn1wR2I;x)p%%bw0LfA|fbK{b{j2w!AStt`mQIANle@i-Sbh{Xsg>rH0F5?vC zed3!SjiRn+9H_D&U4mJgrQ2J?Dq8K*BpsQvovw0Ly2kjjN_>pU5=qM4a;tWGVW1i? zOB}u5=y(Aziw>Ve`o{>E#YkUUMFp6}SYNL_fd01L2l610Hxb4_5^}gkz%1?7>60~^GHHwQ5o2(r3=jBn^SE(sS3;zoP$>xa0h1D zM%&bPSsIwd*t5je1ehft=b%cXJ21<(Dg^#G9hgOBZfn;DPDn3l$!L@U%%X{pq`RpA zv#3;V>v9HW!E(e@49LJNWSTkWJxR%}`HpE{zlweVFiTMzt}jm-n8gUI&n}SGWi(3r zY$KQhvz(V6+eWYkX33SVHtx~_W=Wz47#%49W+@UJqQ5i(W-;p4=NFTJS&X{%_%)=B zjGLs-+a~pZS<>lBM(J9>EO@$)QMwK=OKOgd5m6hM5ZELSPo%dvjI)RF9Hje2qrqon|tW!-Ef`aT_)r8sr#$PqL!3z|YM zr~+m|Q%YW00JE5Fo6=&^fmxUshXFhqm?fI3%%y7qv!EpzT1a3P9qzPN6CIdEgZo7* zmJZB9#_uJw(ZDQp?1tG?G%yRj%XXGj1ZZ6);P5n99;{6);P5nA+0oDqt3L zD<#!+1ej%BD>{l?49tQWc#J+$0L)@FK-1zz1G7X=>PPP(1GAt_nN_O5EEWT|wVp-; zvqX;+I&K7JsjW(mdaIj!Gh!qaJ;wxQDeAPHt*4QdYF<|k zFpFwfRpQn@U=|sh9fX5@z$`MY+X*p!z$`MYI|zsSKF|kqUYd0o3YjG(3oSEX3d~Z@ zniapd2AJhsL`W822ADtA_m}OMFOwDr#FiSYz$`a%j+zn8fLScq_hn9)0khQcvU3bffmx`$vYa|oV3t}|k$AW%FiQ=q zT71eBn5B-DFYas#%tB>Via#_3W~pUS#80ULv+U%Z$U3dMQvFQQJLtOi@Pj?TERB<9 zGuu^RUa*Dky``IawXB1fFi0~o-a4@R4Kr?ovlwUTo-f0GNyKsRi3-ncP}b_4={^Y zYT&Rn3Yf)L|IDr6;ZODM!^XZwz$~%D4Zh!Y0<-KG-@rhlfLTx_Uigqg5spCJ>@o%o z%(AC&3reyHn57Zp$UvchS*~CwsAn%L6WV*#Wb>FMoNt%K?~W|L_Ojx1~a6abi@dNBaV^?8GM*2JZo8@jzQM1nO#Z zy86AQs&ZhK*uo)H<7Hr$ooFk@7j=6&tak4i^#*1!o1vC}Z=_mOC4~(yqkviVSbKJ^ z1!h?@^LP38CSaB(;_8`f6fg@a;7Ul%;wAMvFD?@>3}X~93+hGFL@6*!U?I~Ndj^=L zoHL~Ea|M{CjPpu;>9h%oBSpZO?IZc17>p zF02}u#g6+``{68{Z`t*EHPS@kwjQ$iOPFp?>P+2V&6Y4?`sFl z5>s_9@+7RY9LNjOj*SIoDZ_gYoU$K9j9_fID`a36ZOsEA<-jaw@dpQf+5@xL<4XCx zWDEK*`MTZBQNb=)XR*Ul*=OYA8Gp%tuum8SX0hz&YS#_|vkde0=QY^@vkdVP^5*P- zSq9r2`-`-JSz?JM+;ik$#$9sj0iqQ!i!t|u_S7ISiv{MDF5-Tw7uayU8 zv0JHO*UA4ra3?eQ#kI2JRw^)y?{R2U>3yWLzajHFpHc6YT`C9OBcKfZY2lIB6k<{FpDK2r8i6hm<7Th5H=BJK-wirEPz?USvjI+8o(^( zw#_{(QD7EPGfm7DA+v}!X<{0L%#zt5D{lkL(!}bN7_$IoxyZ_ua90Or3E`d1s?-2x z3F8%J1!w@Xgz_$Az0?3^u_UDR>dFe4MN`W+B0;##B5}kz(L=b+A~Ar_3I}EhWPKAY zlLcn6wC(IE7X@aKXhc+n0kZ@}?CpIe0nDOK_2s?IN|ultwZJ?SGD~l+geC%*MLlF6 z?-wlpE@mmI6Wf4UZn3UqU6;f80keoIX%a32v)t_YA*w42%woYKWSYqWv&dqDSXV`X zS!A)ntWHs27TE-AUX*Z~MIszUy8_H|gLPdZ+ya9$u23cShbHVZ66SBZ8vfaUn zCxqK963%M{k-}{j31^hxpm3W-qI)ebM!3x)(T(CA7H+dhq_5>g3%6M$(owv4;WmrJ zueEJWLS_*?ttn{gdFXbSb@tBt^2zH$W*JJ``6(EfCH&Twp*#~{mYtu%7c=Q!MhZwc>uFiX!%7Y!Zu4>L74}2kC0gko_bJs1G99{ zgG}-&fmwVPC(q7R0JH2`>^p0`6PU$1b;pp02QZ6Gs@sr-2QZ6us@G7w2QbUIuI_0C z6CtyBkWr982WC;-P)WNYWENHKPO=R!izWV4!44B(mUC4hQ9EIq<)-S>ot`$pESBi& z(+g*TS$2-1Qjfwmi^*ICH5{0wb-M3t(N18Noz_;Vr(v5Vbx3L_E)bX{D9L~5xru=} zFiSzc2X+rIOF&dwfr1GzO9i2eu4V$vVp40#_Y-cjnABSFLxkHbCJvQ^b|JHzZ7rYN z1I)5_@%h=-3SgFU;)7{j6JVAK;{EB3CcrEvQ&ye#h0Jo+pnRoG$Sh~;%2#d+nMF0c zTyRUsEUHuGg1bUyQFSgCvl3Yn$gp~s3ZXX=(iB5bpmfAR-r zxhH(S{su5h>%_&V{z70D#P?QME(T^P^fGt20nE}mQ6KfF5SRtFgm2#fW@)|Xx61;U zMdAS@QPlM;0uc+cC79-{;NI)3qKhs~vXQXOf;cN%V{sV~AA?vblnSui8q|I*&{xPT zjDGXuSAkjf@kxdMm;j$YIfLSc`L)!yjo5csRAdoc?MnDpBx#qwu0hmjD1qfgk zbNOqwG+>t7eM1QEtH3Nb`raW%n}Jz^*e$aC7Qif52}c-9=D;k?gh+-80+=PB>oVsG z0+=PBtDaMV0A>m3YUJEO0JB(xh4b?Yh0KB&@Y4zwGD~i?#pE6#v*h9t0|CG+dub2) zF3SS5Sa{yFy&`0m+=Gb50AQBARsQ(n!fh7BoL|>HV3xuq^U+pdmH>Pt!wmtq+TEZ5t*`|@OgS;hR}7FnOYglj@( zksaGhxG7{7*=mcfKq0d*0?dzG1!lP}IK+5q4$NZS9mX#%1ZFYs4&~Pr0<#o;-a8p8 zWEO^!d3vCbSr|U%>HCDtA`A!w37IALs5voM$Sk=A<|~0hW)Uv^?-Mc$?D{WU1!lS4 zH--3hm3P!%mzHm>NUYQO6x{)2}s&J6DyvK+pqeB<-| zm&fk^mZQ8m_TXlCnCyF0bL+*-xaNyzvIb-?kQ!w#pyF;`%#FKxu_Eh*?CZ5Hao5{w z`o76-B$dc+MER*7*SO~5b?IhXk*)d#S*zT;w9S_qlx0Klmt1@+69Z7 zdnbI@>t$OJ|02Sgcqbv1q9A+Jk5JZE&9IU^>QAWdd&AhAdtz;uFMBmQkgc5SigPZZp@q!rHA$%mWPZQC7nHPh0XNTmrt|j_! zVZ_UZ1@Q8+Qlq-FBxK+E@y-!~dtD?xuO$R@HL=np4Aus&(Xa@tkGPrzTYFguS)5|<}styWJrcCc94%RV9d zP}6zs%5Jug>|N3g+4Q}ne`B{Vtm{}5YT&C z_8aMe>^IazD=TA@Zg((h>Uq#bZOejN}wcoZ{zl#lqU}-PJGtl16^`GlkpTEL9Pq{+G?|&Rk#X7t&I%?r(0rd)&+I1paEF zzPYt8zH|O+ePyAA-xrO5YqXti1{G8NZ6L!Z|;0U&qiH z*bbkutmyEECt-Fi*AG|vbz9ChQ?D-~Dq62K6TBE7%-`-p`-}v%Z)Q}Rzg0bYJ+PUh zhk$sspRj}kQACeIc=%cRw?l-f-31y+AVO>Iyr*;D47|fz5$~6NE$K#JQ__RsF~4+Z zvsu0}-0F8kIsE!$b5hyxQTl;Dtt2zwIZ_qQ*H(5gG}`?cW9I3*+r1J3_}2@k{8p@g z?cz5T>X{!e3^o6?hkvi|H2nzT_cehJLm&QM;1ECkk-IEJ;#-;<`O3x58JBdFo)TyS z##qGZ#mqxkZ!VuBUTk)#)=_QVA8Ky8)SU-a19{tYbmHTTE*y$$ywsVeQGX?Gua2;rq`m&ez;T^7 z@e8Y+4lP{XJw6fCth2uOuL0}6Cvk^vuX_z4kdke2DU4j5O~u^qONms@DT z{k96<<7fesP3&|9r8q6?ZE;#xBg^xo7_K_o;~O>`hk9Y@&1 z-CA5m_tmL;As`P7W$84l?-HY^7`(2 zzP|#rkq|=rb#-P7mUT4Z2)_OBM5cn=`mP4Hu#M#Vdg8^%Gl4CU1$kF=&J_sDNd&w4 zp>cxit`iWy5=gAa7Z3QXhp^kan=b|D2ZZ$`51$Dt1{9!;M0fp>zXin;E&Sl(se>zF zD;N3h#m)yOjP4#11ohkK+%5R1lm47noj1iW(E0VO&9xsgZvEjpzh3Yt6F1o53cn9d z7=Jr7(G>YGPi&t_L=ED3ihg~^Ul_l+NGGM2sX5KUV2ArV>^@|}QeX}99X9MwTgduj zFI3ImwcJ?RU(-bLJyzSuP0@=b*&m}`;U?%EqdPi{KF8OVEV+e3!^yLk)@|ByGlUL0- z@8FP&Pse3yZqjx&H9dW)P1?nDb28Fv-=324{LFi3VQP4SZ!g*7_l~yJeS_XVj4Mtj z=V_kInfj)ZFsGYO#)Ctya?SP9QobPqJ}fjX>D0U|{N>d8H1)x#2uHGJ!~3osbB{S1 zHTT|kdCh%{x(K}^Z?7hoYPBSjMlU&ic9K5vL93CgulK$1XUb$NS46L$A+2Y8jQltM zB4Il-PA@uzj8OVO*vf2h(pTfAkYA#=2p`1M^LbBgPCiT6Irr45Zrm|w*PMEd?=hT3 z?Ndjuxs)2%v#RMx8GQlC>Me6cPT_wq->PazzBw9@(tWLgvtFLzuF92Jhj~k9RBE=ML3C#yaKZ`0~|J z*pwLk>g4=%8G4WADxDtw-MNK$Be`MJ%juy?^|AEpOB-vZiVRYIH7}Xz1=IKHIe!qi zF&)}NnXOLgPxiBDuGn6PZoXB0gh9M^StEl4e@O z5NsrGhnSjzr>c)O;dPI_q^X=vRaC^a%veV9|H+(WzLErS(ISlZcu0n6B+HMi=M`oK zDD3Szz&nspG&u{4QfRMxWbp`RBAr?QXX`}S7rjjIG&&fDpP^2i>bMg=bMeY@4 zntrmyHx+CVhEq)QF3>)#etS5nYUq^bTZ>QjkN}bLRMdn|)$1IGr?WYRWer{>Xve+S zghiDy$_1h)+1~N(w0IdjEaizi7*$`egM(7@sU{52_is`2_4HiNZy}7**Qq`Af=`xS zBX;OFblCD&v$;SRqCbi%Dd;o}L-Vf<6~6TBKC-A#wnQhE$$54kT!gxSps~}l9J+mg-ed|N9ZX+T3r6VBAD|X~ zK`>00QQM2p~{dn2K3oDPOKXa7I zPG6Xegl`Qoi#BDat5@6?9F|*$etUjWIpy`qzH@3?q(ikWxA2gLV*9R8Xo5VtwufjY9=xme1}c-pci*U)Au2z7U2z{k+DQuq(uggDjfc zhaKZA*WYBE)c6n#FW>D;*x%mHFs;AJIG|A-oM5Hqa2h`fuUmcTNG#5ccRMZj zGKluJZxuc}SJ!HCKl=c_C^y(@@&Nld`bnRj#;4o1+vV@t())JFozSD-w^HmNP#C8e zX&RsJ2{IV$s2s)_4fi18F6@5%Z3bF{S(tByi{kGtq7>jYn8DNtK9mD6%&a_b+wQ=| zun)?O1y)(cMHc2>)u=^`1yvo0{3{pMH*~DGn6#aZ}Z9+R#Dd( zbymNk@YNGX_yI)^gH}RUBKbZ=8!Z6@Eq}%Ej~4wV{iE^wj-ZaQDXJBI65oTd5&oig zckg~YVCZnI$n`Mt05?(BouuPXavNV>+`vFPl-$8ni@DGf;(dGf(`{Dxv&F9+yzk-* zG0z7MI{4g6&BHvM4RRQ}8{|%fu8``vSJ?1pcY?+ar2T0xDNb;>`)cdDJFi9#Bv)H) z^{Bp)aQE)3fo*QVx*rp6-y7eOa(jFzFRGzq*Woya6y4CA$o|`rui6+A4oKZLcA?y1 zf&)=EQ~Ge&e*0V6Mf3v!)mI;TPCLj|Yre5`t9a~@aJ=JHXvm?C{Kt{t3W6y3JH_b^H{0qwczVb6B-^MMYIqwdoL&wGG>x1p;fK^Gu5 zp(~DSqH9y+=rH@5z!>x=PRa-i2-L~BL7@mT0(0ol~&O*Ua_2)M$J+nsb z6SKlV3VfhCe6wx&(qkyGDW(XUNuV=eKnDs*{tKgU?RDX8m;~ZMu7$2H{Uv06CiZi6 z+g}abbLj5iKM?c|P#n^4b-E?oS?+ioo^V9%&Ea2%mNryBEOJixbuc;qPX+}_Ne(X= zw7-E;lHVPj?hAK1$7a*3r51MfD4Jh$#L%}K!#8L5Baxq2(!2Y6&nT88CEmsi^tLOO zBqw%ZdV32L4>>iPcqi?8(>tW-tP;R~{(KxW!rr9#8S&k*YlY5&1bWs~>blu`b1SyccvOyGakrjkt9&8>QIjMA(a+`#*V; zs*fztm26GDu4z^bn%Ns7)ruiJk7UFvhMy4R^@FbTODImIz#CI<*e>~>G4RIJ94{^3 z028uCV`B&>@oGQlNVqC}R#>H$ZKFq=iO3eb z?%gI;jU*a7Y!tlcJ)zjG>5LJK_xdQ_Bi;C~j0AM0KV0wEEDv-A%1AyX!WYaWL1q6# zyf3m)qjOnKybJHoFC1)O(^ozuHN(W|ahtC!57m;7>zQv8%$&e)=s#LW;YI z5P~;kiQtq;%3adrp1?H`0k#tMXR;*-XqaPps%^Rxp&_#0R@sQzXm08aV<94r;!OyM z2`TuSko+O?auwWmdXDvc?OBBvOEgYYOYERY&=52Q7+=Ldt@)zwu3v!Z(#fZSvDuR>o6o&&NqbV~s zV<@XfJ;I#VLQt%d^pt{Cm>$ar2Ut`QA*c{!;NObKtZ3`1)+|J8W^4A*446L>MFghs zyVnho?iPb*F2$y#oL9j=6qaft3f5z!0x}G;U`%BhN37?y3&aD(h_bL5lkv4Dwg<)z zmIuoMe&sq7(g~v7D%F`sGe2j*|B)U_j}J?u9kHUlA_3EB_QR})nQ$=lIQQzYyxZkg ztnC_XJV}3 zmk%+c)Z9lPBb5Q23*7ov+G{sw&6hOV-Y+NcHANpn<&`(kXyXho>mwTXXtd{y!`4SM z#X=|99B)mK5lz<+`D1waD|jSpHKLQ!CcE6A0CbbYI zW{j;3jxASd%_Vg`lCd_>^q~>|W^C028L_TC&LQhNMF*`L@5ASSzp7@U z?iPWJScj$X&lZ70Bz;T3=~8#+^zTC7X%VHQN=|*q43uc(sF`-z6fN*x;uqmQ3Nsq z6=4ZU{NjlPPO#=wryy*F#c|hk?wmA&BAQn-HrDqFK3bnXwp{%PWW@S+N@xEJq|#=> zt$!b1rc8WX46pp%IcfauugSTjhmXXLm_#Tco@0@mOMhXW^+)O}eoxG4X0x{|xu;_G zA-U{8C1xrnjLKqfRa(F4nW|Z8^GEi6B{&&2QD^xn6p)ckwKMz#3do55*jGX=6J&() z75m~lw2Xfdw$o5RMxcVm;X51;UnYj5<0*%+ez(603ulx-MwDv6F`eXuEjZtwxczCt zkqQoB_3{{Uwti~o5!#u1{8GE<(Hkg0Bl@QZ zT_aqe5oVZvD{mq=1hPPsH0&2dz^VB;wvG}U7Wy}r5+CFPjVNt6NgL`18d17;lJ>YC zXe56lcJf_hsn$XyDgToGCnwU0Q^;AqJ_Tq5`}Q+mgaS13rv+sFMWNcx*p+#<83kyB;(Tg_!F6c=huf-udP#6!(9f;YKS6+{hSZfu+$sIvD)%@2 z`oMRl0F6+7Vn2N+^jsHTXsEwHaV5e(kRO;4CL0GdLPWmft~CG}AtFC;F$O>*MCyD) zBL!&0;7%HKfeSQ3c}lGPv;;IV)UHn1 zu>Qhc=B0@neMvntHW+KES{Ituh_gTQ(~=mr(4h{}&42Yc7I=P4A7`_a+ByHOqjJmw zf<^&9x`ty741nixKbC0&OyGIS2V%(Q=nPsfvlw?|5#K>$FmK?FEaC5K&2tqEPW;eX zBl%N{6WvU?ugLiam%rQv!5@hGeE&)8nM*|X&26u1AmY~QFFL-S)2aiW#~JXKo5$dv zOdU7Po3{)_n{#D2(;ro9Y8UypDX~OPjW1{h)6W2S{{N(k3_8+;2{C>>rIOfXIV-68 zz(p9e{%C=snGd)ro}i>#n=9m5}n(6+$BvuDJU*|!z{@Uqj`MtAmE*}?A z=#RrKw!dZq&l|)OiSMIt1kOc2_+Rr3{|cp*n4lc~b7DTai~>AwF!e()$^@P_aQ-0} zn$u{1J~xIt&F}O|_`ZC7Y>N2H`WJ7xX-p3X->GrG7nkpi0ngX{{;@JPr{Df^4p;Xx zO`m5WBGufi!75z^JTIasmq^Gk-deoMT@qss^G4MwrXOIzhH#{3Nfd?-$rjDF^gs19+eCDw5{xcLCTO&!_Gr|{ z^z0Q0mP$~&SbsR#72%Ian8UMB5 zu-rr%GN1+LxI&fW$X5L}X#&1ykDmn8%-qX>b4BEoc$Xz>ral%zMy=9tu4wbFWu%G1 zxgs^M$+{pMLPT!LLCuT^CvVjTNgAr>G3VT>dy+J(o{v!x85>0>wIfQkkP+K6E{O=w z6)B_!uQmhD716f_yv&S#1}Vc>tV9;hm9?z-45*_)C`l35uo^_*ToEl%!WgpPVcZNK z$ttagvjp>=ry{j7wn4VSM3ESrD{{70JzrMA&gH5O(Gf-A{%lW?NSG{~D{2u(G6F=q zblNy~GFqjG#w;w_nU$4sR1(e=y~Ice{lK{*CrIy65`}X`%vy%Fjs>{@R$+#LB%CV} zg0(&GSHZa=+9Ja{m;qZ;8DB+ka&WE~^;C*<%fh*$7ZIJ|B?9M)7*UsXe-)f7qAOag zri@_viwtL}6%&?2`zBVa$Z0t^SBzP8A}7S*T#;C@U`>g@xgz#Um-jH^6#Z?+Pti|G zgw@UB$ol$yBv%(WSMo5HbhaU!D=tWFPO~zcD=tV~4ow-(6&I>j{Q=TG&Sm90u2dvP z9e;@Q)TLH`Q67I_u*0QR?^X%MY`W8MOb*T!-CJRUKMdhqG1SdXK;9~d)43HiIBN*! zieYeWg5E7JQsG_0*ST>9_1W`U^}f<3TVpTfJ#m>g{0lNi+I_NcuDD>>y;J4^AzFQM z?0o61T8s6iy9Wc;7~aqGbUk3C6qM72*^j#B5~g}9c<>YF@6t^!VH%4`#2JE(xPFntdy(SZek;GsrQztS@UA3X*GYBG0lWt(*mY8aa~y3;*Mqbedb_+L zdJX-Fvd~6c73ETW2Ej%QKPxS79%M&t9Xw;`o(ndjo|H0(HiX&L6{os5UJ5p1$jqlI zFT!j7gJ!Ot^5`8{1m~UdSWcCtmM~Fd2sV;ir4eLE?DTn8RjVQLG>5a-@BZl1y{7@3i zw1(;OeBVJUD50ry{iT{zJ)yt&vh-mt*oeyFSm{O=uo0JEh{eyPU?YaV6_&#XH$@#C zOf&qRk5%3)TC!`mRiB{>*oZMwgUyqMYm7<>Ingf%Y(%A`I58v#Y(&MIyvyFj3cn}k zwTgF1Vj*UI-$4~03Ly_;&k0f)D}lR=8h?^B=LR-X9B32uFA3^Kgv~T16I<66Pgq7R zk*u~t(#Ys;$%Ltk+Z$oo4CTp|6!H^Pmg=RBUC@B@MLIM`*nr~(Hljk6Cb0_Tj0q|$ z(wWl6Ve05Yt)ewiFG&aKp3^EtBu#fqH-EZ^(eWa<*yyyo(;pdujci*$@C!*`BPy+h z6J{>q_+vTgDmM!fZ_FF8|B`;`4mLtcjLxYx0vjnvjL&&v1U9mbX53e!0yi9$yscU~ zY_O3N^hlM@1%y_b4d-7K0}>{L{9hpr*oaX{3C5HY$WNrZ8{u^LXtXs)KvFXT8zBky z=M<>GRmTXc$38DzCSmMZV7tDrgVQPf)X0-!>pDWVwdh-r1{*Pgyw4SBuo38?A(-@C zSFs2y5}c%OFajGfwk8P<(7{GjG`A7WIPgp^>9hOFYK~Dm=1=_$*og5>k|2^UY@MMP zwUofkXHDM)>AG!TBW{5P><($L5n&YQx-{6xpVk?+Zo6aOJ(cbv-UESQA4Mg-kmi<9 zEJ&t@8-a}&|I*>#Bf%BO=$AI$oesef`a`2%dVGWSjhu%nzl&(@3&rr0_(hVc-`RsU zzsA6<*>(&cNA8b8>T{)c8xNd8mlWb}V@wCy(IrLrE{x$o0s7DY+>e3F!_^mE%}CJi$c@wP@E>^r zMBN;x+Yky#Vz3dk6Ju9YipC*%2plyO*+x}C;OH)Gb)K3SIs_jn|6c>nNKw=_bo3ee zf{uOh`2y?#l_gHqmF7aIm=R`-JfgpmQiDbwxs6plhze8bxJR>_^+*4t!#%iU+-6`s z8f+vwL=!4zz(&wV$ar5e*a-SaF+Q05hS8&Sf=;s^SdGRElxn#~3vHyt(Q(TH6!LeIpQE8YgdV8IeX0d( z-dZ1`i5gsZkeh51O+-XzLl$U2t2zp7geh*7SW{cY$@%%N-){0(< zfm!`(NW(*GFm}DWWlA)dYcLMIc$pGSvo#p>mdE0-c_MSj4CB~~;ye>?GQnfycnlGSP}dBoitA|K>H;7D0P&0*&RCQ@yDU1UcN6+If7YZ6sgXIfZ$vP^ht;zpi?-F<5#EzjnJ zsx`{kpE0vk4WqWH`$O^vYG_X25BBNH*mw{+=g+qV^jWCFXlvSK9$6<2qpi#j6Pju7 zo%+*RA5=r?A~I-(P%m?2XJQA_oH?doZU#6-L{g7$!;q>@pcsQy6V` z@=~&5RXsI4ZB2TgiNk2C>gi#-sTVf6dt1ai)L^vT&Wp;bRE5!YO;t6kLcGo2ZKL%{oe-6$XN=d%tu31cFd(jrR5I zF`3glXkH0Eg805ERTyoz3u629WMH(F`MpzcqVF{QQ_gSG6&tpG`v!J*&QCS{?dBp) z&CTaHrRp%+q7*&wghCi?)!iF0<_s8Z)tQYLYX*$A>Sm42=j&m#b^EAp)&!&N10}I9l-E5lx`D@wv7qrZlw;xXgj_Y;4;DqpiBP?Bzs8E~-mCy>N35%4Vnb z6|&peTk*r|VoVpy-G+;N&()jHn4s=o0blWYWjqsu0$+iAQL5YUTh4HKz?FqE@D(Sw zhw3jcgRl7DLkmCng0Ebrp&4E%@RiH7^^C(P@D<<5Gn`TNY&uKbb&tH4&{sCjw3UCq z48Bsi_^Vv$GWg167U#UzY4$GjxS1uFn7<6^;6;!P^ z-?MNiOBg1)0KO7Zco5at1isQ(RS|W(5PSs{=E>hvSP3;W$-Z4@InT=xtyhSfW^_^D zD=5v&M0@a+kL7ZZ_W@rC8~*N_UJAZKP1-Sh1qHr>3b?#t3%+8`a7XQJ0$({B8)DsVoi>-r|_1YeyfG}o}F<-u3%spt4#$lxpXwH5pYGWd%2 z*keN2z>?cPjZXJYc(0Tqfa+)1ngJS0^*%##D8l-*mQDRyiT!Z8{ z9H60qSI)`bJ3zAnUWv%tXg7Htc!e6dJ@1mX@ER10Jjbsk!!^k6Z5=<84A-DPO*F!5 z5O`KNkhQKQYWS;|7RlqCe{ZF-Phu|92UoP)!57(eawpRUL zqdvoM4Jv~IngF}53+{se=iT$AiI_!-oZS$2IYO##yt{VgM=xdVYmj> z{HdFPYmhy$ntfj$u0eJyRqQ4*T!YBYu`4FPE1S5j+NU20uR+{8?GyXq8Wgi)!JX2E zYmoh~YW_nqT!Y9z?LLhWR$mvVHq`GcaZQ42&@9H1$##TmP!jSyw;2i7pd@59mxhFE zP!hGO;XnyogOGQUsZ`*VLnTj>YHNX44vcmr)z$#7n9X%Mjy-~FP<2Nbw9>#e$WeDD zfeO5Gp(AK?))B5jj=?htH630hg?Ala&&1UMuT(YoDw+VV%nGwca1F{o4_Tukxg?BZ zX+J`9miJ%vP1oAka(j&(qc6SHy0fh zGRk)}n+C%_vowTMONPyp8G;kj-qBQUw(4y3p%U0UacrEas*TR8Iv>5ag1q>;z`oh1Rud%Mp43HiY0$y`Y3(qk%Yo+Mk3EPIc_=85Cf*z%T9wP43w7`xT;0T*1$q6IvDCUN! zHA}^>arZVe@2q#eS}Y3X9#;(=#g@=bWQX zjGj*ZRhdkgs7?+a{nt@xx*|i)&(9)pd**x26VZGGrmIJ#ep{xo9MYc^hxg2kmO}=> z=Ny+zHMwV&yF93V7msI_$${`&CGP8eDhI-EN!Z)lAqT>*j=jf9l>p(Fdtnjy5WeZO zMKokV#7MbuzVZ>u&8z#eG!bOk0VCAJorrz5YN@^TEQW-~Zhr{v(prN+W; z-mNTHJdrqs=&FYgJTbl3VGa06KuGAt@XUBwSUl0B-R$`)0gETHlYy*8QCK{Yo!rN| zD+-Gz5}(#4#%6AbO3XBr$kPxmo`fa__MVc3FFh87)4fu%uy`U{Vu`uN%H~|kYLF{I zVfJD}cxSRG8nAd`f$3)DWWnNzEG{S_Dsu-1Qazf4#-9DXqQ0`8R`AK^X3uR=D5O!g zmm4v+z25^1`Jy%&gsVM|qP#OZCBjg=L%oGBWxK<8#aT=Al`J{g?jRnNCqlkx3MIHn zXk`ggKD6zY*0js4zyAb{EM(II6R!mNvl>LN%faFas;#l7Ndgv6BwlC=THp&%k!ZMv z;98F)|5#?RgaZN=PppMmAX!*E`I7~bg~b!OU;mG(J8_44|NqB-PBqln3WIEwLdIUw z&}6G9l-&Qe*jD1c~qi8`XODIHHXUr1Ha$1a%%&4?TGxUzm>D~D~Ue5dT zyS{(Ixvtyw?BRZ1_qhkcx6`jU-!pque%_h-Dq>`&t4w&1ZV7zEjhX4H@V|mgmQJ$% zLBoe9GINIQJyRK#ilp4UC|!92k04EdQIjH_2_K$pB_GMHa)u9249GFL)6VeW$yPEk z_p&a0c;fubgdCR(AD-wQGE6zu4fVTs)uLZBbpP{>@p*KfY^yhp#^7Tu1Et}YQeriV*9+aX+W4)(kk&=29t`Q6UJ zz7Vc7L$!!d0^y2Lici0dE`%#C-weqIb0J(&9J|oojf5+?8ZHywNVt+4;4YB}*+3t~7A|QoMEn!W9qEpkl}c2vXeT%}CC@xqXc8%b*b}f%==FrTrKs) zDy(G#QE9X#?@6toGY?W0?EPBeXYBnNh*_{Lbq;@H;f1LisX;qqtYuo!{+MrA_=^bh z80+`XOsUP!9<`siGyiyc^gACtviI1?mTiBgwJwSu#3#QdC7)>Tn42?n-cnJYbI;jS z{n*Qj14s6Le~gVaH1u3L-L-)+(<83GPXBP&z7g$ie{%FzW;f6v6us??3YDGtDt?G0K(o*K0Ek4b==AYNtM-c|K;rfnKb$=!QE}5C0(%v&IOx z8ny=KE&61=xEg&@#S2^RFEqx$WJI0#JNoNw(&hGMb_cdRSO`DTfsXXXHM0RLW)jLG z82r`TWJ72;>T6E2A*C^M(QwI*>c?`zPy<=|qNiw^x~T=UWv(f9>0V|dcFCXW z#d?pv?=8L~@Mqme-zT*1nXM-gvN{Z5S0qHJfcfPobR)WD*%FpqvS3#Y`?HJPh55DV zS3PX=r2DjoXxY-Dc83)z=g0-X!!k1gAeu4 z1`66sK)qH@AdKw!<~`!_yYe@c)5_m`M%?Vv&FzGC$O4VZ9%XqSoGbjD>>%8qRizAe z(3q@RwoBIXB#x%B>%WsYCSJ%S&XOn&m67RgzlDe*Y~8PXL&wX^(Z1pO+MAhJ6afox zJV~vYWHc{7TG`bdnWg#Fmy2#hHfQ9B_t+YkNtmQD-zW6`-z<$d30g+wdUM_sCalmv z^>u4`1*sZQgZ{caGs87PHDmBMLW{0v(lPkT(1y}mp-2}9DvJh`K!}S zx#BCP4xro&DE?usk;*o8Vymn{?gBc*eW{WSKVHZ7K@$uEH^0DA4+sa*W4Q$7?pjh8 zdyMlL{lGX<<1jlK};4OYDG6UOb|jbp(| zWff90(zmX`g&<=w%BnRV>R~JftYBVyClE%#%BTwpN@Lu}!HT!M zkCp2?u<~yIg(`2bvb=}`Ry+yEZ-bQ!7o7ucZZ`4ySo_TztY9+o-1luE!u-sBZ?IzS zxUZpd?ae$ahER)GA-Cp{v0%m9)t!7N53z!6Bv>7I2UZ3&d?s(qgB2(MwcSQ0jOLRr zB;Q7?q%a3QV!_Jo@`||X0kDGoNjY3Pk~I9UTqLQ;V|Y{RMPMaw*qhow1S?!77WTHl3O521;0;!41=Ylk z0kARRitc*}9dBqlB zB?`Z9x<(zWSX^z8$QFW?LgO9x|3u!E@aV!=3$S8we@tLMJ+2N`Dyp*McYu}SxR5@S zI#@aWpSe8U>dCOGAse_f{l)yd*Ie1s( zTr7;9BkzjvfSlWLihp=(q=ikL#HtXisE3R!Rr0_J&j(6DU?qxrV0us;tXOo{lDl|d zMI9NOfl`pM+{tLcZ6R36UbN_J6vDfb&9I=>3&BeEum!at46Go7Gk+rQN?(LrfCX5I z5>$tE@W2W$NIf|ku_BQ!w2k_CV0Lt=nHORKR-P_(@VeB&O4P(HiD@BNDNK+4=``Kr zpBSq3Jn92(zY|z-iq(?-j9Bp?IPKN~E7tN)$?Zj8#k%OvQ9C|ZDRQ({|BP6f#$Dlm zMy#Y%4izVWl~4G6Gd0#=CE;qrg6tKra>Y3D{-4OZ5*d9ZHUX?8+#egapBc9XD^*oR z@rhvNBQAuGvIZ+3Kh#eXwZMwj>x6_ccvqf{x=1I06;K31B5b{<)Pa ze1_mKv;Rs}0$8!eT&W!bD_&#AKY$gadUOk{eEjq+0jy|c6uCQXA!-pm>`wqI=8jGc zRcmh+VYLXK5G&-?BC<7DNpKw@-zh?@ST~MY9hd%k3k@Ke9D4!KQA(>Izt~4jK7T`Ty_KfR#_w12coxUBrI7+E&fE6ed z(fAr*#f(zPDwYK+>Ui!{jTBf}E)>zg3Nl)=4c?W(&}dq$ELf4fzf-`P8kefQ5`MiZ zBVG=ytjE3SMM;5`^&euUh%3R$%Ga_9JK5$-#Wseu(p4PRZ~Y_k1>`ZrmyvN9uM#BK|5B|&Y! z>sUbLq+$$N2vYjW9dnJP*ON~M9m5gCo>It}4*u6}t z?rrd{@OmRw2FQXHbwNaE2MesQf~1l)z{*LnG|g5Wte7t~vqEIS%J!uWR+ki5QJ;tu zo6^7vEnVX$f2zkZF|_!3R4M+^5wLP3wph9h4^|G39obzBR`e9Mr?mHj6+Onb@!q#! zrQcCcqYMvL=5b2=vSVO{T$x+!T5fdtGLFls(Stb{*YXo0StYQdWPGq?8yT$ZjaG_v zZNYqTZP_`>;*9InT?xNe)gOQGs=oCl{F}EZJyT`XOS3Wa#A2|5l!11RgO%~YEw1pp zK(60+j1M0HD;zr|nyah|d{?l;{4octaP})zxd!TB^^}!rbHR$k&ahH7V^^h%YhIlN zE5gfS*S|z@#Tor0N45})38k*C)jHd)ct=j$TYIw~TTHmD306{C`^kFz{AkxhW+`|2 zKmVfT>NV^%@8^J(T#bVhM?_$SygxnMAlD$xOPUtObuAkn>3SD=Yo~ulVuaf z=D|w;qHFibd9c#Ya7~Mu2P^%uUeC54VC8K@ae!-@;r+`K5pf;4U?n$5 zFR2WElJ2BPTFLhE5?GNm=Z3ic1S^t`+%COeYtt_aB1NW3U_~jtY>Ch5(fhS%oBuq@ z5C4b=SctJO6a!f9`i&8H=K~fMg=R`S7qF->?g@HDfQ9R*q7i^#k>F(b0e}VR6crl* z7JnR>&#|W?(eQy;E69Eg`PKIU#Sd3&_%^tG>3wyszl@VZp z?on+PV6krt^#d$27g4V=H=8t1`HPJJOMV7-gt&zWog!DGYMo{rk9gwl+M8T#KEWTb z+@rK|q5hK}ZFH!Ka)*mxQE6y1>t_R&|Fw+FVRA;M%@4sMOp;0V11v2hX4Xd)z~WaL z7FL}FSXAbak`Z9ZZ#K3$LN2<_McPGVz(VfkqErCOVd{j)As?{hyBL|YwgVQ)fqb{a z)Rzvekw*Avp_L3^Q3-AmS7rg0EFTr$0Kno;>1PkB02ZUpsIaaqz@jqQB(BZ^Ecqt` zCXPt}3wP0|`=kW0a2ZBvF%rPS9X3ikK?W=$rb^GfCBPzz$PX|AEdCP_#1D3nu2_&$A=@p3eQf176VpdWLwQ2^S}ykzeZIISg};rs4WC5 zd)mSTU`6Ai+^cM`B77;10V`!0PevlP5X%Sx*BG#3#fv!6y!Pf3Y#HGtVkM>Z3E2{? z#2ji$x$^|EV%gAU-p>Oog&MIFN2cLn;dyJc2@osONgBxl#0r^N_|X!q2uj1ks|&%3 zfKlf{_slM7(w2_q&gW&x}`8IDOi5f4`0Gc9}WArH&@2|7ly>Z!_qA2DD1KXD=!6+)213=MI-&y5}((Tlo(2X9)-g{3Ii)) zv2^KOV1?{A7PgxXR^%1pDeZk=MV=8S=oNyMK1X?tU0~(27$?o&1y+z%m|{(^VvXan zYUII+X8GKdtTb4WHr8s1L*5npXz5r@u%g+LJi=m)%hz3*ZmQ~w*8(e6_%}k7JXo=s zorOXWSfRhxOiG4#B|$K_NfWGSeoqt(hk+HA9Wp(mvPV-k(d;n`tg!Y=S80M3d1dKZ znoHL577keXcMHcP9@)YHEA))Mk+3a9IsxaZS*;U~nv41QHYGXTS=sR2W)K11s`#%lk51 zBYLxz&0A8@^**|mjfm7xIzvM0Kw!m+IstVcutImyG)aVaMJPT%ceA3tv~P{n#DAW$ zk_IdC!6b1d4Xn_7CaAYfv7nXm)-I?V^Dd`AL$vng&*2ZD!(w7_9UyYIeiY zjHIHEp_v9tGm@IVVa>ET5?B#3<$L0gcSRThqcvc~dg48)g9cV;LGnqvz{>y4&&-m= z&9o3ruo5rspmoWE6>GuFl&LgWkxt*W#Ao#!N(^N_k7~z1Ito^f#xkWl5G#krj_ziH z6%&Q{l=eZeV#0_U?|lbW1|3Z_IuI-KI8}ZJVuf6pUmOTl9^$y%8WXS*SUx9`RRt@m z#z8G{$h)#HS~WHhtOT|skFvPqCSc`W)nI%OSb2bd^A1*GlvN+h&dw8=V1@ZQFew?{ zm4xv@-9WGs_&sra_$XN6+Nsh4!3s1#%pb$grLejCRjUHQ3Y^BZ`C!E%IjkM5s9uzL zm9^O)# z<{+E5aIg&0gKR`_u*}4D528%K$^+`eI|n9MVY&pGB*MEQoIk*Hdq91;uQf6d|5;?E z3RX;lNs`KZu#)cseG#zokkZc`GyyAtou7zZ`C!Flk|e2yzKAjNWXHq@O_^Oq)XvmUm2(o;c)Pc^AP92m~t+C*H?(^b?KzPc%6J|{!9NNIE8(#W=~S>H@f!=@ z%>XOf3Ja8WK3LIa%nN#_!3y6|TO$>*BEc!}Q^AUuR7o$!gOyV_ZeNWySowDi2COI; z?{1k#-jzMk3bA;wf^S(IVfBq`gO$5g{P^8qB?bRx8l??ZQf6l*LGM`-qaapq<_=~E0j`UST!B2XwNM!o#BHOM)Pi) zw{WlU>AP)2aIegeyZI<>u#!TZn08=*6^0AmIPfEDdv zvAB{BR_H$3zNuj46s5m!P#dg3^(3r|4py`$#o}r@SYez@o%n#&a%$o|xq}W?=t0^^sfZP!w1Vv^up$vR(?jrJ zWkK9Q@6rYmX8^0{f*4Pa%1@#&U% z7X@<;U#o0`VGHx%QfxvVbLlqUQ*eGwPWq_4Ei(yM(Wy3{8EW(aIY+H(b#~MAJmB%hySMH zo=}5(R1z+!mh%OBEo#LD5FN4ow1E1MK#QYeqW$|eSSd?N%bVUHX)Y0wcX^H4RRBUZ@7 zoMH#Cl8w{l)NBGP4&~aSg0)~}t?`}~G`uT@dq1y@bpR_4EefNXIOChZ%DwPM@q55Z z7T)$PY7w5ZU6yG-2d7q=DM6`YunBuRz%;|CZ9#D zkO?^-H-VM2rR$(70#-K7A>|XW@<;O?8|Ap7&__t?1nw2pxU@&8O<*O9ibv*XIHEt0 zIU2ZE*3I|-;g&_UhJeKZuOWJ{7OZRvUM(T!fR!AdO}=!nl1(w-3~mA|4xQ>SPy<#r zO|F(iCinrVaOs&RgcD9yGiwdT-zV z5#Rt;vL{sHQggscPSB<#I$}jcUu&Ck;h;qORvPHvyccnmJsG|c`biNk~Od(&{$nITD(*Y|wvV~?w zkgw>lAFZy^0W0fDR!5x$EB1`gySMOLiQvlGD7|Mwwg`%As$zQG|O%m7K;wtph9j zsCW@O2dw0{=$I5D?~0^9$88_g8uAq#yoT_>YOt~{I7du83s%nhtn+mOD~=Qc_TW0O zqSL7!cKR$>SvQ#@jyMZea!xu;sKLE*g0ra8y#el(Fb+c}O%v{wo7TVSLe8dWwSWrv^D?FSouf`m#5X-fv3)H}hnsInb9`dd@epZVmf)!#5V{{X5 z+#IYlhd+)F2P<5>?R%6tSmDlULQw>)l)fe=G2mS(7&p`-f)(QT!f{i`S2pr`)o4Vp zLX<5uH-dachxbUWiU?NBOVpwYz{(y*7zeDV`O5Bsd?jDV6BEHosr_TKh%KU00>_mI zR;=_Qw42u+eOy~g;2~C05+9Sz!3yzEPD<=!#EN+X!`uM!mAC?p@CjwOS9E#L)Y>?R zmFe$LKtZgK2~ZOOE1c4G;SmL3#e9zO!|pLyDQyn7QI0PPeM}FxQG|O%H6E#nfE6wk z4>b|6QtCo9DMa3th5k}EF4Y?H6(U|^`hgl)F%Qm>5DUOcfseT_AFS{w2E0LYutMxq z4?kT1R?H`JBoPH*rSv3!LJjVf6OR{(-5cOu346>SrfI^xa{TcyF-;rp6`l7?^PW89 zU3ve$G=K)Noj1<8nB{E z#S7X0Pd5aAPTYUiO_yp7`HCXGZ0f-(u(D>kZ~|5`eb)HugB3lB0c&s#SW)b}42==6 zvSxC>I3g3Qz|71;Z9nW{4tKaS^9xcC7&`dIIKD<0`PSDqVj`pVf09ajN8b z)D8Sr$X5)HekqZ@g;+Vf^JrHISTRs=NudmY6$6Iz_(sTA{;i21R_5E4`L_@&WMW>i zCs?V*>2hlfz=~)2RZ)R5SW!0iYH>#1m3^O;V?Du&XN%kDChoWaSh*KI5bp(6ZsKj< z!90wz>do2mc~J>iDS7Rg$st=Cs^_PesJ6r@|BI;US*mmSb^q;xe??mI^0Lf zRi0pFc>@QmIJkw~04vJA=v|PnxCpDoo?xZKe!%SL77+~2xO##WE4`yv;a(Z2Eg@7R zR#FlN$Od4=^U(g3*a5_fL4%vQ0pu%jc^Y05C6KS^a-S(f2L(opMBkN@Zy;94guIUi zVC6>XIbuW}STUG$`(X#IllqcoFPjp$R|e=_Hs|49VaBBmpbWsuO)CB!y9C~q5*JUC zgYd4Lo9{1iyGgZ%e8m%ACVHR@Rt$poONe=3CC>-?B4DMOV!$0V04tuImx-tIz>2}- zen|xMMedZGyfsmZ+$#f%p551wdu4#(nO2V6D+9xxX;+bZ z6BpxB^T0}8kU`Qd#EOWnY+DWQ$^l7yUWg}RMUtA=WdK&H1x!(+GFVYgzqO>#?Kz$} zRq#CO2!1Q%D~91;3Zx^!O26OE@U8-|vR+}3Lg9gx^^5_*M#xvfc#i8eA`vT+c3FNT zSb0PuW*2V*D-k%|zMA!5<=+_?up(=`y=4G-SN43Cjok)TwzUk8Z0Z|d4_5Am^WwLI zm7{puY1DeKa&)#_A}RnY1+TXy4a2+gSYWt$8(7)){fWR7@|BH!y|T1zU}c-^6Eh>o zS9JOw$yRLxE9*;Sqq4z@{cz|JumY`=U68K~PDO~fft3P#o>}-7Q32tI>$Ym0L7ZOr zRk&ApwFQI-urfeN(wIT4 zEI`*}2C+gW78S>VmG?MZevK7ai7UT4U7!h8G>zk129S5f>9b~R99W5K86MrlAGZQ4 z&Ed#2jK1|dyzLCi3aq@FEng6ogO&2vaY@7Qu0TUXKMt(KeSb1;3i--Ley=7i4y?q< zJ~20fd_{-28B>b0W0P9LuQd%MCF9Hu5n<+ zN-y#%+$%%1<%IW$m6XIGvK3f~JM<_eb_lV8v`-8mUx_Qyh@U7y?iKzs&9=9QmFe%A z$!`%WWJ1wLE3opm^jvsE5m>RB8~$N81Xf^##-;@Bl_7e(&3U+2mPcr;z{)!+euiBR zR?1!COr9X`%0hp++dHZ?F>1o&n@tnR#Cx z5C>M?Ph5;oEdnb=K~_mKh?Qx&rtN#giX^@$Bo476NiFKK0xRzY%;`o=u%ekhv!u`O zNlBc_d>*BYzYY0HUFergX;rXd;MW$~l?hg09fm?-ffW=3E4Ys2D=bHphALPI5Vx=3 ztAdrSBw|J}8my=w6EG;Sf-Vo9Dp&zlRv62)VBuY1}kV;W3zh5 zS1NlSt*AnS6;#QJs0^1ZUpv!KWw5fs_xBFSR}80A#AvXRY0qkcd?k~h?24||F~n7e z1~2d5!1xSU!BP@gWE5CIAKFTZWg%8j4W?!_NWPMxAv56(_sY%QXDiy2!HO@8&m=2@ z6(bTM<0A^JD3^MKMr42$)ST&(9Sf{vHp8F{+$$`)jEyJUD+E#+3xxtJ7;5JfI}@yA zx}Z&r;a%|%_h-6cs1G4u`FC0dte}Fo!n6!n$?!q>s)Ci}`UwiGpga9TPiKG?)Z|ui zLda&@WR8PIn3_ybwl?h)` zY6e)z2tp;Pf|Zad`U+bWuwo#N&j>+-6+>}qMi&aKs0avCjVr*)|BcTiPMv=qRf4|_ z`AXf9FXyG1h?T=_N4m~~6y6IVrQ^& z5qA@+CSb+6JXlnq2v!u0U0Sg4uGH=QtQhMIR-9W*M=Lnvx?tsAcz?VLSh;}jd5h8o zD;H)1=SAni%K6vMNv80w7>(C$aRw{S-;Ku`Az!)9=~bjTgB534WAl2*S1LJ=6sw%U z3e-}f-~$$42h*?;u%hU@rUUX7L*YfSGgvuq-`@oJ3e-(poxzG#^^st>SNdzu6D}fF zQWE>28Im9Ed}wP*?D7XJx(%l0HAuelzXFQ+P0lmLwi3jO=(}Qa31Wpz$o;4bR!T}e zpe+JcbmvT=e*#v{H@nz)!@bgv6i?t@A;hKiqjbT_1!^aCB~DoSvhJQ&`_R4=xSX zV4l?yzhJckxqJ3r!M?ad?80heKM^ENSnXis6BZwQZJt~$rRA6(*U={?kQ>gq>w|AV z^9dEXxC*qsukZ#I#P|wJpe~UWA!>nwMAr7&k-(PQ4uja1TJ|MTJ+`I6p$rRXYDc=! zTD7Dk45;B2`QY2~W^xbt;Cu2^y(JE056mutM1)OR7gQ&>llQs~9+`S#BepURhKsP@ z?+|;i-&I%Enl+(mwPsKXVUJ*C>mYBbL@!rR(zhZN6z6kN&K)KD_qznp{$QT-{61OL zWjoiPv0OvhwRpEuaz3w8IWNyr?uf;)uQe81FZ}24Q?}Z}>B8~l+KJMC4b|LPy=eF) z4~%S0=rL3uM~rs#HoKSUK}Snh?9=3JR5$el9ejIg-`&nk z45rhYH7L4(4!$$hiVjvi>D_cM(+AUOFj+hG1wC04+Un;ee2IQ@hjhg+M3{}<>AQ55 z(!*B3{%m3&B*tfL@L3{fUCVJZ{8`T)HvH)TD>ej7T99z3=oPxBPW(4IZLO`Z*b8M# zXxjJFlhufcxibY@HmG!OaU==~a_C>RQ(w_ZIqBG?w#yMV-s^v)A!N6)PV>$9Zfmt%ngPjF+;jkJ&+1g83fSX>=qO z)DV`=u%$6JTW*u?v+0MZwUKa_5~XEmR#> zJu4d1T?b$qNi=pL8b%rof7%J@=$*Mw(6*jSciC>7TWDLar3Y+N4inwuxzx%&$!SCP zcrA5hZs5Sbh@Rr+OkLhNL&2tB9?+uvM|oDcLwVM> z$%z$wQ?F@2{qLHIkfX-rGXpDqUicn%(!6o6VSwfX!>!0f%_;2Xd?~B~A?55qwrKwa zry9KfJ9ClW1%3@Kn7c-_7gOk6enWI*)Q@X`y?px^_K?r0EB83|a=r@IzZMs?&ybv* zx80}wrYHtBYhGZ7e8^V^)Mx5C?cQZ!O%+w^Vyfm9FEH~4y@MO>-5E#{p)Y*#Npc_e z2&}nR?0kFH=;7_z+IwXKb$gRBgt~UeeF{kb!yDa1R$;pH?DlQdtIhG|U=`2RmO?q@T)qSM z28(fg#KE+=1c(DH+5wqjsZHg~99(ytGDE2Brl9{KkmE`3S zd^)Q|i7@GP0qa{s@#?%Oip6}r?PvVrc3|Cn(1p*OQx`P6Pk6(wbjUkttgZKh12Mio z&J8ep(sXQmeq0(D_DS2+?fv7zK%-CE=5C)KR|dZN{M?x4Eh)~s<`c87=k}5h*B1k0 zjXpmsL~a=P&38V;j@=M$k<-uT zsUI30*}KP!T$>%Ce&{Lu12;0_+4oW1KX<#uzq2*@@fIyFmS<~T;6kQR>ZZ>q=i2?k zH42%>I=us{C2@tZJMJqHj>RV*};`<**5BbMZtD?wA%Hs>)G&&{@I(d%)p=2x`VC_g z8qyQla^eLgpBJ!WQt!kVh0fcxV^VkSh2JPHxJt*n@G>ve;?uvQIFlU;F>*yx1e(6y zG4YO7>JH`}dv0uc@kfq#-xj$`*@4RHdeK{VcypEIymq)gH4Az?Lb}Ggq#l|3_YRxu z68l1hHBsevBw>Z{uaWwO=S!u$2gIv9{tlZ8a<|`g$$>(*kD>QS`%Nalm1v=w-9;-c3Ts*tDCP5{Z*MU)R(AT(OiM^ zobj+`yhtg)`Oc87i^j;AI1i`ITKXSxhb>-^^NRGWFXwqBT0A9RE_z>?Rdmt%(AcPR zdBwD=R^&%qKzUVFE7B*q)y(^0Z90-tF$8}U}Gxpl+n=$^p3X=|GL?z5v-wmFTx}uQ?v(jTfGKLfqPbk6w z&5ecdD{!$K;0efTFu z$y(_jmb{inYC^1ky|}yQXVr9w);}MSks86Ba7Izrse_@k9pAUN_ptm$ztN=C>ydSt zN1;J9X$_c~j}n{F{v-QBRZdp>gm^Vgj(bjuv5nHgg5z0=v7OYJp~T6RQeThj@9nR= zo8cvEVLswr9(DZ^?A2JP8q2m=kRE3Eiey(ts^bQG$3>be@5q%$PvL1hhIUdiSTb_3 zy=f`nP3cORVwmeYCqe^gUpN@4R~oaSO;zoGhU(MI1cwvM1Kk-C!<0tJ69;>sHS#2v_vg7v>@ZI6hHT~ z++?WOkp{P}+$2%#OKZ!0B{vx+K155-?U3`8E!uf|_4DfyVr|+(Gug{h#Zi>6B#oF~ba$9tEnDyI5Y*fw4=$CR{>0{3A^U=8>_j9~Ob(0E zXbNi7k*PhIQjr=|!IY*Xhvg(^69IprM~cZbH#N9&)u?|?c}c0}-IB8jC!J$)rJ{_? z#q}98vPPIeS^UW<)JoI5cT&D;l>Xj95oTqhI^{|4XOY}WL%I5G;%drZ?<=WE^Ctg# z^Y#~09Z)*?kJL`JiDTm3l5UoQY}zqWu^>jgnYKaW=U!@*csuQy1{}D@#fG#Hjh~Lx zXz?!E8;vFVpU0;3SK4X_bkfYlbs2Z0U|w2dsjv5%6pUX<{n{pO%g_>SSox3o(r_>Q z#B=Y7m8r6`JEB&U)!)6=|KEcz6s1e?BTz0$ZY^!@Q{Zf78z3yY7^Q2;@ z(wMD97AYnDRR^^<7wt$X&z(M4zVVR7=p)Vny+bO62g_lTCAV7St@OmTJfiZ!v9QXU zxla$aC`Ojzo^lTB4XMnO9*Y$PD$QJ~CwKSDxCS4a*CF5Sx70H=PpN5t{PvyRtIIL^ z4QAt$N2H6jxEJ(xT{_!8RGHu3px1G!(>Y-$xfKRxzO5a*+&;9|O{r_|Nu|@S-*$|; zecSnO?}QVowA?#!mEHMo?}T$qDcrSSht;9Llk-x=J1=)0n24EIDQ>$Ae;1vYmoHAf z-03nAdyg2`d0^jdy&c7irO&d${sk0ud$c!3XI)M1 z0X2djXo`GKV<CNOgx!Od{ zn-|A(@x_cXD*G+-=jKKp#77cq{31jtN~Wb052KxT z*&G+WV3o#F%5zS%9{fF2>UW%YA$KP?KyT7gQk46gSlHjH_h!d~L-ji!u--Zr!>aVb zw9xrI#bum`;tiL63f}6Ayo#?~`Z@7dRpe3}a_Q&Gw>w1v#cwYCoa88ooRn-YEsb+j zB>A}^2V-_f%9dVpHt7AbfKk1j5@v4C!P&L{lrD{N()D0e4@xM4aAM}PUzeaN> zVIQwol%WzGP`_FJp5K^=r2x8<$o+fXz{;hN%@J23%EY^C6%Fjr29BNR7__4al^#I z1>@jQi9^0>ZocvTyOdn63-_psX`A2$aThGEd{^1!H@mf=MbNKfl%x`5l+-GCBJ!#1 z$f`3A46mTKzhEb-obvymdn({VctvAYT4hhx?@*^2jh?%H24xyWrm5leQ5M^@r8@t~7VPW^O{ðy=(TIZ6WiN zD=V{jxcoTzV*4A>N|g;QOMhJFj;n0Ak8{E=Z`T|yH++8>nk7mqGx;c$Yb{G9+$_=Y z{9(VqBjmeT>Ms8_OI*0yWQGGMEOtKeK^8_O_@4M4h0Cr}32qiYr1Y_!W$KLJZ|8gB z`&r3G^(xUu4XsNJ+(cp*S59T8Kh>=h+9mlL{3o_p2K;;^;;Y!MyXOD%v1mr6XKjf8 z&jHbxO3%7C{y&FAiz@Jkv;R^*dsyXuPKeRZsIa;$SeI7$)g-=^1#=QvJw`uc!{|tv zMCDgAjCI~?dd1e3NxgHq*D`80GCW$pT=*S zsta%vymgl9d7EQ3k4_)c;OB(LYDO+l%Jg%TVlD5boPSchFTCuQAcojngL@vaz3kR_ zjETP>=J-NF*&WV$%i)%ki%*)lR+a&;@MG^&EThU#POs2Vec8T0Fs7=o&?P1iYtl@9 zP) zJfYO~YI%%iPBHn_ZGqa!J?m=ap73^CCWHyhV*M=VUeoU%k^ z#d+nrH!A-8lgAMz=(6J2i8oNN-dDE6xo6JLunj|+0Z^alXyS<|Dcsd&Su`m3H3x~^w#{;n!$ElO= zWh_IKmzQVGKVfnWEkjy=Tz)df)vzSI3_QAUP;&7}TWoiDe5+r0_?^P!*m|YUu`x|| zM|r8D(;6EDOCvlzn4od8w7Dd}N(|ndlYNA2LA}=5BK5Tz_w3j86FCvY&Ykr*CD($oj%CMiO zaCH7IynQ*cJyWcC)ku98o*Sk_hl~YAXz)+M;z!wGF7#WwVswW{lp~?52i9%9XyW@~OF#ust*Tx#`-6ucR}58S>J(EO|Ok^Ao95ZH{!d@0`3- z?(&Gt=g{)@N!GuEH0|T8e+OyWU$eC2k&a0FD|;@BBAw^1bxeMad|jq5<700e>8nkP zmbH8k{c7pdNU|DEmb;gJu1^rA<4G^pjF6A_z?IK7CHP^yG!G@hJWqmopYXJNgmtOz zT(bBKO(w1V=IwOwAt%5S!b zr$`^RrVa@`?>-hT$cK=YD*fh2S$#~+ZlhrGQf=RuW;c`sU~C6Vhm1wD`-@nM4jGGd zv{vWm&?|mlLus@^`AMa(yOw%cba~6%C-OFN;=44+SmbTu#SdtZvB-DFiCbxqvB-DF zi@W+DW06mfTWaoujD_A~J@H8l8B3q7=C6;WGQphKl%Bq8V$(13Qe)pgn(#A3nsZ6* zd|~h{y{3HizaI$R?1PLYe`V0KBo8Oex0Ma~v>;6#`9yMQ{wb4B`BEnG$>g*7kg=#b zJ;1%>LdK%Xe}JES2N?^qF!1Wz`6Hu(cLpY9?J?LxiQ|vo9XBbpY61gfLG8U5)WO7|TWGp6gB=W6%$XHCUCcF;!v!5K| z$qbnwH4`YOblDrWa!@6oM#4+{c$CQ9cXmcQK=e<`2+?g;bdU~Q== zu6FRa$*uhLCST*tpK>8%$(Ihqe-Lf5S8Wc$&k~C$9B!3~cDy;^alE8GUqA4%s!SmM zqsTxNG8U7oW$!?)O;~qzV~}szzW;g5)aY4OiO4CgJXZm?X)$d`1ve$0iY( zFA1V>I?lxI`xFcitGr@4*FYTz@m3Yk$75C|ou_z?&)|?eW#zODI zKU;7Qu@IBm>6`Hu%1)pG79|4Z*w+dg9WoZ!GU?qDIm+EbP8CXHvQssM9;yVehuiX=R zCl*O``mWs*x^v&%C1#T}9dCup^i=!@wm9@u6MmvYA(m0(G*+drNBOo}N_&s_X!u+6 z@1;43Z`I|1Tq$F z8?m^84jBubfv3(Qzv4k3##85nkg?Dk@zgIu$XMuK@zk#p$XNK-@V{os`Ab~bD``9R z^We`pi2@`26#Q&r>hfMma!xvad9MUA7DmVJIZ69i-J01dUYgDKMGqIvscAcyBYCz? z`p50h)1ez6V>zw)zUnFiG8RVPX-#2O1@2G2hfUFUasloIpKQZeOv%KJMsB{Q{{w&6 z;(JQou%1m>p4b1!)V=sK-T!d|56Ss7XNHi&9L5NvwaT%a&xb5Bg-T_prdG+>FbPpa zQBmaBVag?kZH_r~RJm$4hfY^*bk&rr?)UoLkH>xg2R&+iKJWMQ`+a?1C+(NhE>3{4 zOq|F2eShg$J-h5(8^3~!tmby!5k^WD>sc#!G>>!zVmVCwD*>G`2*#pU@iXCL{Uv(8 z#|b?hN~%iOk9Tuoc&7q1{{D}53tY3QzZak3Tlp_t2p^Z7YB!(!hWP8`H~z~P@mr?e zF0QE+J1LLIJSoq2e7b0jnOT&qH9Sc})SsmB9iI<^u@q}lX5MkZSc)SlGatBMEXA8B zGw-=zEXCAfK3+vwnKHw5QO{9$k~`0Rb28y)D&;W)3s*!>;Rkv9-vFQ2zAdd`!u|Tbf#Zvope322p_TQE22{4w4#gjekWw-X3Md@0r6-q=m zXE~&{b%nC4dp70TL%~p@umU?gBu`!pI<>zUT&JgfvfJd%cmA1Tg_DVh zs*{vCZmb>{%Sp-tH$@MOarq<@j1K7X0adyPhqAvDBUx ztUg(UwwuLh{O@mFF00(-=-S^a@Hy6%nSWA>C;mF+!(L!NNhhVIgRzLvWXP0WFcuMQaU6<0Tu15+ z61A6_4zAW}i|BnKYOii7dRLZwyf>6nm>$0lXU=}fsZPHoR-?`9USl_|khCpbMvSll zEk9}8Yb1hJtq`Z%BE#EF!J)WS?GL&a3nn z;-56zD&Fa|`StdQ{LKF)BC!^!kp!J1N!xp`h~(zaig>I=Y9^Iw=p=dfwu`XxKZ?yL zvRP}uSj1)!?B{F1Sj1+O*yC%!Sj2)w3l%O)+NGoy}k4x3`B4KMOC^kGtOmAIBv1EVb6s4PswMYxDlg-($Mczu$cQ#A4+4nwx z+{wQp_G_m#*qiyM(iKE08oNuUk?i_4dJ+OxiWa+bO|68$nWD*VS`#fHaQ|OziNxx* zRSjMPRF=#boRSh8=)EO!TKS>Gs<5&IKJzT{TUv0eSBF!Z{(If36|}9Hvp=Vw5dAE- zUo^L|Jv#3{m!}AcEUdNbKZ;;1`X6$zScX1lQZYBXQN6AwL6O5#^vFsurOA?l^XgrY zhBP@6VTkKev))(srRm%beLuA#ml{OJj-l(Er)M8I*Qj-LB`9_59eUz&7ZIO@vz?33 zpHN%OO4M1JIlGwMNII1#<4n-t7?4PL_WEck(ha%=V_yGNPJ&5`pcq}@jmO~EHCHi zEvw*YFgJ9zp7HK%tTRmx<4e%vxuM8J#)=ks|5V zsDE4Wd)8#OQon3WHvO31EB#An4{i7JmVBQp>zqR*4w>#j+9>TC+M(Zhw#(GnS3V-A znTpbH$f;IKzF(=d)T3zYGKTn`OJ2{&)YnDGVm2`b^cV6xoN-#5cEuECw_W5 zU(YDjuaR!|$LVk`C>A(d$VbHP7>aknt0&(GS6f;+Yn4R_uu$kSnDf$)R%^|o*v%c# zk5)gNMbJzR83JR`e=EP@JM@g`lXpk|ja<8LvqHN*gQW<@;zCj63@BDH78Lce1y}}r z(ok4mf@7%vsV@;YG2nr-;@fNieolcicy>$ns^gp*-)kt^<&fM;m0%hB)~dtYX?^&7y#BG=8Rg`|;mTO&qW=s&>0LRGUEhC3X}?FKp*+>!p}$W_ zUMbfdYvKHt$-@_Y>J zRU6zB)%dsb;7yr4OT!x7_HwKt=YrA%m3$X#Lj0Jw=#H~;E?PDuD9-jvg0UFh&QIEr z1;(O;E}pb^-b8ZC`ljUM`p&ckx=ch3Qq^`Dq!vwD_J`D+U^E#F_gB!ZN{h#`_0N=hE?NF#R-q%!&4z6@ezYk|Y zm=m$92$ zWkYu*ZeIR&Ltkp8+nAdYDVbqI4Drs#+Hr2@Jz*B+eRr$Wj5$x=(l4f@TgW@RJL{#A zF3s8lhnSs`EOx6jUvMMi`jwRy3dEIa3VFd<*_`*16RzPZF(ihTmjPaz-<9XZ?8uw& zpy+c7h-nN}rD&?4gKWfUQi?7R+$S4yM)H~rdQfkuf}@OM1}$m_44R90hq8_kPiDzM zFA3SNTW>Z~QqO&QgT05qw0}?uuT<+Uu;^b@>QRh#7nt?GQ0h^7>n=d|e^)xKxW#=H z#hFmLmltjDON(@X0mh>AJ5LbLfLe+_g(ak1fyw_0OQ4p5zKK(lccT7cFJ}KI_1NDU zU@REQ|DPh1^Vv(xccN$6p+#i?_$7M{$x7z;)>kH5PxcWvw#jh|HL$SKh%W|1wx zSgcrFOidmbOCEnuVFMA2Wsq#ls?#WleIAZjepxxe1Y==R@Cz~+FqYVsyp_N)FcwU7 z(aO#-FcwT0i=Ya|5;}G-wnrI^MH%quSS%PzY+_zJ28`uD4Lydy8jQt}<$=M|z*si2 z;xYJQFcuqD7^a5?#`34Pq!^3^v)~7vC15Ow@$0!B{ZA zi&mn?pzdbuZ_IbbjP{GQzW%Wf*ZwpCV%eUwJL8oU5DVg|Pf}6_5Q~(&m%kMd%lf`q zDcW`*mJ960wH_uwELCiJ)}RRxOJ!R?e<=cp#RO**Qh#AdlFacuO zPLAvEL;$hun0A`&lA7@H=dP|*-$1tpV&OpWgm4asB_d-~`L+)bOUdPv8S+v1!{!Rjl2moazO{Qp5X@^hGya>cn zljPc8A_c@EHMjv;1;m2ylao5+om5J9?6Z_QBd`lfA0eS)=Af8_+J z&pR&57@oP%>nS7_Whbe>vuY~OPWowVkYzHm0{XEr29L8!;nUM80UN5T+q{8T5WYTG zOCT1**^^Rn#y~87ysa4$%D4@cyN_n%Dg&`>AaCr8l>%as()HuHW~>mmWtb@A%7Iv% z`fR1tjKZtiSm&}Pj45a!7K_1U`Q0}e*VyWHjl2IfD#9|^?-V_ixl3AmzDvjV>$5B8mD0x4UD>a8Rq~dW6 z>jcCytulBGhX-O|lobWmlmW3|=TWGtt3WJ1FD>Txp@3LWk=JpXfmqx}Y=dgbfmnP; zo}(sw(xRwZiw*Sk&`1Kr(vVQfIi~`|5=f{KHkoj!D*ptfZUhxTDnKl#L)%HQR3H}Ap&g_HR3H}A=k26;Di90G48>~j&${UBMe>v0rGbKc3KfXu z(&HWPuYYA^m7N+{55#i4Fpz~(F1cP^X5)3Cz+ir^3?eF(=aTo2>_Y*upuTuvY=Kx# zQzZg%l^p#(mChhsH4w`NW<5&R3y8&wPo|m#0Fa1;Y6TN|0wlF17cyAyUUw#0!I0%TvN7!Hz1jHh2Flh|}V)5bcqJ9YiV)0%H8mSEe zV!66%H~+j0h$ZM($F={E|*GBj<&eXj|`VpDU0-&_d9Vsm$#e2BGd^Ax*fKwJ|& z2E>AOsgkYO1H^KEc1iZ2EfC9|gxV`EIW*1))S0XYVli@gA^W!t5KDZw6W2}?h{ZOl zq8<84>W;DSnm^-!SmKdqNxNCl55Usb%#8zLiAR=@_ORg9N)|?QFb;@iBFROV7TIuU zJjp93(iVtiBFQ6XlPwU-xc{&Nyc4P}-U2HEL_AZ{T)n){X+Pj0z|$bsHd- zctQ#PS|JdNP0vL>G@2AR*tCq%&45^hu@Z_25R37^E7>n`KrHdoXQ7$|h(!}W#EZ#+ z_bGFVZ79`1Ec z;psRamiRxNB|t1Te>zK`nWFH4?dM^-Lz7fB5R2ot2_TlU++L_80b-eNtm3kfKrAW9 z^Yu}*A9dSk11ZQ7AQqd!07u$55R3gF$+36>hy@d0Uu`lE#4?`dId1F-#FAn^nznO% zq0VjmyrbU)5KGDOSatk75KC$25n-yN0*J*CJw%G30kJr~nxk(3V(CCuEGAR|u^4## zMCv31u`IkR6LSP&alF6q?mQ67=EY}@t;0YpRm=CQH>UuxqWvCWih>4ECq-q zMSj?EBM=L2F|XP%1&AeuHtc8*#B!3Rojg+s#F9dbOrE(2#BzeRDS74s5X&)|Cpe6X zur_6;)J5H0-Zr;fNGyw~$@d*MWAqzpZHItZ9P6t|-flhk38}CM#4^lJpbaJiu_WVaxcSIsW(Lx4foulE zl1htDwis#00>Uqr1Y((| zY+Bsp2*lzD;S&cS7W`liQac5RWrDwrrjP=}l0qrta*;qRDU|cvWh4+w3Z@AtY_@yCM$6BJQDri~wTUmE_&mE)K*Z zGvm)@i2||6%mlKZivqF8%mlEJ z9EgPgbtZc1KrB1iouaksKr9jLCedhhAeKn>1JSqYKrG?BhKw!hKrAT1fxcVfKrHI3 z_3T=4AQti8W`bjVKr9)*)mB@9Sen?MGfqfSvxh{b{woL}z+#9~Pz40FAJSS(&P&h0P)V)3d8?AS33#B%PD zXH6&&OF+lo;U``|EM>UPxdAVli?m-xC>b=A#z83D1BQSQtgFalyJBSa*J3*sSgnK&KlB1(DOWKPA3_46H#Ty3X0*n-E7%Td{Y~tB zsxV%H44dbNz|W<*BcR+QB&{QXFpmncztM36duJprq$YH4%2eIR`H-3@;e=ZrADR|g z4pYm`#9i}UAvIxcMaxa}%T$>r*qsS(Rc7cL9N33cE5fGk3TIriS7B3Es4%yfIfa}t zgS#js^;TV^Dx5DNQM=~tLf|(OnzpNaa}!WY*vp2Io)B1{n~?tVy=Jfyg~;DXPAG>l z*UTf>v10^Am-5(~$VevrcS`Ep)7bLU$iatN1_nC=JvHV2arP`Y9)F;I0`R-Dkp4QnR z8M`uq`V;#QGl*_Pl+r1wZBj8(CFDC<^sK8{!F!ClL!G0X^+$}xP%($3mOYy*qd6If zXe+)P`%lg)0m?0$o~6?=M}tq!Huk1uWA2&_x2lE zkBkyge0BbT|63r$kIdiXITOKwvL(cC3w|r&x7q{GnO&S3#CNJNz{RgcDCEKaEi?XG z#BcKu(NO;Yw>~9lnu-SsyK`HJ_+Z|C>6_-*&~0)7L473Y{isGuTA07bwq4=?HmcQE z*?Ytr@{UXUhCmKxA15_iPNn7w&wI*$HbQzLbb9;7&}rA0pzOuG6xEsr9wqw`LtNS_ zm|W6dz%-J!3Ly(qB*BC$m%B35q<2DuqPm@ra7f28W0>%60+-2P3fZ?3gfy=!U9?SG*9#_+gV zci1d6h^Q-=Qv7FKq+xTH)(aKnWq7#hiJw&c8Z;R#>u>Gfw_ZGxsVMHEDjQ@?DD>x5 zLJ~z+yjcdjCBYbgt+*5i#Kt8^D9teA`=|dxMWbSUf>lRb#uw4yrQX*?(CMLY8iZ6CG2LxH4YSr6g)R; z2@s}6AO>>Cf6Co0l2lKv3f{_mRCkH`&CIFsovG7}Pt6?#BP|@Y@_RSCedhn+$d@0x z+3mI0VQSZ2h4N!f-5y_q(^^M<2EVb=^k{b;3H6%QilvInUc(Wfj-!;@d)VNH% zOxP}R7h9IynXp==3%pAjWaiv zj`4FxwuJn;wsg4NYyMFAiQvzwKZ8GqtrU-laW0RznEeh}sog0R?&78@=aGO7L|&Us zkPERx8s6O3*fJX~XXol8H@L6y+LgQ5C>I^MBX-s6eV-SuS#y~zJ)H*@()QO;eCyPU(m^qeZUO>+O(rA|%; zq&}XGh*dAz#kABq{ES@1dd<|)`XX!(IaK$6rK|PjIl1`V4SrP7_kG>bxIg)heQ^67 zpoDV`p)=(Fb)O7umCM`+vE7Zb&#b-S(-F=Cv0Co?$%Mkk);k9@iRa$Q!)BU-xIXXoJ8_~}NEo46~qR7jx#|ke@jX+|=0zXCf<|Y@_QYSVx zPqqaPEOKPzqW23t2VQWr<(f7&Kj+;N_zry6IyKqm-u$f1ci_0(7V4iENoGzFu{7tA z*4>0vx7C}ZCe}f%dj)Sdex9TUalddBicare_2RxIRmK|hVAo)~)Be1RPP8Y&CKFOO&B^J*2Vt0z zyFtg~Bi8Nd6VlD`A5MPC4IdLPLqkbCHOFbF$GfoLZD$e57yl(AGJ~ zEOI*ZSVYI6;HcB9kMAzm*T1{k**Oj|6YBq>A(!qIY)XA|V;OzvZ`Wg&HmAP1wX8c8 zwD_0PmnUft$y*mQo%9|92R>hE8}~@Xnf-Y8uBpD3_KTQKJL9zOy@o~3JB^#Mm9$x6 z7Hu%K@F}0JaV_>Xl(4g?P+{`1@^UOEruu41$GAzVuhO3=$P*`+TaNfXaon}o$#J|i z@sYA^v4CTA$?9YEmCg++P@8hW$?nR>jj3agW2C)uZw*XgBK>ol2A=S#yd!I!oOMdb zAs)?!QzcaI!2?doI@zWlE9+xhXlYL73m*xWmycXJoab<)=UU2@aTr;g)ZY`Dmv`}N z#*LgRK2V}5mzP~G>AhbKr_`lf7@v3AqFVbtT$nb2GGeEmr~GnSE7QR#^nLg(ZY2K_ z?aPtX%dth_O|h3~|2myg+45cxF}O}gn`x|5btG|Wr9cp}7<%b->LK-q?`N*0G>k(# zvD5FTD;47pndnOoj(nc#3j6e=D|&H@6AUX(3h!6L7Ed2H+AXFUB8!Y+Zzq+Vsh113 zODb%PyL|sltlU#YJ2_L~V?R8jpYFYK(myrhD0% z>n2p>5kZ?d`}k> zyJU~5LVL(F_Q7?B)sJa(KVk1$_W&X%Cp5ZWvp3f@^qr7<6VE%vPw$hE!|#Em{U9$U zvsi;1{1r3#V(S*M|CnpU#N=dcAkmGne%fm-Ho%k=E7sK5$0lJ$dK1KAWeZigvV{)T zeZ7nUC@$F}%z_xyY!sCBK@P<5x>j{u()~W@kzH5oAjs%bXL_xRw#N?a-eI~fB-2)I zzcaL^RPOCnIWGdal-+kPi@ZpWXRFR?OUh=gGvMDmGrg6#UiHG67qWBfqyy}&2YU%( z%d(+k4)k6&bde|?)}oBEoih&?9Fs-c(5KlCIGZ!QWYKnl9zicVcik2n7;YJCo6J$U zQ)YPEw*FpEvDyMFS;{`Pxma|;VcBMzwpYFPIGe;EJJQ#y)&9KKN$jKCs_7{kUgsIP zb@o)Bb@tlRM4mdosqfahdzv%oQ?~R0wr}Rcf-SPY_CZ0(=>lPu$xLig>vVk5!@kq9 zx{4BVx;BE=zTbtOy~buYE3y5I5{{{|M^)o~jz+*E*CWb<7aR5WU&M~PS}NaR@{K=K zHXh(WFhJN&lE_gBn1FVYk5xFI>O&o8hHbrTP+AvpqjN8fwewlVRh5bRmyo2E;brfd z!yOmsy>r>dRyJwvbjI8@V=LRVu6(Mw_eO`QepfHjwfs=y78}wX2%KmsLtq3FAe?o7 z`b``z(udc2&se;=kZ`yzGoMyhIy`Rtpz@R1Axv7+@N4fk=+D+^4Z{z-k1GG|^UXT# z-tY-mztZm)yC>%Ulg?CS4a?bEMfA5>d@fV+F9eQky8O=gjDBqPDWF&5bWXc4?yx&2o zQqsqj6QbOn&u}%fQEt63J}KBx99#q#rq_ZnAJtp-!#h}rq&#?VDvDR|5XGln&&GJG%b zPX1x$G2>f&j;|BHTjdsoIJ0{gNbRacW8WcmCa`6Bjj zqF6zEI4)@8ZSr|$r=>lj$kM)j`YFeeLnv5`z`Z5flFl$?ETi`Di6lsmSVqP1lVw!H z!V?Z%$<98{@szn`LD)OHf9h4gom$JCq)YTS{c>ur!V}C7-A*bM52r`m?wow%EFq)g zW>Rp;@@V^X$PAGmT5vC-d)G`1-?!jc#3$qXk)LAthYG&u)u?^BL#}21Wd1MO!Gza1 z|MZ5;222e)N6Y+Y0lTw>>}~6z{@oZxzrW0Yg{-=v1uib;YW5)0RHojtu-!Veu+#cY z|5_Pk1@4A|QcHXciAsLV2~w-QD=4hoPrAr_uLg^A1mz7!MlD*!DuVKoBU@l}B|O-2 z{C1<%-*quD=f#7<)e>cB*)KEH2r@OLuYuT+96w99Wwvqn*GH-_yZ5KMMDU0sU$DvY7y4J5pn>_&;+N&oh?#?=V&-4OTIPRhzuU=Mk+;c1 zoQG<^JIzF1H#QQvlG+~B`b6Zx`kO_uu6Ekb4k8cK=QQr)VzuuiK2;lh-56WnQUm~O z@7qD@C<=1jtljXc`S9{a()~y4>?hjNt|ufcTU5orPTZjNS{N;Pl)~t;pXf~Mc@)PH ztx|v0J+#2&AgU@}b&oB~+ncqtP0hH=Xj?MN?RUPUkFcKA-6=YdNI1ZCBK=jQop`%_ zDs(1+YpmTy>P(2Q|Eoye-ihjKzv=ZSVrLVbxp2$=aA(>B`oO{^`!5|YeNyke99a0< z{)^J)gKrKj&T8GXMnde_qCR3?(G~lg zml|mIX~=wR*9K*=cYm!B8{4cAAKP6dWpAv@+F@_WMw3Q5`ECk2LMQoZD}_s@@l*eBWn)Wubk&Epgf&sog1lkZzCDZk2rN zzT5D&QhwM$hCM?2#!zB+oZ-EIypAGA`-NAP@ts6rqe=9=l^pVug-Cn#SCq)^*p!k- zOnc}ien}vv+<5ezDPh0m_Dabks3Nwncu6?49K^p+q-r1f3aW@>xL(@PFQND~HnzOz znSGDy+n0hjECu`Ld3WrezYIUj_28El)e5600f`-uI8m#5m=}HU(4DSF3rG1?k96#h zy@GDym}S4J(=S&eKXr7yUeK*N!NgbnJoqbiIUSp@eD=}lgTLa#M34FhFyXB!fE7pV zt8F0?04wr0H({RxtUT!>$?tarSUKiDxNVmNz=}@w9D9!hz>2)@k?^-dSjm+`*31H| zOwLBgPaFkUIXcaV3Caan(WzPFQE~xRWO47Y0Ukf+083m8m8Vup%rp zDbEC0k$8cnKNrG^gabB#3$QZP=OfPsSQ+1TLP7@#u!1~#bebuIm4Qf|Hw!{o8Q7%r zW*%VWb6f7f108@Br|uV!?EqN8QYL3z{-jRyjtcTJ0aheV>U>QBSV^*;>{~AZup;52 zBbx%SlEkabERhFTK^7jhW&x~>L&8KEDTEcI?@>Y40Km$hS`i_v4D6F{*aWb$$VVCCavoRW0N@&GHDz1!XatT^TAY>*Eu zaU?(5_8DMBA_X~i#Cif?C07R7^H~Th1NV^VRDhMiS%cgH9e|Y-I%7647hnZ>>K)H8 z7hna6PNB~TVMW3m*_;Zn(%<*L#E5{^XC~341F!OsQ@bjv$gWSlL1!dlkN>{mIqjo|GsH;5MX6;RzLT* z)9S>Hwceq@`o-&uRXj>ON<)U>OzCh+vxsncx}n(cKnct#dvrR`@bs|+myf%Eb9-v& zXY@fCw&eVE3{A#`I(?EF$$o14M&pZ$KKXEzFwz(+uC^o*l0Hd^)K1T$m(dI9)y?#b zMBlsXBdO-Yp>rW~&+6>b{Q_Z49+ zX53;NWB{OWd^za4s&~F$*MBztth71*ME=h_xMdib3>R0csVZ_MIg1kfRsZL?|Au}M z*EpU-HX2>vLzv1j9t_wg&UqRSP`Mk^8Hg4b(Q)2)AY^U2i{i;Wg z0R}V@cHK-zDdUyIL5(QT?Li zm3NH+_X)>^W5|i+OSKn<`%=vChGA#ThhdwUtC$O$gS|Idwd>+)D8=KH->&%R)eM!}@B?ATyqIrAKH2Rd`QN>O9u*5KG8PSZlMD_NnX_rr`KlA=$ zY+;Cy&SG6xy;eO|znn&!-=yRE=V2Fo3zmFT=YgqyR+USq; z=8sD-m5!DYYCbK&R62hH{ow)Dr}}f8g z>R>7rhJ=#gpPj+HhDN85x<;wy1p8wz`}e|BIvR>@g>IOHf0nc=Ufl2YC45<0S8U2hm8ntEM9dbe(A^@g{ib)91S@lv;Sp(2s8gT`$I@o{(INgB+|VCm`c^Z-=|M7 zVJcNuxKAE(gQ=7`fYv3gG`{0@w2q!a|11zeOWa4!CAVpCk{d^3(76RlXuTB2$MZ0i zj!vNm=}2={7xGJT#%O+S%(;SLbWIY*?N8Rn?*aT0j$j@!Q`KQACE&hr#MNOcC3Jrx z<`kUsCp4nt?oVcY?oVi0hikko}R&ix?d-n&MqZjDjZVO9x z6Z;4TERy*53&2ItZ<5<5ncyPo1q6$u{1);A6QS<4tJ?~n#J^Hd zOrTicN$q3IWOZ;6LQ7KD<9TqAis<{RBlGb4sAwXz99_937@z-+KAqI%-h8BMeExU^ zxCr6br{v60aFNkR=zDa5+v*2mQ^7&>-GaA-Pf6`hnBXGnr|%0!ncyM?f1|(Ar_H~0 zO@HC&se_BC=Mq*wa<<$#i3IkntOB@*?)XyD-CS^yTw`76sPU-&IEzzQ zJdt3G<|SRpEx-{h(EOwuxuZD3=XQ>C@uSUmdDDS}_w7(Mjf=9}d7_)ey`$Tz{NhA+ zU#xa9xQOoICCOrN5#84(KC#>D-_2d=%thm7i?8O6iL$)f|#3b5_sESB1)2VmP(@WNUA z%Y$uqAXgW^#Di^jz(!$oGn^#L^nR{qF>JfIQ}6la#jx$-T9)XqxUlUOzdfs0)4^Dom=OM=cpIb8>;kFLXs8O|(hyScY?@20_#vfMT>;Hv=JE^hTbw9vq|3w<@y z+?dws#h8b=r*TL{iOt9*UTf~}ZqHu4`Aq??e-X}=T92R2AC}=B@E--uAD7`=nJA^yd|HNcW$_07!%yM4Qh%n{mICL>59}y3&cL~XzbHufz=d#2}n2!uZaR2b?Q|19)A&@LcI^oj8Sua|JK)138xh=SoWBL<}Cz6};ZEFO%gc!-Esl8Y?8jmH;K)tUaoql zd)XuXf!K`U@pW{EUZ{vs|6hDZ1U@+@qASna-s|C3n|@O4o6K~ytR&W&eK|doc{bfe zOkM^{D3lb0_m+t0sn?6rk{NpoH~1Djn!ImK(`8eevPeYSr%feoV9Jb@z*u(@b};oBYs=VNhH2tq~e9#J8`&# zL^9K3dbf9LS<6?h^>$O^-APaDbuJSdm%=)*yVs~Li&;qWj;F&VB=SbO&ARtB=hB)2 zQ44uq3)!mo+!`-2xP&BmG3mvka0yAZS@bTgflElVMUrmAhDXHd(d$;t*e^I$=}n?7 z@`76eH2b^AX-T?!vm71G{*exskmxUDl1pz9F{t;E$i20232E&2(70-nYcC^uI;Et%XZSd|HoPyGDZ1wn^owO8%=a85QHGrKKjy$Cq_K4MtVu3hLNaOl^M2^VC6o)XqC!Qu zg!IRyNk{YC>qvRV`cP8iQLWACR$R=5OGuIwlULvjmyi@`U*4!QLAQfLJ^RSzuH?%= zf@Vj^P?IxULb=_>b9eON63Xq?-&1uKE+PHH()nlM64HO2`$-YzRh7DwJhVPsLe4W< zoV$u}2|3Sba#|GscKxon*?C5XqdR5G_)Gt@4C&$j;ycpt$vJs`5~O$F63Y6d-4P9g z>(-$&`V!f231uO*lEa5)iLOI~&NSH-x`t|U>JUbMkP0<5xz&ova0#hEH;suiTtYcy zGsY8rxP)B1WiXo=olqh(kxSNx;q{uLRc@tH-3DPb&ISFAS8}bL$EEj4n_ty=Eaez! zeO3Fh%nFs~vQB^CYzf&8!begUs1}d~T0Wa?np>@$d}t_5-)tc_T#@syBH+ABw<<9v z4{%;zEjt{^*~O2_GtxhwLyrCv3W7jTI0ZqB$eY*4tJY={l#=%j!8oh`PdR*f92lo#jM#aqAqKii{OzP* zM!jKZ5xKBYt^GJ7*6_KaouRIJdnn@-N04{K;CT@lUZmvgQ1YXZsg3IVr;`5~RvY^k zPp!{6v}x~ z5>iB}|4aTT9a8^a@<-`@{%57*uJld)o9p7UP8iOp&z|CM&5|(0sa%23;_R$su`8@N z`O^->hk5@o)APh28HEj(&Bt!!wB^AYD0#(hNKHNoW5#)x7i&OMh6vHBdw0F=|r zN&2}~8vIH6G#KIxue(>O#T?IqHAktYnCF}YIIk2%vo>_zVXE|0($+A+8JcmbRi{)! zT{qP!lkVBP4IoSeUBB#bKttC~f(3jQLFl&S0U0 z(^%;-*sKNPie=otm4hr8S6Dc7+^tF&S1=zL&Dt=o5XVZ1HAom&tQKxyJ`BRRqFcYM z(3S|}3JYrvg)=a&U@kHev|wC0RJcs6)`M|nJu4f7roySA>zE7c3Z8^p7(6ZJY>#%Ogxh2xm}3>a7PNe2pF z6JcD*BgGd!Ai}s3`!$brtnh^EgvNIk-HP=e=ZuB|i)`u+(IVCWMmHbE73I#uW2Z1M zu3#isNEH}Y5Ge^`F&G$EFnR@$G2#+~$EGlY3>a6KNWB#2F&I|@3x|bKA2*yP_l5?< zeB8Z@FqW*Z8ZfR9aeCZr4H#F5-N;z?!gKz_Mob)QGD~+L&(Zuks<{$kM+=F4m502o z7R>vLH5sz8?wn_pq_GWgW*wES%&}OtI%JCPSQloI*;uI;yRGm=Y`6ZKLVjprHs+1m z7e=^BEU6H19&?@%uEjmc-&Uv)OHtv*@^=*O#hf1$1XI*f+{Vf=ThyZ&t2PT^n1^a_ z83fgoBV&N`n1{tHE@O7YgJVZ9F9ufwR#lc4E47J-RH}%FXkA7uE6xjz$k@b!_Fb!L zu^xq&FsIZ{GX%D*Xv{CQ2gFabb`w@qyA|t~#%~68y*vx6!hNgpn^~T&92&fX?Ai}F zFIDUtZa0gm+u8p?s@Nyo<*7HB(7#jaKmk|zITY2*Lhd8W#0cj&i>iz24^n>Wmvj-k zd0Xb%_zl>gjowM28J)@-5k<-ycT8^VbL7mdT{OaO*=9@HmLa1|C}R^zJ2JeK3Fp~M zS*l?h5^LJr`pzoM!E$T7Afu==fPVsn;74%@28#gPb>XN8I+B#>eDl z>9{qly`-GFV=`ps9Q*m&dq(tKGbQZtwa1L!8Q(W~SHk|h7I0qbvrkfV2C#{gt~bv= z0|Ee$-NBvX1x%U5yv_O#0bg-O$a`Z=XLn;Z%uZ$vAmr5z5%}{-d-?~Njal`|v>n*c zLN6?SHY-b6Vc_c8Qe|ApZYtTPFG#A!msePMaraJWqp4AyFe2moWTc{0tc=KxeT8d{ zu59pE9`n8|wfRg*QmAuKxYR#czWpyVAb1qjAC^^Xyer5>tw`@A)4qpNzr4&u8k!r=E#;K~&IjsV>exRL>w2f`S&(j3M^58~C3@MN zz~^d%4b7z;heiPBQGT9j$s<9of0Z>{Z>;5PBps)&Lyd1IT~DawG^@lnBy7-XNH|xw zj|!rO8uucdqQ-GDRMfB2O_@w#IOsavf;o>eZ`uA5x3{1Sa2^$Tt=nw=S>2J51A$Rh zOHQPdBiey?cf1URzLo1`+tWG|DmqS%$fGtOQ_GD zQ$DGFQ-;jHQ2=Z~GgX`(6~K96a_5Q)yr!$7&_LcWUti}p(u4YW?UU8ItH{b^|B*i8 z_K_>7+-pd*`&Ej?{7@O-Jj$a1S(@CeQISj;kwOWP4Q1RwHejmo_fQ)GahEun7(3=| zl_xWVtzDXKUs$`5uRTV^Cm?KufYyfnjA9n3_Ua?-Jt9DnNRpw)^{(Q(*;xJsI^1MS5*#Xc*99Z0h&9%mm5 zI}n$od0;Q>Koj96v40oB4rE(X#-A&M9Z1vfk^hzf*nwo_Y^qCUVFwzB&>SC!9cX-V zgUb_H*nw=TD`r<^VFwbXg93732hv18N{WCl`h*fn+S6ESny>@e-W?%_uwVx&e6gq7 zo(nrrp~IfU(stN^xIUUCeAt0ve-`T4zz$@SIzDOS0y~f}^mBd|b|7Ksr~FFQyP7K= zIS*`M2a2OR3_v}My22i^^?-{e>_D2j!dkg7S7l{vx@%_PJ=vUlHWp*NyJVArI@to-Zu2*zZ5sw2 z%L2~ZP%3BZWdY}HEJoXK0}Y(cLUYIw3A+CO;+dO;O}3}ZdqxE(S$Z}gwYDvzbUe41 z(@?l2e&#Hxe!7g5!m`vnrSvdv`eICE&Oe-cvYTu^jqx4`3)xYuE_q56IeB^t?sPz95+nXMbSpmQpayP7ArkV4xLC#9M`Ig8|NlJ zABl)W)^_@i-*<#C*#Zu=IPB=B^lx&_#rjh;cd7alWI;Ts2w4c)qe4QaM;sUDs;g3>Xw(%)wK?+UL2b?>V_ zSuN)tZZ{wNj69JHxS#xKg6~HY|7sZ45~JVwB5z_?7oz7L`E3-AP4r9%G4NqP{rNZ`kyE~&ym?^n&f`Q)@?vE zUKRUbtKk%VYpJM@Ots;-Wm43cWtol2>MHcMUN>>HT7?SzcJE&3m=7np^(}};swARX zEZ{5gA>yrSw`8J%cn}4GY!7^t-ly*_hn3-=>e^T3_8{XQnWLuD0jXw_U0W+e;qi|6 zATp7EAF92ArzWn@3Sj2;2r!7u8glKmT&ECX5H1REPg%DLQzJH2C~7bj&j#N$;rW`|6o-$6G) zIAo4g0anB_Gh+I-cki@DSE_ish4SuXr1d$M$&910_t@Q{s>?DKro7{409M4`n6ukc?c58~N<~D&*!{i2AItJp8$UEGL9Js!qm9ulxPxr8%1G{-U@Lp8Ls&2O z!e~315a|BI22Pgqq$thBiYae`KP!7LA(#aNiw)Y6+hK#Y>ij=^oryn`i`)NC3$hJj z#x_Wul(D6vAyiZ4z;L_MH9LxM3)td)=H$#{Z+K@m zSW4VEb`wSx;*6blX%GBjc8=wamLP9J%7xu?Cl14dNcl+)#Z?&uOBbynt)&a*+uXsj zw3cqbi!N_s=e!ONUYnI=as&>_FtzMd*XP~X2|t>y&#petcf!k%H%d3m;TAlLc2aXp zs(Zk(+{(;0dDH_V?oR7mllwhDm|MK=0*4uRbFNJ5^j#*qv70nD$xu^zxHso?un+ z%)T?jlM;zZC1|NCk4TX%j29Az2|J#~8Il7;csM2Z_VtGKaYxa%H6AH>r`sDP?z`-; z>Z$f{PH506*fQgcukBbt>&sU2-$=Mu@gez~5bP{;PLO1lN+`MrFDBS9UP|N=XwNz# z4&LiPqjBg5=+^_d0iDPvYpwFoM09m^8|Ju3kjNR41@Y%`cjD^emJc}{ARRby0Ow4& zPPp#kkn{$fgLa07h-f0`w#2YA?Ra%9zf59Z4N{Agpo&9^MVN}{7^GMvNWtBOR@u>u z4w7@#8F%LFLAL&@CcjBEx$xtK^>M{GVKXvCqgS2V7tpJk->WBUCBD~qP=)1jS9FB6 zad4MGq(x`sWc$Q1yEcjQpUXq8BtRNT;=GQT1LpwOBjJLAdMV$5lGi~;tC!T(;7@QI zB{H2s>6G({32g0ZgAA9*xH!mz{CeDZV1K)|#Bp@FHr}mWS)xzDSKF^*GC(AqVIl#Y zBQ6KzDMfKhXwscKm45dj_aTWU-K}FWB@7%RD?2dUoe8-*&n!9FTwH=4y6HsZW50)W z=^afHwQ|F_XU8=an&gLd0&45xJ5(9z5~euX(e~cbtT%8NT?a!hSkzEX1kp_i6XS{U}0dLQNYzLH1Jm4E8Km=N$ zwUVy~KboK6j)x)1#0Md_I;JJw%b^o=6mSv*5u@EAPvaucN*zuf+QiB>OvYN5*z!qp zj?M;!YZ4QRH=yC9bQ8J?X*dx{AU>>fA`*nLLI(~~HBux02_18n?}gORpo1KQw~*h9 z#B(QMc1Jq*d_ulLkTdO20D)%5*&_jKFWSO0?H5HL8;f4OJLxL|gEV^e{$zy6m85;> z)q9h{B5=e*;yZy~3_q4!s5bk$eWdF3`szol7~HZ8QUma)2^vP!e?>vZ$h0O-22HIr zBTdWUKBa6SsNY-k%VsD2hRY;G6?Fa0jm+Lwfp=sc<(@@LI~A02ohY|cUJ(jPxVDs> zltf&g^Wc$!veNpLFv4!R*-E@SQv`QA#Rqr0isryPggZ~FMoL8<1Ze7?jB4)EaXO=2 zb7J&_w@x+^1{tdes9IE^JnZ^(wsGh;32=eG)){ULiaEI^@`t0MPL$8$BaWvZeNdTR(O zy4X$@HH+edF5dgH?H@!R`_l!9cR0aT)5(=*U5ZYv+eIs|6`|L=a=*5BIw4JV3nA@<;c2Flp)(VY<`HaVKFXS-egt5&4>z_c$8oKD5+iD?(yo|IZb(23?#)0lcB_wl~s-;~JMfA@^QctjB7 zr$EPu=65GSZpw24TSt@;R1`ZaKbE2gc{rqsZ&-vAGETt9CRa5F&HoMO} zs84`E!?a56K;%8zF)y3+hr8h`!uf!*`iHFZjK?aqufEzHAA?v()9}avo;1Wl)QraO za(o7l()Cn=-VB@?zh9D?IA%1i`P6^zKIgdfJEoh;{7^_{;-y0}(#z-ODnSW9FbP8FNH5H zB2UqER7B4!ZHC&C`<}E!IWp9is2o18w1wrbRPOmKQJh?wcof=3@;2Z1&%5vWG*Ov+ zKk<}`?|FD0`4^N^T3I4-D(-KDw^?xEN9Fx%VGb*Se1pYQc|SxusQcz;pkpB^g@sl@ zr$0zkxh?8OZ&8u*^sj?H5f4f0_^S3-{Gy)~C(;Y4N35zf2d}3PRCXdElIkI%f3-h6 ziYh4Stj8))2%xg>rLdj_mtIu%_^sOVU=JeQ-GB8M@37!8%T8s_`SHz7{^R!rs?wl$ zc1PEGwj{#o7nPWo{Dwq$4ynYv;LMr*r)2UCD}r13O!uCW@$s=1?COY%$HG(eO?F63)vsd zj{9DwJnNGWM?xW#M;W4~#|J47@OJDErhNlY$Y*11rRMv|@7KP;K_a`^^mfuxctI_1 zt(x7TT@d~tRq!?!8tD&3CHL7}Xp+tS@QgHzHmy}~`4CXqklhE1bA^H7O>d&oY0k43 z{%&oL+zag>UrqPUJv%U7QC60bA3m;Y`hj+c{m|4jbu2u5kkXQ|_n?c~yYTVKvKE>R z`J7&61kqV6$a;>B%>aRQmZPjuzvP}axb0HTZO(*1A zGxl3o@Gi@JNV@Q=>52Md*7vszJ0`s`wJ9VNY9z^As2OZ_{#i9*ZMtu0C0XbMog>8` zKW-r_hywgg;`+{*8(R3=86gj%<6=m?8N11T7<`F~b5Xw2b_J z8%Ekh&wus}xe^aEi|Bb(vw=yNS)kTMos7&Z?R05%PncO=PCANa8pF)u%O}ROm0@Pt z6B!pPFAXz`@jzRfwkXUj%J_F}%Azo{DEs+M2CRdbMN~xSQW^kqU7uGEEdQTdsI|Q709BJ=i~7-7RVBN3hG6GEGh-QA+7BqaiZ=< z!rSdY7SZ>+g*oj&7N`|@ZwzEHqV-IM#sgW}(W2=12eB%**Su?M5tZ^(sgtVS?IFp& zdcJKJkVSgEsO7qa)ED~$HW>n0M0c)hjH_${$Omj9(; z#2Cnu^(`OBvg6d)RIC<|CBNEhQ8o+6VyhmsxFHM3qLsNbsu5N%&9m6I&HBy)Svuuy z3;ei1mSj<_)Wm9ms2N8pLdqqS$Hi%#ISSLuGM4l zkYU@@{;YW0)<&5##)6dOb}e`_(n`Fg6hywJbnZWyk)(R9|71q;{K&s2cWT{Fs?@rF zD=&zgn0Jnx(>ba&o7Ai|d#eCa9(Ouj>{@-0o7{`7;ONdC!;#PTDy7D@9BOQjE+kD$++oI)TlLr{zHt&?nKVo|qnnrpK z29?TI+2v?eY3-BMxJ~j3eb@>85uKT~F#%BiS=lMAwQtu7Ea^_=YCYfC>=+#wn$_8> z^(5(`){|SSEC%da$-mDZ3F5*r=Hz1A)c`K6Rwci;T@B*O2+EV6+pY$Zp*Di?Tlfl@Txp{B??%y>pS<`dD3!7btL_)oQn+kaJyIQDbv zF_{fScWkg=z2vsj_@e~@C9w%7bqjFE8$zBY?9wgJ?t3&a&@qZpNeAbVhIHFB{kZ=xj-!)Iu=X9PorHN@mu3<)DW?e~}`~-y5+EZ}l$5UXJ8hnbh9dn8l zAP7vgVVw9+vLmB{Nb+>7uQ**%a%w>D+4*+a`y$=YDbf_8OT(B+E{G#FCyo5pBwYq( zqhuw~8}7BbZfE$X-eW#s$}p&yVIt`~rH~jb*pjN@ij)>n;+g%r@GYuNc#m1D3$M?v z<7X)iOg(ZH{F^~i z!Vg`jTh#sBRLAZ(Q>EpRZ}hdH`FSriUFAOxRQFrjM5*gFHI}K5s=)WmaEvqSPPt&u zEp5-Y9y2@qx*@8%mlrUk$R0CmXF)n-N9!sK>vh)n&phuL3xwAZs>XWx~x@CHhG`aD-(HDZj$X8lo0P5&0~XCv&#G*OU?tr{OFC1({V)nGWz z+Fv>2Y6%Q#SV>QZZ{1KeQvZ$xEk>&Q zMunsBY1>8h`Ka(6OM`2#dSz5N#)2w<^ecg@zFy@c-d<1A&5gglHCnS5ny%>os6o>e zzWNJ)KUFL_UTugHnMAi@dZJ_^BCpY(a*t50P0ZfHNA4oJmzw%)T`PWqBaS*A9`ZYW z^uRi?ay*_;Ff$o0=8MOlESSDi65pVWc`8jlW-dD}Dy0A0Bdp zuA;d28G~~V<(kxuJu}ajpt__~VVlB7x;T&`q8p>eOrISled3$?zJKfq6E6KLYA+rp zT*y1e=VM{QHK8%vAEIEw#fDQtTH;~C6?=yr|8S>;*4_RNszO3)=QxuVV&-b_DM4*~ zVR8)>CS0t=XTBU2CR~(8n2<(?377g0*5U(Sd+Z6m2B(rM=;?|JL+d%j7qDRT_onRn_A;&3<#-07aN^^;KH%!-rtPOeotxuoDS~SG zPgMx07DYGG_&lhVH;48pMmz=85=w!{00~qJjn;q2UJ+D_;&Mx58=_irWm=*ch-&F( zlPu;zwG4aj%e8+BswHBjvD=pfswHgYS+^nyR0|C|7ka%LQ7!ssD25}TT3$1EDAL_P zwYa5)8omY9GQ`9y^4vhRxY>ppj)H1=!`!X-$_-SDTTiIT7^oIV0pu>Z>C1+%Jm?-K zfocg^x!?VS1geFke{rR$8&pfTw|;I5s214zucSG-y&fRV=}%nn-0&)4ntV^v6|YZH zEJ|Izwy-%w?kT7ieKK6GcB&4&(fq;|u?>C5Q$vsc1xd5sqV&z76rW-W4UTAKg z_SFZ~LR)<~?V=B=#Vshbc@$L3n`z_R_cTx~v)oIHV{TCC8ZO96Ep`LdazT)v`rZvx zOSm96^|>3U7MgIFc|{*oi~hxy5gxhLRlbg`@-1roJ)&CJZ`Eb~1l2O)r3Tl?LAA^X zLNcu*LACrDd6Qjc4yq-}Y=}ou2i0O;G|4$A1FFRwI!IEepjy{DvPx zR11}*-uD-%mRU};jAA6HmMFh@j-`xZ6sQ)qjoF!xpjvu()9ST+P%T|vHkl89rUn9d z^$*#gTGVTozP^)q0`Y->h5C3lNL+Q(-)Nn5khsv(HvX}sf!1rKDD&hJNZh{$dg+Dg zAaQ?&24q0uLTF&%l@)A&)BjOd=Yzz3!K1^Oagewl1%{cKmq6nFHPX+^QU{4^Zt+E+ zoCy-w%wjArj8j-ogibhTjnV>!d$82Pmd_#hjvVc*l_$T$DDI%gV{B8khs(lE3PKG z%IvIqs7wO!OZ}h3zd+&+@KiF-n}ftPkC_s9WrD;t58?}Nu|eXhKmSvh%?62^nQs>K z6(sIov|ipGb&$AG!U_6C8IZWH9%9QpKS{r~70DQn7HcNzOD>Z>E0t!aTmQ&;fga4P&`%9hL6X`Y`3NL-7IzbFNn zAaOOe3C#v)LE;umM;CFIK;rhlx?BX6B9xr|M2r5fqeq}tq#kNTjOAum%-+w6Km~~p z1S6jf%pS7vCw=B^)X4qYPZJ3bfPN=w*DMacwARr8i3`mmexGle8>-&t~_JTKD5}IIu?CV5Y^HT&!OUX(GwpU;5jt=4*tKR;5oEp@oyK& z#KIC}2U4ifW}kU-8t@#lP+t?Mmcd_k$wq}QFy z0~TlAQVqUEGiN=D;W<=Hk47z4H(lzT>49}rrUUbH?$pPCxO*&`K7Q=I>_*L}r zWfQBOKTW!!WCWf=8cWeV%k}#Q*i{y1mIUw|V&_{-EY>?9|dOd~1K3@7&U5xs|lbS{5hkyL^9U4c=HB z(;#TTbLeulpnm9Q;fS?`#aFo2{TiA_idP`t%Hi2)xUY_6D8!T=^B)H zNCjTEjk6|WgM$lJd9vacYcc%c*ODOtS9wr9!WP48PA0Fh-?&XIK$d^=DvY>mi`4nh zDU#B#)>ta?V%uTp6-kF)k-B8O!Q{#sgAF52yCp}=CKJ{eNI3AgzvQl`?cOg*LDB;x zxyfS`!quELr$ylU22_tg;|Pqn>qv^ftO3=su2E({vrU@Ix~jHrLIDB<+LL-~f~7}J zjbrL;+Q!yRXg(GFjN$iP1=X@<0TpTAcIi;&)#ZQ7Nci8cLU_QzOktAqOXm8yE$Y&I z;#KxKGtIw6=M*O)2mn1LMJT@QRgk!Aie&hfR~4C8;WCGm1plQ)w8^B-8Zj~K=0s5} zCM2>=j(oPwagDVYb_+`ss~8g9CQc@|9bL0ctYGtf(E?1!Z*9tCAIJ;%h~;gW5zSNd zh`A~fx8{sA|MXQ*Eo&yE_|8`o$Vj`$1gh?!uJq5v|AnMuSJ7*3{^H-Ja(ch{WXl?9 zPtnG;HBx?(+wQ0iND7Pi>qOVkuP&r+_rUHD&r2b!X+XjT)hGwi3Q^&H=oYcN`uJz# zh=l~7aurkyRF4RC+u(EeFYcm3T-#y6g{yXJjKxMaH;Im5CiAa?YFQhj5pt;wRLhzx z62h=HP%Ue&ND42tfol0*ibhb5Ue9H}QQ^k>2FJaOSWVPMF=~Z8_qr^o7E7!;H%S&$ zi)MxdB|ORN;2*N=M7;jN%arIOdCMZD#a}6*FGMx%#iJhJ^`<^(_HT%~kJmX%zwS#7 z__a@b6hGChnV+;|S){QTcirw#1B3+5C@q%C8tJ1bN0V%<8f9}6Xl4wwW{rYNf}e6J zAqf_&WCJZ5r77t&W5xwd*-}{FJorsk&2oHGkkqK*l$~ZUrqFU?x|yi=VNAOfTCwPZm(=`0&)A#B)}J0`2H;0ZK*3G$f&qRl(Q5sNvtkOcx~0H`cl^Ch|W~k z^`;QZr<#9UUl8ASR(7Y@ln=0$v+9+oi&H95mm{BL=A44 ztOgY+qaj-*^;<^kw{8%4C)HX8D7aHc6v$yodgSv-fts!2kbu;mTGo6*McQc=%dhXF z!XYS?#V6OFQ&+84l^1KTU!jg`eivT}e(}Q+ZoFI0qnfu(P1I>kZZP_qT6g6doY&p( zcnemE17NvkDu+r&z|z5y%Sb}N!l~F`Z4F>a4M8eK04&z9QlBgbV6hI6g^CdX%jP0= zL0S@kWwUWwOx$EOHAOk@gG#3p9UxR<4J@fZ%(Q%s~V!Ka&FhmUKSx8hZTtmy<+~ChpIepNa+af3JdAMFz_LYDj-CWyA!7}b0W5M2Nl^^T z>I{d&6^9Y9c-mciEC+XXkw5|E8h}O4_#jd-!cE=aj#P{QSgzU08CwHb4v%0s`AGnl z%`qC3OAG*u+!Z8D0AP{3k}A9eIq71#A1T5J2AtPT`f+&GfC~*q-1($So9EO0*Jc1% zl)kY6EJ==gS+Rxy7IuX^@46Czg^X3_B`E<|3^P(F;b{OCB{mDMe;B}$CQmL>S^}{2 zh#J}_BVd{NVA!98fW?QN=gS7LB#+`}nhgOgUi&7xDF&zO5>jE^509X!xQ&J-XSkgufXY33CEaZueOG9}8 z7Pw*i%zOsG(i3F3kcNO|X+Ce60bogra_=cJ1h5z`WUlN_1F#sXX9$j@0a%nWnUQ8J z083hoVNn_a7TyP?eg*;--ind|1Hh7GxPx^^2?2{&>&CE<9smoHE?@&#SXgvecn|Eh z^iT}h?1D{NtOA9w&>jFw&pE?#Hh_gyfcCnW24G2xGMr%pSa@%gYMBUFc)d!&OaKd$ zh~<4z0GeuA9(;4AK6`L02VUK zFe?whvMco#8^EIEo#vkgU@;tG5LLYZEW9!$4K@N6FRQeEGJqvX;GI@W2C%S46uiRH z04!;NhONm6SZ0z>>TY-1CDBU?IIt;!MR+M-j06EEoZ>{NI9+*ES&malDNHmTh=VPO>^IumW@S4{IqKTmW{@?krz7vEZ5TIi}I0)g|i}Va0I|&Gh)U`5H~mmU|FC0 zD{c1y083oAJn059vBc$wpWX^!IX<9uU~D~rMV^%Vi#UKqzR`X_vmL;4O-+8{CNi;b z^u&Yh04zCmHXUQ~02Zy!IsCqB02c9u%?*j}0I+l%72mcEz@k-<6Y^UJfTiPXXi}-{s-#HGz62}yuI1XU3tJ7^?Tn}K0LyO;B@86+v+R~iUA};Nz)wr?77Qj-H zKVSo3krx%GUjwjs7VJ0*U=eS)7S&-{ecVB-LJPo>=V2H3SRBCe-vuMazY2BR0W9m? zwS>5K085;mxbZds3*-bQ^Z!j1gk1XX#t{IE_!S!=w2d%|#eW(b@o(dw#b{2`K;;g}@Vyb_iIQ7qqV<6U(7*3Th_+ zEViSD({|bbmJ<`dFAm)VuxPj3eP%ulVCf3dUa$qQWG~L&T)qonITGdGRiq7I(O$4$ z*`ESn(N;e$IFbTjQOLX(X~qPwq{L_!*#cOKxgQkz?;`8n+!Y0by8xCWhC7&d6cDhS zZ2dhnqzk~(<*7|A2e917W`~A%0a&^y+U#-w3v`c!hIRp1kjjyA0L%S?Z1Tkv082`g z_Dnf(Qy%w?LhU^O%MEU?LhwBR%ROQN_lp96MWOqIu04R|F7XDEI;bwW_xfaOK1?jf zbf&v*Hieu(G6w(_By#{@fvywC91ydi>!cjOqTrq4p8{afMoLFuVkzd9DQJ`fSZm6H8)_@@a1XOW%N&)tD53 zWfv)TjWU2`S7ZNxCLO?%rnYP1CNi<`^pt~p5wO(Tu*P-)SbRR`2>Q|hEXoUgk@hSA zOWLyc=UXtb6binlL52Xp0>vMhX8|lo@duQCX#VXpmy48sB!5@_S^5EBQ7-D|TP6Zn zq}J_{>iP*5o{qAZFM!1dn?s3Y0a#d$%GSOB79VUbC7K0bVI76q5de!%K@R0N7Jvm} z13tb07N5La&zMA!IOQ|F{L_g57UhW^zH=ggC6TE-(Fb7Zt<$A1NB(aJZa{8=ie z51aE^l%+j=8X*t>U@6NV>;|wv*9kohz~WV4*a=`!Zb*w_SyuNs_*D2HVDYp|e5?#$ zQMTyfQxaifQ8qp+)P;MesZ#DfLL42yl4z%F>i3MO${?R3jU;$Xte)y~!@Bl2l`Ls(r=X?Fv_5oP1-(~?UFC6!>V@&`o zvlUsq>sSCwD7J`~gaxpeWOPx&GXN}D_6%O%8o-hvA6k@$Oe}q(Cia~OSY|(%^uIvB z;!7{^odvLTj^byVO#m#RMa4^T1ptdtJ-`uw+D;%*+Bkz5wLh!W%P#vSY8OcGipNtEVCmwy}~j8EE$0&t(^#1W|vK# z%pzb}DlgbKi-2Y6Nx}131T0In1y^PPES)RCeLq40ETQv#&D&-to@xEhrjf^zSRyj9 z#F~nrv;i#DZ5)w|?EscjlND>N6#*{0S^3|aR3&HqBDFb z6-jB3FxHK{*bZQcONU|+m{@8jS40f916XuMo=ql*7~lXbYmpQI;#gw4B}tCR#1fk$ zavBX_f$kBg909OMl3acf0kB9m8Vr;mjwMb_a>5BFmdZ&zkzhRli*ub#`$CMek<@ z0KgK97P-0Bzg^|@fj=f&M5H}2jS@Awev;*PTnBUjERv!k^tgr83J=l_Z2*f%1Cl$a z)_1^EU;r#c54+gMKPyKpaC}NEfCaiogw=>+S?i7wK0q8xteuFlB7g-m;yjrj2Vjwi zaSyrF4qy?vf)j?d16V|0JNr^QfaQN_8qo)^Z2XoBVA<~2!ie1pV9BjG$-TZ2z;X3dtp~qtR67P>(H*UyvfBz^IW}=(ap-pd%hr~g&;CFhOJ~s51zi9O zVR8QV<(mMO?NRQXMOy(ZTNeyg_9p{awyNt3jwAzEHfCl=K0_Q!a?I8uT>y(a_ru2i zn*bIU?#f1kn*f&WhINcP8xgSBw4MkJ=>)KJdTyoW0$6gegwXI#081xjD?1m!l2hOm z8rlh9={&c!JQu){Q$QeJOa`zdM{S+SC2k^e-)yYS2Cz7Cdp8DW!^DzJByqoN1h8!E zKBj8`V7WQWZPV%s6U))6sjlCfLXJJ%`p56E;eXa)r!Iiy7;9_R@0CQt zPMur10G5s3$^OXzmaRiKiLNlQxO2-ui33F=};jKRmpXr5Ybc@%0V>zS_oJIbC7`D@{vFZ zK(nEI1nbTeClnEL-w^}y_80l}M&>#uoq17F_7A{4oL%Tvx| z&$>h%$c7&bF5e}}Gh311$m1gQA}?{Lle!Eu10$jy)gYuEL&d5(D z<KV7zm2!L^I1!N*-ego#t$5fqo~ z2$v}PZ1_Gzm~ip#$HA|LFyTJEj$pQ3{Q;MQQ?D$|jBHF{KZtymvhCn|+}{MjSu`+) zVCKf$MS#p}6?X-9rhoBLFxhb-eqBEcCC{M4H%3>#Kgu=IM{XOUaI3 zuHKfy`$x$LYIO3^xT>aBHBXspc54QxPAs1Rn zM>;xaW4_M#363keRjALfE zLSAaEjxJ7Dr#dAVH}1-Lf{xkNTFa+OT&d=76NDu|%;0W=B1}D33Z$Di}LPKJa@`cdKJY!Zj&J!c{u}!XkcE z{M`xVI%ZY{m34(34GKjM@32fH`UoKqJ7J1Q=sShnh(p>(^a%rJ4v0~1#dV`eCGfP% zj>~|C5r0a0+3lBpd@YL_S9joXG!JHdZ_1%H~Ol;DNVkmvnB5fpI=_Y=ZL9WIP23HOu2 zrya@U-#hj|BMBaYCV0E{e2M$1$`U@fynArEJI+L62A8*OMu&LEqa^MSxjt^IL=KLq zm7_x}@wgX9AU}>Xk{H4f%j;ObC8#O{Inj;=Kp2GMD)AlzKbtbXTf#zepu~F@VQj}2 zGPOen4V@r2ghY;hLh<%-y|3GaP6rOfZgvtW)*eup>u~VE2}!YA14J_73>0V`Cnq7E z60L11awCq|?y}kOI_`zS3?xnxpxc5FT^3T?5sds=T!ir*8tCu8#yAU8JKWLG8tWpY zc3hVDp>Rce)s?eP;s?QhZB>P%4KoJ!RZ*-0@4!5OLt!2hUOQ5fQea00hjO7@OWAVB z4gHtXYnQ?OyPXD)FdcCj7!x=4U?x-O$u-uBtB(2#xE=__H>{!bT^ z{nPui~$K~{;vee{7K656fq~@ z*9k*QCLO0Pm-!&lmdV2vL59Svl6*6z{5yRR$F9Iug%&8)E}qNouYmo$^2MlZgNkTJ zTZfh3yK)Jwo6itdZg&k6jw@t2wU&fFg7@+VUFUEUPKIth4u&V0y66TGLXV@N3)9Td zow*h*NzmPxO3-zzcVh0WCov_^I)v(tRfKBCdS|9-w$V`4_?^(QE<18Vmmco@iIt|2 z2a9Ui3lF0015U2AcF8yeDeiTOsR%9a%4D!{5tXY4r-SOPndLYfPNC$1&^F<|rDCGQ zN#-n07b!W3F6lrQNw^=n!?lrnkmSU`wQhuf%Axx^72T_JopNwhT-EGLgqW(uy@Efw zG@QOyEN&Atbh$fyf3Ubm(9*TfY5wF&xzJ|%J?@9&3gkw85IqSq zLSiSB*hEPq(p?VR^iH%fY8)OoK)TNfm3Hxi=iI%LGEqO;>7U&rg}cKs!{r*o zDRRjlDG>QwR$t4CDSggbqax;y{k3KoPvl9fP?J=RChk%ySe{&;bh^<*DaV2%8W_LIT1lJSgRMdy~Ns&&_HljYvzot~lIVxSY+w3omj3NAC z9xPG9hgcHvzj=d+fLV(w5{MoI)$>3fZ0ezh2)+3>ch zM5LOlSiEr~{dC9fJQa&!4{_J{GLEEl4SzRMapIohV0Bk>h@X9A_g$5p z>CM8^UM2OfiRkx!<5dFJ^!LdNM&IuUtkd@;7aM&q6}Y6sQ^@Fh38k2|kLj-RV@R0E zy7KsTo2mcnm-WRN|1j5>qFz(3XT%>0RJ#z4E#<*FRR;Y6wuE=Y(Z$+a?XL^FWu-6d zeo48RQN}bh&3F@4(8ptno7%sQ`n`|EJzH2l7f`z7H0b2JtFRY>BYGE_UXGsjRULGm zJ!fh^9F>Sa>ASbk_k)?Mb$VEMK^4zTZ8lXh9EqF2nNgc?(u1-;GEm%|YPp|imTcIs zQd7IIpZ$ibqBi_-?rvEF?YOUEq1y$x{(s@?!01U|2r&9;6kfgnRcjYe111i$n;?hM zZW=sba&Ww2NvqK60!n_*Cv43UM(s(;SonAi52Y57(h@%Ykf*M;oboLkiJhn+v6Hj` zL0yJ5#7@*-31}JH7)R9XKPuhhRsINR>@zfFf5dKMeTqIkyUP@gyMKs2Gb?H8t{@-Y zdUwgYur|r%LhF5K2$5(0S{OW6P#u<%_aN+c#-0ndio3#*pb0zRfEs*pcw=baGhDx) zZK;;ZAgkGg>%OVC)g7+4WFJ$jV)(0_53hbxf800yFVj79;}v09Bh_K4eNVz+ZuuB( zJ=Ua%qd8zR%5c-(a)r=RrYbMhfasg{j4>k`Dt3CG^*V zW|Yl{DOSyWE+nCxJ}fsUJaNr*vs!&d?ZMJ5Z*F|(VGf&07T)W7AUr&Krttm8)#K5% zOJarlKCT{JtzJ?reEtzyV{4XV3crQJ#PV|GN#B+5?^VO=jUCEzV*io)Y=GJkawj%k z3ajg{TH!S*yAIy4q_9Dg8`fE;JQe+@ToHUH*3a+Gjo30NtX_s;NOe0;O5Nc2j>yvX zEGc8mHdQmGhkSf_%6-v%<$$2N1iy22+3iE3V@XY-_PZjVw(nxx5*<@E>z*7H^)K+6 zB(0lO#=Da|CN0+~D$}5XWb(wiFVf+%xyrQhHnb?baw*UG{UTa}8CIeV?ccVpka7E6^j+pKQH&x>VCn`s_aiG10% zeVzS3)H3M|L;TPr-f&{_m1wacvdTHRB7joHC99vqeA=@qe>)uRK|Lfn`661+HPsYzqO|Ks=m~^`?3Af#17+0C1J-^9kX?qWH2`({)*DL zFW+8}+x}eoyBFWROsg(g<+1H&U-A{o)#P<{*t1*@$FYNI^3*)M6V`_tB5$3CKk?eg zvH##@d1;7^q}yV>xqg(iWG_2)-^EBuT&KKU(b-4!_|YkCXc#dg>^2(hIUAOxchDy5 zp>31#=;21AsN0>d?20t|c8~T=ozm(zcI08@XK&;C87q)EQM^ri!GN{^`P|z`p$M&q zF|Adzg958~5GC0`c}!z59&4H2n$xY>HOf5ek_AV3v&?P3?uKe2<5Bim_pE;oUddW# zi`s2+gcKZF*QsiYlG}6pS?82P7M;t3q^5Ny`c7YGqPW}~BN`9AJ&rUV>yiu`%I>RqS2{(xA>p4;QU8t0Fb z7xl7&Qy*%*H(aUdTxJ+&E#$AZOjI@t%l9R zQ%|&>L)2uoeJWGy%Eo>9tIVldEohC+U!_l3WL-&lZu>oOQ6uZeZKGJd*A?EL`y{`m zuG7W(aW+YcrM|^{B;pyclDtEb>Wc5^xKUb@@Ehi%tKTMQGCRxJE@_?WrHef;z^AfP z<#mhv1iz-HGdnT%@Q=W)EUjevVDeo9G*3fRr2a9}xbz|Om2OdxK#Fpoc@fk9D6%O* zWtNB^?uf$lDJl|a109Dja65!Zdu@41QVqf;`cFa<5|FTo>FKKpRZOISWT4a1mr}^o z$F!XOPp?RQ`Og*+3(U9E&{Xb<^`P8g9^n3#8m~9uK}n(wa_unG(^##bCj2;84-qvk<-bSJkP<1SZwI=Ywp39~KbDCT_~ev}Kl zz1-cB!Kd@|eTaGIJaZDDZe)g-ciyuoFW56SApwI)W@2)PM1Rj)#30Xvglvp+@*B($ zk$B#-kg4)XFQ^Xw^jh_!TnxleqFn1%V6HqC<_q6(H89JkR!^>a3xZRB z5LbsEhzFD%)>Hj9I^vG?;O$ox8^se}H#xFdn#QoCIf@@=$r)!TTnK#K;Q0RVy1{|N zTU4z_@u#s~JYqh3)a*=w$I`+4ZKDCY$D5*3dbN!0(RP8x4Wr{el7laObn+htHpvAB z7WpiN=ldq@3T$eK%IR$ge2r(bnvMG!D6O=%pHUO!+%xN*@!PSvP5KR#=5(C^*xGzF zI4BxG(yOVbJWM}sY?}Hh;9~vg8=pOc?|fwP%>$uHG4N~s=&;Ye!S^%2ucy?~`dR-NtJeu&)t9wF zRc0*mpqW3oKfXSamOj{H(V;(86m8a&5h{fvp}(hYZ_i zY>eHTR#GD0%$$ds5EDn+M=OcFoyO%!mB!_xk*`?chIE##YC}K|rqPhiBB?GhqE#gW zbQ|lB=r+|eSSM7w8KJ5=0o9H5$Eus^S*+8lp8|%6!C~>e>Bf7KdySurKn!J!T-6&9 z(0YE!yuNqloAJK7X1C9ED!#``vEjE#FQ$>+B))m4k%R*X1Y^XKhSy)EsgWNfQi zJXUutkX|oHkameQj#iCLH#p?C<&oT&@Yt`$Csc3slA;3}>Q21{$r3RCc;#X50%O_u zO5I9XZ!x3a_)7gsRquPog7J^K=Kh&tu0j5lu~qiW^NV&KVn+TZBmIbC5fd}Tzd#gA zyeSGr0mahO&Ov2_fMWS{rzrk!6et#yiV408Q7mF?Q?ouqu~4^}nsp+Ig?iP*+HAz!rT0#e(X)5KzCH}lxcqk~A*}7DE4GI(s>Jt9*2T&|u>N4rAC{QdYF+6q- z6w3t1gSrO~iX}WUi(Y~P#iE9ah)j(K#exco2%*MO;p>&Vpj0s$zq}L9tw zgd-e%YB3%Z3tss802368Sh2|fcCI7D!&Ep9ibX6V-T+JM2%Fry0Sk&n>_+?uC>F|f z(-&A!EYt-0OHeGlkXm}M8Yq?sVLV+!4HU~oVJh8S4HQd+kV?Od0>xsoI>ys^SPi{a?6{|-XV%5_Y0|LdNDxJ zsE_iWh6)4y5K2jEDhU(|g;JUNRUZ@!i8RlJBRVNnr10=HilA6Xq)$ly1lKgzjdbw> zD3*v(y0?8UD3;vIq^}o1u|$}(P4CYI#iB3P@~j41H}@u@_To#8z`0wE4RCc-9WLBS~;PQx5k*dXEt zTUW!NSl%<&DoVP6VsX=@)V~JBGR~Ax)NupF;s)V~A*N}r(NN3yWN2A8D3)$L{r91u zSUxVQzGBYm z>w30?!1^~R7RmRACPA@0O(HS$L9wL1J~RW0rA3#NqYsKD_mW%8Cq%JyYmmOrA&RBj zo%H=9qFB23k>228!jkfSLMcHYgUh zv%2dgP%OWh_3}QbgJStDU{;`;>3azj3pLCfiJMSjsgmZTDMYbQZOj{&L9xtmLSH3Yb#q_{nBrsST23$)z`8?v9N#UKz^-R#*gr% z)j_eCPfSrN*q~T4>E;t$M6poE%*Q#1VxhK}kMj`4Lj7iLHi;+}s+_v|S5Pdk>0avU z%b-|3aE?NF1W_zs@hl!O6BJ9P{iSFYqF6HfFGaT_iiMqP*7_MyEKn%XIuD9vN+6w? zWe$qP{P+_8J{1%Tv{Lv)<>}3NW>A~eL9wV~B8hXLSU!6uQ}3yRVo`UFBz^$J^2IZq z`dED^s=9uHCz`2h4vNJ*NFdB%gJQ{Ch>RXa6bpOVJZ2IU3rAp;xz`*N%U4=AZ<{(O z7Illi90YeZyrKK5LoQ{9S)RJZvWNIsJdYzOQ^PkCs;`xCPZ_Y9()NGeNPKf1easXM$oe|IQV-WP)Nb|IVQlvq7<_|M)7*WMBE?cAJI& z>&5!wOi(Nqs6Q#!GeNOvT!_YU5yg^8zl`O8Vi~qJ*ZAu)D3;5+7b!P0L9sx*B&whv z6pMxZpHaW}gJLNz7Y0DQWbn1`uHxRypja+9Esnkg#WFN|&cc2f6w52$y~VyOW@DgO zq69eOM3X+2{gDZZMI-kctpiak8fur@5XGV~{P)}xD3(57#bUS1 zpja+{<&VAs#WLorQGEF_D3;48zR3WhSo+&7g886W=9aXItuBLN`G%ce!f1eEi5{Ql zp)^3TM2~;rscV2@i5_3z{U>jd#uwCOf@1kMZ^AgDVgI+%^iRdoZ)m~(8x+g8|5MhT zxI?}8e;?1jFJmx_F&ZK}6EP%}CB>jfQj-`8QPyglrm`FRRuRflNywIjrZC37PMuRu zsTf-aWrmKXIGy{g@Av-Q*Y&&m54tYbc+Au1{drN}&YM_6V);(}FfU;ZiG})NX7zLl zB$g{L=2kD1Kw>EgR++YQfZ+zZ*s%u~?h)CO$!8S&B>>eM*JI@`Ktt{|rt`-cesHuL_hv zVkuGkG4a=CVi`SYy@v~lh0A@Il`3OjX>Gk)_I)gIs_QGaRR&&gj8Qwq3rn2u`i9^^ zVzE|NYsjjzV{81K^Boe4wM5DN(JEfgym!gRACOppePN^FdiVll~yc`68rMR9U%%OycbEDDofwtN$W#Dbdqx#d6SCO&pC$pVmAr2czu zvR|08qoZ=mW?~V@6eE~HVzHil#aAK*iN$bomG6`&Bo;q6ttn zbGHP==tE+WJ|?Iy0Eva~oA8DiB$oAmrpMWh^e#PhTF8|xv$q8`WsJOz(c1W;t`(4V*$p4>gDZ8%h z3>&P%N?)`SJ#W1ycuCm_YC_m161Jt;!nM4az>UX_98`hYOM8 zv{$ypxDiQ~A=3*tBJDL@oIRgpWi<1h9hZ$tqHE8eiWfc3HNDjWuQk z!3~5p3Ty0&7=Fi09?l_=y08bHIVaK@a6Ktp+#P8ykp_tccSnY6Lql~Z$QmV5ESP%# zxft2aw8QH#Rd8C#@Ya{40>zc#>{s3C*(SCl;;=+9Q=A=XJAz!bH56?%o>6Zg%;aV} z*%~4TZEHnacg*NESk4q?2iw+49QM0Hlfs!Ljc+qc9+zW3l7+;A`yj?WPy5DfmR(Nf zZDn3;(10_Ni%Gl#R1DdZ=4!K~c-iJEx>3#zakg(1z7k*XO)#~+ihU_}@xb;O6z=3y z;#Zi!XqWfx)Aq-#=;b63ZO`B$xEE;ZcFN_tFT@(mnNM*qkROwIIA#{^s@$Gro+h&n zcNN*5%+p}LkcGrzvjNZG$^M(f`}nW=>q>?2b{aGmb*vJ%lm?9jv1n(tja>#S4KO*S z&dw=6&-@Ls=n%3My+1^rTS|< z?)d&dg|Z6?X65G+Ua1#jrzFGABwR#{q=sE&qUY_b%-PY1Q54FKZp^kvz$(2R-ITme zgT{g|%A(jap|K!#?YCn>V?oqqjO&mC!cQg`BHH%D>oXlbA<6<0;AQX$)(E%_+5W>p zN!HgVVITX4XuQURtS_wv&u~U0?N`z|pJ{BxfD;sjr}p&SY*oav{3Uz(F7|H3vcfle z`X06b0^R~+k3n#!XwXm+FEuL9mXKS8&S8PVgM5oTB!#SA85uF40-%cMiVV`f3e{iyJF?=qs%|!5c77$ zD&cnu)WY{M6A`GCZbZHv2_1gx7&hFVc?02@@(1FL9Z3cLgXl$#5L!K+q19l)xG5TH zPdO2i!Es5rq(R@_Y){$45J5DjsUob?>rEK&e1hmRb*h~2({UvC73~Hs^^nF@#lvZp z5*p>o+*R7mdJRiNJMELWQMnqP#=9@7u^%x_JC(AHjK-&Br`2i5BL+l0xXuaB5ic@0 zw|UkK8N}6eP9D#KVT8Dv!71YHW~d?7?0N1R3RBCX|Jw1)tHdym_dhCF@18h?kzMU5 z5b3r$_Isw+_Bk9L-e%ZsWi^MuXUSjid!6RB=ZpZ$_{`k`hiQg1iGyI6Lb zskFU2P~Rmp+%a7X@1o%BXPU;c+AqP{vAQGcfCtn4b!5dhxEwsdc)K|@mgeWgW8nnb3-opU1^Kt zs1(-D{qQE|{aH%8{orK_eqU2q=KJ@s_VmOk!4$aMn_eZ%bdnM@7X0~CylIbdcx3t% z9v*3`ZZVkqj9;cjKWNdLdyijEiLPrgn)?T@M*Ch$(Vn9f$n;X}tkA0s1zEjRoAmVW z{_qFlk{>erdzFxWA?e{{YWf2|E95l1s5gDyk1jiHkH_^m;k`4tsg#{P!Fca1{9$4$ zcR%b_z#D?_72Q;mxhTBG+)%+|@`z%(#rC<0f>q{5fdttwz4d^8X6v@Omj#2&l>$9- zZF;L?O(twbzb#lK|9(sj4ni{UW+}t?W+p|i=N;UETq4s|rgt|O^sM8pX$>crXk1@d zjm65^&QpSCSqe&eYM4h7(5sINjuTHR;fD)KdYckdU!{DvCwfxw{}k*SGO|}P zNmcIMaNLG}Rp6CQqN?7d9qy@24J9vSaL=VfW5HjgasASvvEZ+!aD&sKvEYBwV2q-K zU8i)dm$$vAJ=}RRVdq)R38Sf>?3e@*Ei3J*&hm>4OEH;)6nOZ1YDz5Y0Oej~QN1_g zhnNa2NlZo4N_UFIE=cIrA{cgZ*$IN%wFo-<{K6yR+~rIUhE)X?mU_j_i5;hH9*G3r zF3kumnrsq$u$^FAe5>?i++(@o+x0ybVx!I!Zw53LF(V@7FasKkn2|FqPKbniF$R&| zIUIOslaVQw!#Nx7%m_wabR730<(E3e)r!HIjo5G7KHVyX2fIsM<6emUfU^+cZAt|` z*)7(clY-{kl+@1?!`$P_#FA2qkdm%pP7G))Vxg`S2L?2j|K6Q+UlCh&&XHckz> zvF)l02bL<38(LRgISz4G#D2Bdsrt+vG<;pS$DWjCTjaV0>1m84mvH* zy*Ri0V5+PX57t!tQQm_K^H%x8fV@rRzIK4o06JfMXLQ z!voR218HvF`i2Lig9g@gcF_jiyzi`}UWdlwc9EEJ9U2QPR5ZE|jIi6}Pr5}p|7Uo@ ze)QMO#1*%%&MeojuK2Cp9rDmvbc7$Y;JRhQ{ZfnEB%SfZiDEJ(^@Q68-SbY-mj`ya zeSqbP=t~2a-9G4jaf-e+AmRou405zkpY767Lu6IJCdIT)iL8dZ^{t2_0}p=2vvhjy zDHV~6mHxlO6XF4`TWY>LUESN6dpz}xJ5fH*jh;h}NQH+rQ-gFq5-m;+9k4KgSqj*r z7&&NRK5K>FGaGkZ!rf3V&E4=|tLf~H2CLcB>w50Ba@Fp&2V2c%+vIn?elVU*_Dgl% zJOR=9;K*$rFJ#}AU#_6IhB$Grrpma5pmw=Y3&;Vf&{%XX-sf%WKHso?_JNL`jzU?* zhW`DVZm9i^CbRQ82EGrX+^*)lRPc6$qm&gVL`6CrrL@Z(956k7Qyq2cfwtRqoho*b z{AG7)5qlT+N~)UMMgbchkV`$~wo%A7=Uz_T0Bz zOFLofbgZ(>0Hwb2c7scg?GCK2^dN=Oqp<_4vv8&+zu{_+aJg^Ke&wtIN)thAxAa&@xA08h&9dku#q!fhN0pBU1ZuoJLon$2OL;^#;7nlt zRE=anKux$yPo&A6d@eP7Uyp=|_jQ=2KiK1B(mXo8k93o}lJ4nGcO%_s@Z0ZCKTgWy zzD>Wq81 zo=tiKI}*O-1=puCB=bz*Wl|=@LEf33zsUQ0ZYc+xE;cBD|8A@De9Cuals~B;Jm*+3 z{BQ3o=cmjnd-{`Zh2J=KJKVYFiSnD%l?~QiNs=liA#z0~A^mWH^0fQ4xSOm1^vF9Y4amabQP<@?*(Xo~3P0+0Y?{kN)FDQ|~09v6yURPxVSd zV=>vtoq8h)jm2am$M;kcG#2HbBOKSHYmW;%EP{U@XgxIojm1J{DEiPyy#43m3lZ4d zRd}3tq$?7evwGmLdBF4Hza!0Lr6SD+qK}L~W3k8@4ku+oW3g}<3U|xwVb5S5%mn66 zA7Pm~JjM*ey@)|%(*VoTL4{@38)V@!8lFTjad?Iam?i8A$cT)%Q?yDihQ<=Hd1X=z zjU~dn@bQ%qX?A0A{ybfM{sz0MST$08{ttFx@te7&>yIC|7&_!*PDa8`R^-<{mWczb zVmXvxu0(!C{uBA@3}kEgmCR0y^L==`iGt}0jBTVO(tKX&&h*Ve`Ao{_b{O|1;u`;1%-acN6aH#P$m&0 zLl%1TA)Uo=buw(>IyBn4b)HcS58xI%MHmjZ67@%0)!}%gnz>LM98o*mYUq&hw}siv zc!6Kuc(I>T=FwaUA z3P&kbJgwdjn2&Qj7i!6LGG@;l@8Cx9bUvnij_15_XF3bB9&dD)oqcE%cVwEaajW_FF787NNHa zgzAnzg@TLSITLa73Rb5`hd7Kllmh)2$&K@Jb9O>DUx7}aR1ks}VJa*glZBm%H}dpm zp9gW!ScKZ-KOKizPw!HP*gHb-l0u;$@}EwWmV7$pvo{tV5rO*?Ny9MPSZFN#<_b#3 zeF|DyJ2?DL<>?+1M>%V8m*na06CRu|ao^I+G==)v*S=W1YPc z*0}wO)W*rmIM;tf?n=R2XXbP5Tf6iM3@Q6#sppiXEk zoeDypiX_dWD3W{Fxy}nhXKRe=%Q|}ap|S9v5T|=go@;p0u}kQK0S6w{p8SVjO@7?) zqCBs@V}u_Xi^P&H=U^-}mQGiRqvEGzv_sUAJ(W zzPi4WLb}9N&D^-4&YHc3FANXKYfBvIHWyk`mD z4UMH@=85L68_-zvSoiJT$ib|q+-}FO`p{SmSr69G0Tu2#w`t z%R}av95fcqr2Llqn~f#+X+Hh#W@E{{luv)K*;sPFvGn&kT6;rd zX`IQ^bhxqESWc3ODGi#?STye#a)~L>STye#acxqdvB(+SptNo_md%Ti=g?RlGF3FS z@}aTZEI7I{oC}R5*TiW=*WyI|OmS|c6EqelL+#djXe=!=xw%eG&{&*mwObpZv9!$; z<_0^}>Oo^kkQN64jPN*2OaMD6lg4R%lW)gXe@W(jO5}?Xe=bMXNs#M zG#1CJ#~a`^-8`Xe{}>yUZ7I&{!NdPO?kn{=K;~wz}qU z^}s(>@U{8SY^(?N$Siynx^bb(>fR=AJ+R)h2oE63QLnmzA0F^)s5<1x>|qc1XnD9Y zuA5oC`QxIn{b;zG0#p`{VmJFdUHIl)9eC7!6|O~YdNfs)Ur+$y>bg|3zv_Zoj6HWUeaOT6_~~_BFm4r{KA=|xZt*zXt9y7Bei!N1 zJ!*8n0iQ1(Q4h<572sz^_qQXg$ElE76s}Z#I$^(u)t<`dAz4GtZh4W~=`nF=ler4! z9{8qIpo?x2texC|Uq@B+=L5T{Ha}u!-+O#;pM@VA-Q~KKSY4h*b$r&( z!`gcmz7^eYJ!(<)OS&HE56K}4pcmacwd`BE@Egg4OI0vB!i7(nZYK{qsT+QBbiJ$D zf9pbU@t_~>p3#Lb(H_L=m0?}@V&pMW6LIv$W1;F|_t^n?VrbuQ&%Kkh>}J9c(S(gX(C6J^7s1MB&wy@i6iwkj!kV>Phs2>&(k3qx@i+18sA{J z+Mi7flCSV6aG_QhKW9jao1_xOP4xQ8w^sL6Y<(7o5w}KJD9pHJpV)`Z@!V%E^j$$- zR!D(bR+=5hD5u4hi>K;n8#-!hKP~qucoxVyz*XsBco2? zsz#M|O;vp%W1&2WQ7B$kf3xyYeI}z#d@%)%ShRUZ_t~-z?DG?HmN!)xmN)erv)Sjw za@-dzbWXlW;j?^`=h(h|4y=Rw0)$S=M=89QkMiW)W-QywB<@yNa~nUgQOa>h+@rJM zwNd`R4p5Y8!f3zlPNu}KdGLxDAOB6>BcFcA*sW6=f4oyk$WH;jgDVgZaUR6AGe3&| zJZyZpa~s~PQ(nkjfrxyiFm%Y+y;C25q7#!CsQ{lv3WkSxb&SL~ag+|JChY(2k$BQz zT}OxXC@kYxb=Ymj-{R)93~_UviJGvqxRyua%{t>SB{Tle=hrLNJvwfjDu#|m5LCRi3A;ZJ;Fud8< zokg;jP%yC9Q+VXjbeMIt$(`kGFQTAozg^+BN7P}KN0dA3WcMZAcGT`8Ye)WcIvvW5 zA5io7d2c0vn>JvryIutk=2c@#BT5JMk?RL=y2(`-t)%J=kJ(V&YyYbKJOno-2Ndoy z0~P#^qa1F@Xka}ylpp8;W0BrA@azG2Qb*P zb?qDc9I_rT+n?}J+alSYT{nR=Aj6?xoW(E{47s1kREO_VC*BLxgMu1n%02BKEnwfnNd3(+G9y4IDn-zh#B;bNv*zw>eq+`~8Y zWbM4Fo*PK4!r$tdQBUl-j&={)ICAfG!G6Ev8GkVY(654rO7VC5?%^LN>7yG{L(q-E zePvUTIqtTGQms2@5YanlUS*%Mklw0U`IPZhd4sK)SYV9i4WIymJmdNHu3y5UY`U0G~W z=uT7l3{%tBV|KRgETV0sl!0lJ{3Fw**JJx_Pq18UFH5PKM#;l@$?GwDTThm=?Io#? zK^xv*$Hv|Lj53~g%b>6Q&dcj5={7>I2QF1Nlzgg#9hO+MNJ=LP8*M=)TEF*iE?>6VeQhR%M^dIOH#^Y z=aAgvokJ{w=QJD=P4j2`dzPeK8NN9_0Z>j zmEn0k!|11hw0Bj$MML|Ub15^^k|OVIyCQ~o-@MywPSIh+~nbizhz#UyrugB@&2lN&hY^B?Vt-4$dbQ^ zbtds6>FRkSLza!7`g9xr>Z{dA&K$LOQEiQyd7a}h^GA-W!&OxwyM1t`pndOdy>|KC z%1p9lYMPv-dbSN&5Pr>iwooT{hns-Cf8+Nv0Ph(Y3K#Q zz3KW)4R&RwpJnDE0q3$nxIf*LX~M1RffGf4GnKYEeiEN~AWOtdDZj(V1`ZDFgpn#6!kYvS9fXEr+=be(m01;sml|C{+#;9|rw5eugE>y|01@Fql^P#_h<|-!XPp#4M5*R%ePJg+ zq_a@i7a(%3KC`n;7$72yR^q+hYs>1}>nGqW2@sJqH6LT_bz%+d4Hh^j2@sKNG9T;Q z>%i*Y8z68}5+EWOWzM_2!lTMop#YIUPHyL*FhE4OPMLllAQHs6+4)WwAR@f1Ob-Q!1afjZdxZfa z!X?Vo^8k^c$s4h6gaIOa-7#kYBC$_};jE-yl5XOo5|*$zdGaB~h+m*ARz_G|13T_d z;?EOv28i4uhOGGI0YtR&3W;X`BECEB_aD&$h&Wf^Z@u;hi1dHeGARIv+!_i|PfG!a zq%3Jg6aYj_a|Z3sBLO1F@5C8%fQTtKF~vp;AflC8sC@@aw!6c6j}wW(E9-*{Z3j&MF5dP`k9rset<~-39Zl~fJh-dVC6+W zK&0P6ORWeXQb;|s(wYJgN%7S3E&_;X$s?=8oIh!OC^RM(Fm`Z963zic#73EykuOAv z30l?#v>}R5TDX|LRue$PAK%cOWdIO4Mm&qZ)eR8ozOLx*1rTX2=pi3Z1BfWTIx*w} z5b?)9PSjTfh$uFm=<@}Lys|NrXf*(c7|cA*{%FH@c#MHk6!!v%9Me8qR*(n~Nz7Gr z^wK^D5b17IY;OXH1T@gQ#T5Y}ih3tD-m(EA-BpU!UI39}H9-iNK6wj^CpEzUkx*DO z2{izSz;?+P^S_fN2KoRI{if!z9-9-ak_$#te z8bIXNtwT3JL~;GtQZPUyG(4s|RuLegn0yR#79et->zUT>wF!|fhNy&w0YJncy}|f3 z1J2ACf)dw`14PbEslksXK*WI9VEhNe6#t$fDPdv&5HT2PAiQD2Uq$L{Zv{5CRZ6J5`W4tq2fNbnzmb1Be83i@R490V0Yu-(C0YrwGW_y~yryL-gJRJ5n1BjTpza^dq zh`c?6E($sa5c!iBY7##T5E&jaYa9oN{MmP2BRK;gqS6{N(+m)Kk~6yhstQ0v<Zqsnf4zt0x8!zVoBEu_YdgB0*KWjo(gNpznX4P+N&HzMw zA6Gb!14Q1{_&u&228d(`o4t4f5NTwM??~53Gl0nWlHcRHVSvamypH%SK;#{U99cOG z5E(vo?&q0b5K++xNpAv(G~;`Ej;R1dRFEMXL7NcCxE4Ze0*Ex@yL*nP07T#qIYUi& zr=ezi?;NQJAW|d|@-rA9GHh>#`hOcD0Fhx2GtYMbk+-CP$mhcVkzr>u^LGF0ElGxnPt8Mh`c47o^Hwjhzv)XjlKhjyruiE zHVy+shJ(%O-T_43(oe584+BJoFPkmD1Bkq(`mZ(&14M=c%u3z?MBY+PPd~{hp!nG~ zl)W8GI6wVJsw+uHUu8LD+zd1ImJ>pMI+fP3KX5ueqc2nBB0$8~CK(_ijoh^{zF9=N z?n&oa07MAHzl?8m0Ys96FclX7B6G&iyCkInBGM+i`q%)GxvkIQG134L)tQLwcThy$ z?6Vg6t_l!QrB0P*#{)#-%cWB-03rnKze=yizm%RHAm}e{E?uNVNDuC+nFNT;)Hin3 zsRBd@H4F8HT>z1;Lg^`h$mjaZt~P0ah%}nOdksaTZ=autGX@}nF}*a#fFd%mFIeOp z1|WiIx-`}aMWlaUfXGP9tsb*+=wiy2c5c$g|KmNTmKty`YVtg4O@@Xl05c!L9v+JESKty`kg1!t8`NYZT>XinFNS9bpzW_x3 zn!FMJMj9X@&>gz~5Q%>(Eh$n_kD*8U>MBBtWE< zxU}Mz4-hfTYa`A9M5cD!A3S0R5II>9e(UvS5gGhyXwm`@X&qWpPfG)cq%9dnv;ah+ zat7_sqW~hP@1&Va0FfweVw#O1K*TV$O?wU?GF5&r&CSs9S-ReIW6hw^Nq|VJ{&Hj7 zAV6erz_9%yK!n{`HF(w#AYvHYmM{nq8MHK9Yy*fq3!G%1KmkNh87ED{0V0vC33~|? zKm_&ZWK%dmB$D;PUIYaYf#dk7aDYf8>wWhnLx2crE$Yv~%_1_WcJk-UCPW6U4cFTM zBF`|B5v79wkwKhcavMP8*~TBUp@y3f34;@m11Nw9%8!(O0U&Y--`QPa2oQP3>+QBf z0Yp&lB;o~t$R&JNHxUI8K}C{AE&xO>wRX(58=}P9NHY-)|1%Z>5NRXLMm!k=hzwpZ zeDe$-(l#*@v3b|QZf+Z&BR@+6h@}16?9f~5Fj#m!Z7q1K%|X6zw%-bATsD+sP+sX(ng(I zX-xx&q8KLdyu%A=|zoMF18&6rfch~SPSeg=q0jCNl}y$~fO7+SZ`CMZ5x;bK-= zUjZUB_=euB-2f2^aRGm;7a-Dm9qS$q5Lqke@jIRc5W&8>JTwUqnZZ9!*2e-wu#K1d zrZypBYbevY8z8cKW;OdG6p^tm6jnSMAVSezC@V+?h$QD?9iz2B14Mcoe@$9w&o|I} z#jyYptls5~x0^+zw+dSw4G^K!{Dr^*M6ipLnni%fa#?J0=x%_BmFe0T6N<=?ZKRBW z6+pzQX>F_ripYrVWf@f~fQVJp+E_Ofkzw0QG9RM>BAoG8enwdU5$v_Eyu4n32zH&a zvP7-E@A;9*wASDXMo5e_fGFR79fIE zi>59CL>9QWdgrkK5v+GK=mjqn_howDlYexYhS;E#Y{sxGw zu*MybY5)E$U24{Svd+289jC7=iD!dsA*8szXC+o@IAfoT5Bdn z4N2YjYZD?_*Qms=0FgC(ckdB3fQZ_!6Aq`LHGJ-2WdX?`6xhS)Y;m6eS!@T$vS750FfWmnbn3-fXHZob;&wFYxSc9%V-my@=zoU>30T5BG zcj;*31BmdU1$kyrM6~x#Z*k@ai13@rjHyEr(cQba5bVV1Q3y{ckOt=2StQWL=d9{MFbOeprecriU{8|Dc*^W zO^EQPiyH5MB4XK5u*FXRAR<5%E01EZ*3@MQ`SumBO|$s;kZ_y8h&qe50{01+jQ zTgMit_D07Z>yyF9i@GT3`)c_(&oWmWxd;k%?5+N!UAfi0!5%Y$R%y!sn&i*Jk!U-?!n!JibDEoKW4NhD%crn3Pe zxV$8yB0vPO<8*?p0X zw)-Ia-DXAxAR@`Nr`g~DBDmBfZAE|xqTHY6hI33#*F!dj^cmR#L=yGY8g2Sg%ia10 zaP86n5xK^*eP?k15nOPRUElHYi+z^3#Uy}8a-b~xovf7oUD*selkEL ziKGy5p>HSoT;B!Un`D4U(u90OWS;?!-JCS8NWM%{B?r(>;w~mb5lI?Hk+0DN*giB{ zoOCikB#EHNrqGaVf0{eaB^e-+WQAIx_DQhGeNH%|WPnH#U2)~NJ{9(vz7x1mxLNs< zL`SV$?Gs`9_Br6xk^v$~RK=BO8j5|I=85x828iIU%bpc={)GFGWNcHw5a${us4@`e zMU9y2vM)q!5^&ZDGzMjo7A~f)l@1V*$A>esGyo!YL^R%uIiqgRypC|U2Z&@AXp!I2 z03wK24nwj45qW%Mf<6Kuf@pN;LjXifY~W-?10bR?la{>)iij}-g%Gy~h}dbP%Z??m z%k2_!5svoSs`y$Wvk}pr4n+jjKxK*}03rxIhYd?8B96=|M72FY#I8mefdGiW4M~j( zKt!!nSOh=>k!*)S14OXgH?(&9O^9ePXrdY# z01=J!6k{_eBG!xq(QEqwBC1m*qVVz+xjYSGit$b;BD)#sq9z&u5sje~f;kkCJ?uD` z(jaLog&$_Zl?LrTBHx~b1&AnzdobZjgZ2dR#-5}G5K#(uXAUEt+QEYWz9$ksBLE_Z zi}vHH01*}6Qwjed03rx!djbX^f}T2-FpU6+AYAMTssIrcE{VAcLn~@oB%Njg&n(Tg_EH0I!_Pi||m-PbSyaoOpw0E(qwegAjKUj-30RR91 literal 0 HcmV?d00001 diff --git a/course-tests/image_probe.py b/course-tests/image_probe.py new file mode 100644 index 0000000..0a96d17 --- /dev/null +++ b/course-tests/image_probe.py @@ -0,0 +1,26 @@ +"""Read JPEG SOF dimensions without external libraries; not a full image decoder.""" +from client import Failure, check + +def jpeg_size(data): + check(data[:2]==b'\xff\xd8','thumbnail is not JPEG') + i = 2 + sof = {0xc0,0xc1,0xc2,0xc3,0xc5,0xc6,0xc7,0xc9,0xca,0xcb,0xcd,0xce,0xcf} + while i < len(data): + check(data[i]==0xff,'invalid JPEG marker') + while i=2 and i+length<=len(data),'invalid JPEG segment length') + if marker in sof: + check(length>=8,'invalid JPEG SOF') + return (int.from_bytes(data[i+5:i+7],'big'),int.from_bytes(data[i+3:i+5],'big')) + i += length + raise Failure('JPEG dimensions not found') diff --git a/course-tests/load.py b/course-tests/load.py new file mode 100644 index 0000000..ad81d59 --- /dev/null +++ b/course-tests/load.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +"""Small closed-loop HTTP write benchmark; not an open-loop capacity proof.""" +from concurrent.futures import ThreadPoolExecutor +import json +import math +from pathlib import Path +import threading +import time +from client import Client, Failure, LAB, arguments, check, run + + +def main(): + def configure(p): + p.add_argument('--duration',type=float,default=30) + p.add_argument('--concurrency',type=int,default=4) + p.add_argument('--output',default=None,help='Optional JSON metrics, no session tokens') + args = arguments('Synthetic POST message load, closed loop; creates users/messages',configure) + check(1<=args.duration<=3600,'duration must be 1..3600 seconds') + check(1<=args.concurrency<=128,'concurrency must be 1..128') + c = Client(args.base_url,args.ca_file) + alice,bob,eve,chat = c.bootstrap() + barrier = threading.Barrier(args.concurrency) + def worker(worker_id): + local = Client(args.base_url,args.ca_file,timeout=5) + barrier.wait() + deadline = time.monotonic()+args.duration + latencies,errors = [],0 + index = 0 + while time.monotonic()=2,'persistence requires lab2+') + check(args.mode!='failover' or LAB>=3,'failover requires lab3+') + script = Path(args.action_script).expanduser().resolve() + check(script.is_file() and os.access(script,os.X_OK),'action script must exist and be executable (chmod +x)') + check('.example.' not in script.name,'copy and implement the example hook first') + c = Client(args.base_url,args.ca_file,timeout=2) + alice,bob,eve,chat = c.bootstrap() + sent = c.message(alice,chat,'Сохранить при отказе') + original = sent['json'] + target = sent['headers'].get('X-Instance-Id','') + attachment = content = None + if LAB>=4: + attachment,content = c.attachment(alice,chat) + if LAB>=5: + attachment = c.wait_ready(alice,attachment) + env = dict(os.environ) + env['TARGET_INSTANCE_ID'] = target + observed = set() + if args.mode=='failover': + for _ in range(40): + observed.add(c.call('GET','/api/v1/users/me',actor=alice,schema='User')['headers']['X-Instance-Id']) + check(len(observed)>=2,'less than two API instances observed before failure; verify LB/session affinity') + print('Executing explicit '+args.mode+' hook; hook output suppressed to avoid leaking secrets.') + # No shell interpolation and no Docker commands in this test. + proc = subprocess.Popen([str(script)],env=env,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL) + started = time.monotonic() + successes,failures,after_instances = [],0,[] + try: + if args.mode=='persistence': + try: + code = proc.wait(timeout=90) + except subprocess.TimeoutExpired: + raise Failure('restart hook exceeded 90 seconds') from None + check(code==0,'restart hook failed (inspect your script locally)') + deadline = time.monotonic()+90 + while True: + try: + c.call('GET','/health/ready',schema='Health') + c.call('GET','/api/v1/users/me',actor=alice,schema='User') + break + except Failure: + check(time.monotonic()=5,'too few successful writes after stop') + check(target not in after_instances[-5:],'target replica still serves requests; hook must leave it stopped') + print(json.dumps({'observed_instances_before':len(observed),'successful_writes':len(successes),'failed_attempts':failures,'first_recovery_seconds':round(first_recovery,3)},ensure_ascii=False)) + # Existing token, no re-login. Read all confirmations and reject duplicates. + messages = history(c,bob,chat) + ids = [m['id'] for m in messages] + check(len(ids)==len(set(ids)),'duplicated message IDs in history') + check(any(m==original for m in messages),'confirmed pre-failure message lost or altered') + for msg in successes: + check(any(m==msg for m in messages),'confirmed in-flight message lost or altered') + # Bodies are unique in this scenario: an uncertain request may exist once, never twice. + texts = [m['text'] for m in messages] + check(len(texts)==len(set(texts)),'uncertain retry created duplicate message') + if attachment: + c.call('GET','/api/v1/attachments/'+attachment['id'],actor=bob,schema='Attachment') + downloaded = c.call('GET','/api/v1/attachments/'+attachment['id']+'/content',actor=bob) + check(downloaded['bytes']==content,'attachment lost or changed across failure') + print('PASS: '+args.mode+' preserved confirmed data and existing session. Hook scope needs operator evidence.') + finally: + if proc.poll() is None: + proc.terminate() + try: + proc.wait(timeout=5) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait() + +if __name__=='__main__': + run(main) diff --git a/course-tests/smoke.py b/course-tests/smoke.py new file mode 100644 index 0000000..4cbfa55 --- /dev/null +++ b/course-tests/smoke.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +"""One complete public acceptance scenario, cumulative by lab number.""" +import base64 +import hashlib +import uuid +from client import Client, LAB, arguments, check, run, timestamp + + +def main(): + args = arguments('Messenger public smoke; creates synthetic users and messages') + c = Client(args.base_url,args.ca_file) + alice,bob,eve,chat = c.bootstrap() + path = '/api/v1/chats/'+chat['id'] + c.call('POST','/api/v1/users',409,body=alice['registration']) + c.call('GET','/api/v1/users/me',401) + c.call('GET',path,404,actor=eve) + me = c.call('GET','/api/v1/users/me',actor=alice,schema='User')['json'] + check(me==alice['user'],'current user mismatch') + listed = c.call('GET','/api/v1/chats?limit=100',actor=bob,schema='ChatPage')['json'] + check(any(x['id']==chat['id'] for x in listed['items']),'member cannot list chat') + hidden = c.call('GET','/api/v1/chats?limit=100',actor=eve,schema='ChatPage')['json'] + check(all(x['id']!=chat['id'] for x in hidden['items']),'chat leaks to outsider') + key = uuid.uuid4().hex + first = c.message(alice,chat,'Привет, мир 👋',key=key)['json'] + second = c.message(bob,chat,'Сообщение Боба')['json'] + check(first['sender_id']==alice['id'] and second['sender_id']==bob['id'],'sender was not derived from actor') + check(first['chat_id']==second['chat_id']==chat['id'],'wrong chat in message') + check(first['text']=='Привет, мир 👋' and second['text']=='Сообщение Боба','message text changed') + page1 = c.call('GET',path+'/messages?limit=1',actor=bob,schema='MessagePage')['json'] + check(len(page1['items'])==1 and page1['next_cursor'],'first page must have one item and cursor') + from urllib.parse import quote + page2 = c.call('GET',path+'/messages?limit=1&cursor='+quote(page1['next_cursor'],safe=''),actor=alice,schema='MessagePage')['json'] + check(len(page2['items'])==1 and page2['next_cursor'] is None,'second page must finish history') + ordered = sorted([first,second],key=lambda x:(timestamp(x['created_at']),uuid.UUID(x['id']).int)) + check(page1['items']+page2['items']==ordered,'pagination lost, duplicated or reordered a message') + h = {'Idempotency-Key':uuid.uuid4().hex} if LAB>=3 else {} + c.call('POST',path+'/messages',400,actor=alice,body={'text':''},headers=h) + c.call('POST',path+'/messages',404,actor=eve,body={'text':'чужое'},headers=h) + c.call('GET',path+'/messages?limit=0',400,actor=alice) + if LAB>=2: + bad = base64.b64encode((alice['registration']['username']+':wrong-password').encode()).decode() + c.call('POST','/api/v1/auth/sessions',401,headers={'Authorization':'Basic '+bad}) + good = base64.b64encode((alice['registration']['username']+':'+alice['registration']['password']).encode()).decode() + c.call('GET','/api/v1/users/me',401,headers={'Authorization':'Basic '+good}) + c.call('GET','/api/v1/users/me',401,headers={'X-User-Id':alice['id']}) + me = c.call('GET','/api/v1/users/me',actor=alice,headers={'X-User-Id':eve['id']},schema='User')['json'] + check(me['id']==alice['id'],'X-User-Id overrode bearer identity') + if LAB>=3: + replay = c.message(alice,chat,first['text'],key=key)['json'] + check(replay==first,'idempotency replay differs') + c.call('POST',path+'/messages',409,actor=alice,body={'text':'другой body'},headers={'Idempotency-Key':key}) + c.call('POST',path+'/messages',400,actor=alice,body={'text':'без ключа'}) + if LAB>=4: + att,content = c.attachment(alice,chat) + # A queued attachment must already be linkable to its own chat. + attached = c.message(alice,chat,'',attachments=[att['id']])['json'] + check(attached['attachment_ids']==[att['id']],'message lost attachment') + ready = c.wait_ready(bob,att) if LAB>=5 else att + check(ready['error_code'] is None,'ready attachment has an error') + download = c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=bob) + check(download['bytes']==content,'download differs from original') + check(download['headers'].get_content_type()=='image/png','original content type mismatch') + for suffix in ('','/content'): + c.call('GET','/api/v1/attachments/'+att['id']+suffix,404,actor=eve) + if LAB>=5: + variants = [x for x in ready['variants'] if x['name']=='thumbnail'] + check(len(variants)==1,'expected exactly one thumbnail') + v = variants[0] + check((v['width'],v['height'])==(256,160),'thumbnail metadata has wrong dimensions') + thumb = c.call('GET','/api/v1/attachments/'+att['id']+'/content?variant=thumbnail',actor=bob) + check(thumb['headers'].get_content_type()=='image/jpeg','thumbnail content type mismatch') + check(len(thumb['bytes'])==v['size_bytes'] and hashlib.sha256(thumb['bytes']).hexdigest()==v['sha256'],'thumbnail checksum mismatch') + from image_probe import jpeg_size + check(jpeg_size(thumb['bytes'])==(256,160),'actual JPEG dimensions mismatch') + if LAB>=2: + c.call('DELETE','/api/v1/auth/sessions/current',204,actor=alice) + c.call('GET','/api/v1/users/me',401,actor=alice) + c.call('DELETE','/api/v1/auth/sessions/current',401,actor=alice) + print('PASS: lab%d public API scenario; infrastructure and grading evidence remain separate.' % LAB) + +if __name__=='__main__': + run(main) diff --git a/evidence/README.md b/evidence/README.md index 8df7001..0cc9649 100644 --- a/evidence/README.md +++ b/evidence/README.md @@ -1,5 +1,12 @@ -# Доказательства +# Доказательства lab6 -Храните небольшие обезличенные результаты: команды и stdout проверок, JSON/CSV агрегатов нагрузки, EXPLAIN, dashboard JSON, текст разбора отказа. Укажите дату, commit, аппаратные лимиты и профиль нагрузки. +Автор: Каменский Андрей, БПИ2301. -Не добавляйте .env, session tokens, cookies, приватные ключи, credentials, реальные сообщения, raw dumps и signed URLs. Для скриншотов скрывайте секреты. Большие логи/бинарные traces сохраняйте отдельно по правилам преподавателя; одного скриншота без методики недостаточно. +lab6-checks.txt — короткие проверки структуры, OpenAPI, стиля и публичного API. +lab6-signals.txt — метрики, реальные traces/logs и запросы dashboard. +lab6-incidents.txt и lab6-observability-summary.json — контролируемые отказы. +JSON traces/logs относятся к реальным синтетическим запросам, не к статическим +образцам. Tokens/passwords/body изображения в telemetry не сохраняются. + +lab6-worker-incident.json сохраняет проверку очереди/retry; lab6-incidents.txt — +отдельное продолжение DB/backend после исправления префикса route в метриках. diff --git a/evidence/lab6-checks.txt b/evidence/lab6-checks.txt new file mode 100644 index 0000000..f4ac7b1 --- /dev/null +++ b/evidence/lab6-checks.txt @@ -0,0 +1,10 @@ +/Users/andrejkamenskij/PycharmProjects/mtusi2026lab1/.venv/bin/python scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +/Users/andrejkamenskij/PycharmProjects/mtusi2026lab1/.venv/bin/python scripts/validate_openapi.py +PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested. +/Users/andrejkamenskij/PycharmProjects/mtusi2026lab1/.venv/bin/python -m ruff check src +All checks passed! +/Users/andrejkamenskij/PycharmProjects/mtusi2026lab1/.venv/bin/python -m ruff format --check src +64 files already formatted +/Users/andrejkamenskij/PycharmProjects/mtusi2026lab1/.venv/bin/python tests/smoke.py --base-url "http://localhost:8086" +PASS: lab6 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/lab6-correlated-logs.json b/evidence/lab6-correlated-logs.json new file mode 100644 index 0000000..1c4cbb8 --- /dev/null +++ b/evidence/lab6-correlated-logs.json @@ -0,0 +1,56 @@ +[ + { + "event": "job.completed", + "timestamp": "2026-11-17T15:15:07.104258+00:00", + "severity": "INFO", + "service_name": "messenger-worker", + "trace_id": "fd47457d00c7436da8d99589929e44a5", + "span_id": "6d789cf2190bed03", + "outcome": "ready", + "attempt": 1, + "job_type": "attachment.process.v1", + "duration_seconds": 0.2017499170001429 + }, + { + "event": "job.published", + "timestamp": "2026-11-17T15:15:06.902250+00:00", + "severity": "INFO", + "service_name": "messenger-dispatcher", + "trace_id": "fd47457d00c7436da8d99589929e44a5", + "span_id": "3bfbceb139effbd1", + "attempt": 1, + "job_type": "attachment.process.v1" + }, + { + "event": "job.started", + "timestamp": "2026-11-17T15:15:06.902991+00:00", + "severity": "INFO", + "service_name": "messenger-worker", + "trace_id": "fd47457d00c7436da8d99589929e44a5", + "span_id": "6d789cf2190bed03", + "attempt": 1, + "job_type": "attachment.process.v1" + }, + { + "event": "http.completed", + "timestamp": "2026-11-17T15:15:06.675742+00:00", + "severity": "INFO", + "service_name": "messenger-api", + "trace_id": "fd47457d00c7436da8d99589929e44a5", + "span_id": "801bdc6abdbaba59", + "request_id": "0ecdbe17-1712-4864-ae79-38df19ef2c7d", + "method": "POST", + "route": "/api/v1/chats/{chat_id}/attachments", + "status": 202, + "duration_seconds": 0.1755593749999207 + }, + { + "event": "attachment.accepted", + "timestamp": "2026-11-17T15:15:06.675308+00:00", + "severity": "INFO", + "service_name": "messenger-api", + "trace_id": "fd47457d00c7436da8d99589929e44a5", + "span_id": "801bdc6abdbaba59", + "job_type": "attachment.process.v1" + } +] \ No newline at end of file diff --git a/evidence/lab6-db-incident-logs.json b/evidence/lab6-db-incident-logs.json new file mode 100644 index 0000000..50a7f04 --- /dev/null +++ b/evidence/lab6-db-incident-logs.json @@ -0,0 +1,227 @@ +[ + { + "event": "db.error", + "timestamp": "2026-11-17T15:13:12.113656+00:00", + "severity": "INFO", + "service_name": "messenger-api", + "trace_id": "f1ab8757e130d9b3b8e87acd85115012", + "span_id": "10ca75d8e0ce9007", + "error_type": "Error" + }, + { + "event": "db.error", + "timestamp": "2026-11-17T15:13:12.114704+00:00", + "severity": "INFO", + "service_name": "messenger-api", + "trace_id": "69dde9e84908f5b14ab39da9d6b2c30f", + "span_id": "2b1d606367557c42", + "error_type": "Error" + }, + { + "event": "db.error", + "timestamp": "2026-11-17T15:13:12.170591+00:00", + "severity": "INFO", + "service_name": "messenger-api", + "trace_id": "98aa6fecf2eda04174aec162d532413f", + "span_id": "4543089015e17a65", + "error_type": "Error" + }, + { + "event": "db.error", + "timestamp": "2026-11-17T15:13:12.172345+00:00", + "severity": "INFO", + "service_name": "messenger-api", + "trace_id": "d574895b96f583d31bd53d226301bac6", + "span_id": "59cbe017ec796d04", + "error_type": "Error" + }, + { + "event": "db.error", + "timestamp": "2026-11-17T15:13:14.625536+00:00", + "severity": "INFO", + "service_name": "messenger-api", + "trace_id": "4fb197690cd7ed8c34cbaf3af2d2c8c0", + "span_id": "f3b71c918bb75a88", + "error_type": "Error" + }, + { + "event": "db.error", + "timestamp": "2026-11-17T15:13:14.671622+00:00", + "severity": "INFO", + "service_name": "messenger-api", + "trace_id": "22d882d3ea564d808dabc96d24e18a37", + "span_id": "b20632277beb3c6b", + "error_type": "Error" + }, + { + "event": "db.error", + "timestamp": "2026-11-17T15:13:14.681385+00:00", + "severity": "INFO", + "service_name": "messenger-api", + "trace_id": "314dbd0e393abd2865e38d046f87cf27", + "span_id": "37f57af5302ad41a", + "error_type": "Error" + }, + { + "event": "db.error", + "timestamp": "2026-11-17T15:13:14.681488+00:00", + "severity": "INFO", + "service_name": "messenger-api", + "trace_id": "2881ad7651c2ead16b142a1d53525644", + "span_id": "264b3e80956650c9", + "error_type": "Error" + }, + { + "event": "db.error", + "timestamp": "2026-11-17T15:13:17.152802+00:00", + "severity": "INFO", + "service_name": "messenger-api", + "trace_id": "2b68f18c2fd49fdebe5b64e69fb8de33", + "span_id": "a71fac67af31936e", + "error_type": "Error" + }, + { + "event": "db.error", + "timestamp": "2026-11-17T15:13:17.183320+00:00", + "severity": "INFO", + "service_name": "messenger-api", + "trace_id": "40ae54a6e8ba7b0b6b2dfca8de351760", + "span_id": "36b42761727ba91f", + "error_type": "Error" + }, + { + "event": "db.error", + "timestamp": "2026-11-17T15:13:17.191879+00:00", + "severity": "INFO", + "service_name": "messenger-api", + "trace_id": "a3d0183572c94e432db07a2efd9a12d1", + "span_id": "ca24eed9905a9bf9", + "error_type": "Error" + }, + { + "event": "db.error", + "timestamp": "2026-11-17T15:13:17.191979+00:00", + "severity": "INFO", + "service_name": "messenger-api", + "trace_id": "54b3085f4640962ddffd63936435a4d8", + "span_id": "340c3562a22a467e", + "error_type": "Error" + }, + { + "event": "db.error", + "timestamp": "2026-11-17T15:15:12.505367+00:00", + "severity": "INFO", + "service_name": "messenger-api", + "trace_id": "acaaafad8ba06b4e7e105cd3f4355987", + "span_id": "38dff7c61d3f6e4d", + "error_type": "Error" + }, + { + "event": "db.error", + "timestamp": "2026-11-17T15:15:12.515931+00:00", + "severity": "INFO", + "service_name": "messenger-api", + "trace_id": "c9efffd152bab3e6d2355f7127b3e8d1", + "span_id": "6dd8695d5581baf2", + "error_type": "Error" + }, + { + "event": "db.error", + "timestamp": "2026-11-17T15:15:12.575617+00:00", + "severity": "INFO", + "service_name": "messenger-api", + "trace_id": "f7364c5486bdc6b3b14b7ef885f00a95", + "span_id": "8db09be701daf82e", + "error_type": "Error" + }, + { + "event": "db.error", + "timestamp": "2026-11-17T15:15:12.576730+00:00", + "severity": "INFO", + "service_name": "messenger-api", + "trace_id": "ac3e797c761d77bfbf7cda54ff6b02b9", + "span_id": "cdf3e2de745cfc13", + "error_type": "Error" + }, + { + "event": "db.error", + "timestamp": "2026-11-17T15:15:15.027064+00:00", + "severity": "INFO", + "service_name": "messenger-api", + "trace_id": "68e11662b15be879a6794f2450b1347b", + "span_id": "780b4cfe086b291e", + "error_type": "Error" + }, + { + "event": "db.error", + "timestamp": "2026-11-17T15:15:15.074288+00:00", + "severity": "INFO", + "service_name": "messenger-api", + "trace_id": "f03afd499dac3f577a1a3a7508b308e9", + "span_id": "cc1f1e6882bbc154", + "error_type": "Error" + }, + { + "event": "db.error", + "timestamp": "2026-11-17T15:15:15.085689+00:00", + "severity": "INFO", + "service_name": "messenger-api", + "trace_id": "7e42f82236d190f5e5c5208fcb1f6f39", + "span_id": "0be523a4032a2c36", + "error_type": "Error" + }, + { + "event": "db.error", + "timestamp": "2026-11-17T15:15:15.085975+00:00", + "severity": "INFO", + "service_name": "messenger-api", + "trace_id": "829e4fa63c820aa7e3f2de1761f21caf", + "span_id": "4c5aa3f29019e1f6", + "error_type": "Error" + }, + { + "event": "db.error", + "timestamp": "2026-11-17T15:15:17.543116+00:00", + "severity": "INFO", + "service_name": "messenger-api", + "trace_id": "f6a52507f2922e42155d659bba763970", + "span_id": "5e892c5ae6a001d6", + "error_type": "Error" + }, + { + "event": "db.error", + "timestamp": "2026-11-17T15:15:17.590820+00:00", + "severity": "INFO", + "service_name": "messenger-api", + "trace_id": "8f965bae4f6ffa0b3c3cef5c9c52f38e", + "span_id": "6bdafe4d596a25cb", + "error_type": "Error" + }, + { + "event": "db.error", + "timestamp": "2026-11-17T15:15:17.595806+00:00", + "severity": "INFO", + "service_name": "messenger-api", + "trace_id": "2f25a5ca948c2aa0c2ef38bd80230d1f", + "span_id": "17b0d785eca64d68", + "error_type": "Error" + }, + { + "event": "db.error", + "timestamp": "2026-11-17T15:15:17.595865+00:00", + "severity": "INFO", + "service_name": "messenger-api", + "trace_id": "96bf062721f9b68548202ca761ec56e0", + "span_id": "21a167132ebdb621", + "error_type": "Error" + }, + { + "event": "db.error", + "timestamp": "2026-11-17T15:08:12.863991+00:00", + "severity": "INFO", + "service_name": "messenger-api", + "trace_id": "4df9dbc6eaaaee821bbcc649852367dc", + "span_id": "44727a31e3111cd0", + "error_type": "IntegrityError" + } +] \ No newline at end of file diff --git a/evidence/lab6-db-incident-trace.json b/evidence/lab6-db-incident-trace.json new file mode 100644 index 0000000..197d73d --- /dev/null +++ b/evidence/lab6-db-incident-trace.json @@ -0,0 +1,255 @@ +{ + "batches": [ + { + "resource": { + "attributes": [ + { + "key": "telemetry.sdk.language", + "value": { + "stringValue": "python" + } + }, + { + "key": "telemetry.sdk.name", + "value": { + "stringValue": "opentelemetry" + } + }, + { + "key": "telemetry.sdk.version", + "value": { + "stringValue": "1.31.1" + } + }, + { + "key": "service.version", + "value": { + "stringValue": "1.6.0" + } + }, + { + "key": "service.instance.id", + "value": { + "stringValue": "ec467075-05bd-4271-bb1e-a272ae3dd3c3" + } + }, + { + "key": "deployment.environment.name", + "value": { + "stringValue": "lab" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "messenger-api" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "messenger", + "version": "1.6.0" + }, + "spans": [ + { + "traceId": "9zZMVIa9xrOxS374hfAKlQ==", + "spanId": "CNWCiXdj/44=", + "parentSpanId": "jbCb5wHa+C4=", + "name": "db.INSERT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928510074339418", + "endTimeUnixNano": "1794928512575607086", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "INSERT" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "Error" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "9zZMVIa9xrOxS374hfAKlQ==", + "spanId": "jbCb5wHa+C4=", + "name": "POST /api/v1/chats/{chat_id}/messages", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1794928509995947085", + "endTimeUnixNano": "1794928512576277919", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "POST" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/chats/{chat_id}/messages" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + } + ] + } + ] + }, + { + "resource": { + "attributes": [ + { + "key": "telemetry.sdk.language", + "value": { + "stringValue": "python" + } + }, + { + "key": "telemetry.sdk.name", + "value": { + "stringValue": "opentelemetry" + } + }, + { + "key": "telemetry.sdk.version", + "value": { + "stringValue": "1.31.1" + } + }, + { + "key": "service.version", + "value": { + "stringValue": "1.6.0" + } + }, + { + "key": "service.instance.id", + "value": { + "stringValue": "ec467075-05bd-4271-bb1e-a272ae3dd3c3" + } + }, + { + "key": "deployment.environment.name", + "value": { + "stringValue": "lab" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "messenger-api" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "messenger", + "version": "1.6.0" + }, + "spans": [ + { + "traceId": "9zZMVIa9xrOxS374hfAKlQ==", + "spanId": "sbSL3OACRAM=", + "parentSpanId": "jbCb5wHa+C4=", + "name": "db.SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928510069821585", + "endTimeUnixNano": "1794928510070631168", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "9zZMVIa9xrOxS374hfAKlQ==", + "spanId": "26YgITrtuvE=", + "parentSpanId": "jbCb5wHa+C4=", + "name": "db.SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928510071310626", + "endTimeUnixNano": "1794928510071793293", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "9zZMVIa9xrOxS374hfAKlQ==", + "spanId": "20d9NgzWuKs=", + "parentSpanId": "jbCb5wHa+C4=", + "name": "db.INSERT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928510072845626", + "endTimeUnixNano": "1794928510073726543", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "INSERT" + } + } + ], + "status": {} + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/evidence/lab6-incidents.txt b/evidence/lab6-incidents.txt new file mode 100644 index 0000000..b446c07 --- /dev/null +++ b/evidence/lab6-incidents.txt @@ -0,0 +1,5 @@ +{"case": "upload_to_worker_trace_and_logs", "result": "PASS", "trace_id": "fd47457d00c7436da8d99589929e44a5", "spans": 29} +{"case": "metrics_labels_and_dashboard_queries", "result": "PASS", "metrics": 13, "panels": 18} +{"case": "incident_DB_firing", "result": "PASS", "errors": 12, "db_error_spans": 1, "diagnosis": "db.INSERT timeout, matching db.error event; API readiness remains available"} +{"case": "incident_DB_resolved", "result": "PASS"} +{"case": "telemetry_backend_outage_business_survives", "result": "PASS", "messages": 15, "attachment_ready": true, "api_memory": {"bytes": 93831168, "limit": 536870912}, "collector_rss_bytes": 207998976.0, "exporter_failed_spans": 32.0, "max_exporter_queue": 9.0} diff --git a/evidence/lab6-observability-summary.json b/evidence/lab6-observability-summary.json new file mode 100644 index 0000000..c17a352 --- /dev/null +++ b/evidence/lab6-observability-summary.json @@ -0,0 +1,40 @@ +{ + "results": [ + { + "case": "upload_to_worker_trace_and_logs", + "result": "PASS", + "trace_id": "fd47457d00c7436da8d99589929e44a5", + "spans": 29 + }, + { + "case": "metrics_labels_and_dashboard_queries", + "result": "PASS", + "metrics": 13, + "panels": 18 + }, + { + "case": "incident_DB_firing", + "result": "PASS", + "errors": 12, + "db_error_spans": 1, + "diagnosis": "db.INSERT timeout, matching db.error event; API readiness remains available" + }, + { + "case": "incident_DB_resolved", + "result": "PASS" + }, + { + "case": "telemetry_backend_outage_business_survives", + "result": "PASS", + "messages": 15, + "attachment_ready": true, + "api_memory": { + "bytes": 93831168, + "limit": 536870912 + }, + "collector_rss_bytes": 207998976.0, + "exporter_failed_spans": 32.0, + "max_exporter_queue": 9.0 + } + ] +} \ No newline at end of file diff --git a/evidence/lab6-retry-trace.json b/evidence/lab6-retry-trace.json new file mode 100644 index 0000000..93b98e6 --- /dev/null +++ b/evidence/lab6-retry-trace.json @@ -0,0 +1,1421 @@ +{ + "batches": [ + { + "resource": { + "attributes": [ + { + "key": "telemetry.sdk.language", + "value": { + "stringValue": "python" + } + }, + { + "key": "telemetry.sdk.name", + "value": { + "stringValue": "opentelemetry" + } + }, + { + "key": "telemetry.sdk.version", + "value": { + "stringValue": "1.31.1" + } + }, + { + "key": "service.version", + "value": { + "stringValue": "1.6.0" + } + }, + { + "key": "service.instance.id", + "value": { + "stringValue": "891b2083-7cf2-4250-afc1-66fd5bf10832" + } + }, + { + "key": "deployment.environment.name", + "value": { + "stringValue": "lab" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "messenger-api" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "messenger", + "version": "1.6.0" + }, + "spans": [ + { + "traceId": "bLIOEvl3RbWDTh1BU+NjrQ==", + "spanId": "ORoZhlAWEcc=", + "parentSpanId": "EjRWeJCrze8=", + "name": "POST /chats/{chat_id}/attachments", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1794928348113712135", + "endTimeUnixNano": "1794928348132234927", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "POST" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/chats/{chat_id}/attachments" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "202" + } + } + ], + "status": {} + }, + { + "traceId": "bLIOEvl3RbWDTh1BU+NjrQ==", + "spanId": "4FISSRMyjyA=", + "parentSpanId": "ORoZhlAWEcc=", + "name": "db.SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928348116561010", + "endTimeUnixNano": "1794928348117130677", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "bLIOEvl3RbWDTh1BU+NjrQ==", + "spanId": "Kt/dbCAsNLo=", + "parentSpanId": "ORoZhlAWEcc=", + "name": "db.SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928348120973302", + "endTimeUnixNano": "1794928348121575052", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "bLIOEvl3RbWDTh1BU+NjrQ==", + "spanId": "tr2onCZDvXo=", + "parentSpanId": "ORoZhlAWEcc=", + "name": "db.SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928348121955052", + "endTimeUnixNano": "1794928348122239343", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "bLIOEvl3RbWDTh1BU+NjrQ==", + "spanId": "br9EhJbwD70=", + "parentSpanId": "ORoZhlAWEcc=", + "name": "db.INSERT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928348122602802", + "endTimeUnixNano": "1794928348122931635", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "INSERT" + } + } + ], + "status": {} + }, + { + "traceId": "bLIOEvl3RbWDTh1BU+NjrQ==", + "spanId": "07LdghSDqyg=", + "parentSpanId": "ORoZhlAWEcc=", + "name": "db.SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928348123745843", + "endTimeUnixNano": "1794928348123956802", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "bLIOEvl3RbWDTh1BU+NjrQ==", + "spanId": "+AGOk4l4hAo=", + "parentSpanId": "ORoZhlAWEcc=", + "name": "db.SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928348124235593", + "endTimeUnixNano": "1794928348124534385", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "bLIOEvl3RbWDTh1BU+NjrQ==", + "spanId": "0ji8ZdZwL3w=", + "parentSpanId": "ORoZhlAWEcc=", + "name": "s3.PUT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928348125035052", + "endTimeUnixNano": "1794928348128364302", + "attributes": [ + { + "key": "storage.operation", + "value": { + "stringValue": "PUT" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "200" + } + } + ], + "status": {} + }, + { + "traceId": "bLIOEvl3RbWDTh1BU+NjrQ==", + "spanId": "bDIABkUa/CY=", + "parentSpanId": "ORoZhlAWEcc=", + "name": "db.INSERT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928348129024635", + "endTimeUnixNano": "1794928348129832718", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "INSERT" + } + } + ], + "status": {} + }, + { + "traceId": "bLIOEvl3RbWDTh1BU+NjrQ==", + "spanId": "LZ1mdyNsUAE=", + "parentSpanId": "ORoZhlAWEcc=", + "name": "db.INSERT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928348130217677", + "endTimeUnixNano": "1794928348130606343", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "INSERT" + } + } + ], + "status": {} + }, + { + "traceId": "bLIOEvl3RbWDTh1BU+NjrQ==", + "spanId": "r7fZ3UZGz3M=", + "parentSpanId": "ORoZhlAWEcc=", + "name": "db.UPDATE", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928348130731010", + "endTimeUnixNano": "1794928348131034385", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "UPDATE" + } + } + ], + "status": {} + } + ] + } + ] + }, + { + "resource": { + "attributes": [ + { + "key": "telemetry.sdk.language", + "value": { + "stringValue": "python" + } + }, + { + "key": "telemetry.sdk.name", + "value": { + "stringValue": "opentelemetry" + } + }, + { + "key": "telemetry.sdk.version", + "value": { + "stringValue": "1.31.1" + } + }, + { + "key": "service.version", + "value": { + "stringValue": "1.6.0" + } + }, + { + "key": "service.instance.id", + "value": { + "stringValue": "500f4be3-830d-47c7-9ee4-09a03a1c55f0" + } + }, + { + "key": "deployment.environment.name", + "value": { + "stringValue": "lab" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "messenger-dispatcher" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "messenger", + "version": "1.6.0" + }, + "spans": [ + { + "traceId": "bLIOEvl3RbWDTh1BU+NjrQ==", + "spanId": "yh6jZziVMCI=", + "parentSpanId": "ORoZhlAWEcc=", + "name": "queue.publish", + "kind": "SPAN_KIND_PRODUCER", + "startTimeUnixNano": "1794928348216678718", + "endTimeUnixNano": "1794928348217857885", + "status": {} + } + ] + } + ] + }, + { + "resource": { + "attributes": [ + { + "key": "telemetry.sdk.language", + "value": { + "stringValue": "python" + } + }, + { + "key": "telemetry.sdk.name", + "value": { + "stringValue": "opentelemetry" + } + }, + { + "key": "telemetry.sdk.version", + "value": { + "stringValue": "1.31.1" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + }, + { + "key": "service.version", + "value": { + "stringValue": "1.6.0" + } + }, + { + "key": "service.instance.id", + "value": { + "stringValue": "225630f3-f0d0-4ba4-a7d3-6e46167c64a9" + } + }, + { + "key": "deployment.environment.name", + "value": { + "stringValue": "lab" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "messenger", + "version": "1.6.0" + }, + "spans": [ + { + "traceId": "bLIOEvl3RbWDTh1BU+NjrQ==", + "spanId": "a5Qvy18kFnI=", + "parentSpanId": "yh6jZziVMCI=", + "flags": 768, + "name": "queue.consume", + "kind": "SPAN_KIND_CONSUMER", + "startTimeUnixNano": "1794928375544746675", + "endTimeUnixNano": "1794928380586867344", + "attributes": [ + { + "key": "messaging.system", + "value": { + "stringValue": "redis" + } + }, + { + "key": "job.type", + "value": { + "stringValue": "attachment.process.v1" + } + }, + { + "key": "job.attempt", + "value": { + "intValue": "1" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "bLIOEvl3RbWDTh1BU+NjrQ==", + "spanId": "V2RqMnO14U8=", + "parentSpanId": "a5Qvy18kFnI=", + "flags": 256, + "name": "s3.GET", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928375576314467", + "endTimeUnixNano": "1794928380580369761", + "attributes": [ + { + "key": "storage.operation", + "value": { + "stringValue": "GET" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "ReadTimeoutError" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "bLIOEvl3RbWDTh1BU+NjrQ==", + "spanId": "iJQ81PnQZ6I=", + "parentSpanId": "a5Qvy18kFnI=", + "flags": 256, + "name": "db.SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928380581861344", + "endTimeUnixNano": "1794928380582697303", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "bLIOEvl3RbWDTh1BU+NjrQ==", + "spanId": "zbrW95wHjRM=", + "parentSpanId": "a5Qvy18kFnI=", + "flags": 256, + "name": "db.UPDATE", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928380583697178", + "endTimeUnixNano": "1794928380584050511", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "UPDATE" + } + } + ], + "status": {} + }, + { + "traceId": "bLIOEvl3RbWDTh1BU+NjrQ==", + "spanId": "q8nGCITS+Es=", + "parentSpanId": "a5Qvy18kFnI=", + "flags": 256, + "name": "db.UPDATE", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928380584809636", + "endTimeUnixNano": "1794928380585355261", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "UPDATE" + } + } + ], + "status": {} + } + ] + } + ] + }, + { + "resource": { + "attributes": [ + { + "key": "telemetry.sdk.language", + "value": { + "stringValue": "python" + } + }, + { + "key": "telemetry.sdk.name", + "value": { + "stringValue": "opentelemetry" + } + }, + { + "key": "telemetry.sdk.version", + "value": { + "stringValue": "1.31.1" + } + }, + { + "key": "service.version", + "value": { + "stringValue": "1.6.0" + } + }, + { + "key": "service.instance.id", + "value": { + "stringValue": "225630f3-f0d0-4ba4-a7d3-6e46167c64a9" + } + }, + { + "key": "deployment.environment.name", + "value": { + "stringValue": "lab" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "messenger", + "version": "1.6.0" + }, + "spans": [ + { + "traceId": "bLIOEvl3RbWDTh1BU+NjrQ==", + "spanId": "tUn2q5YA2dM=", + "parentSpanId": "a5Qvy18kFnI=", + "name": "db.SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928375565352134", + "endTimeUnixNano": "1794928375566138509", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "bLIOEvl3RbWDTh1BU+NjrQ==", + "spanId": "NcYU9EJzEaA=", + "parentSpanId": "a5Qvy18kFnI=", + "name": "db.SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928375567159842", + "endTimeUnixNano": "1794928375567566925", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "bLIOEvl3RbWDTh1BU+NjrQ==", + "spanId": "9MEws/MDr44=", + "parentSpanId": "a5Qvy18kFnI=", + "name": "db.SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928375568003717", + "endTimeUnixNano": "1794928375568206800", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "bLIOEvl3RbWDTh1BU+NjrQ==", + "spanId": "mM8SGlmdFQ4=", + "parentSpanId": "a5Qvy18kFnI=", + "name": "db.UPDATE", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928375568957884", + "endTimeUnixNano": "1794928375569330675", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "UPDATE" + } + } + ], + "status": {} + }, + { + "traceId": "bLIOEvl3RbWDTh1BU+NjrQ==", + "spanId": "BhVvZmgXS5Y=", + "parentSpanId": "a5Qvy18kFnI=", + "name": "db.UPDATE", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928375569614592", + "endTimeUnixNano": "1794928375569877675", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "UPDATE" + } + } + ], + "status": {} + }, + { + "traceId": "bLIOEvl3RbWDTh1BU+NjrQ==", + "spanId": "z4dCqy3Zu8E=", + "parentSpanId": "a5Qvy18kFnI=", + "name": "db.SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928375570814425", + "endTimeUnixNano": "1794928375570973092", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "bLIOEvl3RbWDTh1BU+NjrQ==", + "spanId": "O6Ph4KKzAOc=", + "parentSpanId": "a5Qvy18kFnI=", + "name": "db.SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928375571153384", + "endTimeUnixNano": "1794928375571285884", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + } + ] + } + ] + }, + { + "resource": { + "attributes": [ + { + "key": "telemetry.sdk.language", + "value": { + "stringValue": "python" + } + }, + { + "key": "telemetry.sdk.name", + "value": { + "stringValue": "opentelemetry" + } + }, + { + "key": "telemetry.sdk.version", + "value": { + "stringValue": "1.31.1" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "messenger-dispatcher" + } + }, + { + "key": "service.version", + "value": { + "stringValue": "1.6.0" + } + }, + { + "key": "service.instance.id", + "value": { + "stringValue": "500f4be3-830d-47c7-9ee4-09a03a1c55f0" + } + }, + { + "key": "deployment.environment.name", + "value": { + "stringValue": "lab" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "messenger", + "version": "1.6.0" + }, + "spans": [ + { + "traceId": "bLIOEvl3RbWDTh1BU+NjrQ==", + "spanId": "pt5rvA0wId0=", + "parentSpanId": "a5Qvy18kFnI=", + "flags": 768, + "name": "queue.publish", + "kind": "SPAN_KIND_PRODUCER", + "startTimeUnixNano": "1794928382403461220", + "endTimeUnixNano": "1794928382405164595", + "status": {} + } + ] + } + ] + }, + { + "resource": { + "attributes": [ + { + "key": "telemetry.sdk.language", + "value": { + "stringValue": "python" + } + }, + { + "key": "telemetry.sdk.name", + "value": { + "stringValue": "opentelemetry" + } + }, + { + "key": "telemetry.sdk.version", + "value": { + "stringValue": "1.31.1" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + }, + { + "key": "service.version", + "value": { + "stringValue": "1.6.0" + } + }, + { + "key": "service.instance.id", + "value": { + "stringValue": "225630f3-f0d0-4ba4-a7d3-6e46167c64a9" + } + }, + { + "key": "deployment.environment.name", + "value": { + "stringValue": "lab" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "messenger", + "version": "1.6.0" + }, + "spans": [ + { + "traceId": "bLIOEvl3RbWDTh1BU+NjrQ==", + "spanId": "RM9Ap7OJJjY=", + "parentSpanId": "pt5rvA0wId0=", + "flags": 768, + "name": "queue.consume", + "kind": "SPAN_KIND_CONSUMER", + "startTimeUnixNano": "1794928382405238595", + "endTimeUnixNano": "1794928383334647721", + "attributes": [ + { + "key": "messaging.system", + "value": { + "stringValue": "redis" + } + }, + { + "key": "job.type", + "value": { + "stringValue": "attachment.process.v1" + } + }, + { + "key": "job.attempt", + "value": { + "intValue": "2" + } + } + ], + "status": {} + }, + { + "traceId": "bLIOEvl3RbWDTh1BU+NjrQ==", + "spanId": "EcKpHL4zK/k=", + "parentSpanId": "RM9Ap7OJJjY=", + "flags": 256, + "name": "s3.GET", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928382410023262", + "endTimeUnixNano": "1794928383157912596", + "attributes": [ + { + "key": "storage.operation", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "200" + } + } + ], + "status": {} + }, + { + "traceId": "bLIOEvl3RbWDTh1BU+NjrQ==", + "spanId": "ANuPwb8Mwb8=", + "parentSpanId": "RM9Ap7OJJjY=", + "flags": 256, + "name": "image.transform", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1794928383158795387", + "endTimeUnixNano": "1794928383323845929", + "status": {} + }, + { + "traceId": "bLIOEvl3RbWDTh1BU+NjrQ==", + "spanId": "yc7hYRrrwjo=", + "parentSpanId": "RM9Ap7OJJjY=", + "flags": 256, + "name": "s3.PUT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928383324263762", + "endTimeUnixNano": "1794928383327018971", + "attributes": [ + { + "key": "storage.operation", + "value": { + "stringValue": "PUT" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "200" + } + } + ], + "status": {} + }, + { + "traceId": "bLIOEvl3RbWDTh1BU+NjrQ==", + "spanId": "Pp8L6QtJ5k4=", + "parentSpanId": "RM9Ap7OJJjY=", + "flags": 256, + "name": "s3.PUT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928383327363554", + "endTimeUnixNano": "1794928383329042512", + "attributes": [ + { + "key": "storage.operation", + "value": { + "stringValue": "PUT" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "200" + } + } + ], + "status": {} + }, + { + "traceId": "bLIOEvl3RbWDTh1BU+NjrQ==", + "spanId": "tvr/gpGRHTY=", + "parentSpanId": "RM9Ap7OJJjY=", + "flags": 256, + "name": "s3.PUT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928383329292512", + "endTimeUnixNano": "1794928383330662554", + "attributes": [ + { + "key": "storage.operation", + "value": { + "stringValue": "PUT" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "200" + } + } + ], + "status": {} + }, + { + "traceId": "bLIOEvl3RbWDTh1BU+NjrQ==", + "spanId": "BSZky8Gs3Dk=", + "parentSpanId": "RM9Ap7OJJjY=", + "flags": 256, + "name": "s3.PUT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928383330904054", + "endTimeUnixNano": "1794928383332100637", + "attributes": [ + { + "key": "storage.operation", + "value": { + "stringValue": "PUT" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "200" + } + } + ], + "status": {} + }, + { + "traceId": "bLIOEvl3RbWDTh1BU+NjrQ==", + "spanId": "0fhbUkQheN4=", + "parentSpanId": "RM9Ap7OJJjY=", + "flags": 256, + "name": "db.SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928383332434179", + "endTimeUnixNano": "1794928383332651554", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "bLIOEvl3RbWDTh1BU+NjrQ==", + "spanId": "LJwkydWdwDs=", + "parentSpanId": "RM9Ap7OJJjY=", + "flags": 256, + "name": "db.UPDATE", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928383333278971", + "endTimeUnixNano": "1794928383333652887", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "UPDATE" + } + } + ], + "status": {} + }, + { + "traceId": "bLIOEvl3RbWDTh1BU+NjrQ==", + "spanId": "W33R8I7jDZg=", + "parentSpanId": "RM9Ap7OJJjY=", + "flags": 256, + "name": "db.UPDATE", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928383333862429", + "endTimeUnixNano": "1794928383334055512", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "UPDATE" + } + } + ], + "status": {} + } + ] + } + ] + }, + { + "resource": { + "attributes": [ + { + "key": "telemetry.sdk.language", + "value": { + "stringValue": "python" + } + }, + { + "key": "telemetry.sdk.name", + "value": { + "stringValue": "opentelemetry" + } + }, + { + "key": "telemetry.sdk.version", + "value": { + "stringValue": "1.31.1" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + }, + { + "key": "service.version", + "value": { + "stringValue": "1.6.0" + } + }, + { + "key": "service.instance.id", + "value": { + "stringValue": "225630f3-f0d0-4ba4-a7d3-6e46167c64a9" + } + }, + { + "key": "deployment.environment.name", + "value": { + "stringValue": "lab" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "messenger", + "version": "1.6.0" + }, + "spans": [ + { + "traceId": "bLIOEvl3RbWDTh1BU+NjrQ==", + "spanId": "zYsu8frY4w4=", + "parentSpanId": "RM9Ap7OJJjY=", + "flags": 256, + "name": "db.SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928382406530970", + "endTimeUnixNano": "1794928382406896137", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "bLIOEvl3RbWDTh1BU+NjrQ==", + "spanId": "fOhUE9Tn6yM=", + "parentSpanId": "RM9Ap7OJJjY=", + "flags": 256, + "name": "db.SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928382407290179", + "endTimeUnixNano": "1794928382407458595", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "bLIOEvl3RbWDTh1BU+NjrQ==", + "spanId": "S0LcV/Hc0dA=", + "parentSpanId": "RM9Ap7OJJjY=", + "flags": 256, + "name": "db.SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928382407651220", + "endTimeUnixNano": "1794928382407765345", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "bLIOEvl3RbWDTh1BU+NjrQ==", + "spanId": "F7TweHMUmTo=", + "parentSpanId": "RM9Ap7OJJjY=", + "flags": 256, + "name": "db.UPDATE", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928382408011554", + "endTimeUnixNano": "1794928382408135095", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "UPDATE" + } + } + ], + "status": {} + }, + { + "traceId": "bLIOEvl3RbWDTh1BU+NjrQ==", + "spanId": "ACExqAe7JtA=", + "parentSpanId": "RM9Ap7OJJjY=", + "flags": 256, + "name": "db.UPDATE", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928382408208679", + "endTimeUnixNano": "1794928382408322554", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "UPDATE" + } + } + ], + "status": {} + }, + { + "traceId": "bLIOEvl3RbWDTh1BU+NjrQ==", + "spanId": "m3ZkyxvIqGg=", + "parentSpanId": "RM9Ap7OJJjY=", + "flags": 256, + "name": "db.SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928382408997179", + "endTimeUnixNano": "1794928382409151887", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "bLIOEvl3RbWDTh1BU+NjrQ==", + "spanId": "MHg3tIi1aEQ=", + "parentSpanId": "RM9Ap7OJJjY=", + "flags": 256, + "name": "db.SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928382409331887", + "endTimeUnixNano": "1794928382409449970", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/evidence/lab6-signals.txt b/evidence/lab6-signals.txt new file mode 100644 index 0000000..b225308 --- /dev/null +++ b/evidence/lab6-signals.txt @@ -0,0 +1,2 @@ +{"case": "upload_to_worker_trace_and_logs", "result": "PASS", "trace_id": "6127dd4877af4ee8a4c0411427c57ec5", "spans": 29} +{"case": "metrics_labels_and_dashboard_queries", "result": "PASS", "metrics": 13, "panels": 18} diff --git a/evidence/lab6-summary.json b/evidence/lab6-summary.json new file mode 100644 index 0000000..0b3f67f --- /dev/null +++ b/evidence/lab6-summary.json @@ -0,0 +1,71 @@ +{ + "author": "Каменский Андрей, БПИ2301", + "grade_target": 4, + "public_contract_tests_unchanged": true, + "results": [ + { + "case": "upload_to_worker_trace_and_logs", + "result": "PASS", + "trace_id": "c4674a56c0144140973ffadebfb865b4", + "spans": 29 + }, + { + "case": "metrics_labels_and_dashboard_queries", + "result": "PASS", + "metrics": 13, + "panels": 18 + }, + { + "case": "incident_worker_firing", + "result": "PASS", + "queue_depth": 1.0, + "oldest_seconds": 22.580556 + }, + { + "case": "retry_keeps_upload_trace", + "result": "PASS", + "trace_id": "6cb20e12f97745b5834e1d4153e363ad", + "consumer_spans": 2 + }, + { + "case": "incident_worker_resolved", + "result": "PASS" + }, + { + "case": "upload_to_worker_trace_and_logs", + "result": "PASS", + "trace_id": "fd47457d00c7436da8d99589929e44a5", + "spans": 29 + }, + { + "case": "metrics_labels_and_dashboard_queries", + "result": "PASS", + "metrics": 13, + "panels": 18 + }, + { + "case": "incident_DB_firing", + "result": "PASS", + "errors": 12, + "db_error_spans": 1, + "diagnosis": "db.INSERT timeout, matching db.error event; API readiness remains available" + }, + { + "case": "incident_DB_resolved", + "result": "PASS" + }, + { + "case": "telemetry_backend_outage_business_survives", + "result": "PASS", + "messages": 15, + "attachment_ready": true, + "api_memory": { + "bytes": 93831168, + "limit": 536870912 + }, + "collector_rss_bytes": 207998976.0, + "exporter_failed_spans": 32.0, + "max_exporter_queue": 9.0 + } + ] +} diff --git a/evidence/lab6-trace.json b/evidence/lab6-trace.json new file mode 100644 index 0000000..e87e4ba --- /dev/null +++ b/evidence/lab6-trace.json @@ -0,0 +1,1003 @@ +{ + "batches": [ + { + "resource": { + "attributes": [ + { + "key": "telemetry.sdk.language", + "value": { + "stringValue": "python" + } + }, + { + "key": "telemetry.sdk.name", + "value": { + "stringValue": "opentelemetry" + } + }, + { + "key": "telemetry.sdk.version", + "value": { + "stringValue": "1.31.1" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "messenger-api" + } + }, + { + "key": "service.version", + "value": { + "stringValue": "1.6.0" + } + }, + { + "key": "service.instance.id", + "value": { + "stringValue": "ec467075-05bd-4271-bb1e-a272ae3dd3c3" + } + }, + { + "key": "deployment.environment.name", + "value": { + "stringValue": "lab" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "messenger", + "version": "1.6.0" + }, + "spans": [ + { + "traceId": "/UdFfQDHQ22o2ZWJkp5EpQ==", + "spanId": "gBvcar26ulk=", + "parentSpanId": "EjRWeJCrze8=", + "flags": 768, + "name": "POST /api/v1/chats/{chat_id}/attachments", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1794928506500115041", + "endTimeUnixNano": "1794928506675791500", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "POST" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/chats/{chat_id}/attachments" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "202" + } + } + ], + "status": {} + }, + { + "traceId": "/UdFfQDHQ22o2ZWJkp5EpQ==", + "spanId": "kQEe3FRNyYM=", + "parentSpanId": "gBvcar26ulk=", + "flags": 256, + "name": "db.SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928506594712500", + "endTimeUnixNano": "1794928506667958000", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "/UdFfQDHQ22o2ZWJkp5EpQ==", + "spanId": "pzZzbS6ryYE=", + "parentSpanId": "gBvcar26ulk=", + "flags": 256, + "name": "s3.PUT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928506668485791", + "endTimeUnixNano": "1794928506671840708", + "attributes": [ + { + "key": "storage.operation", + "value": { + "stringValue": "PUT" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "200" + } + } + ], + "status": {} + }, + { + "traceId": "/UdFfQDHQ22o2ZWJkp5EpQ==", + "spanId": "bFI69nFO4Sw=", + "parentSpanId": "gBvcar26ulk=", + "flags": 256, + "name": "db.INSERT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928506672592125", + "endTimeUnixNano": "1794928506673296416", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "INSERT" + } + } + ], + "status": {} + }, + { + "traceId": "/UdFfQDHQ22o2ZWJkp5EpQ==", + "spanId": "XKy+x7V6jO0=", + "parentSpanId": "gBvcar26ulk=", + "flags": 256, + "name": "db.INSERT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928506673925875", + "endTimeUnixNano": "1794928506674299541", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "INSERT" + } + } + ], + "status": {} + }, + { + "traceId": "/UdFfQDHQ22o2ZWJkp5EpQ==", + "spanId": "v0OJJaDrnw4=", + "parentSpanId": "gBvcar26ulk=", + "flags": 256, + "name": "db.UPDATE", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928506674742041", + "endTimeUnixNano": "1794928506674945666", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "UPDATE" + } + } + ], + "status": {} + } + ] + } + ] + }, + { + "resource": { + "attributes": [ + { + "key": "telemetry.sdk.language", + "value": { + "stringValue": "python" + } + }, + { + "key": "telemetry.sdk.name", + "value": { + "stringValue": "opentelemetry" + } + }, + { + "key": "telemetry.sdk.version", + "value": { + "stringValue": "1.31.1" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "messenger-api" + } + }, + { + "key": "service.version", + "value": { + "stringValue": "1.6.0" + } + }, + { + "key": "service.instance.id", + "value": { + "stringValue": "ec467075-05bd-4271-bb1e-a272ae3dd3c3" + } + }, + { + "key": "deployment.environment.name", + "value": { + "stringValue": "lab" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "messenger", + "version": "1.6.0" + }, + "spans": [ + { + "traceId": "/UdFfQDHQ22o2ZWJkp5EpQ==", + "spanId": "oa5+bdDcECI=", + "parentSpanId": "gBvcar26ulk=", + "flags": 256, + "name": "db.SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928506585396541", + "endTimeUnixNano": "1794928506585931041", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "/UdFfQDHQ22o2ZWJkp5EpQ==", + "spanId": "qliUreo3mc8=", + "parentSpanId": "gBvcar26ulk=", + "flags": 256, + "name": "db.SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928506591477791", + "endTimeUnixNano": "1794928506591835958", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "/UdFfQDHQ22o2ZWJkp5EpQ==", + "spanId": "1FU3fzCjZGc=", + "parentSpanId": "gBvcar26ulk=", + "flags": 256, + "name": "db.SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928506592016125", + "endTimeUnixNano": "1794928506592194208", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "/UdFfQDHQ22o2ZWJkp5EpQ==", + "spanId": "zXtP2PH9gRE=", + "parentSpanId": "gBvcar26ulk=", + "flags": 256, + "name": "db.INSERT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928506593026833", + "endTimeUnixNano": "1794928506593331875", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "INSERT" + } + } + ], + "status": {} + }, + { + "traceId": "/UdFfQDHQ22o2ZWJkp5EpQ==", + "spanId": "v/AWWHDj3DQ=", + "parentSpanId": "gBvcar26ulk=", + "flags": 256, + "name": "db.SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928506593960791", + "endTimeUnixNano": "1794928506594177666", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + } + ] + } + ] + }, + { + "resource": { + "attributes": [ + { + "key": "telemetry.sdk.language", + "value": { + "stringValue": "python" + } + }, + { + "key": "telemetry.sdk.name", + "value": { + "stringValue": "opentelemetry" + } + }, + { + "key": "telemetry.sdk.version", + "value": { + "stringValue": "1.31.1" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "messenger-dispatcher" + } + }, + { + "key": "service.version", + "value": { + "stringValue": "1.6.0" + } + }, + { + "key": "service.instance.id", + "value": { + "stringValue": "a332c142-9ba3-47c0-ab2c-a62df73b9e51" + } + }, + { + "key": "deployment.environment.name", + "value": { + "stringValue": "lab" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "messenger", + "version": "1.6.0" + }, + "spans": [ + { + "traceId": "/UdFfQDHQ22o2ZWJkp5EpQ==", + "spanId": "O/vOsTnv+9E=", + "parentSpanId": "gBvcar26ulk=", + "flags": 768, + "name": "queue.publish", + "kind": "SPAN_KIND_PRODUCER", + "startTimeUnixNano": "1794928506899106708", + "endTimeUnixNano": "1794928506902470042", + "status": {} + } + ] + } + ] + }, + { + "resource": { + "attributes": [ + { + "key": "telemetry.sdk.language", + "value": { + "stringValue": "python" + } + }, + { + "key": "telemetry.sdk.name", + "value": { + "stringValue": "opentelemetry" + } + }, + { + "key": "telemetry.sdk.version", + "value": { + "stringValue": "1.31.1" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + }, + { + "key": "service.version", + "value": { + "stringValue": "1.6.0" + } + }, + { + "key": "service.instance.id", + "value": { + "stringValue": "3860555f-ed56-4900-ae4a-5e7b82712fdf" + } + }, + { + "key": "deployment.environment.name", + "value": { + "stringValue": "lab" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "messenger", + "version": "1.6.0" + }, + "spans": [ + { + "traceId": "/UdFfQDHQ22o2ZWJkp5EpQ==", + "spanId": "bXic8hkL7QM=", + "parentSpanId": "O/vOsTnv+9E=", + "flags": 768, + "name": "queue.consume", + "kind": "SPAN_KIND_CONSUMER", + "startTimeUnixNano": "1794928506902902167", + "endTimeUnixNano": "1794928507104380250", + "attributes": [ + { + "key": "messaging.system", + "value": { + "stringValue": "redis" + } + }, + { + "key": "job.type", + "value": { + "stringValue": "attachment.process.v1" + } + }, + { + "key": "job.attempt", + "value": { + "intValue": "1" + } + } + ], + "status": {} + }, + { + "traceId": "/UdFfQDHQ22o2ZWJkp5EpQ==", + "spanId": "90NGO4xtu0c=", + "parentSpanId": "bXic8hkL7QM=", + "flags": 256, + "name": "image.transform", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1794928506949619833", + "endTimeUnixNano": "1794928507089492833", + "status": {} + }, + { + "traceId": "/UdFfQDHQ22o2ZWJkp5EpQ==", + "spanId": "0s1HlZQhrAA=", + "parentSpanId": "bXic8hkL7QM=", + "flags": 256, + "name": "s3.PUT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928507089944500", + "endTimeUnixNano": "1794928507093279292", + "attributes": [ + { + "key": "storage.operation", + "value": { + "stringValue": "PUT" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "200" + } + } + ], + "status": {} + }, + { + "traceId": "/UdFfQDHQ22o2ZWJkp5EpQ==", + "spanId": "KvsmUhqSTMg=", + "parentSpanId": "bXic8hkL7QM=", + "flags": 256, + "name": "s3.PUT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928507093675667", + "endTimeUnixNano": "1794928507095736417", + "attributes": [ + { + "key": "storage.operation", + "value": { + "stringValue": "PUT" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "200" + } + } + ], + "status": {} + }, + { + "traceId": "/UdFfQDHQ22o2ZWJkp5EpQ==", + "spanId": "Ej/sw/9+/wM=", + "parentSpanId": "bXic8hkL7QM=", + "flags": 256, + "name": "s3.PUT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928507096028958", + "endTimeUnixNano": "1794928507098908333", + "attributes": [ + { + "key": "storage.operation", + "value": { + "stringValue": "PUT" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "200" + } + } + ], + "status": {} + }, + { + "traceId": "/UdFfQDHQ22o2ZWJkp5EpQ==", + "spanId": "vswUS8KNY9M=", + "parentSpanId": "bXic8hkL7QM=", + "flags": 256, + "name": "s3.PUT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928507099219750", + "endTimeUnixNano": "1794928507101115292", + "attributes": [ + { + "key": "storage.operation", + "value": { + "stringValue": "PUT" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "200" + } + } + ], + "status": {} + }, + { + "traceId": "/UdFfQDHQ22o2ZWJkp5EpQ==", + "spanId": "b4Unm18GieA=", + "parentSpanId": "bXic8hkL7QM=", + "flags": 256, + "name": "db.SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928507101459833", + "endTimeUnixNano": "1794928507101694792", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "/UdFfQDHQ22o2ZWJkp5EpQ==", + "spanId": "TtuyKW00R88=", + "parentSpanId": "bXic8hkL7QM=", + "flags": 256, + "name": "db.UPDATE", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928507102243167", + "endTimeUnixNano": "1794928507103020792", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "UPDATE" + } + } + ], + "status": {} + }, + { + "traceId": "/UdFfQDHQ22o2ZWJkp5EpQ==", + "spanId": "JOUymSFGSfM=", + "parentSpanId": "bXic8hkL7QM=", + "flags": 256, + "name": "db.UPDATE", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928507103145167", + "endTimeUnixNano": "1794928507103502958", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "UPDATE" + } + } + ], + "status": {} + } + ] + } + ] + }, + { + "resource": { + "attributes": [ + { + "key": "telemetry.sdk.language", + "value": { + "stringValue": "python" + } + }, + { + "key": "telemetry.sdk.name", + "value": { + "stringValue": "opentelemetry" + } + }, + { + "key": "telemetry.sdk.version", + "value": { + "stringValue": "1.31.1" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + }, + { + "key": "service.version", + "value": { + "stringValue": "1.6.0" + } + }, + { + "key": "service.instance.id", + "value": { + "stringValue": "3860555f-ed56-4900-ae4a-5e7b82712fdf" + } + }, + { + "key": "deployment.environment.name", + "value": { + "stringValue": "lab" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "messenger", + "version": "1.6.0" + }, + "spans": [ + { + "traceId": "/UdFfQDHQ22o2ZWJkp5EpQ==", + "spanId": "najEho7C/68=", + "parentSpanId": "bXic8hkL7QM=", + "flags": 256, + "name": "db.SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928506930007417", + "endTimeUnixNano": "1794928506930772792", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "/UdFfQDHQ22o2ZWJkp5EpQ==", + "spanId": "ZUHBi7/YiIg=", + "parentSpanId": "bXic8hkL7QM=", + "flags": 256, + "name": "db.SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928506931914333", + "endTimeUnixNano": "1794928506932305167", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "/UdFfQDHQ22o2ZWJkp5EpQ==", + "spanId": "NSA+BKEPKwA=", + "parentSpanId": "bXic8hkL7QM=", + "flags": 256, + "name": "db.SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928506932751875", + "endTimeUnixNano": "1794928506932943708", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "/UdFfQDHQ22o2ZWJkp5EpQ==", + "spanId": "kPC7XrqHWRo=", + "parentSpanId": "bXic8hkL7QM=", + "flags": 256, + "name": "db.UPDATE", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928506933712375", + "endTimeUnixNano": "1794928506934148708", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "UPDATE" + } + } + ], + "status": {} + }, + { + "traceId": "/UdFfQDHQ22o2ZWJkp5EpQ==", + "spanId": "RvxyZJmfiKo=", + "parentSpanId": "bXic8hkL7QM=", + "flags": 256, + "name": "db.UPDATE", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928506934736625", + "endTimeUnixNano": "1794928506935103917", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "UPDATE" + } + } + ], + "status": {} + }, + { + "traceId": "/UdFfQDHQ22o2ZWJkp5EpQ==", + "spanId": "DAb3TnaXk5g=", + "parentSpanId": "bXic8hkL7QM=", + "flags": 256, + "name": "db.SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928506936975917", + "endTimeUnixNano": "1794928506937283792", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "/UdFfQDHQ22o2ZWJkp5EpQ==", + "spanId": "CIEq/klKqCU=", + "parentSpanId": "bXic8hkL7QM=", + "flags": 256, + "name": "db.SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928506937616625", + "endTimeUnixNano": "1794928506937826250", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "/UdFfQDHQ22o2ZWJkp5EpQ==", + "spanId": "IHEueDpnxnk=", + "parentSpanId": "bXic8hkL7QM=", + "flags": 256, + "name": "s3.GET", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794928506947425375", + "endTimeUnixNano": "1794928506949196375", + "attributes": [ + { + "key": "storage.operation", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "200" + } + } + ], + "status": {} + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/evidence/lab6-worker-incident.json b/evidence/lab6-worker-incident.json new file mode 100644 index 0000000..9ecc72f --- /dev/null +++ b/evidence/lab6-worker-incident.json @@ -0,0 +1,32 @@ +{ + "results": [ + { + "case": "upload_to_worker_trace_and_logs", + "result": "PASS", + "trace_id": "c4674a56c0144140973ffadebfb865b4", + "spans": 29 + }, + { + "case": "metrics_labels_and_dashboard_queries", + "result": "PASS", + "metrics": 13, + "panels": 18 + }, + { + "case": "incident_worker_firing", + "result": "PASS", + "queue_depth": 1.0, + "oldest_seconds": 22.580556 + }, + { + "case": "retry_keeps_upload_trace", + "result": "PASS", + "trace_id": "6cb20e12f97745b5834e1d4153e363ad", + "consumer_spans": 2 + }, + { + "case": "incident_worker_resolved", + "result": "PASS" + } + ] +} \ No newline at end of file diff --git a/infra/README.md b/infra/README.md index 5a39e86..000faed 100644 --- a/infra/README.md +++ b/infra/README.md @@ -1,7 +1,8 @@ -# Инфраструктура студента +# Инфраструктура lab6 -Здесь разместите конфигурации выбранных сервисов; основную топологию — в корневом compose.yaml/stack.yaml. `compose.example.yaml` — только worksheet ролей и намеренно не запускается. `contracts/infrastructure.json` задаёт минимальные роли, не готовые образы/пароли. +Автор: Каменский Андрей, БПИ2301. -В REPORT.md сопоставьте роль → service name → версия/digest → health/readiness → volume → сеть/порт → способ передачи credentials. Приведите bootstrap (schema, bucket, broker topology, а с lab6 provisioning) и безопасное повторение этих команд. - -Приложение строится из исходников. Не используйте заранее существующий образ преподавательского сервера вместо своей реализации. Restart hook перезапускает все соответствующие stores с volumes; failover hook останавливает только заданную API-реплику. +compose.yaml включает приложение, PostgreSQL, Redis-сессии, отдельный Streams broker, +MinIO, worker/dispatcher, HAProxy и наблюдаемость. infra/observability содержит +Collector, Prometheus/alerts, Loki, Tempo и импортируемые Grafana datasources/dashboard. +Все настройки воспроизводимы без ручного создания панелей. Данные сохраняются в volumes. diff --git a/infra/haproxy.cfg b/infra/haproxy.cfg new file mode 100644 index 0000000..5a96573 --- /dev/null +++ b/infra/haproxy.cfg @@ -0,0 +1,43 @@ +global + log stdout format raw local0 + maxconn 2048 + stats socket /tmp/haproxy.sock mode 600 level admin + stats socket ipv4@127.0.0.1:9999 level admin + +defaults + log global + mode http + option httplog + timeout connect 1s + timeout client 15s + timeout server 12s + timeout http-request 7s + timeout http-keep-alive 2s + timeout check 1s + # Не повторяем POST автоматически: неизвестный результат повторяет клиент + # с исходным Idempotency-Key. Соединения к недоступному API не ждут долго. + retries 0 + +resolvers docker + nameserver dns 127.0.0.11:53 + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + hold valid 1s + hold obsolete 1s + hold nx 1s + hold timeout 1s + hold refused 1s + accepted_payload_size 8192 + +frontend public + bind :8080 + default_backend api_pool + +backend api_pool + balance roundrobin + option httpchk + http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host api + http-check expect status 200 + default-server inter 1s fastinter 500ms downinter 1s fall 2 rise 2 + server-template api 1-16 api:8080 check resolvers docker resolve-prefer ipv4 init-addr none diff --git a/infra/minio/Dockerfile b/infra/minio/Dockerfile new file mode 100644 index 0000000..398ae8c --- /dev/null +++ b/infra/minio/Dockerfile @@ -0,0 +1,20 @@ +# MinIO community distribution is source-only; pin the reviewed release commit. +FROM golang:1.24.8-alpine3.22 AS build +RUN apk add --no-cache git +WORKDIR /source +RUN git init . \ + && git remote add origin https://github.com/minio/minio.git \ + && git fetch --depth 1 origin 9e49d5e7a648f00e26f2246f4dc28e6b07f8c84a \ + && git checkout --detach FETCH_HEAD +ENV CGO_ENABLED=0 GOMAXPROCS=2 +RUN go build -p 2 -trimpath -o /minio . + +FROM alpine:3.22.2 +RUN apk add --no-cache ca-certificates \ + && addgroup -S minio && adduser -S -G minio minio \ + && mkdir /data && chown minio:minio /data +COPY --from=build /minio /usr/local/bin/minio +USER minio +EXPOSE 9000 +ENTRYPOINT ["minio"] +CMD ["server", "/data", "--console-address", ":9001"] diff --git a/infra/observability/alerts.yaml b/infra/observability/alerts.yaml new file mode 100644 index 0000000..eb3c671 --- /dev/null +++ b/infra/observability/alerts.yaml @@ -0,0 +1,23 @@ +groups: + - name: lab6 + interval: 2s + rules: + - alert: ApiHighErrorRate + expr: | + (sum(increase(http_server_requests_total{route=~"/api/v1/.*",status_class="5xx"}[30s])) + / clamp_min(sum(increase(http_server_requests_total{route=~"/api/v1/.*"}[30s])), 1) > 0.05) + and sum(increase(http_server_requests_total{route=~"/api/v1/.*"}[30s])) >= 3 + for: 5s + labels: + severity: warning + annotations: + summary: "Business API 5xx ratio exceeds 5% over 30 seconds" + description: "Exclude health and client errors. Inspect DB spans, db.error logs and pool waiting. See RUNBOOK.md." + - alert: QueueLagGrowing + expr: max(queue_oldest_age_seconds) > 15 and max(queue_depth) > 0 + for: 5s + labels: + severity: warning + annotations: + summary: "Oldest accepted job has waited more than 15 seconds" + description: "Inspect queue depth, worker capacity/busy and job traces. Restore worker and verify lag returns to zero. See RUNBOOK.md." diff --git a/infra/observability/collector.yaml b/infra/observability/collector.yaml new file mode 100644 index 0000000..621ca1b --- /dev/null +++ b/infra/observability/collector.yaml @@ -0,0 +1,67 @@ +receivers: + otlp: + protocols: + http: + endpoint: 0.0.0.0:4318 +processors: + memory_limiter: + check_interval: 1s + limit_mib: 192 + spike_limit_mib: 32 + batch: + timeout: 1s + send_batch_size: 128 + send_batch_max_size: 256 +exporters: + prometheus: + endpoint: 0.0.0.0:8889 + resource_to_telemetry_conversion: + enabled: true + otlphttp/tempo: + endpoint: http://tempo:4318 + timeout: 2s + sending_queue: + enabled: true + num_consumers: 2 + queue_size: 64 + retry_on_failure: + initial_interval: 1s + max_interval: 3s + max_elapsed_time: 10s + otlphttp/loki: + endpoint: http://loki:3100/otlp + timeout: 2s + sending_queue: + enabled: true + num_consumers: 2 + queue_size: 64 + retry_on_failure: + initial_interval: 1s + max_interval: 3s + max_elapsed_time: 10s +extensions: + health_check: + endpoint: 0.0.0.0:13133 +service: + extensions: [health_check] + telemetry: + metrics: + readers: + - pull: + exporter: + prometheus: + host: 0.0.0.0 + port: 8888 + pipelines: + traces: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlphttp/tempo] + logs: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlphttp/loki] + metrics: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [prometheus] diff --git a/infra/observability/grafana/dashboards/messenger.json b/infra/observability/grafana/dashboards/messenger.json new file mode 100644 index 0000000..e4cd41c --- /dev/null +++ b/infra/observability/grafana/dashboards/messenger.json @@ -0,0 +1,519 @@ +{ + "uid": "messenger-lab6", + "title": "Мессенджер · Lab6 · Каменский Андрей", + "schemaVersion": 39, + "version": 1, + "refresh": "5s", + "time": { + "from": "now-15m", + "to": "now" + }, + "tags": [ + "lab6", + "messenger" + ], + "panels": [ + { + "id": 1, + "title": "Business API requests / second", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 0, + "w": 12, + "h": 8 + }, + "targets": [ + { + "expr": "sum(rate(http_server_requests_total{route=~\"/api/v1/.*\"}[1m]))", + "legendFormat": "RPS", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps" + }, + "overrides": [] + } + }, + { + "id": 2, + "title": "Business API 5xx ratio", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 0, + "w": 12, + "h": 8 + }, + "targets": [ + { + "expr": "sum(rate(http_server_requests_total{route=~\"/api/v1/.*\",status_class=\"5xx\"}[1m])) / clamp_min(sum(rate(http_server_requests_total{route=~\"/api/v1/.*\"}[1m])), 0.00001)", + "legendFormat": "5xx / all business requests", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit" + }, + "overrides": [] + } + }, + { + "id": 3, + "title": "API latency p50 (histogram, all replicas)", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 8, + "w": 12, + "h": 8 + }, + "targets": [ + { + "expr": "histogram_quantile(0.5, sum by (le) (rate(http_server_duration_seconds_bucket{route=~\"/api/v1/.*\"}[1m])))", + "legendFormat": "p50", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + } + }, + { + "id": 4, + "title": "API latency p95 (histogram, all replicas)", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 8, + "w": 12, + "h": 8 + }, + "targets": [ + { + "expr": "histogram_quantile(0.95, sum by (le) (rate(http_server_duration_seconds_bucket{route=~\"/api/v1/.*\"}[1m])))", + "legendFormat": "p95", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + } + }, + { + "id": 5, + "title": "API latency p99 (histogram, all replicas)", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 16, + "w": 12, + "h": 8 + }, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum by (le) (rate(http_server_duration_seconds_bucket{route=~\"/api/v1/.*\"}[1m])))", + "legendFormat": "p99", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + } + }, + { + "id": 6, + "title": "Jobs per second by outcome", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 16, + "w": 12, + "h": 8 + }, + "targets": [ + { + "expr": "sum by(outcome)(rate(worker_jobs_total[1m]))", + "legendFormat": "{{outcome}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops" + }, + "overrides": [] + } + }, + { + "id": 7, + "title": "Worker processing p95", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 24, + "w": 12, + "h": 8 + }, + "targets": [ + { + "expr": "histogram_quantile(0.95, sum by(le)(rate(worker_job_duration_seconds_bucket[1m])))", + "legendFormat": "p95", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + } + }, + { + "id": 8, + "title": "Acceptance to ready p95", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 24, + "w": 12, + "h": 8 + }, + "targets": [ + { + "expr": "histogram_quantile(0.95, sum by(le)(rate(attachment_ready_duration_seconds_bucket[1m])))", + "legendFormat": "p95", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + } + }, + { + "id": 9, + "title": "Accepted unfinished jobs (includes outbox)", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 32, + "w": 12, + "h": 8 + }, + "targets": [ + { + "expr": "max(queue_depth)", + "legendFormat": "jobs", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + } + }, + { + "id": 10, + "title": "Oldest unfinished job age", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 32, + "w": 12, + "h": 8 + }, + "targets": [ + { + "expr": "max(queue_oldest_age_seconds)", + "legendFormat": "age", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + } + }, + { + "id": 11, + "title": "DB pool used / waiting", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 40, + "w": 12, + "h": 8 + }, + "targets": [ + { + "expr": "sum by(service_name)(db_pool_used)", + "legendFormat": "{{service_name}}", + "refId": "A" + }, + { + "expr": "sum by(service_name)(db_pool_waiting)", + "legendFormat": "{{service_name}} waiting", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + } + }, + { + "id": 12, + "title": "Worker saturation", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 40, + "w": 12, + "h": 8 + }, + "targets": [ + { + "expr": "sum(worker_busy) / clamp_min(sum(worker_capacity),1)", + "legendFormat": "busy / capacity", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit" + }, + "overrides": [] + } + }, + { + "id": 13, + "title": "Process resident memory", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 48, + "w": 12, + "h": 8 + }, + "targets": [ + { + "expr": "sum by(service_name)(process_memory_rss_bytes)", + "legendFormat": "{{service_name}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes" + }, + "overrides": [] + } + }, + { + "id": 14, + "title": "Process CPU cores", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 48, + "w": 12, + "h": 8 + }, + "targets": [ + { + "expr": "sum by(service_name)(rate(process_cpu_time_seconds_total[1m]))", + "legendFormat": "{{service_name}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + } + }, + { + "id": 15, + "title": "API availability SLI (5m), SLO 99%", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 56, + "w": 12, + "h": 8 + }, + "targets": [ + { + "expr": "1 - (sum(increase(http_server_requests_total{route=~\"/api/v1/.*\",status_class=\"5xx\"}[5m])) or vector(0)) / clamp_min(sum(increase(http_server_requests_total{route=~\"/api/v1/.*\"}[5m])),1)", + "legendFormat": "SLI", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit" + }, + "overrides": [] + } + }, + { + "id": 16, + "title": "Collector export failures / second", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 56, + "w": 12, + "h": 8 + }, + "targets": [ + { + "expr": "sum by(exporter)(rate(otelcol_exporter_send_failed_spans_total[1m]))", + "legendFormat": "{{exporter}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops" + }, + "overrides": [] + } + }, + { + "id": 17, + "title": "Local firing alerts", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 64, + "w": 12, + "h": 8 + }, + "targets": [ + { + "expr": "ALERTS{alertstate=\"firing\"}", + "legendFormat": "{{alertname}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + } + }, + { + "id": 99, + "title": "Structured API / worker logs — trace_id links to Tempo", + "type": "logs", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "x": 0, + "y": 72, + "w": 24, + "h": 10 + }, + "targets": [ + { + "expr": "{service_name=~\"messenger-(api|worker)\"}", + "refId": "A" + } + ] + } + ] +} diff --git a/infra/observability/grafana/provisioning/dashboards/provider.yaml b/infra/observability/grafana/provisioning/dashboards/provider.yaml new file mode 100644 index 0000000..2079565 --- /dev/null +++ b/infra/observability/grafana/provisioning/dashboards/provider.yaml @@ -0,0 +1,10 @@ +apiVersion: 1 +providers: + - name: messenger + orgId: 1 + folder: Lab6 + type: file + disableDeletion: true + updateIntervalSeconds: 10 + options: + path: /var/lib/grafana/dashboards diff --git a/infra/observability/grafana/provisioning/datasources/sources.yaml b/infra/observability/grafana/provisioning/datasources/sources.yaml new file mode 100644 index 0000000..dc4dcd3 --- /dev/null +++ b/infra/observability/grafana/provisioning/datasources/sources.yaml @@ -0,0 +1,34 @@ +apiVersion: 1 +datasources: + - name: Prometheus + uid: prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + jsonData: + timeInterval: 2s + - name: Loki + uid: loki + type: loki + access: proxy + url: http://loki:3100 + jsonData: + derivedFields: + - name: TraceID + matcherRegex: '"trace_id":\s*"([0-9a-f]{32})"' + datasourceUid: tempo + url: '$${__value.raw}' + - name: Tempo + uid: tempo + type: tempo + access: proxy + url: http://tempo:3200 + jsonData: + tracesToLogsV2: + datasourceUid: loki + spanStartTimeShift: '-1m' + spanEndTimeShift: '1m' + filterByTraceID: false + customQuery: true + query: '{service_name=~"messenger-.*"} | json | trace_id="$${__trace.traceId}"' diff --git a/infra/observability/loki.yaml b/infra/observability/loki.yaml new file mode 100644 index 0000000..5db0331 --- /dev/null +++ b/infra/observability/loki.yaml @@ -0,0 +1,30 @@ +auth_enabled: false +server: + http_listen_port: 3100 +common: + path_prefix: /loki + instance_addr: 127.0.0.1 + replication_factor: 1 + ring: + kvstore: + store: inmemory + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules +schema_config: + configs: + - from: 2024-01-01 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h +limits_config: + allow_structured_metadata: true + retention_period: 168h +compactor: + working_directory: /loki/compactor + retention_enabled: true + delete_request_store: filesystem diff --git a/infra/observability/prometheus.yaml b/infra/observability/prometheus.yaml new file mode 100644 index 0000000..ea5d22d --- /dev/null +++ b/infra/observability/prometheus.yaml @@ -0,0 +1,12 @@ +global: + scrape_interval: 2s + evaluation_interval: 2s +rule_files: + - /etc/prometheus/alerts.yaml +scrape_configs: + - job_name: messenger + static_configs: + - targets: [otel-collector:8889] + - job_name: collector + static_configs: + - targets: [otel-collector:8888] diff --git a/infra/observability/tempo.yaml b/infra/observability/tempo.yaml new file mode 100644 index 0000000..c34ea23 --- /dev/null +++ b/infra/observability/tempo.yaml @@ -0,0 +1,21 @@ +server: + http_listen_port: 3200 +distributor: + receivers: + otlp: + protocols: + http: + endpoint: 0.0.0.0:4318 +ingester: + trace_idle_period: 2s + max_block_duration: 1m +compactor: + compaction: + block_retention: 168h +storage: + trace: + backend: local + wal: + path: /var/tempo/wal + local: + path: /var/tempo/blocks diff --git a/infra/otel/README.md b/infra/otel/README.md deleted file mode 100644 index c46fe89..0000000 --- a/infra/otel/README.md +++ /dev/null @@ -1,7 +0,0 @@ -# Конфигурация наблюдаемости — реализовать - -Добавьте `collector.yaml` с OTLP receivers, processors и exporters выбранной версии Collector. Укажите явные bind endpoints, внутренние сети, memory limits/batch и поведение при backpressure по уровню. Сопоставьте все три pipeline с реально развёрнутыми backend. - -Добавьте конфигурации storage, Grafana provisioning/datasources, provisioning/dashboards и импортируемые dashboard JSON. На 4 — alert rules и runbooks. Пустой dashboard или debug exporter, который только печатает данные в stdout, не заменяют хранилища/доказательства из `contracts/observability.md`. - -Не публикуйте OTLP/metrics/административные ports в общую сеть без необходимости. В lab7 реализуйте TLS/mTLS по trust matrix; экспорт Authorization и message bodies запрещён. diff --git a/infra/s3/api-policy.json b/infra/s3/api-policy.json new file mode 100644 index 0000000..996faf7 --- /dev/null +++ b/infra/s3/api-policy.json @@ -0,0 +1,29 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:ListBucket", + "s3:GetBucketLocation" + ], + "Resource": [ + "arn:aws:s3:::${bucket}" + ] + }, + { + "Effect": "Allow", + "Action": [ + "s3:GetObject", + "s3:PutObject", + "s3:AbortMultipartUpload", + "s3:ListMultipartUploadParts" + ], + "Resource": [ + "arn:aws:s3:::${bucket}/originals/*", + "arn:aws:s3:::${bucket}/staging/*", + "arn:aws:s3:::${bucket}/variants/*" + ] + } + ] +} diff --git a/infra/s3/reconciler-policy.json b/infra/s3/reconciler-policy.json new file mode 100644 index 0000000..47a8a31 --- /dev/null +++ b/infra/s3/reconciler-policy.json @@ -0,0 +1,28 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:ListBucket", + "s3:GetBucketLocation", + "s3:GetBucketVersioning" + ], + "Resource": [ + "arn:aws:s3:::${bucket}" + ] + }, + { + "Effect": "Allow", + "Action": [ + "s3:GetObject", + "s3:DeleteObject" + ], + "Resource": [ + "arn:aws:s3:::${bucket}/originals/*", + "arn:aws:s3:::${bucket}/staging/*", + "arn:aws:s3:::${bucket}/variants/*" + ] + } + ] +} diff --git a/infra/s3/worker-policy.json b/infra/s3/worker-policy.json new file mode 100644 index 0000000..36ac92e --- /dev/null +++ b/infra/s3/worker-policy.json @@ -0,0 +1,30 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:ListBucket", + "s3:GetBucketLocation" + ], + "Resource": [ + "arn:aws:s3:::${bucket}" + ] + }, + { + "Effect": "Allow", + "Action": [ + "s3:GetObject", + "s3:PutObject", + "s3:AbortMultipartUpload", + "s3:ListMultipartUploadParts", + "s3:DeleteObject" + ], + "Resource": [ + "arn:aws:s3:::${bucket}/originals/*", + "arn:aws:s3:::${bucket}/staging/*", + "arn:aws:s3:::${bucket}/variants/*" + ] + } + ] +} diff --git a/lab.json b/lab.json index b365b06..85d21da 100644 --- a/lab.json +++ b/lab.json @@ -4,5 +4,5 @@ "contract_version": "1.6.0", "default_base_url": "http://localhost:8080", "grading": "3; 4 includes 3; 5 includes 4 plus one completed research track", - "scaffold_has_application": false + "scaffold_has_application": true } diff --git a/migrations/env.py b/migrations/env.py new file mode 100644 index 0000000..16eaf7e --- /dev/null +++ b/migrations/env.py @@ -0,0 +1,34 @@ +import asyncio + +from alembic import context +from sqlalchemy import pool +from sqlalchemy.ext.asyncio import create_async_engine + +from src.core.db.models import Base +from src.core.settings import settings + +url = settings.sqlalchemy_database_url +target_metadata = Base.metadata + + +def run_migrations(connection): + context.configure(connection=connection, target_metadata=target_metadata, compare_type=True) + with context.begin_transaction(): + context.run_migrations() + + +async def run_online(): + engine = create_async_engine(url, poolclass=pool.NullPool) + try: + async with engine.connect() as connection: + await connection.run_sync(run_migrations) + finally: + await engine.dispose() + + +if context.is_offline_mode(): + context.configure(url=url, target_metadata=target_metadata, literal_binds=True) + with context.begin_transaction(): + context.run_migrations() +else: + asyncio.run(run_online()) diff --git a/migrations/script.py.mako b/migrations/script.py.mako new file mode 100644 index 0000000..5449f62 --- /dev/null +++ b/migrations/script.py.mako @@ -0,0 +1,17 @@ +"""${message}""" +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + + +def upgrade(): + ${upgrades if upgrades else "pass"} + + +def downgrade(): + ${downgrades if downgrades else "pass"} diff --git a/migrations/versions/001_initial.py b/migrations/versions/001_initial.py new file mode 100644 index 0000000..b8cef40 --- /dev/null +++ b/migrations/versions/001_initial.py @@ -0,0 +1,60 @@ +"""Пользователи, чаты, участники, сообщения и сессии.""" +from alembic import op +import sqlalchemy as sa + +revision = '001' +down_revision = None +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table( + 'users', + sa.Column('id', sa.Uuid(), primary_key=True), + sa.Column('username', sa.String(32), nullable=False, unique=True), + sa.Column('display_name', sa.String(100), nullable=False), + sa.Column('password_hash', sa.Text(), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), nullable=False), + sa.CheckConstraint("username ~ '^[a-z0-9_]{3,32}$'"), + sa.CheckConstraint('length(trim(display_name)) > 0'), + ) + op.create_table( + 'chats', + sa.Column('id', sa.Uuid(), primary_key=True), + sa.Column('title', sa.String(100), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), nullable=False), + sa.CheckConstraint('length(trim(title)) > 0'), + ) + op.create_table( + 'chat_members', + sa.Column('chat_id', sa.Uuid(), sa.ForeignKey('chats.id', ondelete='CASCADE'), primary_key=True), + sa.Column('user_id', sa.Uuid(), sa.ForeignKey('users.id'), primary_key=True), + ) + op.create_index('chat_members_user_idx', 'chat_members', ['user_id', 'chat_id']) + op.create_table( + 'messages', + sa.Column('id', sa.Uuid(), primary_key=True), + sa.Column('chat_id', sa.Uuid(), sa.ForeignKey('chats.id'), nullable=False), + sa.Column('sender_id', sa.Uuid(), sa.ForeignKey('users.id'), nullable=False), + sa.Column('text', sa.String(4000), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), nullable=False), + sa.ForeignKeyConstraint(['chat_id', 'sender_id'], ['chat_members.chat_id', 'chat_members.user_id']), + sa.CheckConstraint('length(trim(text)) > 0'), + ) + op.create_index('messages_history_idx', 'messages', ['chat_id', 'created_at', 'id']) + op.create_table( + 'sessions', + sa.Column('token_hash', sa.Text(), primary_key=True), + sa.Column('user_id', sa.Uuid(), sa.ForeignKey('users.id', ondelete='CASCADE'), nullable=False), + sa.Column('expires_at', sa.DateTime(timezone=True), nullable=False), + ) + op.create_index('sessions_expiry_idx', 'sessions', ['expires_at']) + + +def downgrade(): + op.drop_table('sessions') + op.drop_table('messages') + op.drop_table('chat_members') + op.drop_table('chats') + op.drop_table('users') diff --git a/migrations/versions/002_redis_sessions.py b/migrations/versions/002_redis_sessions.py new file mode 100644 index 0000000..109638b --- /dev/null +++ b/migrations/versions/002_redis_sessions.py @@ -0,0 +1,22 @@ +"""Сессии перенесены в Redis.""" +from alembic import op +import sqlalchemy as sa + +revision = '002' +down_revision = '001' +branch_labels = None +depends_on = None + + +def upgrade(): + op.drop_table('sessions') + + +def downgrade(): + op.create_table( + 'sessions', + sa.Column('token_hash', sa.Text(), primary_key=True), + sa.Column('user_id', sa.Uuid(), sa.ForeignKey('users.id', ondelete='CASCADE'), nullable=False), + sa.Column('expires_at', sa.DateTime(timezone=True), nullable=False), + ) + op.create_index('sessions_expiry_idx', 'sessions', ['expires_at']) diff --git a/migrations/versions/003_admin.py b/migrations/versions/003_admin.py new file mode 100644 index 0000000..7b57ee8 --- /dev/null +++ b/migrations/versions/003_admin.py @@ -0,0 +1,16 @@ +"""Признак системного администратора.""" +from alembic import op +import sqlalchemy as sa + +revision = '003' +down_revision = '002' +branch_labels = None +depends_on = None + + +def upgrade(): + op.add_column('users', sa.Column('is_admin', sa.Boolean(), nullable=False, server_default=sa.false())) + + +def downgrade(): + op.drop_column('users', 'is_admin') diff --git a/migrations/versions/004_idempotency.py b/migrations/versions/004_idempotency.py new file mode 100644 index 0000000..017366a --- /dev/null +++ b/migrations/versions/004_idempotency.py @@ -0,0 +1,25 @@ +"""Общая идемпотентность сообщений, совместимая с предыдущей схемой.""" + +from alembic import op +import sqlalchemy as sa + +revision = "004" +down_revision = "003" +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table( + "message_requests", + sa.Column("sender_id", sa.Uuid(), sa.ForeignKey("users.id"), primary_key=True), + sa.Column("chat_id", sa.Uuid(), sa.ForeignKey("chats.id"), primary_key=True), + sa.Column("key", sa.String(128), primary_key=True), + sa.Column("request_text", sa.String(4000), nullable=False), + sa.Column("message_id", sa.Uuid(), sa.ForeignKey("messages.id"), nullable=True), + sa.Column("expires_at", sa.DateTime(timezone=True), nullable=False), + ) + + +def downgrade(): + op.drop_table("message_requests") diff --git a/migrations/versions/005_message_unicode.py b/migrations/versions/005_message_unicode.py new file mode 100644 index 0000000..762803d --- /dev/null +++ b/migrations/versions/005_message_unicode.py @@ -0,0 +1,55 @@ +"""Сохранять весь допустимый Unicode сообщений, включая U+0000, без потерь.""" + +import sqlalchemy as sa +from alembic import op + +revision = "005" +down_revision = "004" +branch_labels = None +depends_on = None + + +def upgrade(): + op.drop_constraint("messages_text_check", "messages", type_="check") + op.alter_column( + "messages", + "text", + existing_type=sa.String(4000), + type_=sa.LargeBinary(), + existing_nullable=False, + postgresql_using="convert_to(text, 'UTF8')", + ) + op.alter_column( + "message_requests", + "request_text", + existing_type=sa.String(4000), + type_=sa.LargeBinary(), + existing_nullable=False, + postgresql_using="convert_to(request_text, 'UTF8')", + ) + op.create_check_constraint( + "messages_text_bytes_check", "messages", "octet_length(text) BETWEEN 1 AND 16000" + ) + + +def downgrade(): + # convert_from откажет при наличии U+0000: не удаляем символы или строки + # молча. Откат возможен только если все данные представимы в PostgreSQL text. + op.drop_constraint("messages_text_bytes_check", "messages", type_="check") + op.alter_column( + "messages", + "text", + existing_type=sa.LargeBinary(), + type_=sa.String(4000), + existing_nullable=False, + postgresql_using="convert_from(text, 'UTF8')", + ) + op.alter_column( + "message_requests", + "request_text", + existing_type=sa.LargeBinary(), + type_=sa.String(4000), + existing_nullable=False, + postgresql_using="convert_from(request_text, 'UTF8')", + ) + op.create_check_constraint("messages_text_check", "messages", "length(trim(text)) > 0") diff --git a/migrations/versions/006_unicode_names.py b/migrations/versions/006_unicode_names.py new file mode 100644 index 0000000..e60df49 --- /dev/null +++ b/migrations/versions/006_unicode_names.py @@ -0,0 +1,40 @@ +"""Сохранение допустимого Unicode имён и названий, включая U+0000.""" + +import sqlalchemy as sa +from alembic import op + +revision = "006" +down_revision = "005" +branch_labels = None +depends_on = None + + +def upgrade(): + for table, column in [("users", "display_name"), ("chats", "title")]: + op.drop_constraint(f"{table}_{column}_check", table, type_="check") + op.alter_column( + table, + column, + existing_type=sa.String(100), + type_=sa.LargeBinary(), + existing_nullable=False, + postgresql_using=f"convert_to({column}, 'UTF8')", + ) + op.create_check_constraint( + f"{table}_{column}_bytes_check", table, f"octet_length({column}) BETWEEN 1 AND 400" + ) + + +def downgrade(): + # PostgreSQL откажет при U+0000 и откатит транзакцию вместо потери символов. + for table, column in [("users", "display_name"), ("chats", "title")]: + op.drop_constraint(f"{table}_{column}_bytes_check", table, type_="check") + op.alter_column( + table, + column, + existing_type=sa.LargeBinary(), + type_=sa.String(100), + existing_nullable=False, + postgresql_using=f"convert_from({column}, 'UTF8')", + ) + op.create_check_constraint(f"{table}_{column}_check", table, f"length(trim({column})) > 0") diff --git a/migrations/versions/007_attachments.py b/migrations/versions/007_attachments.py new file mode 100644 index 0000000..e468209 --- /dev/null +++ b/migrations/versions/007_attachments.py @@ -0,0 +1,65 @@ +"""Private S3 attachment metadata and ordered message links.""" + +from alembic import op +import sqlalchemy as sa + +revision = "007" +down_revision = "006" +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table( + "attachments", + sa.Column("id", sa.Uuid(), primary_key=True), + sa.Column("chat_id", sa.Uuid(), sa.ForeignKey("chats.id"), nullable=False), + sa.Column("owner_id", sa.Uuid(), sa.ForeignKey("users.id"), nullable=False), + sa.Column("filename", sa.String(255), nullable=False), + sa.Column("object_key", sa.Text(), nullable=False, unique=True), + sa.Column("content_type", sa.String(32), nullable=False), + sa.Column("size_bytes", sa.BigInteger(), nullable=False), + sa.Column("sha256", sa.String(64), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + sa.UniqueConstraint("chat_id", "id"), + sa.CheckConstraint("size_bytes BETWEEN 1 AND 10485760"), + sa.CheckConstraint("content_type IN ('image/png', 'image/jpeg')"), + ) + op.create_index("attachments_chat_idx", "attachments", ["chat_id"]) + op.create_unique_constraint("messages_chat_id_id_key", "messages", ["chat_id", "id"]) + op.create_table( + "message_attachments", + sa.Column("message_id", sa.Uuid(), primary_key=True), + sa.Column("attachment_id", sa.Uuid(), primary_key=True), + sa.Column("chat_id", sa.Uuid(), nullable=False), + sa.Column("position", sa.Integer(), nullable=False), + sa.ForeignKeyConstraint( + ["chat_id", "message_id"], ["messages.chat_id", "messages.id"], ondelete="CASCADE" + ), + sa.ForeignKeyConstraint( + ["chat_id", "attachment_id"], ["attachments.chat_id", "attachments.id"] + ), + sa.UniqueConstraint("message_id", "position"), + sa.CheckConstraint("position BETWEEN 0 AND 9"), + ) + op.add_column("message_requests", sa.Column( + "request_attachment_ids", sa.Text(), nullable=False, server_default="" + )) + op.drop_constraint("messages_text_bytes_check", "messages", type_="check") + op.create_check_constraint( + "messages_text_bytes_check", "messages", "octet_length(text) BETWEEN 0 AND 16000" + ) + + +def downgrade(): + # Refuse downgrade if attachment-only messages cannot satisfy the old schema. + op.drop_constraint("messages_text_bytes_check", "messages", type_="check") + op.create_check_constraint( + "messages_text_bytes_check", "messages", "octet_length(text) BETWEEN 1 AND 16000" + ) + op.drop_column("message_requests", "request_attachment_ids") + op.drop_table("message_attachments") + op.drop_constraint("messages_chat_id_id_key", "messages", type_="unique") + op.drop_index("attachments_chat_idx", "attachments") + op.drop_table("attachments") diff --git a/migrations/versions/008_reconciliation.py b/migrations/versions/008_reconciliation.py new file mode 100644 index 0000000..19afaf2 --- /dev/null +++ b/migrations/versions/008_reconciliation.py @@ -0,0 +1,37 @@ +"""Durable upload intents, missing-content marker and reconciliation observations.""" + +import sqlalchemy as sa +from alembic import op + +revision = "008" +down_revision = "007" +branch_labels = None +depends_on = None + + +def upgrade(): + op.add_column("attachments", sa.Column("storage_missing_since", sa.DateTime(timezone=True))) + op.create_table( + "upload_intents", + sa.Column("object_key", sa.Text(), primary_key=True), + sa.Column("state", sa.String(16), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False, + server_default=sa.func.now()), + sa.CheckConstraint("state IN ('pending', 'committed', 'abandoned')"), + ) + op.create_table( + "storage_findings", + sa.Column("object_key", sa.Text(), primary_key=True), + sa.Column("kind", sa.String(16), primary_key=True), + sa.Column("fingerprint", sa.Text()), + sa.Column("first_seen", sa.DateTime(timezone=True), nullable=False), + sa.Column("last_seen", sa.DateTime(timezone=True), nullable=False), + sa.Column("resolved_at", sa.DateTime(timezone=True)), + sa.CheckConstraint("kind IN ('orphan', 'missing')"), + ) + + +def downgrade(): + op.drop_table("storage_findings") + op.drop_table("upload_intents") + op.drop_column("attachments", "storage_missing_since") diff --git a/migrations/versions/009_pipeline.py b/migrations/versions/009_pipeline.py new file mode 100644 index 0000000..e3cc77e --- /dev/null +++ b/migrations/versions/009_pipeline.py @@ -0,0 +1,67 @@ +"""Asynchronous image pipeline, durable jobs/outbox and replay audit.""" +import sqlalchemy as sa +from alembic import op + +revision = "009" +down_revision = "008" +branch_labels = None +depends_on = None + + +def upgrade(): + op.add_column("attachments", sa.Column("status", sa.String(16), nullable=False, server_default="ready")) + op.add_column("attachments", sa.Column("error_code", sa.String(64))) + op.add_column("attachments", sa.Column("staging_key", sa.Text())) + op.add_column("attachments", sa.Column("variants", sa.JSON(), nullable=False, server_default="[]")) + op.create_check_constraint("attachments_status_check", "attachments", "status IN ('queued', 'processing', 'ready', 'failed')") + op.create_table( + "attachment_jobs", + sa.Column("id", sa.Uuid(), primary_key=True), + sa.Column("attachment_id", sa.Uuid(), sa.ForeignKey("attachments.id"), nullable=False), + sa.Column("pipeline_version", sa.Integer(), nullable=False), + sa.Column("state", sa.String(16), nullable=False), + sa.Column("attempt", sa.Integer(), nullable=False), + sa.Column("cycle_attempt", sa.Integer(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("next_run_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("published_at", sa.DateTime(timezone=True)), + sa.Column("lease_until", sa.DateTime(timezone=True)), + sa.Column("claim_token", sa.Uuid()), + sa.Column("error_code", sa.String(64)), + sa.Column("fault", sa.String(32)), + sa.UniqueConstraint("attachment_id", "pipeline_version"), + sa.CheckConstraint("state IN ('queued', 'processing', 'ready', 'failed')"), + sa.CheckConstraint("attempt >= 1 AND cycle_attempt >= 1"), + sa.CheckConstraint("pipeline_version = 1"), + ) + op.create_index("attachment_jobs_dispatch_idx", "attachment_jobs", ["state", "next_run_at"]) + op.create_table( + "job_replays", + sa.Column("id", sa.Uuid(), primary_key=True), + sa.Column("job_id", sa.Uuid(), sa.ForeignKey("attachment_jobs.id"), nullable=False), + sa.Column("operator", sa.String(100), nullable=False), + sa.Column("reason", sa.String(500), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("attempt", sa.Integer(), nullable=False), + ) + + # Existing lab4 originals remain readable by the worker as their staging + # source. Retention must never delete this shared source/final key. + op.execute(""" + INSERT INTO attachment_jobs + (id, attachment_id, pipeline_version, state, attempt, cycle_attempt, + created_at, updated_at, next_run_at) + SELECT gen_random_uuid(), id, 1, 'queued', 1, 1, + clock_timestamp(), clock_timestamp(), clock_timestamp() + FROM attachments + """) + op.execute("UPDATE attachments SET staging_key=object_key, status='queued', updated_at=clock_timestamp()") + + +def downgrade(): + op.drop_table("job_replays") + op.drop_table("attachment_jobs") + op.drop_constraint("attachments_status_check", "attachments") + for name in ("variants", "staging_key", "error_code", "status"): + op.drop_column("attachments", name) diff --git a/migrations/versions/010_trace_context.py b/migrations/versions/010_trace_context.py new file mode 100644 index 0000000..0a7f5c2 --- /dev/null +++ b/migrations/versions/010_trace_context.py @@ -0,0 +1,15 @@ +"""Persist W3C context across durable outbox, retry and process restarts.""" +import sqlalchemy as sa +from alembic import op +revision = "010" +down_revision = "009" +branch_labels = None +depends_on = None + +def upgrade(): + op.add_column('attachment_jobs', sa.Column('traceparent', sa.String(128))) + op.add_column('attachment_jobs', sa.Column('tracestate', sa.String(512))) + +def downgrade(): + op.drop_column('attachment_jobs','tracestate') + op.drop_column('attachment_jobs','traceparent') diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..6c70477 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,7 @@ +[tool.ruff] +target-version = "py313" +line-length = 100 +include = ["src/**/*.py"] + +[tool.ruff.lint] +select = ["E4", "E7", "E9", "F", "I"] diff --git a/requirements-dev.txt b/requirements-dev.txt index 6b103dd..b36fdc9 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,2 +1,2 @@ -# Only for full OpenAPI/JSON Schema validation; HTTP tests use stdlib. openapi-spec-validator==0.7.2 +ruff==0.14.2 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..a93addf --- /dev/null +++ b/requirements.txt @@ -0,0 +1,25 @@ +annotated-doc==0.0.5 +annotated-types==0.8.0 +anyio==4.15.1 +click==8.5.0 +fastapi==0.141.1 +h11==0.16.0 +idna==3.20 +pydantic==2.13.5 +pydantic-core==2.46.5 +starlette==1.7.0 +typing-extensions==4.16.0 +typing-inspection==0.4.4 +uvicorn==0.53.0 +asyncpg==0.30.0 +redis==5.2.1 +argon2-cffi==25.1.0 +SQLAlchemy[asyncio]==2.0.44 +alembic==1.17.0 +pydantic-settings==2.15.0 +minio==7.2.20 +Pillow==13.0.0 + +opentelemetry-api==1.31.1 +opentelemetry-sdk==1.31.1 +opentelemetry-exporter-otlp-proto-http==1.31.1 diff --git a/scripts/replicas.py b/scripts/replicas.py new file mode 100644 index 0000000..e8cbc36 --- /dev/null +++ b/scripts/replicas.py @@ -0,0 +1,56 @@ +"""Управление только API-контейнерами текущего Compose-проекта.""" + +import json +import os +import subprocess +import time +import urllib.request +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def docker(*args): + return subprocess.check_output(["docker", *args], cwd=ROOT, text=True).strip() + + +def compose(*args): + return docker("compose", "--env-file", os.environ.get("ENV_FILE", ".env"), *args) + + +def containers(): + return compose("ps", "--all", "-q", "api").split() + + +def describe(container): + # Никогда не выводим Config.Env: там credentials. + return json.loads(docker("inspect", "--format", "{{json .State}}", container)) + + +def instance_id(container): + code = "import urllib.request; r=urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2); print(r.headers['X-Instance-Id'])" + return docker("exec", container, "python", "-c", code) + + +def wait_ready(container, timeout=60): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + state = describe(container) + if state.get("Health", {}).get("Status") == "healthy": + return instance_id(container) + time.sleep(0.5) + raise RuntimeError("API container did not become healthy: " + container[:12]) + + +def wait_visible(instance, timeout=30): + base = os.environ.get("BASE_URL", "http://localhost:8080").rstrip("/") + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + try: + with urllib.request.urlopen(base + "/health/ready", timeout=2) as response: + if response.headers.get("X-Instance-Id") == instance: + return + except OSError: + pass + time.sleep(0.1) + raise RuntimeError("New replica was not discovered by HAProxy") diff --git a/scripts/restart.example.sh b/scripts/restart.example.sh deleted file mode 100755 index 967e951..0000000 --- a/scripts/restart.example.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/sh -set -eu -# TODO: copy to restart.sh and implement using YOUR compose/stack services. -# Restart API and ALL used state stores (DB, sessions, S3, broker as applicable). -# Preserve persistent volumes and schema/data; never down -v / flush / re-seed. -# Allow operator unseal if required in lab7; do not print credentials. -# Exit 0 only after the intended action; test waits for API readiness separately. -printf '%s -' 'TODO: implement restart.sh for your infrastructure' >&2 -exit 2 diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 0000000..5da325c --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,7 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +# Полный restart работающих ролей; volumes не удаляются. +docker compose --env-file "${ENV_FILE:-.env}" restart postgres redis minio broker +docker compose --env-file "${ENV_FILE:-.env}" up -d --wait --force-recreate api +docker compose --env-file "${ENV_FILE:-.env}" restart reconciler dispatcher worker lb diff --git a/scripts/setup-env.py b/scripts/setup-env.py new file mode 100644 index 0000000..2ff4f50 --- /dev/null +++ b/scripts/setup-env.py @@ -0,0 +1,28 @@ +"""Create local synthetic credentials without overwriting an existing .env.""" +import argparse +import os +import secrets +from pathlib import Path + +root=Path(__file__).resolve().parents[1] +parser=argparse.ArgumentParser(description=__doc__) +parser.add_argument('--output',type=Path,default=root/'.env') +args=parser.parse_args() +values={ + 'GRAFANA_ADMIN_PASSWORD':secrets.token_hex(24), + 'POSTGRES_PASSWORD':secrets.token_hex(24), + 'ADMIN_PASSWORD':secrets.token_hex(24), + 'MINIO_ROOT_PASSWORD':secrets.token_hex(24), + 'S3_SECRET_ACCESS_KEY':secrets.token_hex(24), + 'S3_RECONCILE_SECRET_ACCESS_KEY':secrets.token_hex(24), + 'S3_WORKER_SECRET_ACCESS_KEY':secrets.token_hex(24), +} +lines=(root/'.env.example').read_text().splitlines() +content='\n'.join(key+'='+values[key] if '=' in line and (key:=line.split('=',1)[0]) in values else line for line in lines)+'\n' +try: + descriptor=os.open(args.output,os.O_WRONLY|os.O_CREAT|os.O_EXCL,0o600) +except FileExistsError: + parser.exit(1,'Файл уже существует; его настройки и пароли сохранены.\n') +with os.fdopen(descriptor,'w') as stream: + stream.write(content) +print('Создан файл локальных настроек с новыми паролями; значения не выводятся.') diff --git a/scripts/stop-one.example.sh b/scripts/stop-one.example.sh deleted file mode 100755 index ae149e8..0000000 --- a/scripts/stop-one.example.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/sh -set -eu -# TODO: copy to stop-one.sh, map TARGET_INSTANCE_ID to ONE API container/task. -# Stop that instance and leave it stopped; no stop of DB/LB/volumes. -# Return quickly (within 10 s); the test sends requests during this script. -# Restore the replica explicitly after the test. Never print credentials. -: "${TARGET_INSTANCE_ID:?test provides target API instance alias}" -printf '%s -' 'TODO: implement stop-one.sh for your infrastructure' >&2 -exit 2 diff --git a/scripts/stop-one.py b/scripts/stop-one.py new file mode 100644 index 0000000..0430d67 --- /dev/null +++ b/scripts/stop-one.py @@ -0,0 +1,34 @@ +"""Сопоставить X-Instance-Id реальному контейнеру и оставить его остановленным.""" + +import json +import os +import time + +from replicas import containers, describe, docker, instance_id + +target = os.environ["TARGET_INSTANCE_ID"] +running = [c for c in containers() if describe(c)["Running"]] +if len(running) < 2: + raise SystemExit("Need at least two running API containers") +for container in running: + if instance_id(container) == target: + before = describe(container) + started = time.monotonic() + docker("stop", "--time", "10", container) + after = describe(container) + assert not after["Running"] + print( + json.dumps( + { + "container": container[:12], + "instance_id": target, + "started_at": before["StartedAt"], + "finished_at": after["FinishedAt"], + "stop_seconds": round(time.monotonic() - started, 3), + "exit_code": after["ExitCode"], + } + ) + ) + break +else: + raise SystemExit("Requested instance not found in this Compose project") diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 0000000..383f91e --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +exec "${PYTHON:-python3}" scripts/stop-one.py diff --git a/scripts/test-persistence.py b/scripts/test-persistence.py new file mode 100644 index 0000000..5e2c0c9 --- /dev/null +++ b/scripts/test-persistence.py @@ -0,0 +1,46 @@ +"""Исходный persistence-сценарий плюс доказательство рестарта API и всех хранилищ.""" +import argparse +import json +from pathlib import Path +import subprocess +import sys + +ROOT = Path(__file__).resolve().parents[1] + + +def states(): + result = {} + for service in ('api', 'postgres', 'redis', 'minio', 'broker', 'worker', 'dispatcher', 'reconciler', 'lb'): + ids = subprocess.check_output(['docker', 'compose', 'ps', '-q', service], cwd=ROOT, text=True).split() + if not ids: + raise RuntimeError('Не запущен сервис ' + service) + result[service] = {} + for container in ids: + state = json.loads(subprocess.check_output(['docker', 'inspect', '--format', '{{json .State}}', container], cwd=ROOT, text=True)) + result[service][container] = {'StartedAt': state['StartedAt'], 'Pid': state['Pid']} + return result + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--base-url', default='http://localhost:8080') + parser.add_argument('--action-script', required=True) + parser.add_argument('--ca-file') + args = parser.parse_args() + before = states() + print('ДО: ' + json.dumps(before), flush=True) + command = [sys.executable, str(ROOT / 'tests/resilience.py'), '--mode', 'persistence', + '--action-script', args.action_script, '--base-url', args.base_url] + if args.ca_file: + command += ['--ca-file', args.ca_file] + subprocess.run(command, cwd=ROOT, check=True) + after = states() + print('ПОСЛЕ: ' + json.dumps(after), flush=True) + for service in before: + if before[service] == after[service]: + raise AssertionError('Сервис не перезапущен: ' + service) + print('PASS: StartedAt изменился у API, PostgreSQL, Redis, MinIO, broker, worker, dispatcher, reconciler и LB; прежний токен и данные сохранены') + + +if __name__ == '__main__': + main() diff --git a/scripts/verify-lab6.py b/scripts/verify-lab6.py new file mode 100644 index 0000000..a64c16f --- /dev/null +++ b/scripts/verify-lab6.py @@ -0,0 +1,20 @@ +"""Short acceptance: structure, public API and real telemetry. Incidents are opt-in.""" +import argparse +import os +import subprocess +import sys +from pathlib import Path + +ROOT=Path(__file__).resolve().parents[1] +p=argparse.ArgumentParser(description=__doc__) +p.add_argument('--incidents',action='store_true') +args=p.parse_args() + +def run(command):subprocess.run(command,cwd=ROOT,check=True) + +if not (ROOT/'.env').exists():run([sys.executable,'scripts/setup-env.py']) +run(['make','check','validate','lint','PYTHON='+sys.executable]) +run(['docker','compose','up','-d','--build','--wait']) +base=os.environ.get('BASE_URL','http://localhost:8086') +run([sys.executable,'tests/smoke.py','--base-url',base]) +run([sys.executable,'tests/observability.py','--base-url',base]+(['--incidents'] if args.incidents else [])) diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/__main__.py b/src/__main__.py new file mode 100644 index 0000000..da5d2b9 --- /dev/null +++ b/src/__main__.py @@ -0,0 +1,13 @@ +import uvicorn + +from src.core.settings import settings + +if __name__ == "__main__": + uvicorn.run( + "src.main:app", + host=settings.app_host, + port=settings.app_port, + workers=settings.app_workers, + log_level=settings.log_level.lower(), + timeout_graceful_shutdown=settings.shutdown_timeout_seconds, + ) diff --git a/src/api/__init__.py b/src/api/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/api/dependencies.py b/src/api/dependencies.py new file mode 100644 index 0000000..d04ea93 --- /dev/null +++ b/src/api/dependencies.py @@ -0,0 +1,106 @@ +import base64 +import binascii +import re +from typing import Annotated + +from fastapi import Depends, Request +from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer +from fastapi.security.http import HTTPBase, HTTPBasicCredentials +from sqlalchemy.ext.asyncio import AsyncSession + +from src.core.db.session import get_db_session +from src.core.errors import ApiError +from src.core.settings import settings +from src.service.auth import AuthService +from src.service.chat import ChatService +from src.service.dto import UserDTO +from src.service.exceptions import InvalidSessionException +from src.service.message import MessageService +from src.service.user import UserService + +bearer_scheme = HTTPBearer( + scheme_name="bearerAuth", + description="Вставьте токен из ответа на вход, без префикса Bearer.", + auto_error=False, +) +basic_scheme = HTTPBase(scheme="basic", scheme_name="basicAuth", auto_error=False) + + +async def get_basic_credentials( + request: Request, + credentials: Annotated[HTTPAuthorizationCredentials | None, Depends(basic_scheme)], +) -> HTTPBasicCredentials: + if await request.body(): + raise ApiError(400, "invalid_request", "Вход не принимает тело запроса") + try: + authorization = request.headers.getlist("authorization") + if len(authorization) != 1 or len(authorization[0]) > settings.authorization_max_length: + raise ValueError + if credentials is None or credentials.scheme.lower() != "basic": + raise ValueError + decoded = base64.b64decode(credentials.credentials, validate=True).decode("utf-8") + username, separator, password = decoded.partition(":") + if ( + not separator + or re.fullmatch(r"[a-z0-9_]{3,32}", username) is None + or not 12 <= len(password) <= 128 + or not password.strip() + ): + raise ValueError + except (ValueError, UnicodeError, binascii.Error): + raise ApiError(401, "unauthorized", "Неверный логин или пароль") from None + return HTTPBasicCredentials(username=username, password=password) + + +def get_user_service( + db_session: Annotated[AsyncSession, Depends(get_db_session, scope="function")], +) -> UserService: + return UserService(db_session=db_session) + + +def get_chat_service( + db_session: Annotated[AsyncSession, Depends(get_db_session, scope="function")], +) -> ChatService: + return ChatService(db_session=db_session) + + +def get_message_service( + db_session: Annotated[AsyncSession, Depends(get_db_session, scope="function")], +) -> MessageService: + return MessageService(db_session=db_session) + + +def get_auth_service( + db_session: Annotated[AsyncSession, Depends(get_db_session, scope="function")], +) -> AuthService: + return AuthService( + db_session=db_session, + session_ttl_seconds=settings.session_ttl_seconds, + ) + + +def get_bearer_token( + request: Request, + credentials: Annotated[HTTPAuthorizationCredentials | None, Depends(bearer_scheme)], +) -> str: + if credentials is None or len(request.headers.getlist("authorization")) != 1: + raise ApiError(401, "unauthorized", "Требуется Bearer-токен") + token = credentials.credentials + if ( + credentials.scheme.lower() != "bearer" + or not token + or len(token) > 512 + or any(c.isspace() for c in token) + ): + raise ApiError(401, "unauthorized", "Требуется Bearer-токен") + return token + + +async def get_current_user( + auth: Annotated[AuthService, Depends(get_auth_service)], + token: Annotated[str, Depends(get_bearer_token)], +) -> UserDTO: + try: + return await auth.get_user(token=token) + except InvalidSessionException as error: + raise ApiError(401, "unauthorized", str(error)) from error diff --git a/src/api/router.py b/src/api/router.py new file mode 100644 index 0000000..7171012 --- /dev/null +++ b/src/api/router.py @@ -0,0 +1,17 @@ +from fastapi import APIRouter + +from src.api.v1.routers.attachments import router as attachments_router +from src.api.v1.routers.auth.router import router as auth_router +from src.api.v1.routers.chats.router import router as chats_router +from src.api.v1.routers.messages.router import router as messages_router +from src.api.v1.routers.users.router import router as users_router + +router = APIRouter(prefix="/api/v1", tags=["Версия 1"]) + +router.include_router(users_router) +router.include_router(chats_router) +router.include_router(messages_router) + +router.include_router(auth_router) + +router.include_router(attachments_router) diff --git a/src/api/v1/__init__.py b/src/api/v1/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/api/v1/routers/__init__.py b/src/api/v1/routers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/api/v1/routers/attachments.py b/src/api/v1/routers/attachments.py new file mode 100644 index 0000000..a5ead39 --- /dev/null +++ b/src/api/v1/routers/attachments.py @@ -0,0 +1,134 @@ +from datetime import datetime +from typing import Annotated, Literal +from uuid import UUID + +from anyio import to_thread +from fastapi import APIRouter, Depends, Query, Request +from fastapi.responses import StreamingResponse +from pydantic import BaseModel, Field +from sqlalchemy.ext.asyncio import AsyncSession + +from src.api.dependencies import get_current_user +from src.core.db.models import Attachment +from src.core.db.session import get_db_session +from src.core.errors import ApiError +from src.core.settings import settings +from src.core.storage import STORAGE_ERRORS, get_storage, unavailable +from src.service.attachment import AttachmentService +from src.service.dto import UserDTO + +router = APIRouter(tags=["Вложения"]) + + +class VariantOut(BaseModel): + name: Literal["thumbnail", "crop", "optimized"] + content_type: Literal["image/jpeg", "image/webp"] + size_bytes: int + sha256: str + width: int + height: int + + +class AttachmentOut(BaseModel): + id: UUID + chat_id: UUID + owner_id: UUID + filename: str + content_type: str + size_bytes: int + sha256: str + status: Literal["queued", "processing", "ready", "failed"] + variants: list[VariantOut] = Field(default_factory=list) + error_code: str | None = None + created_at: datetime + updated_at: datetime + + +def to_output(attachment: Attachment) -> AttachmentOut: + return AttachmentOut.model_validate(attachment, from_attributes=True) + + +def get_service( + session: Annotated[AsyncSession, Depends(get_db_session, scope="function")], +) -> AttachmentService: + return AttachmentService(session) + + +@router.post( + "/chats/{chat_id}/attachments", + status_code=202, + openapi_extra={ + "requestBody": { + "required": True, + "content": { + mime: {"schema": {"type": "string", "format": "binary"}} + for mime in ("image/png", "image/jpeg") + }, + } + }, +) +async def upload_attachment( + chat_id: UUID, + request: Request, + user: Annotated[UserDTO, Depends(get_current_user)], + service: Annotated[AttachmentService, Depends(get_service)], + filename: Annotated[str, Query(min_length=1, max_length=255)], +) -> AttachmentOut: + if len(request.query_params.getlist("filename")) != 1: + raise ApiError(400, "invalid_request", "Неоднозначное имя файла") + return to_output(await service.upload(request, chat_id, user.id, filename)) + + +@router.get("/attachments/{attachment_id}") +async def get_attachment( + attachment_id: UUID, + user: Annotated[UserDTO, Depends(get_current_user)], + service: Annotated[AttachmentService, Depends(get_service)], +) -> AttachmentOut: + return to_output(await service.get(attachment_id, user.id)) + + +@router.get("/attachments/{attachment_id}/content") +async def download_attachment( + attachment_id: UUID, + request: Request, + user: Annotated[UserDTO, Depends(get_current_user)], + service: Annotated[AttachmentService, Depends(get_service)], + variant: str = "original", +): + attachment = await service.get(attachment_id, user.id) + if len(request.query_params.getlist("variant")) > 1: + raise ApiError(400, "invalid_request", "Неоднозначный вариант") + if variant not in {"original", "thumbnail", "crop", "optimized"}: + raise ApiError(400, "invalid_request", "Некорректный вариант") + if attachment.status != "ready": + raise ApiError(409, "conflict", "Вложение ещё не готово") + key, mime, size = attachment.object_key, attachment.content_type, attachment.size_bytes + if variant != "original": + item = next((v for v in attachment.variants if v["name"] == variant), None) + if item is None: + raise ApiError(404, "not_found", "Вариант не найден") + key = f"variants/{attachment.id}/v1/{variant}" + mime, size = item["content_type"], item["size_bytes"] + try: + response = await to_thread.run_sync( + lambda: get_storage().get_object(settings.s3_bucket, key) + ) + except STORAGE_ERRORS as error: + raise unavailable(error) from error + + async def content(): + try: + while chunk := await to_thread.run_sync(response.read, 64 * 1024): + yield chunk + finally: + response.close() + response.release_conn() + + return StreamingResponse( + content(), + media_type=mime, + headers={ + "Content-Length": str(size), + }, + ) diff --git a/src/api/v1/routers/auth/__init__.py b/src/api/v1/routers/auth/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/api/v1/routers/auth/callback.py b/src/api/v1/routers/auth/callback.py new file mode 100644 index 0000000..45334b8 --- /dev/null +++ b/src/api/v1/routers/auth/callback.py @@ -0,0 +1,31 @@ +import logging + +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + +from src.core.db.repo.auth import fetch_user_credentials +from src.core.security import verify_password_async +from src.core.settings import settings +from src.service.user import UserService + +logger = logging.getLogger(__name__) + + +async def create_admin(session_factory: async_sessionmaker[AsyncSession]) -> None: + async with session_factory.begin() as session: + admin = await UserService(db_session=session).create_admin( + username=settings.admin_username, + display_name=settings.admin_display_name, + password=settings.admin_password.get_secret_value(), + ) + if admin is None and settings.app_env == "production": + credentials = await fetch_user_credentials(session, settings.admin_username) + if credentials is None or not await verify_password_async( + settings.admin_password.get_secret_value(), credentials.password_hash + ): + raise RuntimeError( + "Пароль существующего администратора не совпадает с настройкой production" + ) + if admin is None: + logger.info("Администратор уже существует") + else: + logger.info("Администратор создан: пользователь=%s", admin.id) diff --git a/src/api/v1/routers/auth/router.py b/src/api/v1/routers/auth/router.py new file mode 100644 index 0000000..49076ec --- /dev/null +++ b/src/api/v1/routers/auth/router.py @@ -0,0 +1,50 @@ +from dataclasses import asdict +from typing import Annotated + +from fastapi import APIRouter, Depends, Response +from fastapi.security import HTTPBasicCredentials + +from src.api.dependencies import ( + get_auth_service, + get_basic_credentials, + get_bearer_token, + get_current_user, +) +from src.api.v1.routers.auth.schemas import SessionOut +from src.api.v1.routers.users.schemas import UserOut +from src.core.errors import ApiError +from src.service.auth import AuthService +from src.service.dto import UserDTO +from src.service.exceptions import InvalidCredentialsException + +router = APIRouter(prefix="/auth/sessions", tags=["Сессии"]) + + +@router.post("", status_code=201, response_model=SessionOut) +async def login( + credentials: Annotated[HTTPBasicCredentials, Depends(get_basic_credentials)], + auth: Annotated[AuthService, Depends(get_auth_service)], +) -> SessionOut: + try: + result = await auth.login( + username=credentials.username, + password=credentials.password, + ) + except InvalidCredentialsException as error: + raise ApiError(401, "unauthorized", str(error)) from error + return SessionOut( + token=result.token, + token_type=result.token_type, + expires_at=result.expires_at, + user=UserOut(**asdict(result.user)), + ) + + +@router.delete("/current", status_code=204) +async def logout( + current_user: Annotated[UserDTO, Depends(get_current_user)], + token: Annotated[str, Depends(get_bearer_token)], + auth: Annotated[AuthService, Depends(get_auth_service)], +) -> Response: + await auth.logout(token=token) + return Response(status_code=204) diff --git a/src/api/v1/routers/auth/schemas.py b/src/api/v1/routers/auth/schemas.py new file mode 100644 index 0000000..cdeff56 --- /dev/null +++ b/src/api/v1/routers/auth/schemas.py @@ -0,0 +1,13 @@ +from datetime import datetime +from typing import Literal + +from pydantic import BaseModel + +from src.api.v1.routers.users.schemas import UserOut + + +class SessionOut(BaseModel): + token: str + token_type: Literal["Bearer"] + expires_at: datetime + user: UserOut diff --git a/src/api/v1/routers/chats/__init__.py b/src/api/v1/routers/chats/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/api/v1/routers/chats/router.py b/src/api/v1/routers/chats/router.py new file mode 100644 index 0000000..d071839 --- /dev/null +++ b/src/api/v1/routers/chats/router.py @@ -0,0 +1,86 @@ +from typing import Annotated +from uuid import UUID + +from fastapi import APIRouter, Depends, Query, status + +from src.api.dependencies import get_chat_service, get_current_user +from src.api.v1.routers.chats.schemas import ChatOut, CreateChatIn, GetChatOut +from src.api.v1.schemas import PaginationQuery +from src.core.errors import ApiError +from src.core.pagination import decode_cursor, encode_cursor +from src.service.chat import ChatService +from src.service.dto import ChatDTO, UserDTO +from src.service.exceptions import ChatNotFoundException, InvalidChatMembersException + +router = APIRouter(prefix="/chats", tags=["Чаты"]) + + +def _to_chat_out(chat: ChatDTO) -> ChatOut: + return ChatOut( + id=chat.id, + title=chat.title, + member_ids=list(chat.member_ids), + created_at=chat.created_at, + ) + + +@router.post("", status_code=status.HTTP_201_CREATED) +async def create_chat( + chat_in: CreateChatIn, + current_user: Annotated[UserDTO, Depends(get_current_user)], + chat_service: Annotated[ChatService, Depends(get_chat_service)], +) -> ChatOut: + try: + chat = await chat_service.create_chat( + title=chat_in.title, + member_ids=chat_in.member_ids, + current_user_id=current_user.id, + ) + except InvalidChatMembersException as error: + raise ApiError(400, "invalid_request", str(error)) from error + return _to_chat_out(chat) + + +@router.get("", status_code=status.HTTP_200_OK) +async def get_chats( + pagination: Annotated[PaginationQuery, Query()], + current_user: Annotated[UserDTO, Depends(get_current_user)], + chat_service: Annotated[ChatService, Depends(get_chat_service)], +) -> GetChatOut: + after = None + if pagination.cursor is not None: + after = decode_cursor( + pagination.cursor, + expected_kind="chats", + expected_owner_id=current_user.id, + ) + chats = await chat_service.get_chats( + current_user_id=current_user.id, limit=pagination.limit + 1, after=after + ) + + page = chats[: pagination.limit] + next_cursor = None + if len(chats) > pagination.limit: + last = page[-1] + next_cursor = encode_cursor("chats", current_user.id, last.created_at, last.id) + + return GetChatOut( + items=[_to_chat_out(chat) for chat in page], + next_cursor=next_cursor, + ) + + +@router.get("/{chat_id}", summary="Получить доступный чат") +async def get_chat( + chat_id: UUID, + current_user: Annotated[UserDTO, Depends(get_current_user)], + chat_service: Annotated[ChatService, Depends(get_chat_service)], +) -> ChatOut: + try: + chat = await chat_service.get_chat( + chat_id=chat_id, + current_user_id=current_user.id, + ) + except ChatNotFoundException as error: + raise ApiError(404, "not_found", str(error)) from error + return _to_chat_out(chat) diff --git a/src/api/v1/routers/chats/schemas.py b/src/api/v1/routers/chats/schemas.py new file mode 100644 index 0000000..b51b7f8 --- /dev/null +++ b/src/api/v1/routers/chats/schemas.py @@ -0,0 +1,42 @@ +from datetime import datetime +from uuid import UUID + +from pydantic import BaseModel, ConfigDict, Field, field_validator + + +class CreateChatIn(BaseModel): + model_config = ConfigDict(extra="forbid") + + title: str = Field(min_length=1, max_length=100) + member_ids: list[UUID] = Field(min_length=1, max_length=100) + + @field_validator("title") + @classmethod + def title_must_not_be_blank(cls, value: str) -> str: + value.encode("utf-8") + if not value.strip(): + raise ValueError("title не может состоять только из пробелов") + return value + + @field_validator("member_ids") + @classmethod + def member_ids_must_be_unique(cls, value: list[UUID]) -> list[UUID]: + if len(value) != len(set(value)): + raise ValueError("member_ids не должен содержать дубликаты") + return value + + +class ChatOut(BaseModel): + model_config = ConfigDict(extra="forbid") + + id: UUID + title: str + member_ids: list[UUID] + created_at: datetime + + +class GetChatOut(BaseModel): + model_config = ConfigDict(extra="forbid") + + items: list[ChatOut] + next_cursor: str | None diff --git a/src/api/v1/routers/messages/__init__.py b/src/api/v1/routers/messages/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/api/v1/routers/messages/router.py b/src/api/v1/routers/messages/router.py new file mode 100644 index 0000000..68f9265 --- /dev/null +++ b/src/api/v1/routers/messages/router.py @@ -0,0 +1,96 @@ +import re +from typing import Annotated +from uuid import UUID + +from fastapi import APIRouter, Depends, Header, Query, Request, status + +from src.api.dependencies import get_current_user, get_message_service +from src.api.v1.routers.messages.schemas import ( + CreateMessageIn, + MessageOut, + MessagePageOut, +) +from src.api.v1.schemas import PaginationQuery +from src.core.errors import ApiError +from src.core.pagination import decode_cursor, encode_cursor +from src.service.dto import MessageDTO, UserDTO +from src.service.exceptions import ChatNotFoundException, IdempotencyConflictException +from src.service.message import MessageService + +router = APIRouter(prefix="/chats", tags=["Сообщения"]) + + +def _to_message_out(message: MessageDTO) -> MessageOut: + return MessageOut( + id=message.id, + chat_id=message.chat_id, + sender_id=message.sender_id, + text=message.text, + attachment_ids=list(message.attachment_ids), + created_at=message.created_at, + ) + + +@router.post("/{chat_id}/messages", status_code=status.HTTP_201_CREATED) +async def create_message( + chat_id: UUID, + message_in: CreateMessageIn, + request: Request, + current_user: Annotated[UserDTO, Depends(get_current_user)], + message_service: Annotated[MessageService, Depends(get_message_service)], + idempotency_key: Annotated[str, Header()], +) -> MessageOut: + if ( + len(request.headers.getlist("idempotency-key")) != 1 + or re.fullmatch(r"[A-Za-z0-9._:-]{1,128}", idempotency_key) is None + ): + raise ApiError(400, "invalid_request", "Требуется корректный Idempotency-Key") + try: + message = await message_service.create_message( + chat_id=chat_id, + sender_id=current_user.id, + text=message_in.text, + attachment_ids=message_in.attachment_ids, + idempotency_key=idempotency_key, + ) + except ChatNotFoundException as error: + raise ApiError(404, "not_found", str(error)) from error + except IdempotencyConflictException as error: + raise ApiError(409, "conflict", "Ключ уже использован для другого сообщения") from error + return _to_message_out(message) + + +@router.get("/{chat_id}/messages", status_code=status.HTTP_200_OK) +async def get_messages( + chat_id: UUID, + pagination: Annotated[PaginationQuery, Query()], + current_user: Annotated[UserDTO, Depends(get_current_user)], + message_service: Annotated[MessageService, Depends(get_message_service)], +) -> MessagePageOut: + after = None + if pagination.cursor is not None: + after = decode_cursor( + pagination.cursor, + expected_kind="messages", + expected_owner_id=chat_id, + ) + try: + messages = await message_service.get_messages( + chat_id=chat_id, + current_user_id=current_user.id, + limit=pagination.limit + 1, + after=after, + ) + except ChatNotFoundException as error: + raise ApiError(404, "not_found", str(error)) from error + + page = messages[: pagination.limit] + next_cursor = None + if len(messages) > pagination.limit: + last = page[-1] + next_cursor = encode_cursor("messages", chat_id, last.created_at, last.id) + + return MessagePageOut( + items=[_to_message_out(message) for message in page], + next_cursor=next_cursor, + ) diff --git a/src/api/v1/routers/messages/schemas.py b/src/api/v1/routers/messages/schemas.py new file mode 100644 index 0000000..1a0c821 --- /dev/null +++ b/src/api/v1/routers/messages/schemas.py @@ -0,0 +1,43 @@ +from datetime import datetime +from uuid import UUID + +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator + + +class CreateMessageIn(BaseModel): + model_config = ConfigDict(extra="forbid") + + text: str = Field(max_length=4000) + attachment_ids: list[UUID] = Field(default_factory=list, max_length=10) + + @field_validator("text") + @classmethod + def text_must_not_be_blank(cls, value: str) -> str: + value.encode("utf-8") + return value + + @model_validator(mode="after") + def validate_message(self): + if not self.text.strip() and not self.attachment_ids: + raise ValueError("Требуется текст или вложение") + if len(set(self.attachment_ids)) != len(self.attachment_ids): + raise ValueError("Вложения не должны повторяться") + return self + + +class MessageOut(BaseModel): + model_config = ConfigDict(extra="forbid") + + id: UUID + chat_id: UUID + sender_id: UUID + text: str + attachment_ids: list[UUID] + created_at: datetime + + +class MessagePageOut(BaseModel): + model_config = ConfigDict(extra="forbid") + + items: list[MessageOut] + next_cursor: str | None diff --git a/src/api/v1/routers/users/__init__.py b/src/api/v1/routers/users/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/api/v1/routers/users/router.py b/src/api/v1/routers/users/router.py new file mode 100644 index 0000000..cbe3726 --- /dev/null +++ b/src/api/v1/routers/users/router.py @@ -0,0 +1,44 @@ +from typing import Annotated + +from fastapi import APIRouter, Depends, status + +from src.api.dependencies import get_current_user, get_user_service +from src.api.v1.routers.users.schemas import CreateUserIn, UserOut +from src.core.errors import ApiError +from src.service.dto import UserDTO +from src.service.exceptions import UserAlreadyExistsException +from src.service.user import UserService + +router = APIRouter(prefix="/users", tags=["Пользователи"]) + + +def _to_user_out(user: UserDTO) -> UserOut: + return UserOut( + id=user.id, + username=user.username, + display_name=user.display_name, + created_at=user.created_at, + ) + + +@router.post("", status_code=status.HTTP_201_CREATED) +async def create_user( + user_in: CreateUserIn, + user_service: Annotated[UserService, Depends(get_user_service)], +) -> UserOut: + try: + user = await user_service.create_user( + password=user_in.password, + username=user_in.username, + display_name=user_in.display_name, + ) + except UserAlreadyExistsException as error: + raise ApiError(409, "conflict", str(error)) from error + return _to_user_out(user) + + +@router.get("/me", status_code=status.HTTP_200_OK) +async def get_me( + current_user: Annotated[UserDTO, Depends(get_current_user)], +) -> UserOut: + return _to_user_out(current_user) diff --git a/src/api/v1/routers/users/schemas.py b/src/api/v1/routers/users/schemas.py new file mode 100644 index 0000000..7cac9da --- /dev/null +++ b/src/api/v1/routers/users/schemas.py @@ -0,0 +1,29 @@ +from datetime import datetime +from uuid import UUID + +from pydantic import BaseModel, ConfigDict, Field, field_validator + + +class CreateUserIn(BaseModel): + model_config = ConfigDict(extra="forbid") + + username: str = Field(pattern=r"^[a-z0-9_]{3,32}$") + password: str = Field(min_length=12, max_length=128) + display_name: str = Field(min_length=1, max_length=100) + + @field_validator("display_name", "password") + @classmethod + def required_string_must_not_be_blank(cls, value: str) -> str: + value.encode("utf-8") + if not value.strip(): + raise ValueError("Обязательная строка не может состоять только из пробелов") + return value + + +class UserOut(BaseModel): + model_config = ConfigDict(extra="forbid") + + id: UUID + username: str + display_name: str + created_at: datetime diff --git a/src/api/v1/schemas.py b/src/api/v1/schemas.py new file mode 100644 index 0000000..6310689 --- /dev/null +++ b/src/api/v1/schemas.py @@ -0,0 +1,21 @@ +import re + +from pydantic import BaseModel, ConfigDict, Field, field_validator + +from src.core.settings import settings + + +class PaginationQuery(BaseModel): + model_config = ConfigDict(extra="forbid") + + limit: int = Field(default=50, ge=1, le=100) + cursor: str | None = Field(default=None, min_length=1, max_length=settings.cursor_max_length) + + @field_validator("limit", mode="before") + @classmethod + def limit_must_be_integer(cls, value): + if isinstance(value, bool) or isinstance(value, float): + raise ValueError("limit должен быть целым числом") + if isinstance(value, str) and re.fullmatch(r"[0-9]+", value) is None: + raise ValueError("limit должен содержать десятичные цифры") + return value diff --git a/src/core/__init__.py b/src/core/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/core/cache.py b/src/core/cache.py new file mode 100644 index 0000000..19444bf --- /dev/null +++ b/src/core/cache.py @@ -0,0 +1,68 @@ +"""Клиент Redis и необязательный кэш профилей пользователей.""" + +import logging + +from redis.asyncio import Redis +from redis.exceptions import RedisError + +from src.core.settings import settings + +logger = logging.getLogger(__name__) +cache: Redis | None = None + + +def open_cache(): + global cache + cache = Redis.from_url( + settings.redis_url.get_secret_value(), + decode_responses=True, + max_connections=settings.redis_max_connections, + socket_connect_timeout=settings.redis_connect_timeout_seconds, + socket_timeout=settings.redis_read_timeout_seconds, + retry_on_timeout=settings.redis_retry_on_timeout, + ) + + +async def close_cache(): + global cache + if cache is not None: + await cache.aclose() + cache = None + + +async def get(key: str) -> str | None: + if cache is None: + return None + try: + return await cache.get(key) + except UnicodeError: + logger.warning("Повреждённый UTF-8 в кэше профиля; читаем PostgreSQL") + await delete(key) + return None + except (RedisError, OSError): + logger.warning("Redis недоступен; читаем профиль из PostgreSQL") + return None + + +async def set(key: str, value: str): + if cache is None: + return + try: + await cache.set(key, value, ex=settings.profile_cache_ttl_seconds) + except (RedisError, OSError): + logger.warning("Redis недоступен; пропускаем запись кэша") + + +async def delete(key: str) -> None: + if cache is None: + return + try: + await cache.delete(key) + except (RedisError, OSError): + logger.warning("Redis недоступен; пропускаем удаление кэша профиля") + + +def get_client() -> Redis: + if cache is None: + raise RedisError("Клиент Redis не настроен") + return cache diff --git a/src/core/callback.py b/src/core/callback.py new file mode 100644 index 0000000..3cf0cff --- /dev/null +++ b/src/core/callback.py @@ -0,0 +1,24 @@ +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager + +from fastapi import FastAPI + +from src.api.v1.routers.auth.callback import create_admin +from src.core import readiness +from src.core.db import session as database + +STARTUP_CALLBACKS = (create_admin,) + + +@asynccontextmanager +async def lifespan(app: FastAPI) -> AsyncIterator[None]: + async with database.database_lifespan(app): + session_factory = database.session_factory + if session_factory is None: + raise RuntimeError("Фабрика сессий не запущена") + for callback in STARTUP_CALLBACKS: + await callback(session_factory) + try: + yield + finally: + await readiness.close() diff --git a/src/core/db/__init__.py b/src/core/db/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/core/db/models.py b/src/core/db/models.py new file mode 100644 index 0000000..2b57a4e --- /dev/null +++ b/src/core/db/models.py @@ -0,0 +1,207 @@ +from datetime import datetime +from uuid import UUID + +from sqlalchemy import ( + JSON, + BigInteger, + Boolean, + CheckConstraint, + DateTime, + ForeignKey, + ForeignKeyConstraint, + Index, + Integer, + String, + Text, + UniqueConstraint, + false, +) +from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship + +from src.core.db.types import Utf8Text + + +class Base(DeclarativeBase): + pass + + +class User(Base): + __tablename__ = "users" + __table_args__ = ( + CheckConstraint("username ~ '^[a-z0-9_]{3,32}$'"), + CheckConstraint( + "octet_length(display_name) BETWEEN 1 AND 400", name="users_display_name_bytes_check" + ), + ) + + id: Mapped[UUID] = mapped_column(primary_key=True) + username: Mapped[str] = mapped_column(String(32), unique=True) + display_name: Mapped[str] = mapped_column(Utf8Text()) + password_hash: Mapped[str] = mapped_column(Text) + is_admin: Mapped[bool] = mapped_column(Boolean, default=False, server_default=false()) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True)) + + +class Chat(Base): + __tablename__ = "chats" + __table_args__ = ( + CheckConstraint("octet_length(title) BETWEEN 1 AND 400", name="chats_title_bytes_check"), + ) + + id: Mapped[UUID] = mapped_column(primary_key=True) + title: Mapped[str] = mapped_column(Utf8Text()) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True)) + members: Mapped[list["ChatMember"]] = relationship(order_by="ChatMember.user_id") + + +class ChatMember(Base): + __tablename__ = "chat_members" + __table_args__ = (Index("chat_members_user_idx", "user_id", "chat_id"),) + + chat_id: Mapped[UUID] = mapped_column( + ForeignKey("chats.id", ondelete="CASCADE"), primary_key=True + ) + user_id: Mapped[UUID] = mapped_column(ForeignKey("users.id"), primary_key=True) + + +class Message(Base): + __tablename__ = "messages" + __table_args__ = ( + ForeignKeyConstraint( + ["chat_id", "sender_id"], ["chat_members.chat_id", "chat_members.user_id"] + ), + CheckConstraint("octet_length(text) BETWEEN 0 AND 16000", name="messages_text_bytes_check"), + UniqueConstraint("chat_id", "id"), + Index("messages_history_idx", "chat_id", "created_at", "id"), + ) + + id: Mapped[UUID] = mapped_column(primary_key=True) + chat_id: Mapped[UUID] = mapped_column(ForeignKey("chats.id")) + sender_id: Mapped[UUID] = mapped_column(ForeignKey("users.id")) + attachments: Mapped[list["MessageAttachment"]] = relationship( + lazy="selectin", order_by="MessageAttachment.position" + ) + text: Mapped[str] = mapped_column(Utf8Text()) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True)) + + +class MessageRequest(Base): + # Таблица обслуживает только POST /chats/{chat_id}/messages: метод и путь + # однозначно заданы таблицей и chat_id. Общая область ключа: sender/chat/key. + __tablename__ = "message_requests" + + sender_id: Mapped[UUID] = mapped_column(ForeignKey("users.id"), primary_key=True) + chat_id: Mapped[UUID] = mapped_column(ForeignKey("chats.id"), primary_key=True) + key: Mapped[str] = mapped_column(String(128), primary_key=True) + request_text: Mapped[str] = mapped_column(Utf8Text()) + request_attachment_ids: Mapped[str] = mapped_column(Text, default="", server_default="") + message_id: Mapped[UUID | None] = mapped_column(ForeignKey("messages.id")) + expires_at: Mapped[datetime] = mapped_column(DateTime(timezone=True)) + + +class Attachment(Base): + __tablename__ = "attachments" + __table_args__ = ( + UniqueConstraint("chat_id", "id"), + CheckConstraint("size_bytes BETWEEN 1 AND 10485760"), + CheckConstraint( + "status IN ('queued', 'processing', 'ready', 'failed')", name="attachments_status_check" + ), + CheckConstraint("content_type IN ('image/png', 'image/jpeg')"), + Index("attachments_chat_idx", "chat_id"), + ) + + id: Mapped[UUID] = mapped_column(primary_key=True) + chat_id: Mapped[UUID] = mapped_column(ForeignKey("chats.id")) + owner_id: Mapped[UUID] = mapped_column(ForeignKey("users.id")) + filename: Mapped[str] = mapped_column(String(255)) + object_key: Mapped[str] = mapped_column(Text, unique=True) + content_type: Mapped[str] = mapped_column(String(32)) + size_bytes: Mapped[int] = mapped_column(BigInteger) + sha256: Mapped[str] = mapped_column(String(64)) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True)) + updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True)) + storage_missing_since: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + status: Mapped[str] = mapped_column(String(16), default="ready", server_default="ready") + error_code: Mapped[str | None] = mapped_column(String(64)) + staging_key: Mapped[str | None] = mapped_column(Text) + variants: Mapped[list] = mapped_column(JSON, default=list, server_default="[]") + + +class AttachmentJob(Base): + """A job is also its transactional outbox: published_at=NULL means send it.""" + + __tablename__ = "attachment_jobs" + __table_args__ = ( + UniqueConstraint("attachment_id", "pipeline_version"), + CheckConstraint("state IN ('queued', 'processing', 'ready', 'failed')"), + CheckConstraint("attempt >= 1 AND cycle_attempt >= 1"), + CheckConstraint("pipeline_version = 1"), + Index("attachment_jobs_dispatch_idx", "state", "next_run_at"), + ) + id: Mapped[UUID] = mapped_column(primary_key=True) + attachment_id: Mapped[UUID] = mapped_column(ForeignKey("attachments.id")) + pipeline_version: Mapped[int] = mapped_column(Integer, default=1) + state: Mapped[str] = mapped_column(String(16), default="queued") + attempt: Mapped[int] = mapped_column(Integer, default=1) + cycle_attempt: Mapped[int] = mapped_column(Integer, default=1) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True)) + updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True)) + next_run_at: Mapped[datetime] = mapped_column(DateTime(timezone=True)) + published_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + lease_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + claim_token: Mapped[UUID | None] = mapped_column() + error_code: Mapped[str | None] = mapped_column(String(64)) + traceparent: Mapped[str | None] = mapped_column(String(128)) + tracestate: Mapped[str | None] = mapped_column(String(512)) + fault: Mapped[str | None] = mapped_column(String(32)) + + +class JobReplay(Base): + __tablename__ = "job_replays" + id: Mapped[UUID] = mapped_column(primary_key=True) + job_id: Mapped[UUID] = mapped_column(ForeignKey("attachment_jobs.id")) + operator: Mapped[str] = mapped_column(String(100)) + reason: Mapped[str] = mapped_column(String(500)) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True)) + attempt: Mapped[int] = mapped_column(Integer) + + +class MessageAttachment(Base): + __tablename__ = "message_attachments" + __table_args__ = ( + ForeignKeyConstraint( + ["chat_id", "message_id"], ["messages.chat_id", "messages.id"], ondelete="CASCADE" + ), + ForeignKeyConstraint( + ["chat_id", "attachment_id"], ["attachments.chat_id", "attachments.id"] + ), + UniqueConstraint("message_id", "position"), + CheckConstraint("position BETWEEN 0 AND 9"), + ) + + message_id: Mapped[UUID] = mapped_column(primary_key=True) + attachment_id: Mapped[UUID] = mapped_column(primary_key=True) + chat_id: Mapped[UUID] = mapped_column() + position: Mapped[int] = mapped_column(Integer) + + +class UploadIntent(Base): + __tablename__ = "upload_intents" + __table_args__ = (CheckConstraint("state IN ('pending', 'committed', 'abandoned')"),) + + object_key: Mapped[str] = mapped_column(Text, primary_key=True) + state: Mapped[str] = mapped_column(String(16)) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True)) + + +class StorageFinding(Base): + __tablename__ = "storage_findings" + __table_args__ = (CheckConstraint("kind IN ('orphan', 'missing')"),) + + object_key: Mapped[str] = mapped_column(Text, primary_key=True) + kind: Mapped[str] = mapped_column(String(16), primary_key=True) + fingerprint: Mapped[str | None] = mapped_column(Text) + first_seen: Mapped[datetime] = mapped_column(DateTime(timezone=True)) + last_seen: Mapped[datetime] = mapped_column(DateTime(timezone=True)) + resolved_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) diff --git a/src/core/db/repo/__init__.py b/src/core/db/repo/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/core/db/repo/auth.py b/src/core/db/repo/auth.py new file mode 100644 index 0000000..2497013 --- /dev/null +++ b/src/core/db/repo/auth.py @@ -0,0 +1,16 @@ +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from src.core.db.models import User +from src.core.db.repo.user import to_user +from src.service.dto import UserCredentialsDTO + + +async def fetch_user_credentials( + session: AsyncSession, + username: str, +) -> UserCredentialsDTO | None: + user = await session.scalar(select(User).where(User.username == username)) + if user is None: + return None + return UserCredentialsDTO(user=to_user(user), password_hash=user.password_hash) diff --git a/src/core/db/repo/chat.py b/src/core/db/repo/chat.py new file mode 100644 index 0000000..aa88770 --- /dev/null +++ b/src/core/db/repo/chat.py @@ -0,0 +1,59 @@ +from datetime import datetime +from uuid import UUID + +from sqlalchemy import select, tuple_ +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.orm import selectinload + +from src.core.db.models import Chat, ChatMember +from src.service.dto import ChatDTO + + +def to_chat(chat: Chat) -> ChatDTO: + return ChatDTO( + chat.id, chat.title, tuple(member.user_id for member in chat.members), chat.created_at + ) + + +async def fetch_chat_by_id(session: AsyncSession, chat_id: UUID) -> ChatDTO | None: + chat = await session.scalar( + select(Chat).options(selectinload(Chat.members)).where(Chat.id == chat_id), + ) + return to_chat(chat) if chat else None + + +async def fetch_chats( + session: AsyncSession, + user_id: UUID, + limit: int, + after: tuple[datetime, UUID] | None = None, +) -> list[ChatDTO]: + query = ( + select(Chat) + .options(selectinload(Chat.members)) + .join(ChatMember) + .where(ChatMember.user_id == user_id) + ) + if after is not None: + query = query.where(tuple_(Chat.created_at, Chat.id) > after) + chats = await session.scalars(query.order_by(Chat.created_at, Chat.id).limit(limit)) + return [to_chat(chat) for chat in chats] + + +async def create_chat( + session: AsyncSession, + chat_id: UUID, + title: str, + member_ids: list[UUID], + created_at: datetime, +) -> ChatDTO: + chat = Chat( + id=chat_id, + title=title, + created_at=created_at, + members=[ChatMember(user_id=user_id) for user_id in member_ids], + ) + session.add(chat) + # ORM сохраняет чат и участников в транзакции текущего запроса. + await session.flush() + return to_chat(chat) diff --git a/src/core/db/repo/message.py b/src/core/db/repo/message.py new file mode 100644 index 0000000..35036a6 --- /dev/null +++ b/src/core/db/repo/message.py @@ -0,0 +1,151 @@ +from datetime import datetime +from uuid import UUID + +from sqlalchemy import or_, select, tuple_ +from sqlalchemy.ext.asyncio import AsyncSession + +from src.core.db.models import Attachment, Message, MessageAttachment, MessageRequest +from src.service.dto import MessageDTO + + +def to_message(message: Message) -> MessageDTO: + return MessageDTO( + message.id, + message.chat_id, + message.sender_id, + message.text, + message.created_at, + tuple(link.attachment_id for link in message.attachments), + ) + + +def history_query(chat_id: UUID, limit: int, after: tuple[datetime, UUID] | None = None): + query = select(Message).where(Message.chat_id == chat_id) + if after is not None: + query = query.where(tuple_(Message.created_at, Message.id) > after) + return query.order_by(Message.created_at, Message.id).limit(limit) + + +async def fetch_messages_by_chat_id( + session: AsyncSession, + chat_id: UUID, + limit: int, + after: tuple[datetime, UUID] | None = None, +) -> list[MessageDTO]: + messages = await session.scalars(history_query(chat_id, limit, after)) + return [to_message(message) for message in messages] + + +async def create_message( + session: AsyncSession, + message_id: UUID, + chat_id: UUID, + sender_id: UUID, + text: str, + created_at: datetime, + attachment_ids: list[UUID] | None = None, +) -> MessageDTO: + message = Message( + id=message_id, + chat_id=chat_id, + sender_id=sender_id, + text=text, + created_at=created_at, + attachments=[ + MessageAttachment(chat_id=chat_id, attachment_id=aid, position=i) + for i, aid in enumerate(attachment_ids or []) + ], + ) + session.add(message) + await session.flush() + return to_message(message) + + +async def create_idempotent_message( + session: AsyncSession, + message_id: UUID, + chat_id: UUID, + sender_id: UUID, + text: str, + created_at: datetime, + idempotency_key: str, + attachment_ids: list[UUID] | None = None, +) -> MessageDTO: + from sqlalchemy import func, update + from sqlalchemy import text as sql_text + from sqlalchemy.dialects.postgresql import insert + + from src.core.errors import ApiError + from src.service.exceptions import IdempotencyConflictException + + attachment_ids = attachment_ids or [] + request_attachments = ",".join(str(aid) for aid in attachment_ids) + if attachment_ids: + found = set( + await session.scalars( + select(Attachment.id).where( + Attachment.chat_id == chat_id, Attachment.id.in_(attachment_ids) + ) + ) + ) + if found != set(attachment_ids): + raise ApiError(404, "not_found", "Вложение не найдено") + + identity = ( + MessageRequest.sender_id == sender_id, + MessageRequest.chat_id == chat_id, + MessageRequest.key == idempotency_key, + ) + # UNIQUE/UPSERT сериализует конкурентные повторы даже на разных процессах. + # Проигравший ждёт COMMIT; при rollback он сам получает право на создание. + values = dict( + sender_id=sender_id, + chat_id=chat_id, + key=idempotency_key, + request_text=text, + request_attachment_ids=request_attachments, + message_id=None, + expires_at=func.clock_timestamp() + sql_text("interval '24 hours'"), + ) + claim = ( + insert(MessageRequest) + .values(**values) + .on_conflict_do_update( + index_elements=["sender_id", "chat_id", "key"], + set_={ + k: v + for k, v in values.items() + if k in {"request_text", "request_attachment_ids", "message_id", "expires_at"} + }, + where=MessageRequest.expires_at <= func.clock_timestamp(), + ) + .returning(MessageRequest.key) + ) + owned = await session.scalar(claim) + if owned is None: + existing = (await session.execute(select(MessageRequest).where(*identity))).scalar_one() + if existing.request_text != text or existing.request_attachment_ids != request_attachments: + raise IdempotencyConflictException + message = await session.get(Message, existing.message_id) + if message is None: + raise RuntimeError("Нарушена атомарность результата идемпотентности") + return to_message(message) + + if attachment_ids: + missing = await session.scalar( + select(Attachment.id) + .where( + Attachment.id.in_(attachment_ids), + or_(Attachment.storage_missing_since.is_not(None), Attachment.status == "failed"), + ) + .limit(1) + ) + if missing is not None: + raise ApiError(409, "conflict", "Оригинал вложения недоступен") + + message = await create_message( + session, message_id, chat_id, sender_id, text, created_at, attachment_ids + ) + await session.execute(update(MessageRequest).where(*identity).values(message_id=message.id)) + # COMMIT сообщения и ключа выполняется вместе в get_db_session до HTTP 201. + return message diff --git a/src/core/db/repo/user.py b/src/core/db/repo/user.py new file mode 100644 index 0000000..6f10eed --- /dev/null +++ b/src/core/db/repo/user.py @@ -0,0 +1,120 @@ +import json +import re +from dataclasses import asdict +from datetime import datetime, timezone +from uuid import UUID + +from sqlalchemy import func, select +from sqlalchemy.dialects.postgresql import insert +from sqlalchemy.ext.asyncio import AsyncSession + +from src.core import cache +from src.core.db.models import User +from src.service.dto import UserDTO + + +def to_user(user: User) -> UserDTO: + return UserDTO(user.id, user.username, user.display_name, user.created_at) + + +async def fetch_user_by_id(session: AsyncSession, user_id: UUID) -> UserDTO | None: + key = f"user:v1:{user_id}" + cached = await cache.get(key) + if cached is not None: + try: + data = json.loads(cached) + if ( + not isinstance(data, dict) + or set(data) != {"id", "username", "display_name", "created_at"} + or not all(isinstance(value, str) for value in data.values()) + ): + raise ValueError + created_at = datetime.fromisoformat(data["created_at"]) + if created_at.tzinfo is None: + raise ValueError + data["display_name"].encode("utf-8") + user = UserDTO( + UUID(data["id"]), + data["username"], + data["display_name"], + created_at.astimezone(timezone.utc), + ) + if ( + user.id != user_id + or re.fullmatch(r"[a-z0-9_]{3,32}", user.username) is None + or not 1 <= len(user.display_name) <= 100 + or not user.display_name.strip() + or user.created_at.tzinfo is None + ): + raise ValueError + return user + except (ValueError, KeyError, TypeError, OverflowError): + await cache.delete(key) + model = await session.get(User, user_id) + user = to_user(model) if model else None + if user: + await cache.set(key, json.dumps(asdict(user), default=str)) + return user + + +async def fetch_user_by_username(session: AsyncSession, username: str) -> UserDTO | None: + model = await session.scalar(select(User).where(User.username == username)) + return to_user(model) if model else None + + +async def create_user( + session: AsyncSession, + user_id: UUID, + username: str, + display_name: str, + created_at: datetime, + password_hash: str, +) -> UserDTO: + user = User( + id=user_id, + username=username, + display_name=display_name, + created_at=created_at, + password_hash=password_hash, + ) + session.add(user) + await session.flush() + return to_user(user) + + +async def users_exist(session: AsyncSession, user_ids: list[UUID]) -> bool: + count = await session.scalar( + select(func.count()).select_from(User).where(User.id.in_(user_ids)) + ) + return count == len(user_ids) + + +async def create_admin( + session: AsyncSession, + user_id: UUID, + username: str, + display_name: str, + password_hash: str, + created_at: datetime, +) -> UserDTO | None: + user = await session.scalar( + insert(User) + .values( + id=user_id, + username=username, + display_name=display_name, + password_hash=password_hash, + created_at=created_at, + is_admin=True, + ) + .on_conflict_do_nothing(index_elements=[User.username]) + .returning(User) + ) + return to_user(user) if user else None + + +async def admin_exists(session: AsyncSession, username: str) -> bool: + user_id = await session.scalar( + select(User.id).where(User.username == username, User.is_admin.is_(True)) + ) + return user_id is not None diff --git a/src/core/db/session.py b/src/core/db/session.py new file mode 100644 index 0000000..ef34790 --- /dev/null +++ b/src/core/db/session.py @@ -0,0 +1,105 @@ +from contextlib import asynccontextmanager + +from opentelemetry import trace +from opentelemetry.trace import SpanKind, Status, StatusCode +from sqlalchemy import event +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine +from sqlalchemy.pool import AsyncAdaptedQueuePool + +from src.core.cache import close_cache, open_cache +from src.core.settings import settings +from src.observability import signals + +session_factory: async_sessionmaker[AsyncSession] | None = None + + +class ObservedPool(AsyncAdaptedQueuePool): + waiting = 0 + + def _do_get(self): + self.waiting += 1 + try: + return super()._do_get() + finally: + self.waiting -= 1 + + +def create_engine(): + engine = create_async_engine( + settings.sqlalchemy_database_url, + poolclass=ObservedPool, + pool_size=settings.database_pool_size, + max_overflow=settings.database_max_overflow, + pool_timeout=settings.database_pool_timeout_seconds, + pool_pre_ping=settings.database_pool_pre_ping, + hide_parameters=True, + connect_args={ + "timeout": settings.database_connect_timeout_seconds, + "command_timeout": settings.database_command_timeout_seconds, + "server_settings": {"statement_timeout": str(settings.database_statement_timeout_ms)}, + }, + ) + + signals.register_pool(engine) + + @event.listens_for(engine.sync_engine, "before_cursor_execute") + def before(conn, cursor, statement, parameters, context, executemany): + operation = statement.lstrip().split(None, 1)[0].upper() if statement.strip() else "OTHER" + if operation not in {"SELECT", "INSERT", "UPDATE", "DELETE", "BEGIN", "COMMIT", "ROLLBACK"}: + operation = "OTHER" + span = signals.tracer.start_span( + "db." + operation, + kind=SpanKind.CLIENT, + attributes={"db.system": "postgresql", "db.operation.name": operation}, + ) + scope = trace.use_span( + span, end_on_exit=True, record_exception=False, set_status_on_exception=False + ) + scope.__enter__() + context._telemetry_scope = scope + + @event.listens_for(engine.sync_engine, "after_cursor_execute") + def after(conn, cursor, statement, parameters, context, executemany): + scope = getattr(context, "_telemetry_scope", None) + if scope: + scope.__exit__(None, None, None) + context._telemetry_scope = None + + @event.listens_for(engine.sync_engine, "handle_error") + def failed(exception_context): + context = exception_context.execution_context + scope = getattr(context, "_telemetry_scope", None) + if scope: + trace.get_current_span().set_status(Status(StatusCode.ERROR)) + trace.get_current_span().set_attribute( + "error.type", type(exception_context.original_exception).__name__ + ) + scope.__exit__(None, None, None) + context._telemetry_scope = None + signals.event("db.error", error_type=type(exception_context.original_exception).__name__) + + return engine + + +@asynccontextmanager +async def database_lifespan(app): + global session_factory + engine = create_engine() + session_factory = async_sessionmaker(engine, expire_on_commit=False) + open_cache() + try: + yield + finally: + await close_cache() + await engine.dispose() + session_factory = None + + +async def get_db_session(): + if session_factory is None: + raise RuntimeError("Фабрика сессий не запущена") + async with session_factory() as session: + # Одна транзакция на запрос: аутентификация и бизнес-операция вместе. + # Depends(scope='function') завершает COMMIT до отправки ответа. + async with session.begin(): + yield session diff --git a/src/core/db/types.py b/src/core/db/types.py new file mode 100644 index 0000000..36501fd --- /dev/null +++ b/src/core/db/types.py @@ -0,0 +1,15 @@ +"""Unicode-строки в UTF-8 bytea: PostgreSQL text не допускает U+0000.""" + +from sqlalchemy import LargeBinary +from sqlalchemy.types import TypeDecorator + + +class Utf8Text(TypeDecorator[str]): + impl = LargeBinary + cache_ok = True + + def process_bind_param(self, value: str | None, dialect) -> bytes | None: + return None if value is None else value.encode("utf-8") + + def process_result_value(self, value: bytes | None, dialect) -> str | None: + return None if value is None else value.decode("utf-8") diff --git a/src/core/errors.py b/src/core/errors.py new file mode 100644 index 0000000..70a62c9 --- /dev/null +++ b/src/core/errors.py @@ -0,0 +1,29 @@ +from fastapi import Request +from fastapi.responses import JSONResponse + + +class ApiError(Exception): + def __init__(self, status_code: int, code: str, message: str) -> None: + self.status_code = status_code + self.code = code + self.message = message + super().__init__(message) + + +def error_response( + request: Request, + status_code: int, + code: str, + message: str, +) -> JSONResponse: + request_id = getattr(request.state, "request_id", "неизвестен") + return JSONResponse( + status_code=status_code, + content={ + "error": { + "code": code, + "message": message, + "request_id": request_id, + } + }, + ) diff --git a/src/core/pagination.py b/src/core/pagination.py new file mode 100644 index 0000000..d49e7b7 --- /dev/null +++ b/src/core/pagination.py @@ -0,0 +1,43 @@ +import base64 +import json +from datetime import datetime, timezone +from uuid import UUID + +from src.core.errors import ApiError + + +def encode_cursor(kind: str, owner_id: UUID, created_at: datetime, item_id: UUID) -> str: + payload = { + "kind": kind, + "owner_id": str(owner_id), # id чата + "created_at": created_at.isoformat(), # его время + "item_id": str(item_id), # id сообщения + } + raw = json.dumps(payload, separators=(",", ":")).encode("utf-8") + return base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=") + + +def decode_cursor( + cursor: str, expected_kind: str, expected_owner_id: UUID +) -> tuple[datetime, UUID]: + try: + padding = "=" * (-len(cursor) % 4) + raw = base64.b64decode(cursor + padding, altchars=b"-_", validate=True) + payload = json.loads(raw.decode("utf-8")) + if ( + not isinstance(payload, dict) + or set(payload) != {"kind", "owner_id", "created_at", "item_id"} + or not all(isinstance(value, str) for value in payload.values()) + ): + raise ValueError + if payload["kind"] != expected_kind: + raise ValueError + if UUID(payload["owner_id"]) != expected_owner_id: + raise ValueError + created_at = datetime.fromisoformat(payload["created_at"]) + item_id = UUID(payload["item_id"]) + if created_at.tzinfo is None: + raise ValueError + return created_at.astimezone(timezone.utc), item_id + except (KeyError, TypeError, ValueError, UnicodeError, OverflowError): + raise ApiError(400, "invalid_request", "Некорректный курсор") from None diff --git a/src/core/readiness.py b/src/core/readiness.py new file mode 100644 index 0000000..133c3db --- /dev/null +++ b/src/core/readiness.py @@ -0,0 +1,89 @@ +"""Ограниченная проверка обязательного пути, включая повреждение схемы. + +Отмена asyncpg может ждать ответа замороженного PostgreSQL. HTTP readiness +не должна ждать завершения такого rollback; незавершённая проба остаётся одной +на процесс и до её завершения сервер считается неготовым. +""" + +import asyncio + +from anyio import to_thread +from sqlalchemy import select + +from src.core import cache +from src.core.db import session as database +from src.core.db.models import ( + Attachment, + Chat, + ChatMember, + Message, + MessageAttachment, + MessageRequest, + StorageFinding, + UploadIntent, + User, +) +from src.core.errors import ApiError +from src.core.settings import settings +from src.core.storage import STORAGE_ERRORS, get_storage, unavailable + +_pending: asyncio.Task[None] | None = None + + +async def _probe() -> None: + factory = database.session_factory + if factory is None: + raise ApiError(503, "unavailable", "Хранилище временно недоступно") + async with factory() as session: + for model in ( + User, + Chat, + ChatMember, + Message, + MessageRequest, + Attachment, + StorageFinding, + UploadIntent, + MessageAttachment, + ): + await session.execute(select(model).limit(0)) + await cache.get_client().ping() + try: + exists = await to_thread.run_sync( + lambda: get_storage().bucket_exists(settings.s3_bucket) + ) + except STORAGE_ERRORS as error: + raise unavailable(error) from error + if not exists: + raise ApiError(503, "unavailable", "Хранилище файлов не готово") + + +def _consume_result(task: asyncio.Task[None]) -> None: + if not task.cancelled(): + task.exception() + + +async def check() -> None: + global _pending + if _pending is None or _pending.done(): + _pending = asyncio.create_task(_probe()) + _pending.add_done_callback(_consume_result) + task = _pending + # wait(), в отличие от wait_for(), не ждёт завершения отмены драйвером. + done, _ = await asyncio.wait({task}, timeout=settings.healthcheck_timeout_seconds) + if not done: + if not task.cancelling(): + task.cancel() + raise ApiError(503, "unavailable", "Проверка готовности превысила время ожидания") + if task.cancelled(): + raise ApiError(503, "unavailable", "Хранилище временно недоступно") + task.result() + + +async def close() -> None: + global _pending + if _pending is not None and not _pending.done(): + if not _pending.cancelling(): + _pending.cancel() + await asyncio.wait({_pending}, timeout=settings.healthcheck_timeout_seconds) + _pending = None diff --git a/src/core/request_deadline.py b/src/core/request_deadline.py new file mode 100644 index 0000000..b79939a --- /dev/null +++ b/src/core/request_deadline.py @@ -0,0 +1,77 @@ +"""Bound the time to response headers, including slow DB cancellation/rollback. + +A timed-out handler is cancelled but not awaited: asyncpg can wait for a frozen +server during rollback. Such handlers retain their slot until they actually end. +This prevents an outage from accumulating unbounded background handlers. +""" + +import asyncio +import logging +import re + +from starlette.requests import Request + +from src.core.errors import error_response +from src.core.settings import settings + +logger = logging.getLogger(__name__) + + +class RequestDeadlineMiddleware: + def __init__(self, app): + self.app = app + self.pending = set() + + def completed(self, task): + self.pending.discard(task) + if not task.cancelled(): + error = task.exception() + if error is not None: + logger.warning("Обработчик запроса завершился с ошибкой: %s", type(error).__name__) + + async def __call__(self, scope, receive, send): + if scope["type"] != "http" or not scope["path"].startswith("/api/v1/"): + await self.app(scope, receive, send) + return + if len(self.pending) >= settings.http_concurrency_limit: + await error_response(Request(scope), 503, "unavailable", "Сервис перегружен")( + scope, receive, send + ) + return + started = asyncio.Event() + expired = False + + async def guarded_send(message): + if expired: + return + if message["type"] == "http.response.start": + started.set() + await send(message) + + task = asyncio.create_task(self.app(scope, receive, guarded_send)) + self.pending.add(task) + task.add_done_callback(self.completed) + start_waiter = asyncio.create_task(started.wait()) + timeout = settings.http_request_timeout_seconds + if scope["method"] == "POST" and re.fullmatch( + r"/api/v1/chats/[^/]+/attachments/?", scope["path"] + ): + # File transfer has its own byte/time bounds; allow its receive and + # S3 stages before applying the final response-header deadline. + timeout += settings.upload_timeout_seconds + 2 * settings.s3_timeout_seconds + try: + done, _ = await asyncio.wait({task, start_waiter}, timeout=timeout) + if done: + await task + else: + expired = True + task.cancel() + await error_response( + Request(scope), 503, "unavailable", "Истекло время обработки запроса" + )(scope, receive, send) + except asyncio.CancelledError: + expired = True + task.cancel() + raise + finally: + start_waiter.cancel() diff --git a/src/core/request_limits.py b/src/core/request_limits.py new file mode 100644 index 0000000..ddf7b63 --- /dev/null +++ b/src/core/request_limits.py @@ -0,0 +1,132 @@ +"""Ограниченный приём тела до валидации FastAPI, включая chunked-запросы.""" + +import asyncio +import json +import re + +from anyio import CapacityLimiter, WouldBlock +from starlette.requests import Request +from starlette.types import ASGIApp, Message, Receive, Scope, Send + +from src.core.errors import error_response +from src.core.settings import settings + + +def unique_object(pairs: list[tuple[str, object]]) -> dict[str, object]: + result = {} + for key, value in pairs: + if key in result: + raise ValueError("Повторяющийся ключ JSON") + result[key] = value + return result + + +def reject_constant(value: str) -> None: + raise ValueError("Недопустимое значение JSON") + + +class RequestLimitsMiddleware: + def __init__(self, app: ASGIApp) -> None: + self.app = app + self.limiter = CapacityLimiter(settings.http_concurrency_limit) + + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: + if scope["type"] != "http" or not scope["path"].startswith("/api/v1/"): + await self.app(scope, receive, send) + return + try: + self.limiter.acquire_nowait() + except WouldBlock: + await error_response(Request(scope), 503, "unavailable", "Сервис перегружен")( + scope, receive, send + ) + return + try: + await self.handle_request(scope, receive, send) + finally: + self.limiter.release() + + async def handle_request(self, scope: Scope, receive: Receive, send: Send) -> None: + request = Request(scope) + if len(request.headers.getlist("content-type")) > 1 or any( + len(request.query_params.getlist(name)) > 1 for name in ("limit", "cursor") + ): + await error_response( + request, 400, "invalid_request", "Неоднозначные параметры запроса" + )(scope, receive, send) + return + # Binary uploads enforce their own streaming limit after authentication. + # Do not buffer them or apply the JSON-only 64 KiB limit. + upload = ( + request.method == "POST" + and re.fullmatch(r"/api/v1/chats/[^/]+/attachments/?", scope["path"]) is not None + ) + if upload: + await self.app(scope, receive, send) + return + content_type = request.headers.get("content-type", "").split(";", 1)[0].strip().lower() + json_body = ( + request.method == "POST" and scope["path"].rstrip("/") != "/api/v1/auth/sessions" + ) + if json_body and content_type != "application/json": + await error_response( + request, 415, "unsupported_media_type", "Требуется application/json" + )(scope, receive, send) + return + try: + length = int(request.headers.get("content-length", "0")) + if length < 0: + raise ValueError + except ValueError: + await error_response(request, 400, "invalid_request", "Неверная длина тела")( + scope, receive, send + ) + return + body = bytearray() + too_large = length > settings.http_max_body_bytes + if not too_large: + try: + async with asyncio.timeout(settings.http_body_timeout_seconds): + while True: + message = await receive() + if message["type"] == "http.disconnect": + return + chunk = message.get("body", b"") + if len(body) + len(chunk) > settings.http_max_body_bytes: + too_large = True + break + body.extend(chunk) + if not message.get("more_body", False): + break + except TimeoutError: + await error_response(request, 408, "invalid_request", "Истекло время приёма тела")( + scope, receive, send + ) + return + if too_large: + await error_response(request, 413, "payload_too_large", "Превышен размер тела запроса")( + scope, receive, send + ) + return + if json_body: + try: + json.loads( + body.decode("utf-8"), + object_pairs_hook=unique_object, + parse_constant=reject_constant, + ) + except (ValueError, UnicodeError, RecursionError): + await error_response(request, 400, "invalid_request", "Некорректный JSON UTF-8")( + scope, receive, send + ) + return + delivered = False + + async def buffered_receive() -> Message: + nonlocal delivered + if delivered: + return await receive() + delivered = True + return {"type": "http.request", "body": bytes(body), "more_body": False} + + await self.app(scope, buffered_receive, send) diff --git a/src/core/security.py b/src/core/security.py new file mode 100644 index 0000000..210710c --- /dev/null +++ b/src/core/security.py @@ -0,0 +1,42 @@ +"""Пароли: Argon2id. Токены: SHA-256 от случайного секрета.""" + +import hashlib + +from anyio import CapacityLimiter, to_thread +from argon2 import PasswordHasher, Type +from argon2.exceptions import InvalidHashError, VerificationError + +from src.core.settings import settings + +password_hasher = PasswordHasher( + time_cost=settings.argon2_time_cost, + memory_cost=settings.argon2_memory_cost, + parallelism=settings.argon2_parallelism, + salt_len=settings.argon2_salt_len, + hash_len=settings.argon2_hash_len, + type=Type.ID, +) +password_limiter = CapacityLimiter(settings.password_hash_concurrency) + + +async def hash_password_async(password: str) -> str: + return await to_thread.run_sync(hash_password, password, limiter=password_limiter) + + +async def verify_password_async(password: str, stored: str) -> bool: + return await to_thread.run_sync(verify_password, password, stored, limiter=password_limiter) + + +def hash_password(password: str) -> str: + return password_hasher.hash(password) + + +def verify_password(password: str, stored: str) -> bool: + try: + return password_hasher.verify(stored, password) + except (VerificationError, InvalidHashError): + return False + + +def hash_token(token: str) -> str: + return hashlib.sha256(token.encode("utf-8")).hexdigest() diff --git a/src/core/session_store.py b/src/core/session_store.py new file mode 100644 index 0000000..1d7083f --- /dev/null +++ b/src/core/session_store.py @@ -0,0 +1,55 @@ +"""Сессии в Redis: абсолютный срок жизни, без продления при чтении.""" + +import json +from dataclasses import asdict +from datetime import datetime, timezone +from uuid import UUID + +from redis.exceptions import ResponseError + +from src.core import cache +from src.service.dto import StoredSessionDTO + + +def session_key(token_hash: str) -> str: + return f"session:v1:{token_hash}" + + +async def create_session(token_hash: str, session: StoredSessionDTO) -> None: + await cache.get_client().set( + session_key(token_hash), + json.dumps(asdict(session), default=str), + pxat=int(session.expires_at.timestamp() * 1000), + ) + + +async def fetch_session(token_hash: str) -> StoredSessionDTO | None: + key = session_key(token_hash) + try: + raw = await cache.get_client().get(key) + if raw is None: + return None + data = json.loads(raw) + if ( + not isinstance(data, dict) + or set(data) != {"user_id", "expires_at"} + or not all(isinstance(value, str) for value in data.values()) + ): + raise ValueError + session = StoredSessionDTO( + UUID(data["user_id"]), datetime.fromisoformat(data["expires_at"]) + ) + if session.expires_at.tzinfo is None or session.expires_at <= datetime.now(timezone.utc): + raise ValueError + return session + except ResponseError as error: + if not str(error).startswith("WRONGTYPE"): + raise + except (ValueError, KeyError, TypeError, UnicodeError): + pass + await cache.get_client().delete(key) + return None + + +async def delete_session(token_hash: str) -> None: + await cache.get_client().delete(session_key(token_hash)) diff --git a/src/core/settings.py b/src/core/settings.py new file mode 100644 index 0000000..cbe3b65 --- /dev/null +++ b/src/core/settings.py @@ -0,0 +1,130 @@ +from typing import Literal, Self + +from pydantic import Field, SecretStr, field_validator, model_validator +from pydantic_settings import BaseSettings, SettingsConfigDict + + +class AppSettings(BaseSettings): + model_config = SettingsConfigDict( + env_file=".env", + env_file_encoding="utf-8", + extra="ignore", + hide_input_in_errors=True, + ) + + otel_enabled: bool = True + otel_exporter_otlp_endpoint: str = "http://otel-collector:4318" + otel_service_name: str = "messenger-api" + + app_host: str = "0.0.0.0" + app_port: int = Field(default=8080, ge=1, le=65535) + app_env: Literal["lab", "production"] = "lab" + app_workers: int = Field(default=1, ge=1) + shutdown_timeout_seconds: int = Field(default=10, gt=0) + healthcheck_timeout_seconds: float = Field(default=2, gt=0) + http_max_body_bytes: int = Field(default=65536, ge=65536) + http_request_timeout_seconds: float = Field(default=5, gt=0) + http_body_timeout_seconds: float = Field(default=5, gt=0) + http_concurrency_limit: int = Field(default=64, gt=0) + authorization_max_length: int = Field(default=2048, ge=1024) + cursor_max_length: int = Field(default=1024, ge=512) + password_hash_concurrency: int = Field(default=2, gt=0) + + s3_endpoint: str = "http://minio:9000" + s3_region: str = "us-east-1" + s3_bucket: str = "messenger-attachments" + s3_access_key_id: SecretStr = SecretStr("messenger-api") + s3_secret_access_key: SecretStr = SecretStr("local-messenger-api-password") + minio_root_user: SecretStr = SecretStr("messenger") + minio_root_password: SecretStr = SecretStr("local-messenger-s3-password") + s3_reconcile_access_key_id: SecretStr = SecretStr("messenger-reconciler") + s3_reconcile_secret_access_key: SecretStr = SecretStr("local-messenger-reconciler-password") + reconcile_grace_seconds: int = Field(default=300, ge=1) + s3_timeout_seconds: float = Field(default=5, gt=0) + max_upload_bytes: int = Field(default=10485760, gt=0, le=10485760) + max_image_pixels: int = Field(default=40000000, gt=0, le=40000000) + upload_concurrency: int = Field(default=1, gt=0) + upload_timeout_seconds: float = Field(default=30, gt=0) + + s3_worker_access_key_id: SecretStr = SecretStr("messenger-worker") + s3_worker_secret_access_key: SecretStr = SecretStr("local-messenger-worker-password") + + broker_url: SecretStr = SecretStr("redis://broker:6379/0") + worker_concurrency: int = Field(default=1, ge=1, le=8) + job_max_attempts: int = Field(default=5, ge=1, le=20) + job_timeout_seconds: int = Field(default=30, ge=1, le=120) + job_lease_seconds: int = Field(default=120, ge=5) + job_retry_base_seconds: float = Field(default=2, gt=0) + job_retry_cap_seconds: float = Field(default=60, gt=0) + staging_retention_seconds: int = Field(default=604800, ge=1) + worker_memory_mb: int = Field(default=768, ge=256) + pipeline_fault_injection: bool = False + + database_url: SecretStr + database_pool_size: int = Field(default=10, gt=0) + database_max_overflow: int = Field(default=0, ge=0) + database_pool_timeout_seconds: float = Field(default=5, gt=0) + database_connect_timeout_seconds: float = Field(default=5, gt=0) + database_command_timeout_seconds: float = Field(default=5, gt=0) + database_statement_timeout_ms: int = Field(default=5000, gt=0) + database_pool_pre_ping: bool = True + + redis_url: SecretStr + redis_max_connections: int = Field(default=10, gt=0) + redis_connect_timeout_seconds: float = Field(default=0.3, gt=0) + redis_read_timeout_seconds: float = Field(default=0.3, gt=0) + redis_retry_on_timeout: bool = False + profile_cache_ttl_seconds: int = Field(default=60, gt=0) + + admin_username: str = Field(default="admin", pattern=r"^[a-z0-9_]{3,32}$") + admin_password: SecretStr = Field( + default=SecretStr("admin-password-123"), + min_length=12, + max_length=128, + ) + admin_display_name: str = Field(default="Администратор", min_length=1, max_length=100) + + session_ttl_seconds: int = Field(default=3600, gt=0) + session_token_bytes: int = Field(default=32, ge=32) + argon2_time_cost: int = Field(default=3, gt=0) + argon2_memory_cost: int = Field(default=65536, ge=8192) + argon2_parallelism: int = Field(default=4, gt=0) + argon2_salt_len: int = Field(default=16, ge=16) + argon2_hash_len: int = Field(default=32, ge=32) + + log_level: Literal["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"] = "INFO" + log_format: str = "%(asctime)s %(levelname)s %(name)s: %(message)s" + + @field_validator("admin_password") + @classmethod + def admin_password_must_not_be_blank(cls, value: SecretStr) -> SecretStr: + if not value.get_secret_value().strip(): + raise ValueError("Пароль администратора не может быть пробельным") + value.get_secret_value().encode("utf-8") + return value + + @field_validator("admin_display_name") + @classmethod + def admin_display_name_must_be_valid(cls, value: str) -> str: + if not value.strip() or "\x00" in value: + raise ValueError("Некорректное имя администратора") + return value + + @model_validator(mode="after") + def production_must_use_custom_password(self) -> Self: + if self.app_env == "production" and self.admin_password.get_secret_value() == ( + "admin-password-123" + ): + raise ValueError("Для production требуется собственный ADMIN_PASSWORD") + return self + + @property + def sqlalchemy_database_url(self) -> str: + return self.database_url.get_secret_value().replace( + "postgresql://", + "postgresql+asyncpg://", + 1, + ) + + +settings = AppSettings() diff --git a/src/core/storage.py b/src/core/storage.py new file mode 100644 index 0000000..8e6c900 --- /dev/null +++ b/src/core/storage.py @@ -0,0 +1,96 @@ +"""S3 clients with separate roles, bounded socket timeouts and request metrics.""" + +from collections import Counter +from dataclasses import dataclass, field +from functools import lru_cache +from urllib.parse import parse_qs, urlsplit + +import urllib3 +from minio import Minio +from minio.error import InvalidResponseError, S3Error, ServerError +from opentelemetry.trace import SpanKind, Status, StatusCode + +from src.core.errors import ApiError +from src.core.settings import settings +from src.observability import signals + + +@dataclass +class StorageMetrics: + requests: Counter = field(default_factory=Counter) + + +class StoragePool(urllib3.PoolManager): + def __init__(self, *args, metrics=None, **kwargs): + super().__init__(*args, **kwargs) + self.metrics = metrics + + def urlopen(self, method, url, *args, **kwargs): + if self.metrics is not None: + query = parse_qs(urlsplit(url).query, keep_blank_values=True) + operation = ( + "LIST" + if "list-type" in query + else ("VERSIONING" if "versioning" in query else method) + ) + self.metrics.requests[operation] += 1 + with signals.tracer.start_as_current_span( + "s3." + method, + kind=SpanKind.CLIENT, + attributes={"storage.operation": method}, + record_exception=False, + set_status_on_exception=False, + ) as span: + try: + response = super().urlopen(method, url, *args, **kwargs) + span.set_attribute("http.response.status_code", response.status) + if response.status >= 500: + span.set_status(Status(StatusCode.ERROR)) + return response + except Exception as error: + span.set_status(Status(StatusCode.ERROR)) + span.set_attribute("error.type", type(error).__name__) + raise + + +def make_storage(*, role="api", metrics=None) -> Minio: + endpoint = urlsplit(settings.s3_endpoint) + if endpoint.scheme not in {"http", "https"} or not endpoint.netloc: + raise ValueError("S3_ENDPOINT должен быть HTTP(S) URL") + credentials = { + "api": (settings.s3_access_key_id, settings.s3_secret_access_key), + "reconciler": ( + settings.s3_reconcile_access_key_id, + settings.s3_reconcile_secret_access_key, + ), + "worker": (settings.s3_worker_access_key_id, settings.s3_worker_secret_access_key), + "bootstrap": (settings.minio_root_user, settings.minio_root_password), + } + access_key, secret_key = credentials[role] + return Minio( + endpoint.netloc, + access_key=access_key.get_secret_value(), + secret_key=secret_key.get_secret_value(), + secure=endpoint.scheme == "https", + region=settings.s3_region, + http_client=StoragePool( + metrics=metrics, + timeout=urllib3.Timeout( + connect=settings.s3_timeout_seconds, read=settings.s3_timeout_seconds + ), + retries=False, + maxsize=8, + ), + ) + + +@lru_cache(maxsize=1) +def get_storage() -> Minio: + return make_storage() + + +def unavailable(error: Exception) -> ApiError: + return ApiError(503, "unavailable", "Хранилище файлов временно недоступно") + + +STORAGE_ERRORS = (S3Error, InvalidResponseError, ServerError, urllib3.exceptions.HTTPError, OSError) diff --git a/src/core/storage_bootstrap.py b/src/core/storage_bootstrap.py new file mode 100644 index 0000000..f2fe5ef --- /dev/null +++ b/src/core/storage_bootstrap.py @@ -0,0 +1,96 @@ +"""Подготовка приватного bucket и отдельных пользователей MinIO.""" + +import json +from pathlib import Path +from string import Template +from urllib.parse import urlsplit + +import urllib3 +from minio import Minio +from minio.credentials import StaticProvider +from minio.minioadmin import MinioAdmin +from pydantic import SecretStr + +from src.core.settings import settings +from src.core.storage import make_storage + +POLICY_DIRECTORY = Path(__file__).resolve().parents[2] / "infra" / "s3" + + +def create_admin_client() -> MinioAdmin: + endpoint = urlsplit(settings.s3_endpoint) + return MinioAdmin( + endpoint=endpoint.netloc, + secure=endpoint.scheme == "https", + region=settings.s3_region, + credentials=StaticProvider( + settings.minio_root_user.get_secret_value(), + settings.minio_root_password.get_secret_value(), + ), + http_client=urllib3.PoolManager( + timeout=urllib3.Timeout( + connect=settings.s3_timeout_seconds, read=settings.s3_timeout_seconds + ), + retries=False, + ), + ) + + +def ensure_private_bucket(storage: Minio) -> None: + if not storage.bucket_exists(settings.s3_bucket): + storage.make_bucket(settings.s3_bucket, location=settings.s3_region) + storage.delete_bucket_policy(settings.s3_bucket) + + +def load_policy(filename: str) -> dict: + template = Template((POLICY_DIRECTORY / filename).read_text(encoding="utf-8")) + return json.loads(template.substitute(bucket=settings.s3_bucket)) + + +def configure_user( + admin: MinioAdmin, + *, + access_key: SecretStr, + secret_key: SecretStr, + policy_name: str, + policy: dict, +) -> None: + username = access_key.get_secret_value() + if username == settings.minio_root_user.get_secret_value(): + raise ValueError("Пользователь приложения не должен совпадать с владельцем MinIO") + admin.user_add(username, secret_key.get_secret_value()) + admin.policy_add(policy_name, policy=policy) + admin.policy_set(policy_name, user=username) + + +def main() -> None: + storage = make_storage(role="bootstrap") + admin = create_admin_client() + ensure_private_bucket(storage) + + configure_user( + admin, + access_key=settings.s3_access_key_id, + secret_key=settings.s3_secret_access_key, + policy_name="messenger-api", + policy=load_policy("api-policy.json"), + ) + configure_user( + admin, + access_key=settings.s3_reconcile_access_key_id, + secret_key=settings.s3_reconcile_secret_access_key, + policy_name="messenger-reconciler", + policy=load_policy("reconciler-policy.json"), + ) + configure_user( + admin, + access_key=settings.s3_worker_access_key_id, + secret_key=settings.s3_worker_secret_access_key, + policy_name="messenger-worker", + policy=load_policy("worker-policy.json"), + ) + print("MinIO настроен: приватный bucket, пользователи API и reconciler") + + +if __name__ == "__main__": + main() diff --git a/src/core/upload_coordination.py b/src/core/upload_coordination.py new file mode 100644 index 0000000..b177d67 --- /dev/null +++ b/src/core/upload_coordination.py @@ -0,0 +1,21 @@ +"""One transaction-scoped PostgreSQL lock for all actions on an S3 key. + +All writers must register an intent and use this protocol. The lock is released +by PostgreSQL on commit, rollback, lost connection or process death. +""" + +import hashlib + +from sqlalchemy import func, select +from sqlalchemy.ext.asyncio import AsyncSession + + +def lock_id(key: str) -> int: + return int.from_bytes(hashlib.sha256(key.encode()).digest()[:8], "big", signed=True) + + +async def lock_object(session: AsyncSession, key: str, *, wait: bool = False) -> bool: + if wait: + await session.execute(select(func.pg_advisory_xact_lock(lock_id(key)))) + return True + return bool(await session.scalar(select(func.pg_try_advisory_xact_lock(lock_id(key))))) diff --git a/src/main.py b/src/main.py new file mode 100644 index 0000000..fe83ccb --- /dev/null +++ b/src/main.py @@ -0,0 +1,156 @@ +import logging +import time +import uuid + +from fastapi import FastAPI, Request +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse +from opentelemetry.trace import SpanKind, Status, StatusCode +from redis.exceptions import RedisError +from sqlalchemy.exc import DBAPIError +from sqlalchemy.exc import TimeoutError as DatabaseTimeoutError +from starlette.exceptions import HTTPException as StarletteHTTPException + +from src.api.router import router as api_router +from src.core import readiness +from src.core.callback import lifespan +from src.core.errors import ApiError, error_response +from src.core.request_deadline import RequestDeadlineMiddleware +from src.core.request_limits import RequestLimitsMiddleware +from src.core.settings import settings +from src.observability import signals + +logging.basicConfig(level=settings.log_level, format=settings.log_format) + +signals.setup("messenger-api") + +INSTANCE_ID = str(uuid.uuid4()) + +app = FastAPI(lifespan=lifespan) +app.add_middleware(RequestLimitsMiddleware) +app.add_middleware(RequestDeadlineMiddleware) +app.include_router(api_router) +logger = logging.getLogger(__name__) + + +@app.middleware("http") +async def add_request_id(request: Request, call_next): + with signals.tracer.start_as_current_span( + "HTTP", + context=signals.parent( + { + "traceparent": request.headers.get("traceparent"), + "tracestate": request.headers.get("tracestate"), + } + ), + kind=SpanKind.SERVER, + record_exception=False, + set_status_on_exception=False, + ) as span: + request_id = str(uuid.uuid4()) + request.state.request_id = request_id + started_at = time.perf_counter() + + try: + response = await call_next(request) + except Exception: + logger.exception( + "Необработанная ошибка запроса", + extra={"request_id": request_id}, + ) + + response = JSONResponse( + status_code=500, + content={ + "error": { + "code": "internal_error", + "message": "Внутренняя ошибка сервера", + "request_id": request_id, + } + }, + ) + + response.headers["X-Request-Id"] = request_id + response.headers["X-Instance-Id"] = INSTANCE_ID + if request.url.path.startswith("/api/v1/"): + response.headers["Cache-Control"] = "no-store" + response.headers["X-Content-Type-Options"] = "nosniff" + route = getattr(request.scope.get("route"), "path", "unmatched") + if request.url.path.startswith("/api/v1/") and not route.startswith("/api/v1/"): + route = "/api/v1/" + route.lstrip("/") + attributes = { + "method": request.method + if request.method in {"GET", "POST", "DELETE", "PUT", "PATCH", "OPTIONS", "HEAD"} + else "OTHER", + "route": route, + "status_class": str(response.status_code // 100) + "xx", + } + elapsed = time.perf_counter() - started_at + signals.http_requests.add(1, attributes) + signals.http_duration.record(elapsed, attributes) + span.update_name(attributes["method"] + " " + route) + span.set_attributes( + { + "http.request.method": attributes["method"], + "http.route": route, + "http.response.status_code": response.status_code, + } + ) + if response.status_code >= 500: + span.set_status(Status(StatusCode.ERROR)) + response.headers["X-Trace-Id"] = format(span.get_span_context().trace_id, "032x") + signals.event( + "http.completed", + request_id=request_id, + method=attributes["method"], + route=route, + status=response.status_code, + duration_seconds=elapsed, + ) + return response + + +@app.exception_handler(ApiError) +async def handle_api_error(request: Request, exc: ApiError) -> JSONResponse: + response = error_response(request, exc.status_code, exc.code, exc.message) + if exc.status_code == 401: + response.headers["WWW-Authenticate"] = ( + 'Basic realm="messenger", charset="UTF-8"' + if request.url.path == "/api/v1/auth/sessions" + else 'Bearer realm="messenger"' + ) + return response + + +@app.exception_handler(RequestValidationError) +async def handle_validation_error(request: Request, exc: RequestValidationError) -> JSONResponse: + return error_response(request, 400, "invalid_request", "Некорректные параметры запроса") + + +@app.exception_handler(StarletteHTTPException) +async def handle_http_error(request: Request, exc: StarletteHTTPException) -> JSONResponse: + code = "not_found" if exc.status_code == 404 else "invalid_request" + message = "Объект не найден" if exc.status_code == 404 else "Некорректный запрос" + return error_response(request, exc.status_code, code, message) + + +@app.get("/health/live") +async def get_health(): + return { + "status": "ok", + } + + +@app.get("/health/ready") +async def get_ready(): + await readiness.check() + return {"status": "ok"} + + +async def handle_database_error(request: Request, exc: Exception) -> JSONResponse: + logger.warning("Хранилище временно недоступно: тип_ошибки=%s", type(exc).__name__) + return error_response(request, 503, "unavailable", "Хранилище временно недоступно") + + +for error_type in (RedisError, DBAPIError, DatabaseTimeoutError, OSError, TimeoutError): + app.add_exception_handler(error_type, handle_database_error) diff --git a/src/maintenance/__init__.py b/src/maintenance/__init__.py new file mode 100644 index 0000000..42c06c7 --- /dev/null +++ b/src/maintenance/__init__.py @@ -0,0 +1 @@ +"""Operator maintenance commands; no public HTTP endpoints.""" diff --git a/src/maintenance/reconcile.py b/src/maintenance/reconcile.py new file mode 100644 index 0000000..f537182 --- /dev/null +++ b/src/maintenance/reconcile.py @@ -0,0 +1,361 @@ +"""Bounded, repeatable S3/PostgreSQL reconciliation. Default mode is dry-run.""" + +import argparse +import asyncio +import hashlib +import json +import resource +import time +from collections import Counter +from datetime import timedelta +from uuid import UUID + +from anyio import to_thread +from minio.error import S3Error +from sqlalchemy import event, func, or_, select +from sqlalchemy.ext.asyncio import async_sessionmaker + +from src.core.db.models import Attachment, StorageFinding, UploadIntent +from src.core.db.session import create_engine +from src.core.settings import settings +from src.core.storage import StorageMetrics, make_storage +from src.core.upload_coordination import lock_object + + +class Reconciler: + def __init__( + self, + factory, + storage, + *, + apply=False, + grace_seconds=300, + prefix="originals/", + sample_limit=50, + ): + if not prefix.startswith("originals/") or not prefix.endswith("/"): + raise ValueError("Scope must be originals/ or a narrower prefix ending with /") + if grace_seconds < 1: + raise ValueError("Grace period must be positive") + self.factory = factory + self.storage = storage + self.apply = apply + self.grace = timedelta(seconds=grace_seconds) + self.prefix = prefix + self.sample_limit = sample_limit + self.counts = Counter() + self.samples = [] + self.content_bytes_read = 0 + + def note(self, action, key): + self.counts[action] += 1 + if len(self.samples) < self.sample_limit: + self.samples.append({"action": action, "object_key": key}) + + async def stat(self, key): + try: + return await to_thread.run_sync( + lambda: self.storage.stat_object(settings.s3_bucket, key) + ) + except S3Error as error: + # Bucket absence, AccessDenied, timeout and server failure must abort. + if error.code not in {"NoSuchKey", "NoSuchObject"}: + raise + return None + + async def observe(self, session, key, kind, now, fingerprint=None): + finding = await session.get(StorageFinding, (key, kind)) + if ( + finding is None + or finding.resolved_at is not None + or (kind == "orphan" and finding.fingerprint != fingerprint) + ): + if not self.apply: + return None + if finding is None: + finding = StorageFinding(object_key=key, kind=kind) + session.add(finding) + finding.first_seen = now + finding.resolved_at = None + finding.fingerprint = fingerprint + if self.apply: + finding.last_seen = now + return finding + + async def resolve(self, session, key, kind, now): + if self.apply: + finding = await session.get(StorageFinding, (key, kind)) + if finding is not None and finding.resolved_at is None: + finding.resolved_at = now + finding.last_seen = now + + async def orphan(self, key): + async with self.factory.begin() as session: + if not await lock_object(session, key): + self.note("protected_active", key) + return + now = await session.scalar(select(func.clock_timestamp())) + attachment = await session.scalar( + select(Attachment.id).where( + or_(Attachment.object_key == key, Attachment.staging_key == key) + ) + ) + if attachment is not None: + self.note("protected_referenced", key) + await self.resolve(session, key, "orphan", now) + return + obj = await self.stat(key) + if obj is None: + await self.resolve(session, key, "orphan", now) + return + # Opaque ETag identifies a changed object; it is NOT treated as SHA-256. + fingerprint = f"{obj.etag}:{obj.size}:{obj.last_modified.isoformat()}" + finding = await self.observe(session, key, "orphan", now, fingerprint) + intent = await session.get(UploadIntent, key) + cutoff = now - self.grace + if obj.last_modified > cutoff or (intent is not None and intent.created_at > cutoff): + self.note("protected_fresh", key) + return + if not self.apply: + self.note("would_delete_orphan", key) + return + # Registration, PUT, publication and cleanup all take this same lock. + # Tombstone also prevents a delayed pending uploader from publishing. + if intent is not None: + intent.state = "abandoned" + await session.flush() + await to_thread.run_sync(lambda: self.storage.remove_object(settings.s3_bucket, key)) + finding.resolved_at = now + self.note("deleted_orphan", key) + + async def content_matches(self, attachment): + def verify(): + response = self.storage.get_object(settings.s3_bucket, attachment.object_key) + digest = hashlib.sha256() + size = 0 + try: + while chunk := response.read(65536): + size += len(chunk) + if size > attachment.size_bytes: + return False, size + digest.update(chunk) + finally: + response.close() + response.release_conn() + return size == attachment.size_bytes and digest.hexdigest() == attachment.sha256, size + + valid, size = await to_thread.run_sync(verify) + self.content_bytes_read += size + return valid + + async def check_attachment(self, attachment_id, key): + async with self.factory.begin() as session: + if not await lock_object(session, key): + self.note("protected_active", key) + return + attachment = await session.get(Attachment, attachment_id) + if attachment is None or attachment.object_key != key or attachment.status != "ready": + return + now = await session.scalar(select(func.clock_timestamp())) + obj = await self.stat(key) + if obj is None: + await self.observe(session, key, "missing", now) + if attachment.storage_missing_since is None: + if self.apply: + attachment.storage_missing_since = now + self.note("marked_missing" if self.apply else "would_mark_missing", key) + else: + self.note("missing_unchanged", key) + return + if attachment.storage_missing_since is not None: + if obj.size != attachment.size_bytes or not await self.content_matches(attachment): + self.note("restore_checksum_mismatch", key) + return + if self.apply: + attachment.storage_missing_since = None + self.note("recovered" if self.apply else "would_recover", key) + else: + self.note("healthy", key) + await self.resolve(session, key, "missing", now) + + async def stale_intent(self, key): + async with self.factory.begin() as session: + if not await lock_object(session, key): + self.note("protected_active_intent", key) + return + intent = await session.get(UploadIntent, key) + if intent is None or intent.state != "pending": + return + now = await session.scalar(select(func.clock_timestamp())) + if intent.created_at > now - self.grace: + return + attached = await session.scalar( + select(Attachment.id).where( + or_(Attachment.object_key == key, Attachment.staging_key == key) + ) + ) + if attached is not None: + if self.apply: + intent.state = "committed" + self.note("intent_has_attachment", key) + elif await self.stat(key) is None: + await self.resolve(session, key, "orphan", now) + if self.apply: + intent.state = "abandoned" + self.note("abandoned_empty_intent" if self.apply else "would_abandon_intent", key) + + async def run(self): + # SDK lazily fetches ListObjectsV2 pages; never collect the whole bucket. + objects = self.storage.list_objects(settings.s3_bucket, prefix=self.prefix, recursive=True) + sentinel = object() + while True: + obj = await to_thread.run_sync(lambda: next(objects, sentinel)) + if obj is sentinel: + break + self.counts["objects_scanned"] += 1 + await self.orphan(obj.object_name) + + after: UUID | None = None + while True: + async with self.factory() as session: + query = ( + select(Attachment.id, Attachment.object_key) + .where(Attachment.object_key.startswith(self.prefix, autoescape=True)) + .order_by(Attachment.id) + .limit(200) + ) + if after is not None: + query = query.where(Attachment.id > after) + batch = (await session.execute(query)).all() + if not batch: + break + for aid, key in batch: + self.counts["attachments_scanned"] += 1 + await self.check_attachment(aid, key) + after = batch[-1].id + + # Pending attempts without any visible object also need bounded cleanup. + after_key = "" + while True: + async with self.factory() as session: + batch = list( + await session.scalars( + select(UploadIntent.object_key) + .where( + UploadIntent.state == "pending", + UploadIntent.object_key > after_key, + UploadIntent.object_key.startswith(self.prefix, autoescape=True), + ) + .order_by(UploadIntent.object_key) + .limit(200) + ) + ) + if not batch: + break + for key in batch: + self.counts["pending_intents_scanned"] += 1 + await self.stale_intent(key) + after_key = batch[-1] + + after_key = "" + while True: + async with self.factory() as session: + batch = list( + await session.scalars( + select(StorageFinding.object_key) + .where( + StorageFinding.kind == "orphan", + StorageFinding.resolved_at.is_(None), + StorageFinding.first_seen <= func.clock_timestamp() - self.grace, + StorageFinding.object_key > after_key, + StorageFinding.object_key.startswith(self.prefix, autoescape=True), + ) + .order_by(StorageFinding.object_key) + .limit(200) + ) + ) + if not batch: + break + for key in batch: + await self.orphan(key) + after_key = batch[-1] + + return { + "mode": "apply" if self.apply else "dry-run", + "bucket": settings.s3_bucket, + "prefix": self.prefix, + "grace_seconds": self.grace.total_seconds(), + "counts": dict(self.counts), + "samples": self.samples, + "content_bytes_read": self.content_bytes_read, + } + + +async def execute(args): + engine = create_engine() + db_statements = 0 + + @event.listens_for(engine.sync_engine, "before_cursor_execute") + def count_statements(*_): + nonlocal db_statements + db_statements += 1 + + storage_metrics = StorageMetrics() + storage = make_storage(role="reconciler", metrics=storage_metrics) + factory = async_sessionmaker(engine, expire_on_commit=False) + try: + while True: + started = time.perf_counter() + db_statements = 0 + storage_metrics.requests.clear() + report = {"mode": "apply" if args.apply else "dry-run", "prefix": args.prefix} + try: + # This implementation intentionally supports an unversioned bucket. + versioning = await to_thread.run_sync( + lambda: storage.get_bucket_versioning(settings.s3_bucket) + ) + if versioning.status in {"Enabled", "Suspended"}: + raise ValueError("Versioned buckets require version-aware reconciliation") + report = await Reconciler( + factory, + storage, + apply=args.apply, + grace_seconds=args.grace_seconds, + prefix=args.prefix, + ).run() + report["result"] = "ok" + except Exception as error: + # Do not print connection strings, SDK bodies or credentials. + report.update(result="error", error_type=type(error).__name__) + report.update( + elapsed_seconds=round(time.perf_counter() - started, 6), + db_statements=db_statements, + s3_requests=dict(storage_metrics.requests), + process_peak_rss_kib=resource.getrusage(resource.RUSAGE_SELF).ru_maxrss, + ) + print(json.dumps(report, ensure_ascii=False), flush=True) + if not args.interval: + return 0 if report["result"] == "ok" else 1 + await asyncio.sleep(args.interval) + finally: + await engine.dispose() + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--apply", action="store_true", help="Apply safe repairs; otherwise dry-run" + ) + parser.add_argument("--prefix", default="originals/") + parser.add_argument("--grace-seconds", type=int, default=settings.reconcile_grace_seconds) + parser.add_argument( + "--interval", type=int, default=0, help="Repeat every N seconds; 0 runs once" + ) + args = parser.parse_args() + if args.interval < 0: + parser.error("interval must be nonnegative") + raise SystemExit(asyncio.run(execute(args))) + + +if __name__ == "__main__": + main() diff --git a/src/observability/__init__.py b/src/observability/__init__.py new file mode 100644 index 0000000..7015ac2 --- /dev/null +++ b/src/observability/__init__.py @@ -0,0 +1 @@ +"""Bounded OTLP signals; telemetry is never a business dependency.""" diff --git a/src/observability/signals.py b/src/observability/signals.py new file mode 100644 index 0000000..9ecd3b9 --- /dev/null +++ b/src/observability/signals.py @@ -0,0 +1,182 @@ +"""Low-cardinality metrics, sanitized events and explicit W3C propagation.""" + +import atexit +import json +import logging +import os +import time +from datetime import datetime, timezone +from pathlib import Path +from uuid import uuid4 + +from opentelemetry import metrics, trace +from opentelemetry._logs import set_logger_provider +from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter +from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter +from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter +from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler +from opentelemetry.sdk._logs.export import BatchLogRecordProcessor +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader +from opentelemetry.sdk.metrics.view import ExplicitBucketHistogramAggregation, View +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import SpanLimits, TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor +from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator + +from src.core.settings import settings + +tracer = trace.get_tracer("messenger", "1.6.0") +meter = metrics.get_meter("messenger", "1.6.0") +http_requests = meter.create_counter("http.server.requests") +http_duration = meter.create_histogram("http.server.duration", unit="s") +jobs = meter.create_counter("worker.jobs") +job_duration = meter.create_histogram("worker.job.duration", unit="s") +ready_duration = meter.create_histogram("attachment.ready.duration", unit="s") +worker_busy = meter.create_up_down_counter("worker.busy") +queue_depth = meter.create_gauge("queue.depth") +queue_age = meter.create_gauge("queue.oldest.age", unit="s") +db_available = meter.create_gauge("db.available") +propagator = TraceContextTextMapPropagator() +configured = False +logger = logging.getLogger("messenger.events") +logger.setLevel(logging.INFO) +logger.propagate = False + + +def carrier(): + result = {} + propagator.inject(result) + return result + + +def parent(carrier): + return propagator.extract( + {k: v for k, v in carrier.items() if v and k in {"traceparent", "tracestate"}} + ) + + +def event(name, **fields): + # Callers pass explicit safe fields. Never forward exception text, HTTP headers, + # SQL statements/parameters, URL, filename or arbitrary application objects. + context = trace.get_current_span().get_span_context() + body = dict( + event=name, + timestamp=datetime.now(timezone.utc).isoformat(), + severity="INFO", + service_name=settings.otel_service_name, + trace_id=format(context.trace_id, "032x"), + span_id=format(context.span_id, "016x"), + **fields, + ) + logger.info(json.dumps(body, ensure_ascii=False), extra={"event": name}) + + +def setup(service=None): + global configured + if configured: + return + configured = True + if service: + settings.otel_service_name = service + stdout = logging.StreamHandler() + stdout.setFormatter(logging.Formatter("%(message)s")) + logger.addHandler(stdout) + if not settings.otel_enabled: + return + resource = Resource.create( + { + "service.name": settings.otel_service_name, + "service.version": "1.6.0", + "service.instance.id": str(uuid4()), + "deployment.environment.name": "lab", + } + ) + endpoint = settings.otel_exporter_otlp_endpoint.rstrip("/") + spans = TracerProvider( + resource=resource, span_limits=SpanLimits(max_attributes=32, max_events=8) + ) + spans.add_span_processor( + BatchSpanProcessor( + OTLPSpanExporter(endpoint=endpoint + "/v1/traces", timeout=1), + max_queue_size=256, + max_export_batch_size=32, + schedule_delay_millis=500, + export_timeout_millis=1500, + ) + ) + trace.set_tracer_provider(spans) + logs = LoggerProvider(resource=resource) + logs.add_log_record_processor( + BatchLogRecordProcessor( + OTLPLogExporter(endpoint=endpoint + "/v1/logs", timeout=1), + max_queue_size=256, + max_export_batch_size=32, + schedule_delay_millis=500, + export_timeout_millis=1500, + ) + ) + set_logger_provider(logs) + logger.addHandler(LoggingHandler(level=logging.INFO, logger_provider=logs)) + reader = PeriodicExportingMetricReader( + OTLPMetricExporter(endpoint=endpoint + "/v1/metrics", timeout=1), + export_interval_millis=2000, + export_timeout_millis=1500, + ) + provider = MeterProvider( + resource=resource, + metric_readers=[reader], + views=[ + View( + instrument_type=metrics.Histogram, + aggregation=ExplicitBucketHistogramAggregation( + boundaries=( + 0.005, + 0.01, + 0.025, + 0.05, + 0.1, + 0.25, + 0.5, + 1, + 2.5, + 5, + 10, + 30, + 60, + 120, + ) + ), + ) + ], + ) + metrics.set_meter_provider(provider) + + def rss(_): + resident = int(Path("/proc/self/statm").read_text().split()[1]) * os.sysconf("SC_PAGE_SIZE") + yield metrics.Observation(resident) + + meter.create_observable_gauge("process.memory.rss", callbacks=[rss], unit="By") + meter.create_observable_counter( + "process.cpu.time", + callbacks=[lambda _: [metrics.Observation(time.process_time())]], + unit="s", + ) + if settings.otel_service_name == "messenger-worker": + meter.create_observable_gauge( + "worker.capacity", + callbacks=[lambda _: [metrics.Observation(settings.worker_concurrency)]], + ) + # Explicit bounds apply to the asynchronous queues; dropping telemetry is + # preferable to delaying business requests. Shutdown runs outside requests. + atexit.register(lambda: spans.shutdown()) + + +def register_pool(engine): + pool = engine.sync_engine.pool + meter.create_observable_gauge( + "db.pool.used", callbacks=[lambda _: [metrics.Observation(pool.checkedout())]] + ) + meter.create_observable_gauge( + "db.pool.waiting", callbacks=[lambda _: [metrics.Observation(getattr(pool, "waiting", 0))]] + ) diff --git a/src/pipeline/__init__.py b/src/pipeline/__init__.py new file mode 100644 index 0000000..e43076b --- /dev/null +++ b/src/pipeline/__init__.py @@ -0,0 +1 @@ +"""Durable asynchronous attachment processing.""" diff --git a/src/pipeline/admin.py b/src/pipeline/admin.py new file mode 100644 index 0000000..7405713 --- /dev/null +++ b/src/pipeline/admin.py @@ -0,0 +1,113 @@ +"""Local operator CLI: failed registry and audited, explicit replay.""" + +import argparse +import asyncio +import json +from functools import lru_cache +from uuid import UUID, uuid4 + +from anyio import to_thread +from sqlalchemy import select +from sqlalchemy.ext.asyncio import async_sessionmaker + +from src.core.db.models import Attachment, AttachmentJob, JobReplay +from src.core.db.session import create_engine +from src.core.settings import settings +from src.core.storage import make_storage +from src.pipeline.common import now + + +@lru_cache(maxsize=1) +def worker_storage(): + return make_storage(role="worker") + + +async def execute(args): + engine = create_engine() + factory = async_sessionmaker(engine, expire_on_commit=False) + try: + async with factory.begin() as session: + if args.command == "failed": + jobs = await session.scalars( + select(AttachmentJob) + .where(AttachmentJob.state == "failed") + .order_by(AttachmentJob.updated_at) + .limit(100) + ) + print( + json.dumps( + [ + dict( + job_id=str(j.id), + attachment_id=str(j.attachment_id), + attempt=j.attempt, + error_code=j.error_code, + ) + for j in jobs + ] + ) + ) + return + job = await session.scalar( + select(AttachmentJob).where(AttachmentJob.id == args.job_id).with_for_update() + ) + if job is None or job.state != "failed": + raise ValueError("Replay accepts only an existing failed job") + attachment = await session.scalar( + select(Attachment).where(Attachment.id == job.attachment_id).with_for_update() + ) + if not attachment.staging_key: + raise ValueError("Staging retention expired; replay unavailable") + await to_thread.run_sync( + lambda: worker_storage().stat_object(settings.s3_bucket, attachment.staging_key) + ) + when = await now(session) + job.attempt += 1 + job.cycle_attempt = 1 + job.state = attachment.status = "queued" + job.error_code = attachment.error_code = None + job.published_at = job.claim_token = job.lease_until = None + job.next_run_at = job.updated_at = attachment.updated_at = when + session.add( + JobReplay( + id=uuid4(), + job_id=job.id, + operator=args.operator, + reason=args.reason, + created_at=when, + attempt=job.attempt, + ) + ) + print(json.dumps(dict(job_id=str(job.id), status="queued", attempt=job.attempt))) + finally: + await engine.dispose() + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest="command", required=True) + sub.add_parser("failed") + replay = sub.add_parser("replay") + replay.add_argument("job_id", type=UUID) + replay.add_argument("--operator", required=True) + replay.add_argument("--reason", required=True) + args = parser.parse_args() + if args.command == "replay" and ( + not 1 <= len(args.operator.strip()) <= 100 or not 1 <= len(args.reason.strip()) <= 500 + ): + parser.error("operator: 1–100; reason: 1–500 nonblank characters") + try: + asyncio.run(execute(args)) + except Exception as error: + print( + json.dumps( + dict( + error=type(error).__name__, message="Replay rejected or dependency unavailable" + ) + ) + ) + raise SystemExit(1) from None + + +if __name__ == "__main__": + main() diff --git a/src/pipeline/common.py b/src/pipeline/common.py new file mode 100644 index 0000000..d3f9557 --- /dev/null +++ b/src/pipeline/common.py @@ -0,0 +1,58 @@ +"""Shared state transitions; every acknowledgement follows a DB commit.""" + +import random +from datetime import timedelta + +from redis.asyncio import Redis +from sqlalchemy import func, select + +from src.core.settings import settings +from src.observability import signals + +STREAM = "attachment-jobs-v1" +GROUP = "image-workers-v1" + + +def broker(): + return Redis.from_url( + settings.broker_url.get_secret_value(), + decode_responses=True, + socket_connect_timeout=2, + socket_timeout=3, + ) + + +async def now(session): + return await session.scalar(select(func.clock_timestamp())) + + +def retry(job, attachment, when, code, permanent=False): + context = signals.carrier() + if context.get("traceparent"): + job.traceparent = context["traceparent"] + job.tracestate = context.get("tracestate") + signals.event( + "job.retry" + if not permanent and job.cycle_attempt < settings.job_max_attempts + else "job.failed", + attempt=job.attempt, + error_code=code, + ) + job.claim_token = None + job.lease_until = None + job.updated_at = attachment.updated_at = when + job.error_code = code + if permanent or job.cycle_attempt >= settings.job_max_attempts: + job.state = attachment.status = "failed" + attachment.error_code = code if permanent else "retry_exhausted" + else: + cap = min( + settings.job_retry_cap_seconds, + settings.job_retry_base_seconds * 2 ** (job.cycle_attempt - 1), + ) + job.next_run_at = when + timedelta(seconds=random.uniform(cap / 2, cap)) + job.attempt += 1 + job.cycle_attempt += 1 + job.state = attachment.status = "queued" + job.published_at = None + attachment.error_code = None diff --git a/src/pipeline/dispatcher.py b/src/pipeline/dispatcher.py new file mode 100644 index 0000000..2ea800e --- /dev/null +++ b/src/pipeline/dispatcher.py @@ -0,0 +1,197 @@ +"""Transactional outbox relay, expired-lease recovery and staging retention.""" + +import asyncio +import json +import logging +import os +from datetime import datetime, timedelta, timezone +from functools import lru_cache + +from anyio import to_thread +from opentelemetry.trace import SpanKind +from sqlalchemy import func, or_, select +from sqlalchemy.ext.asyncio import async_sessionmaker + +from src.core.db.models import Attachment, AttachmentJob, UploadIntent +from src.core.db.session import create_engine +from src.core.settings import settings +from src.core.storage import make_storage +from src.core.upload_coordination import lock_object +from src.observability import signals +from src.pipeline.common import STREAM, broker, now, retry + + +@lru_cache(maxsize=1) +def worker_storage(): + return make_storage(role="worker") + + +logger = logging.getLogger(__name__) + + +async def dispatch(factory, client): + async with factory.begin() as session: + when = await now(session) + jobs = list( + await session.scalars( + select(AttachmentJob) + .where(AttachmentJob.state == "processing", AttachmentJob.lease_until < when) + .with_for_update(skip_locked=True) + .limit(20) + ) + ) + for job in jobs: + attachment = await session.get(Attachment, job.attachment_id) + retry(job, attachment, when, "worker_lost") + # Bounded batch; row locks serialize multiple relays. XADD may succeed while + # COMMIT fails: that creates a duplicate, never a missing durable intention. + async with factory.begin() as session: + when = await now(session) + jobs = list( + await session.scalars( + select(AttachmentJob) + .where( + AttachmentJob.state == "queued", + AttachmentJob.published_at.is_(None), + AttachmentJob.next_run_at <= when, + ) + .order_by(AttachmentJob.next_run_at) + .with_for_update(skip_locked=True) + .limit(20) + ) + ) + for job in jobs: + payload = dict( + schema_version=1, + job_id=str(job.id), + type="attachment.process.v1", + attachment_id=str(job.attachment_id), + pipeline_version=1, + attempt=job.attempt, + created_at=job.created_at.isoformat().replace("+00:00", "Z"), + ) + if settings.pipeline_fault_injection and job.fault == "before_publish": + os._exit(81) + with signals.tracer.start_as_current_span( + "queue.publish", + context=signals.parent( + {"traceparent": job.traceparent, "tracestate": job.tracestate} + ), + kind=SpanKind.PRODUCER, + record_exception=False, + set_status_on_exception=False, + ): + payload.update(signals.carrier()) + await client.xadd(STREAM, {"job": json.dumps(payload)}) + signals.event( + "job.published", attempt=job.attempt, job_type="attachment.process.v1" + ) + if settings.pipeline_fault_injection and job.fault == "after_publish": + os._exit(82) + job.published_at = when + + +async def cleanup(factory): + storage = worker_storage() + async with factory.begin() as session: + when = await now(session) + attachments = list( + await session.scalars( + select(Attachment) + .where( + Attachment.staging_key.is_not(None), + or_( + Attachment.status == "ready", + (Attachment.status == "failed") + & ( + Attachment.updated_at + < when - timedelta(seconds=settings.staging_retention_seconds) + ), + ), + ) + .with_for_update(skip_locked=True) + .limit(20) + ) + ) + for attachment in attachments: + # Serialize deletion with replay, which locks the attachment too. + if attachment.staging_key != attachment.object_key: + await to_thread.run_sync( + lambda: storage.remove_object(settings.s3_bucket, attachment.staging_key) + ) + attachment.staging_key = None + # Intents also cover a crash after staging PUT, before metadata/job COMMIT. + async with factory.begin() as session: + when = await now(session) + intents = list( + await session.scalars( + select(UploadIntent) + .where( + UploadIntent.object_key.startswith("staging/"), + UploadIntent.state == "pending", + UploadIntent.created_at + < when - timedelta(seconds=settings.reconcile_grace_seconds), + ) + .with_for_update(skip_locked=True) + .limit(20) + ) + ) + for intent in intents: + if not await lock_object(session, intent.object_key): + continue + attached = await session.scalar( + select(Attachment.id).where(Attachment.staging_key == intent.object_key) + ) + if attached is None: + await to_thread.run_sync( + lambda: storage.remove_object(settings.s3_bucket, intent.object_key) + ) + intent.state = "abandoned" + + +async def main(): + signals.setup("messenger-dispatcher") + engine = create_engine() + factory = async_sessionmaker(engine, expire_on_commit=False) + client = broker() + ticks = 0 + try: + while True: + try: + await dispatch(factory, client) + async with factory() as session: + count, oldest = ( + await session.execute( + select(func.count(), func.min(AttachmentJob.created_at)).where( + AttachmentJob.state.in_(["queued", "processing"]) + ) + ) + ).one() + attrs = {"queue": STREAM} + signals.queue_depth.set(count, attrs) + signals.queue_age.set( + max(0, (datetime.now(timezone.utc) - oldest).total_seconds()) + if oldest + else 0, + attrs, + ) + signals.db_available.set(1) + except Exception as error: + signals.db_available.set(0) + signals.event("relay.error", error_type=type(error).__name__) + logger.warning("relay unavailable: %s", type(error).__name__) + if ticks % 10 == 0: + try: + await cleanup(factory) + except Exception as error: + logger.warning("cleanup unavailable: %s", type(error).__name__) + ticks += 1 + await asyncio.sleep(0.5) + finally: + await client.aclose() + await engine.dispose() + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO) + asyncio.run(main()) diff --git a/src/pipeline/images.py b/src/pipeline/images.py new file mode 100644 index 0000000..ee3bb5d --- /dev/null +++ b/src/pipeline/images.py @@ -0,0 +1,95 @@ +"""Isolated image transformation subprocess; originals are never re-encoded.""" + +import hashlib +import json +import os +import resource +import sys +import time +import warnings +from pathlib import Path + +from PIL import Image, ImageOps + + +def dimensions(width, height, limit): + # Contract requires floor, unlike Pillow thumbnail's closest-aspect rounding. + largest = max(width, height) + if largest <= limit: + return width, height + return max(1, width * limit // largest), max(1, height * limit // largest) + + +def transform(source: Path, output: Path, max_pixels: int): + Image.MAX_IMAGE_PIXELS = max_pixels + with warnings.catch_warnings(): + warnings.simplefilter("error", Image.DecompressionBombWarning) + with Image.open(source) as raw: + if raw.format not in {"PNG", "JPEG"}: + raise ValueError("invalid format") + if raw.width * raw.height > max_pixels: + raise ValueError("pixel limit") + raw.verify() + with Image.open(source) as raw: + raw.load() + oriented = ImageOps.exif_transpose(raw) + # All generated variants use an RGB canvas, so no EXIF/XMP/ICC is inherited. + rgba = oriented.convert("RGBA") + image = Image.new("RGB", rgba.size, "white") + image.paste(rgba, mask=rgba.getchannel("A")) + del rgba + variants = [] + for name, size, fmt, quality in ( + ("thumbnail", dimensions(*image.size, 256), "JPEG", 85), + ("crop", (128, 128), "JPEG", 85), + ("optimized", dimensions(*image.size, 1280), "WEBP", 80), + ): + result = ( + ImageOps.fit(image, size, Image.Resampling.LANCZOS, centering=(0.5, 0.5)) + if name == "crop" + else image.resize(size, Image.Resampling.LANCZOS) + ) + options = {"method": 4} if fmt == "WEBP" else {"optimize": True} + target = output / name + result.save(target, fmt, quality=quality, **options) + data = target.read_bytes() + variants.append( + dict( + name=name, + width=result.width, + height=result.height, + content_type="image/webp" if fmt == "WEBP" else "image/jpeg", + size_bytes=len(data), + sha256=hashlib.sha256(data).hexdigest(), + ) + ) + result.close() + image.close() + (output / "manifest.json").write_text(json.dumps(variants)) + + +def main(): + source, output, max_pixels, memory_mb, seconds = sys.argv[1:] + # Linux worker subprocess: hard virtual-memory and CPU ceilings plus parent wall timeout. + memory = int(memory_mb) * 1024 * 1024 + resource.setrlimit(resource.RLIMIT_AS, (memory, memory)) + resource.setrlimit(resource.RLIMIT_CPU, (int(seconds), int(seconds) + 1)) + if os.environ.get("PIPELINE_FAULT_INJECTION", "").lower() == "true": + time.sleep(float(os.environ.get("PIPELINE_TEST_IMAGE_DELAY", "0"))) + try: + transform(Path(source), Path(output), int(max_pixels)) + except ( + Image.DecompressionBombError, + Image.DecompressionBombWarning, + ValueError, + OSError, + SyntaxError, + EOFError, + ): + raise SystemExit(2) from None + except MemoryError: + raise SystemExit(3) from None + + +if __name__ == "__main__": + main() diff --git a/src/pipeline/worker.py b/src/pipeline/worker.py new file mode 100644 index 0000000..3188f41 --- /dev/null +++ b/src/pipeline/worker.py @@ -0,0 +1,364 @@ +"""Redis Streams consumer with bounded prefetch, leases and idempotent effects.""" + +import asyncio +import hashlib +import json +import logging +import os +import socket +import sys +import time +from datetime import datetime, timedelta, timezone +from functools import lru_cache +from pathlib import Path +from tempfile import TemporaryDirectory +from typing import Literal +from uuid import UUID, uuid4 + +from anyio import to_thread +from opentelemetry.trace import SpanKind, Status, StatusCode +from pydantic import BaseModel, ConfigDict, Field +from redis.exceptions import ResponseError +from sqlalchemy import select +from sqlalchemy.ext.asyncio import async_sessionmaker + +from src.core.db.models import Attachment, AttachmentJob +from src.core.db.session import create_engine +from src.core.settings import settings +from src.core.storage import make_storage +from src.observability import signals +from src.pipeline.common import GROUP, STREAM, broker, now, retry + + +@lru_cache(maxsize=1) +def worker_storage(): + return make_storage(role="worker") + + +logger = logging.getLogger(__name__) + + +class Envelope(BaseModel): + model_config = ConfigDict(extra="forbid", strict=True) + schema_version: Literal[1] + job_id: str + type: Literal["attachment.process.v1"] + attachment_id: str + pipeline_version: Literal[1] + attempt: int = Field(ge=1) + created_at: str + traceparent: str | None = None + tracestate: str | None = None + + +class PermanentError(Exception): + pass + + +async def transform(source, directory, *, test_delay=False): + process = await asyncio.create_subprocess_exec( + sys.executable, + "-m", + "src.pipeline.images", + str(source), + str(directory), + str(settings.max_image_pixels), + str(settings.worker_memory_mb), + str(settings.job_timeout_seconds), + env=dict( + os.environ, + PIPELINE_TEST_IMAGE_DELAY=str(settings.job_timeout_seconds + 2) if test_delay else "0", + ), + stdout=asyncio.subprocess.DEVNULL, + stderr=asyncio.subprocess.DEVNULL, + ) + try: + await asyncio.wait_for(process.wait(), settings.job_timeout_seconds) + finally: + if process.returncode is None: + process.kill() + await process.wait() + if process.returncode == 2: + raise PermanentError("invalid_image") + if process.returncode: + raise PermanentError("image_resource_limit") + return json.loads((directory / "manifest.json").read_text()) + + +def download(attachment, source): + response = worker_storage().get_object(settings.s3_bucket, attachment.staging_key) + digest, size = hashlib.sha256(), 0 + try: + with source.open("wb") as output: + while chunk := response.read(65536): + size += len(chunk) + if size > settings.max_upload_bytes: + raise PermanentError("payload_too_large") + digest.update(chunk) + output.write(chunk) + finally: + response.close() + response.release_conn() + if size != attachment.size_bytes or digest.hexdigest() != attachment.sha256: + raise PermanentError("source_checksum_mismatch") + + +async def verify_ready(attachment): + objects = [(attachment.object_key, attachment.size_bytes)] + [ + (f"variants/{attachment.id}/v1/{item['name']}", item["size_bytes"]) + for item in attachment.variants + ] + if {item["name"] for item in attachment.variants} != {"thumbnail", "crop", "optimized"}: + # Legacy lab4 attachments have no job and never reach this branch. + raise RuntimeError("Incomplete committed pipeline") + for key, size in objects: + stat = await to_thread.run_sync( + lambda: worker_storage().stat_object(settings.s3_bucket, key) + ) + if stat.size != size: + raise RuntimeError("Committed result is missing") + + +async def _process_job(factory, message): + token = uuid4() + async with factory.begin() as session: + job = await session.scalar( + select(AttachmentJob) + .where(AttachmentJob.id == UUID(message.job_id)) + .with_for_update(skip_locked=True) + ) + if job is None: + # Missing jobs are not valid work; busy jobs must remain pending. + exists = await session.get(AttachmentJob, UUID(message.job_id)) + return exists is None + if ( + str(job.attachment_id) != message.attachment_id + or job.pipeline_version != message.pipeline_version + ): + raise PermanentError("envelope_mismatch") + attachment = await session.get(Attachment, job.attachment_id) + if job.state == "ready": + await verify_ready(attachment) + return "noop" + if job.state == "failed" or message.attempt < job.attempt: + return "noop" + if message.attempt != job.attempt: + raise PermanentError("attempt_mismatch") + when = await now(session) + if job.state == "processing" or when < job.next_run_at: + return False + job.state = attachment.status = "processing" + job.claim_token = token + job.lease_until = when + timedelta(seconds=settings.job_lease_seconds) + job.updated_at = attachment.updated_at = when + # Keep the job row locked during bounded side effects. Recovery uses SKIP + # LOCKED, so a live attempt and its replacement cannot publish concurrently. + # A process crash releases the lock; its durable lease is recovered by relay. + async with factory.begin() as session: + job = await session.scalar( + select(AttachmentJob) + .where(AttachmentJob.id == UUID(message.job_id)) + .with_for_update(skip_locked=True) + ) + if job is None or job.claim_token != token or job.state != "processing": + return False + attachment = await session.get(Attachment, job.attachment_id) + try: + if settings.pipeline_fault_injection and job.fault == "temporary": + raise OSError("injected temporary failure") + if settings.pipeline_fault_injection and job.fault == "permanent": + raise PermanentError("invalid_image") + with TemporaryDirectory(prefix="image-job-") as temporary: + directory = Path(temporary) + source = directory / "original" + await to_thread.run_sync(download, attachment, source) + with signals.tracer.start_as_current_span( + "image.transform", record_exception=False + ): + variants = await transform( + source, + directory, + test_delay=settings.pipeline_fault_injection and job.fault == "timeout", + ) + for name, mime in [("original", attachment.content_type)] + [ + (v["name"], v["content_type"]) for v in variants + ]: + path = directory / name + key = ( + attachment.object_key + if name == "original" + else f"variants/{attachment.id}/v1/{name}" + ) + + def put(): + with path.open("rb") as data: + worker_storage().put_object( + settings.s3_bucket, + key, + data, + path.stat().st_size, + content_type=mime, + num_parallel_uploads=1, + ) + + await to_thread.run_sync(put) + if settings.pipeline_fault_injection and job.fault == "after_objects": + os._exit(83) + attachment.storage_missing_since = None + attachment.variants = variants + attachment.status = job.state = "ready" + attachment.error_code = job.error_code = None + job.claim_token = job.lease_until = None + job.updated_at = attachment.updated_at = await now(session) + except Exception as error: + code = ( + str(error) + if isinstance(error, PermanentError) + else ( + "processing_timeout" + if isinstance(error, TimeoutError) + else "processing_unavailable" + ) + ) + retry(job, attachment, await now(session), code, isinstance(error, PermanentError)) + crash_after_commit = ( + settings.pipeline_fault_injection + and job.fault == "after_commit" + and job.state == "ready" + ) + if crash_after_commit: + os._exit(84) + return job.state + + +async def process_job(factory, message): + started = time.perf_counter() + with signals.tracer.start_as_current_span( + "queue.consume", + context=signals.parent( + {"traceparent": message.traceparent, "tracestate": message.tracestate} + ), + kind=SpanKind.CONSUMER, + attributes={ + "messaging.system": "redis", + "job.type": message.type, + "job.attempt": message.attempt, + }, + record_exception=False, + set_status_on_exception=False, + ) as span: + signals.worker_busy.add(1) + signals.event("job.started", attempt=message.attempt, job_type=message.type) + try: + result = await _process_job(factory, message) + outcome = {"queued": "retry", "ready": "ready", "failed": "failed"}.get(result) + if outcome: + attributes = {"job_type": message.type, "outcome": outcome} + signals.jobs.add(1, attributes) + signals.job_duration.record(time.perf_counter() - started, attributes) + if outcome != "ready": + span.set_status(Status(StatusCode.ERROR)) + else: + created = datetime.fromisoformat(message.created_at.replace("Z", "+00:00")) + signals.ready_duration.record( + max(0, (datetime.now(timezone.utc) - created).total_seconds()), + {"job_type": message.type}, + ) + signals.event( + "job.completed", + outcome=outcome, + attempt=message.attempt, + job_type=message.type, + duration_seconds=time.perf_counter() - started, + ) + return result + except Exception as error: + span.set_status(Status(StatusCode.ERROR)) + span.set_attribute("error.type", type(error).__name__) + signals.event("job.error", error_type=type(error).__name__, attempt=message.attempt) + raise + finally: + signals.worker_busy.add(-1) + + +async def acknowledge(client, message_id): + # Only after result/terminal error or a durable retry intention is committed. + # XDEL follows XACK; no active/unacknowledged work is trimmed. + async with client.pipeline(transaction=True) as transaction: + transaction.xack(STREAM, GROUP, message_id) + transaction.xdel(STREAM, message_id) + await transaction.execute() + + +async def consume(factory, index): + client = broker() + consumer = f"{socket.gethostname()}-{os.getpid()}-{index}" + cursor = "0-0" + try: + while True: + try: + try: + await client.xgroup_create(STREAM, GROUP, id="0", mkstream=True) + except ResponseError as error: + if "BUSYGROUP" not in str(error): + raise + # One uncompleted message per consumer: concurrency=prefetch=N. + claimed = await client.xautoclaim( + STREAM, + GROUP, + consumer, + min_idle_time=1000, + start_id=cursor, + count=1, + ) + cursor, messages = claimed[0], claimed[1] + if not messages: + batches = await client.xreadgroup( + GROUP, consumer, {STREAM: ">"}, count=1, block=1000 + ) + messages = batches[0][1] if batches else [] + for message_id, fields in messages: + try: + message = Envelope.model_validate_json(fields["job"]) + UUID(message.job_id) + UUID(message.attachment_id) + except (ValueError, KeyError): + # A bounded rejection record, never arbitrary payload/credentials. + await client.xadd( + "attachment-rejected-v1", + {"message_id": message_id, "error_code": "invalid_envelope"}, + maxlen=1000, + ) + await acknowledge(client, message_id) + continue + try: + done = await process_job(factory, message) + except PermanentError: + await client.xadd( + "attachment-rejected-v1", + {"message_id": message_id, "error_code": "envelope_mismatch"}, + maxlen=1000, + ) + done = True + if done: + await acknowledge(client, message_id) + except Exception as error: + logger.warning("consumer unavailable: %s", type(error).__name__) + await asyncio.sleep(1) + finally: + await client.aclose() + + +async def main(): + signals.setup("messenger-worker") + engine = create_engine() + factory = async_sessionmaker(engine, expire_on_commit=False) + try: + await asyncio.gather(*(consume(factory, i) for i in range(settings.worker_concurrency))) + finally: + await engine.dispose() + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO) + asyncio.run(main()) diff --git a/src/service/__init__.py b/src/service/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/service/attachment.py b/src/service/attachment.py new file mode 100644 index 0000000..b1d22c8 --- /dev/null +++ b/src/service/attachment.py @@ -0,0 +1,209 @@ +"""Validate originals without re-encoding; keep only metadata in PostgreSQL.""" + +import asyncio +import hashlib +import logging +import unicodedata +import warnings +from datetime import datetime, timezone +from tempfile import TemporaryFile +from uuid import UUID, uuid4 + +from anyio import CapacityLimiter, WouldBlock, to_thread +from fastapi import Request +from PIL import Image, UnidentifiedImageError +from sqlalchemy import func, select +from sqlalchemy.ext.asyncio import AsyncSession +from starlette.requests import ClientDisconnect + +from src.core.db.models import Attachment, AttachmentJob, ChatMember, UploadIntent +from src.core.errors import ApiError +from src.core.settings import settings +from src.core.storage import STORAGE_ERRORS, get_storage, unavailable +from src.core.upload_coordination import lock_object +from src.observability import signals + +logger = logging.getLogger(__name__) +_upload_limiter = CapacityLimiter(settings.upload_concurrency) +_decode_limiter = CapacityLimiter(1) +# The explicit dimensions check below is authoritative; convert Pillow warnings +# to errors as an additional guard before decoding malicious image headers. +Image.MAX_IMAGE_PIXELS = settings.max_image_pixels + + +def validate_filename(filename: str) -> None: + if not 1 <= len(filename) <= 255 or any( + c in "/\\" or unicodedata.category(c) == "Cc" or 0xD800 <= ord(c) <= 0xDFFF + for c in filename + ): + raise ApiError(400, "invalid_request", "Некорректное имя файла") + + +def validate_image(file, content_type: str) -> None: + try: + with warnings.catch_warnings(): + warnings.simplefilter("error", Image.DecompressionBombWarning) + file.seek(0) + with Image.open(file) as image: + actual_type = {"PNG": "image/png", "JPEG": "image/jpeg"}.get(image.format) + if actual_type != content_type: + raise ApiError(415, "unsupported_media_type", "Тип изображения не совпадает") + if image.width * image.height > settings.max_image_pixels: + raise ApiError(413, "payload_too_large", "Превышен размер изображения") + image.verify() + file.seek(0) + with Image.open(file) as image: + image.load() + except (Image.DecompressionBombError, Image.DecompressionBombWarning) as error: + raise ApiError(413, "payload_too_large", "Превышен размер изображения") from error + except (UnidentifiedImageError, OSError, ValueError, SyntaxError, EOFError) as error: + raise ApiError(415, "unsupported_media_type", "Повреждённое изображение") from error + finally: + file.seek(0) + + +async def require_chat_member(session: AsyncSession, chat_id: UUID, user_id: UUID) -> None: + member = await session.scalar( + select(ChatMember.user_id).where( + ChatMember.chat_id == chat_id, ChatMember.user_id == user_id + ) + ) + if member is None: + raise ApiError(404, "not_found", "Объект не найден") + + +class AttachmentService: + def __init__(self, session: AsyncSession): + self.session = session + + async def get(self, attachment_id: UUID, user_id: UUID) -> Attachment: + attachment = await self.session.scalar( + select(Attachment) + .join(ChatMember, ChatMember.chat_id == Attachment.chat_id) + .where(Attachment.id == attachment_id, ChatMember.user_id == user_id) + ) + if attachment is None: + raise ApiError(404, "not_found", "Вложение не найдено") + if attachment.storage_missing_since is not None and attachment.status == "ready": + raise ApiError(503, "unavailable", "Оригинал вложения временно недоступен") + return attachment + + async def upload( + self, request: Request, chat_id: UUID, user_id: UUID, filename: str + ) -> Attachment: + await require_chat_member(self.session, chat_id, user_id) + validate_filename(filename) + content_type = request.headers.get("content-type", "").strip().lower() + if content_type not in {"image/png", "image/jpeg"}: + raise ApiError(415, "unsupported_media_type", "Требуется PNG или JPEG") + try: + length = int(request.headers.get("content-length", "0")) + if length < 0: + raise ValueError + except ValueError: + raise ApiError(400, "invalid_request", "Некорректная длина тела") from None + if length > settings.max_upload_bytes: + raise ApiError(413, "payload_too_large", "Превышен размер файла") + try: + _upload_limiter.acquire_nowait() + except WouldBlock: + raise ApiError(503, "unavailable", "Слишком много загрузок") from None + try: + return await self._upload(request, chat_id, user_id, filename, content_type) + finally: + _upload_limiter.release() + + async def _upload(self, request, chat_id, user_id, filename, content_type): + size = 0 + digest = hashlib.sha256() + with TemporaryFile() as file: + try: + async with asyncio.timeout(settings.upload_timeout_seconds): + async for chunk in request.stream(): + size += len(chunk) + if size > settings.max_upload_bytes: + raise ApiError(413, "payload_too_large", "Превышен размер файла") + digest.update(chunk) + await to_thread.run_sync(file.write, chunk) + except ClientDisconnect: + raise ApiError(400, "invalid_request", "Передача файла прервана") from None + except TimeoutError: + raise ApiError(408, "invalid_request", "Истекло время приёма файла") from None + await to_thread.run_sync(validate_image, file, content_type, limiter=_decode_limiter) + attachment_id = uuid4() + object_key = f"staging/{attachment_id}" + # Durable registration is independent of the HTTP transaction. If the + # process dies after PUT, the reconciler still knows this key. + async with AsyncSession(bind=self.session.bind) as registration: + async with registration.begin(): + await lock_object(registration, object_key, wait=True) + registration.add( + UploadIntent( + object_key=object_key, + state="pending", + created_at=await registration.scalar(select(func.clock_timestamp())), + ) + ) + await lock_object(self.session, object_key, wait=True) + intent = await self.session.get(UploadIntent, object_key) + if intent is None or intent.state != "pending": + raise ApiError(503, "unavailable", "Попытка загрузки уже закрыта") + storage = get_storage() + try: + await to_thread.run_sync( + lambda: storage.put_object( + settings.s3_bucket, + object_key, + file, + size, + content_type=content_type, + num_parallel_uploads=1, + ) + ) + except STORAGE_ERRORS as error: + raise unavailable(error) from error + now = datetime.now(timezone.utc) + attachment = Attachment( + id=attachment_id, + chat_id=chat_id, + owner_id=user_id, + filename=filename, + object_key=f"originals/{attachment_id}", + staging_key=object_key, + status="queued", + variants=[], + content_type=content_type, + size_bytes=size, + sha256=digest.hexdigest(), + created_at=now, + updated_at=now, + ) + try: + self.session.add(attachment) + await self.session.flush() + self.session.add( + AttachmentJob( + id=uuid4(), + attachment_id=attachment_id, + pipeline_version=1, + traceparent=signals.carrier().get("traceparent"), + tracestate=signals.carrier().get("tracestate"), + state="queued", + attempt=1, + cycle_attempt=1, + created_at=now, + updated_at=now, + next_run_at=now, + ) + ) + # Intent completion and Attachment publication are one DB commit. + # Keep the advisory lock from before PUT through this commit. + intent.state = "committed" + await self.session.commit() + except Exception: + # A failed COMMIT can have an unknown outcome. Keep the object rather + # than deleting bytes that a committed Attachment might reference. + logger.warning("Метаданные загрузки не подтверждены; требуется сверка S3/БД") + raise + signals.event("attachment.accepted", job_type="attachment.process.v1") + return attachment diff --git a/src/service/auth.py b/src/service/auth.py new file mode 100644 index 0000000..425f285 --- /dev/null +++ b/src/service/auth.py @@ -0,0 +1,74 @@ +import logging +import secrets +from datetime import datetime, timedelta, timezone + +from sqlalchemy.ext.asyncio import AsyncSession + +from src.core import session_store +from src.core.db.repo import auth as auth_repo +from src.core.db.repo import user as user_repo +from src.core.security import hash_password, hash_token, verify_password_async +from src.core.settings import settings +from src.service.dto import AuthSessionDTO, StoredSessionDTO, UserDTO +from src.service.exceptions import InvalidCredentialsException, InvalidSessionException + +logger = logging.getLogger(__name__) +# Для неизвестного логина тоже выполняем Argon2, чтобы не отвечать заметно быстрее. +_DUMMY_PASSWORD_HASH = hash_password("dummy-password-for-timing") + + +class AuthService: + def __init__( + self, + db_session: AsyncSession, + *, + session_ttl_seconds: int, + ) -> None: + self._db_session = db_session + self._session_ttl = timedelta(seconds=session_ttl_seconds) + + async def login(self, *, username: str, password: str) -> AuthSessionDTO: + credentials = await auth_repo.fetch_user_credentials( + session=self._db_session, + username=username, + ) + password_hash = credentials.password_hash if credentials else _DUMMY_PASSWORD_HASH + password_valid = await verify_password_async(password, password_hash) + if credentials is None or not password_valid: + logger.warning("Вход отклонён: неверный логин или пароль") + raise InvalidCredentialsException("Неверный логин или пароль") + + user = credentials.user + token = secrets.token_urlsafe(settings.session_token_bytes) + expires_at = datetime.now(timezone.utc) + self._session_ttl + await session_store.create_session( + token_hash=hash_token(token), + session=StoredSessionDTO(user_id=user.id, expires_at=expires_at), + ) + logger.info("Сессия создана: пользователь=%s", user.id) + return AuthSessionDTO( + token=token, + expires_at=expires_at, + user=user, + ) + + async def get_user(self, *, token: str) -> UserDTO: + session = await session_store.fetch_session(token_hash=hash_token(token)) + if session is None: + logger.info("Доступ отклонён: сессия отсутствует или истекла") + raise InvalidSessionException("Нет действующей сессии") + + user = await user_repo.fetch_user_by_id( + session=self._db_session, + user_id=session.user_id, + ) + if user is None: + logger.warning("Доступ отклонён: пользователь сессии не найден") + raise InvalidSessionException("Нет действующей сессии") + + logger.debug("Сессия подтверждена: пользователь=%s", user.id) + return user + + async def logout(self, *, token: str) -> None: + await session_store.delete_session(token_hash=hash_token(token)) + logger.info("Сессия завершена") diff --git a/src/service/chat.py b/src/service/chat.py new file mode 100644 index 0000000..97ce519 --- /dev/null +++ b/src/service/chat.py @@ -0,0 +1,61 @@ +from datetime import datetime, timezone +from uuid import UUID, uuid4 + +from sqlalchemy.ext.asyncio import AsyncSession + +from src.core.db.repo import chat as chat_repo +from src.core.db.repo import user as user_repo +from src.service.dto import ChatDTO +from src.service.exceptions import ChatNotFoundException, InvalidChatMembersException + + +class ChatService: + def __init__(self, db_session: AsyncSession) -> None: + self._db_session = db_session + + async def create_chat( + self, + *, + title: str, + member_ids: list[UUID], + current_user_id: UUID, + ) -> ChatDTO: + if current_user_id not in member_ids: + raise InvalidChatMembersException("Создатель должен быть в списке member_ids") + if not await user_repo.users_exist( + session=self._db_session, + user_ids=member_ids, + ): + raise InvalidChatMembersException("Указан неизвестный пользователь") + + return await chat_repo.create_chat( + session=self._db_session, + chat_id=uuid4(), + title=title, + member_ids=member_ids, + created_at=datetime.now(timezone.utc), + ) + + async def get_chat(self, *, chat_id: UUID, current_user_id: UUID) -> ChatDTO: + chat = await chat_repo.fetch_chat_by_id( + session=self._db_session, + chat_id=chat_id, + ) + self._check_access(chat=chat, user_id=current_user_id) + return chat + + async def get_chats( + self, + *, + current_user_id: UUID, + limit: int, + after: tuple[datetime, UUID] | None = None, + ) -> list[ChatDTO]: + return await chat_repo.fetch_chats( + session=self._db_session, user_id=current_user_id, limit=limit, after=after + ) + + @staticmethod + def _check_access(*, chat: ChatDTO | None, user_id: UUID) -> None: + if chat is None or user_id not in chat.member_ids: + raise ChatNotFoundException("Чат не найден") diff --git a/src/service/dto.py b/src/service/dto.py new file mode 100644 index 0000000..ed799ff --- /dev/null +++ b/src/service/dto.py @@ -0,0 +1,50 @@ +from dataclasses import dataclass, field +from datetime import datetime +from typing import Literal +from uuid import UUID + + +@dataclass(frozen=True, slots=True) +class UserDTO: + id: UUID + username: str + display_name: str + created_at: datetime + + +@dataclass(frozen=True, slots=True) +class ChatDTO: + id: UUID + title: str + member_ids: tuple[UUID, ...] + created_at: datetime + + +@dataclass(frozen=True, slots=True) +class MessageDTO: + id: UUID + chat_id: UUID + sender_id: UUID + text: str + created_at: datetime + attachment_ids: tuple[UUID, ...] = () + + +@dataclass(frozen=True, slots=True) +class AuthSessionDTO: + token: str = field(repr=False) + expires_at: datetime + user: UserDTO + token_type: Literal["Bearer"] = "Bearer" + + +@dataclass(frozen=True, slots=True) +class StoredSessionDTO: + user_id: UUID + expires_at: datetime + + +@dataclass(frozen=True, slots=True) +class UserCredentialsDTO: + user: UserDTO + password_hash: str = field(repr=False) diff --git a/src/service/exceptions.py b/src/service/exceptions.py new file mode 100644 index 0000000..4c682db --- /dev/null +++ b/src/service/exceptions.py @@ -0,0 +1,30 @@ +class IdempotencyConflictException(Exception): + pass + + +class ServiceException(Exception): + pass + + +class UserAlreadyExistsException(ServiceException): + pass + + +class UserNotFoundException(ServiceException): + pass + + +class InvalidChatMembersException(ServiceException): + pass + + +class ChatNotFoundException(ServiceException): + pass + + +class InvalidCredentialsException(ServiceException): + pass + + +class InvalidSessionException(ServiceException): + pass diff --git a/src/service/message.py b/src/service/message.py new file mode 100644 index 0000000..e10ffb0 --- /dev/null +++ b/src/service/message.py @@ -0,0 +1,64 @@ +from datetime import datetime, timezone +from uuid import UUID, uuid4 + +from sqlalchemy.ext.asyncio import AsyncSession + +from src.core.db.repo import chat as chat_repo +from src.core.db.repo import message as message_repo +from src.service.dto import ChatDTO, MessageDTO +from src.service.exceptions import ChatNotFoundException + + +class MessageService: + def __init__(self, db_session: AsyncSession) -> None: + self._db_session = db_session + + async def create_message( + self, + *, + chat_id: UUID, + sender_id: UUID, + text: str, + idempotency_key: str, + attachment_ids: list[UUID] | None = None, + ) -> MessageDTO: + chat = await chat_repo.fetch_chat_by_id( + session=self._db_session, + chat_id=chat_id, + ) + self._check_access(chat=chat, user_id=sender_id) + return await message_repo.create_idempotent_message( + session=self._db_session, + message_id=uuid4(), + idempotency_key=idempotency_key, + chat_id=chat_id, + sender_id=sender_id, + text=text, + attachment_ids=attachment_ids or [], + created_at=datetime.now(timezone.utc), + ) + + async def get_messages( + self, + *, + chat_id: UUID, + current_user_id: UUID, + limit: int, + after: tuple[datetime, UUID] | None = None, + ) -> list[MessageDTO]: + chat = await chat_repo.fetch_chat_by_id( + session=self._db_session, + chat_id=chat_id, + ) + self._check_access(chat=chat, user_id=current_user_id) + return await message_repo.fetch_messages_by_chat_id( + session=self._db_session, + chat_id=chat_id, + limit=limit, + after=after, + ) + + @staticmethod + def _check_access(*, chat: ChatDTO | None, user_id: UUID) -> None: + if chat is None or user_id not in chat.member_ids: + raise ChatNotFoundException("Чат не найден") diff --git a/src/service/user.py b/src/service/user.py new file mode 100644 index 0000000..05b6882 --- /dev/null +++ b/src/service/user.py @@ -0,0 +1,63 @@ +from datetime import datetime, timezone +from uuid import UUID, uuid4 + +from sqlalchemy.exc import IntegrityError +from sqlalchemy.ext.asyncio import AsyncSession + +from src.core.db.repo import user as user_repo +from src.core.security import hash_password_async +from src.service.dto import UserDTO +from src.service.exceptions import UserAlreadyExistsException, UserNotFoundException + + +class UserService: + def __init__(self, db_session: AsyncSession) -> None: + self._db_session = db_session + + async def create_user(self, *, username: str, display_name: str, password: str) -> UserDTO: + password_hash = await hash_password_async(password) + try: + return await user_repo.create_user( + session=self._db_session, + user_id=uuid4(), + username=username, + display_name=display_name, + created_at=datetime.now(timezone.utc), + password_hash=password_hash, + ) + except IntegrityError as error: + if getattr(error.orig, "sqlstate", None) != "23505": + raise + raise UserAlreadyExistsException("Пользователь с таким именем уже существует") from None + + async def create_admin( + self, + *, + username: str, + display_name: str, + password: str, + ) -> UserDTO | None: + password_hash = await hash_password_async(password) + user = await user_repo.create_admin( + session=self._db_session, + user_id=uuid4(), + username=username, + display_name=display_name, + password_hash=password_hash, + created_at=datetime.now(timezone.utc), + ) + if user is None and not await user_repo.admin_exists( + session=self._db_session, + username=username, + ): + raise UserAlreadyExistsException("Имя администратора занято обычным пользователем") + return user + + async def get_user_by_id(self, *, user_id: UUID) -> UserDTO: + user = await user_repo.fetch_user_by_id( + session=self._db_session, + user_id=user_id, + ) + if user is None: + raise UserNotFoundException("Пользователь не найден") + return user diff --git a/tests/api_regressions.py b/tests/api_regressions.py new file mode 100644 index 0000000..667bb92 --- /dev/null +++ b/tests/api_regressions.py @@ -0,0 +1,199 @@ +"""Replay concrete reviewer cases through real HTTP and controlled dependency freezes.""" +import io +import json +import struct +import subprocess +import time +import zlib +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +from client import Client, arguments, check, run + +ROOT=Path(__file__).resolve().parents[1] + + +def compose(*args): + return subprocess.check_output(['docker','compose',*args],cwd=ROOT,text=True,stderr=subprocess.DEVNULL).strip() + + +def png(width,height): + # Complete PNG, including all scanlines and CRCs, without allocating a bitmap. + def chunk(kind,data): + return struct.pack('>I',len(data))+kind+data+struct.pack('>I',zlib.crc32(kind+data)) + compressor=zlib.compressobj() + packed=io.BytesIO() + row=b'\0'*(width+1) + for _ in range(height): packed.write(compressor.compress(row)) + packed.write(compressor.flush()) + return (b'\x89PNG\r\n\x1a\n'+chunk(b'IHDR',struct.pack('>IIBBBBB',width,height,8,0,0,0,0)) + +chunk(b'IDAT',packed.getvalue())+chunk(b'IEND',b'')) + + +DIRECT='''import json,sys,time,urllib.request,urllib.error +p=json.load(sys.stdin);h={} +if p.get('token'):h['Authorization']='Bearer '+p['token'] +r=urllib.request.Request('http://127.0.0.1:8080'+p['path'],headers=h) +t=time.monotonic() +try:r=urllib.request.urlopen(r,timeout=10) +except urllib.error.HTTPError as e:r=e +with r: + data=json.load(r) + if r.code>=400: + assert data['error']['request_id']==r.headers['X-Request-Id'] + assert data['error']['code']=='unavailable' + print(json.dumps({'status':r.code,'seconds':round(time.monotonic()-t,4), + 'request_id_present':bool(r.headers.get('X-Request-Id')),'instance_id_present':bool(r.headers.get('X-Instance-Id'))})) +''' +LOCKER='''import asyncio,sys,asyncpg +from src.core.settings import settings +async def f(): + c=await asyncpg.connect(settings.database_url.get_secret_value()) + await c.execute('BEGIN; LOCK TABLE messages IN ACCESS EXCLUSIVE MODE') + print('LOCKED',flush=True) + await asyncio.to_thread(sys.stdin.readline) + await c.execute('ROLLBACK');await c.close() +asyncio.run(f()) +''' +PENDING='''import asyncio,asyncpg +from src.core.settings import settings +async def f(): + c=await asyncpg.connect(settings.database_url.get_secret_value()) + print(await c.fetchval("SELECT count(*) FROM pg_stat_activity WHERE wait_event_type='Lock' AND query LIKE '%FROM messages%'")) + await c.close() +asyncio.run(f()) +''' + + +def main(): + args=arguments('API reviewer regression scenarios') + c=Client(args.base_url,args.ca_file,timeout=15) + alice,bob,_,chat=c.bootstrap() + history='/api/v1/chats/'+chat['id']+'/messages' + c.message(alice,chat,'Контроль до отказов') + results=[] + for name in ('review123\n','review123\r','review123\x00','review123\t'): + c.call('POST','/api/v1/users',400,body={'username':name,'display_name':'Review','password':'valid-password-123'}) + results.append({'case':'username_control_characters','result':'PASS','status':400}) + data=(ROOT/'tests/fixtures/sample.png').read_bytes() + upload='/api/v1/chats/'+chat['id']+'/attachments' + for label,filename,payload,mime,status in [ + ('truncated_png','sample.png',data[:-12],'image/png',415), + ('header_only_png','sample.png',data[:33],'image/png',415), + ('mismatched_mime','sample.png',data,'image/jpeg',415), + ('DEL_filename','a%7F.png',data,'image/png',400), + ('missing_filename',None,data,'image/png',400), + ]: + c.call('POST',upload+('' if filename is None else '?filename='+filename),status, + actor=alice,raw=payload,headers={'Content-Type':mime}) + results.append({'case':label,'result':'PASS','status':status}) + for width,height,status in ((8000,5000,202),(8000,5001,413),(20000,10000,413)): + payload=png(width,height) + reply=c.call('POST',upload+'?filename=dimensions.png',status,actor=alice,raw=payload, + headers={'Content-Type':'image/png'},schema='Attachment' if status==202 else None) + if status==202: + c.wait_ready(alice, reply['json']) + check(c.call('GET','/api/v1/attachments/'+reply['json']['id']+'/content',actor=bob)['bytes']==payload, + '40 MP original changed') + results.append({'case':'pixels','dimensions':[width,height],'result':'PASS','status':status}) + api=compose('ps','-q','api').split()[0] + # Actual application credentials must not permit creating unrelated buckets. + code='''from src.core.storage import get_storage +from minio.error import S3Error +import uuid +s=get_storage();b='forbidden-'+uuid.uuid4().hex +try:s.make_bucket(b) +except S3Error as e:assert e.code=='AccessDenied';print('PASS') +else:s.remove_bucket(b);raise AssertionError('API can create foreign buckets') +''' + check(subprocess.check_output(['docker','exec',api,'python','-c',code],text=True).strip()=='PASS', + 'API has excessive S3 rights') + results.append({'case':'API_cannot_create_foreign_bucket','result':'PASS'}) + def direct(path,token=None): + reply=subprocess.run(['docker','exec','-i',api,'python','-c',DIRECT], + input=json.dumps({'path':path,'token':token}),text=True,capture_output=True,timeout=13,check=True) + return json.loads(reply.stdout) + transfer_code="""import asyncio,http.client,json,sys,time +from sqlalchemy import func,select +from sqlalchemy.ext.asyncio import async_sessionmaker +from src.core.db.session import create_engine +from src.core.db.models import Attachment,UploadIntent +p=json.load(sys.stdin) +async def counts(): + e=create_engine();f=async_sessionmaker(e) + try: + async with f() as s: + return [await s.scalar(select(func.count()).select_from(m)) for m in (Attachment,UploadIntent)] + finally:await e.dispose() +before=asyncio.run(counts()) +c=http.client.HTTPConnection('127.0.0.1',8080,timeout=8) +c.putrequest('POST',p['path']) +c.putheader('Authorization','Bearer '+p['token']) +c.putheader('Content-Type','image/png');c.putheader('Content-Length','1000');c.endheaders() +c.send(b'\\x89PNG\\r\\n\\x1a\\nshort');c.close();time.sleep(.3) +assert before==asyncio.run(counts()),'interrupted client created durable state' +c=http.client.HTTPConnection('127.0.0.1',8080,timeout=8) +c.request('POST',p['path'],body=(b'x'*65536 for _ in range(161)), + headers={'Authorization':'Bearer '+p['token'],'Content-Type':'image/png'},encode_chunked=True) +r=c.getresponse();body=json.loads(r.read()) +assert r.status==413 and body['error']['code']=='payload_too_large' +assert body['error']['request_id']==r.getheader('X-Request-Id') +c.close();assert before==asyncio.run(counts()),'oversized chunked body created durable state' +print('PASS') +""" + transfer=subprocess.run(['docker','exec','-i',api,'python','-c',transfer_code], + input=json.dumps({'path':upload+'?filename=interrupted.png','token':alice['token']}), + text=True,capture_output=True,timeout=20,check=True) + check(transfer.stdout.strip()=='PASS','transfer limits failed') + results.append({'case':'client_abort_and_oversized_chunked_body','result':'PASS', + 'oversized_status':413,'durable_state_unchanged':True}) + direct(history,alice['token']) + # Start an actual history SELECT, hold it on a table lock, then freeze PG. + locker=subprocess.Popen(['docker','exec','-i',api,'python','-u','-c',LOCKER], + stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE,text=True) + check(locker.stdout.readline().strip()=='LOCKED','test table lock was not acquired') + paused=False + pool=ThreadPoolExecutor(max_workers=2) + query=pool.submit(direct,history,alice['token']) + try: + deadline=time.monotonic()+2 + while True: + pending=int(subprocess.check_output(['docker','exec',api,'python','-c',PENDING],text=True)) + if pending:break + check(time.monotonic()deadline:raise + time.sleep(.2) + results.append({'case':'PostgreSQL_resume_without_API_restart','result':'PASS'}) + print(json.dumps({'result':'PASS','checks':results},ensure_ascii=False,indent=2)) + + +if __name__=='__main__': + run(main) diff --git a/tests/attachment_failover.py b/tests/attachment_failover.py new file mode 100644 index 0000000..94225a3 --- /dev/null +++ b/tests/attachment_failover.py @@ -0,0 +1,45 @@ +"""Stop the exact API replica which accepted the image; preserve original bytes.""" +import os +import subprocess +import time +from pathlib import Path + +from client import Client, Failure, arguments, check, run + + +def main(): + args = arguments('Attachment download after stopping the uploading replica') + c = Client(args.base_url, args.ca_file) + alice, bob, _, chat = c.bootstrap() + content = (Path(__file__).parent / 'fixtures/sample.png').read_bytes() + uploaded = c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=failover.png', + 202, actor=alice, raw=content, headers={'Content-Type': 'image/png'}, + schema='Attachment') + att = uploaded['json'] + c.message(alice, chat, '', attachments=[att['id']]) + c.wait_ready(alice, att) + target = uploaded['headers']['X-Instance-Id'] + subprocess.run(['scripts/stop-one.sh'], check=True, + env=dict(os.environ, TARGET_INSTANCE_ID=target), + stdout=subprocess.DEVNULL) + deadline = time.monotonic() + 15 + try: + while True: + try: + downloaded = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=bob) + break + except Failure: + if time.monotonic() >= deadline: + raise + time.sleep(.2) + check(downloaded['bytes'] == content, 'original lost after uploader replica stopped') + check(downloaded['headers']['X-Instance-Id'] != target, 'download used stopped replica') + print('PASS: exact uploading replica stopped; another replica returns identical original') + finally: + # Restore the two-replica development stack for subsequent work. + subprocess.run(['docker', 'compose', 'up', '-d', '--wait', 'api'], check=True, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + + +if __name__ == '__main__': + run(main) diff --git a/tests/attachments.py b/tests/attachments.py new file mode 100644 index 0000000..35ed5cb --- /dev/null +++ b/tests/attachments.py @@ -0,0 +1,99 @@ +"""Lab4 black-box checks against the running Compose stack.""" +import hashlib +import io +import struct +import zlib +from concurrent.futures import ThreadPoolExecutor +import uuid +from pathlib import Path + +from PIL import Image +from client import Client, arguments, check, run + + +def main(): + args = arguments('Attachment originals, ACL, validation and message limits') + c = Client(args.base_url, args.ca_file) + alice, bob, eve, chat = c.bootstrap() + path = '/api/v1/chats/' + chat['id'] + content = (Path(__file__).parent / 'fixtures/sample.png').read_bytes() + + def upload(data=content, name='photo.png', mime='image/png', status=202, actor=alice): + return c.call('POST', path + '/attachments?filename=' + name, status, + actor=actor, raw=data, headers={'Content-Type': mime}, + schema='Attachment' if status == 202 else None)['json'] + + first = upload() + second = upload() # Same display name must not overwrite the first object. + check(first['id'] != second['id'], 'same filename overwrote attachment') + jpg = io.BytesIO() + Image.new('RGB', (31, 17), 'red').save(jpg, format='JPEG') + jpeg = upload(jpg.getvalue(), 'photo.jpg', 'image/jpeg') + for att, original in [(first, content), (second, content), (jpeg, jpg.getvalue())]: + check(att['sha256'] == hashlib.sha256(original).hexdigest(), 'incorrect digest') + att = c.wait_ready(alice, att) + check({v['name'] for v in att['variants']} == {'thumbnail', 'crop', 'optimized'}, 'incorrect ready metadata') + for suffix in ('', '/content'): + c.call('GET', '/api/v1/attachments/' + att['id'] + suffix, 404, actor=eve) + downloaded = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=bob) + check(downloaded['bytes'] == original, 'original changed') + + ids = [first['id'], second['id'], jpeg['id']] + key = uuid.uuid4().hex + message = c.message(alice, chat, '', key=key, attachments=ids)['json'] + check(message['attachment_ids'] == ids, 'attachment order lost') + check(c.message(alice, chat, '', key=key, attachments=ids)['json'] == message, + 'idempotency replay changed message') + c.call('POST', path + '/messages', 409, actor=alice, + body={'text': '', 'attachment_ids': ids[:1]}, headers={'Idempotency-Key': key}) + page = c.call('GET', path + '/messages', actor=bob, schema='MessagePage')['json'] + check(any(m == message for m in page['items']), 'history lost attachments') + # The contract permits referencing the same accessible chat attachment again. + c.message(bob, chat, 'same image', attachments=[first['id']]) + + for body in ({'text': '', 'attachment_ids': []}, + {'text': '', 'attachment_ids': [first['id']] * 2}, + {'text': '', 'attachment_ids': [str(uuid.uuid4()) for _ in range(11)]}): + c.call('POST', path + '/messages', 400, actor=alice, body=body, + headers={'Idempotency-Key': uuid.uuid4().hex}) + c.call('POST', path + '/messages', 404, actor=alice, + body={'text': '', 'attachment_ids': [str(uuid.uuid4())]}, + headers={'Idempotency-Key': uuid.uuid4().hex}) + other = c.call('POST', '/api/v1/chats', 201, actor=alice, + body={'title': 'Другой чат', 'member_ids': [alice['id'], bob['id']]}, + schema='Chat')['json'] + c.call('POST', '/api/v1/chats/' + other['id'] + '/messages', 404, actor=alice, + body={'text': '', 'attachment_ids': ids}, + headers={'Idempotency-Key': uuid.uuid4().hex}) + upload(actor=eve, status=404) + upload(name='..%2Fphoto.png', status=400) + upload(name='a%00b.png', status=400) + upload(data=b'not a PNG', status=415) + upload(data=content[:30], status=415) + upload(mime='image/jpeg', status=415) + upload(mime='application/octet-stream', status=415) + # Oversized dimensions with a valid PNG header; no huge bitmap allocation. + header = struct.pack('>IIBBBBB', 10000, 5000, 8, 2, 0, 0, 0) + bomb = (content[:8] + struct.pack('>I', len(header)) + b'IHDR' + header + + struct.pack('>I', zlib.crc32(b'IHDR' + header)) + content[33:]) + upload(data=bomb, status=413) + upload(data=b'x' * (10485760 + 1), status=413) + c.call('POST', path + '/attachments?filename=a.png', 401, + raw=content, headers={'Content-Type': 'image/png'}) + + # Exactly 10 distinct attachments must remain valid, including after reading history. + ten = ids + [upload()['id'] for _ in range(7)] + check(len(c.message(alice, chat, '', attachments=ten)['json']['attachment_ids']) == 10, + '10 attachments rejected') + replay_key = uuid.uuid4().hex + with ThreadPoolExecutor(max_workers=8) as pool: + responses = list(pool.map(lambda _: Client(args.base_url, args.ca_file).message( + alice, chat, 'Concurrent image', key=replay_key, attachments=ids + ), range(8))) + check(all(r['json'] == responses[0]['json'] for r in responses), + 'concurrent attachment retries differ') + print('PASS: PNG/JPEG bytes, metadata, ACL, chat boundaries, limits and idempotency') + + +if __name__ == '__main__': + run(main) diff --git a/tests/idempotency.py b/tests/idempotency.py new file mode 100644 index 0000000..7e45699 --- /dev/null +++ b/tests/idempotency.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 +"""Конкурентная идемпотентность, область ключей и повторная авторизация.""" + +import json +import threading +import uuid +from concurrent.futures import ThreadPoolExecutor + +from client import Client, arguments, check, run + + +def main(): + args = arguments("Lab3 concurrency and idempotency") + c = Client(args.base_url, args.ca_file) + alice, bob, eve, chat = c.bootstrap() + path = "/api/v1/chats/" + chat["id"] + "/messages" + key = uuid.uuid4().hex + clients = [Client(args.base_url, args.ca_file) for _ in range(16)] + barrier = threading.Barrier(len(clients)) + + def send(local): + barrier.wait() + return local.message(alice, chat, "Одна отправка 👋", key=key) + + with ThreadPoolExecutor(max_workers=len(clients)) as pool: + responses = list(pool.map(send, clients)) + instances = {r["headers"]["X-Instance-Id"] for r in responses} + check(len(instances) >= 2, "Parallel repeats did not reach >=2 replicas") + original = responses[0]["json"] + check(all(r["json"] == original for r in responses), "Concurrent results differ") + messages = c.call("GET", path, actor=bob, schema="MessagePage")["json"]["items"] + check(messages == [original], "Concurrent retries duplicated message") + # Missing attachment_ids and [] are semantically equal; raw JSON ordering too. + replay = c.call( + "POST", + path, + 201, + actor=alice, + raw=b'{"attachment_ids":[],"text":"' + "Одна отправка 👋".encode() + b'"}', + headers={"Idempotency-Key": key, "Content-Type": "application/json"}, + schema="Message", + ) + check(replay["json"] == original, "Equivalent JSON did not replay") + c.call( + "POST", + path, + 409, + actor=alice, + body={"text": "Другой текст"}, + headers={"Idempotency-Key": key}, + ) + other = c.message(bob, chat, "Другой отправитель", key=key)["json"] + check(other["id"] != original["id"], "Key incorrectly shared across users") + chat2 = c.call( + "POST", + "/api/v1/chats", + 201, + actor=alice, + body={"title": "Другой чат", "member_ids": [alice["id"]]}, + schema="Chat", + )["json"] + check( + c.message(alice, chat2, "Другой чат", key=key)["json"]["id"] != original["id"], + "Key scope includes wrong chat", + ) + for invalid in [None, "", "bad key", "x" * 129, "bad/key"]: + c.call( + "POST", + path, + 400, + actor=alice, + body={"text": "invalid"}, + headers={} if invalid is None else {"Idempotency-Key": invalid}, + ) + # Validation/access failures must not consume a key. + retry_key = uuid.uuid4().hex + c.call( + "POST", path, 400, actor=alice, body={"text": ""}, headers={"Idempotency-Key": retry_key} + ) + c.message(alice, chat, "После 400", key=retry_key) + c.call( + "POST", + path, + 404, + actor=eve, + body={"text": original["text"]}, + headers={"Idempotency-Key": key}, + ) + c.call("DELETE", "/api/v1/auth/sessions/current", 204, actor=alice) + c.call( + "POST", + path, + 401, + actor=alice, + body={"text": original["text"]}, + headers={"Idempotency-Key": key}, + ) + for route in ["/health/live", "/health/ready", "/missing"]: + c.call("GET", route, 404 if route == "/missing" else 200) + print( + json.dumps( + { + "parallel_repeats": len(responses), + "instances": sorted(instances), + "messages_created": 1, + "checks": "normalization, conflict, key scope, invalid keys, access, logout, headers", + "result": "PASS", + }, + ensure_ascii=False, + indent=2, + ) + ) + + +if __name__ == "__main__": + run(main) diff --git a/tests/lab5.py b/tests/lab5.py new file mode 100644 index 0000000..800a4fd --- /dev/null +++ b/tests/lab5.py @@ -0,0 +1,291 @@ +"""Lab5 HTTP checks and real, opt-in Compose fault injection on an isolated stand.""" +import hashlib +import io +import json +import os +import subprocess +import tempfile +import time +import uuid +from pathlib import Path + +from PIL import Image +from client import Client, Failure, arguments, check, run + +ROOT = Path(__file__).resolve().parents[1] +RESULTS = [] + + +def passed(name, **details): + result = dict(case=name, result='PASS', **details) + RESULTS.append(result) + print(json.dumps(result, ensure_ascii=False), flush=True) + + +def wait(predicate, description, seconds=40): + deadline = time.monotonic() + seconds + while time.monotonic() < deadline: + result = predicate() + if result: + return result + time.sleep(.25) + raise AssertionError('Timeout: ' + description) + + +def main(): + def configure(parser): + parser.add_argument('--faults', action='store_true') + parser.add_argument('--project', default=os.environ.get('COMPOSE_PROJECT_NAME', '')) + args = arguments(__doc__, configure) + c = Client(args.base_url, args.ca_file) + alice, bob, eve, chat = c.bootstrap() + base = '/api/v1/attachments/' + + def upload(data, mime='image/png'): + att = c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=image.png', + 202, actor=alice, raw=data, headers={'Content-Type': mime}, schema='Attachment')['json'] + check(att['status'] == 'queued' and att['variants'] == [], '202 must be a queued snapshot') + return att + + def fixture(size, mode='RGB', color='red', fmt='PNG', exif=None): + data = io.BytesIO() + options = {'exif': exif} if exif else {} + Image.new(mode, size, color).save(data, fmt, **options) + return data.getvalue() + + cases = [((320, 200), (256, 160)), ((7, 3), (7, 3)), ((1001, 667), (256, 170)), + ((1600, 1000), (256, 160))] + for size, thumb_size in cases: + data = fixture(size) + att = c.wait_ready(alice, upload(data)) + variants = {v['name']: v for v in att['variants']} + check(set(variants) == {'thumbnail', 'crop', 'optimized'}, 'all variants required') + check(c.call('GET', base + att['id'] + '/content', actor=bob)['bytes'] == data, 'original changed') + for name, item in variants.items(): + response = c.call('GET', base + att['id'] + '/content?variant=' + name, actor=bob) + encoded = response['bytes'] + check(hashlib.sha256(encoded).hexdigest() == item['sha256'], 'variant checksum') + check(len(encoded) == item['size_bytes'], 'variant size') + with Image.open(io.BytesIO(encoded)) as image: + image.load() + check(image.size == (item['width'], item['height']), 'reported dimensions') + check(not image.getexif() and 'icc_profile' not in image.info, 'metadata leaked') + check(image.format == ('WEBP' if name == 'optimized' else 'JPEG'), 'format') + if name == 'thumbnail': check(image.size == thumb_size, 'floor/no-upscale') + if name == 'crop': check(image.size == (128, 128), 'crop dimensions') + if name == 'optimized': + check(max(image.size) <= 1280 and image.width <= size[0] and image.height <= size[1], 'optimized bounds') + c.call('GET', base + att['id'] + '/content?variant=' + name, 404, actor=eve) + passed('variants', original=list(size), thumbnail=list(thumb_size)) + + alpha = upload(fixture((20, 10), 'RGBA', (255, 0, 0, 0))) + alpha = c.wait_ready(alice, alpha) + encoded = c.call('GET', base + alpha['id'] + '/content?variant=thumbnail', actor=bob)['bytes'] + with Image.open(io.BytesIO(encoded)) as image: + check(min(image.getpixel((0, 0))) >= 250, 'alpha must be white') + exif = Image.Exif(); exif[274] = 6; exif[270] = 'private metadata' + rotated = c.wait_ready(alice, upload(fixture((200, 100), fmt='JPEG', exif=exif), 'image/jpeg')) + thumbnail = next(v for v in rotated['variants'] if v['name'] == 'thumbnail') + check((thumbnail['width'], thumbnail['height']) == (100, 200), 'EXIF orientation') + stripes = Image.new('RGB', (300, 100), 'red') + stripes.paste('green', (100, 0, 200, 100)); stripes.paste('blue', (200, 0, 300, 100)) + buf = io.BytesIO(); stripes.save(buf, 'PNG') + cropped = c.wait_ready(alice, upload(buf.getvalue())) + data = c.call('GET', base + cropped['id'] + '/content?variant=crop', actor=bob)['bytes'] + with Image.open(io.BytesIO(data)) as image: + red, green, blue = image.getpixel((64, 64)) + check(green > red + 50 and green > blue + 50, 'crop must be centered') + passed('alpha_EXIF_metadata_center_crop') + + if not args.faults: + return + check(bool(args.project) and args.project != ROOT.name, 'Fault tests require an explicitly named isolated project') + override = Path(tempfile.mkdtemp(prefix='lab5-faults-')) / 'override.yaml' + override.write_text('''services: + worker: + environment: &faults + PIPELINE_FAULT_INJECTION: "true" + JOB_LEASE_SECONDS: "5" + JOB_MAX_ATTEMPTS: "3" + JOB_TIMEOUT_SECONDS: "1" + JOB_RETRY_BASE_SECONDS: "0.5" + JOB_RETRY_CAP_SECONDS: "2" + dispatcher: + environment: *faults +''') + command = ['docker', 'compose', '--env-file', os.environ.get('ENV_FILE', '.env'), '-p', args.project, '-f', str(ROOT / 'compose.yaml'), '-f', str(override)] + + def compose(*parts, input=None, expected=0): + result = subprocess.run(command + list(parts), input=input, text=True, capture_output=True, timeout=100) + check(result.returncode == expected, 'Compose action failed: ' + ' '.join(parts[:3]) + ' ' + result.stderr[-400:]) + return result.stdout.strip() + + def sql(query): + return compose('exec', '-T', 'postgres', 'sh', '-c', + 'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -At -v ON_ERROR_STOP=1', input=query) + + def job(att): + return json.loads(sql("SELECT row_to_json(j) FROM attachment_jobs j WHERE attachment_id='" + att['id'] + "';")) + + def set_fault(att, value): + sql("UPDATE attachment_jobs SET fault=" + ("'" + value + "'" if value else 'NULL') + " WHERE attachment_id='" + att['id'] + "';") + + def exited(service, code): + ids = compose('ps', '-aq', service).split() + if not ids: return False + inspect = subprocess.check_output(['docker', 'inspect', '--format', '{{json .State}}', ids[0]], text=True) + state = json.loads(inspect) + return state['Status'] == 'exited' and state['ExitCode'] == code + + def pending(): + return int(compose('exec', '-T', 'broker', 'redis-cli', 'XPENDING', 'attachment-jobs-v1', 'image-workers-v1').splitlines()[0]) + + def objects(att): + code = '''import json,sys +from src.core.storage import make_storage +from src.core.settings import settings +s=make_storage(role='worker');i=sys.argv[1] +print(json.dumps({o.object_name:o.etag for prefix in ['originals/'+i,'variants/'+i+'/'] for o in s.list_objects(settings.s3_bucket,prefix=prefix,recursive=True)}))''' + return json.loads(compose('exec', '-T', 'dispatcher', 'python', '-c', code, att['id'])) + + def inject(att): + j = job(att) + payload = dict(schema_version=1, job_id=j['id'], type='attachment.process.v1', attachment_id=att['id'], + pipeline_version=1, attempt=j['attempt'], created_at=j['created_at']) + compose('exec', '-T', 'broker', 'redis-cli', 'XADD', 'attachment-jobs-v1', '*', 'job', json.dumps(payload)) + + try: + compose('up', '-d', '--no-deps', '--force-recreate', 'worker', 'dispatcher') + compose('stop', 'worker') + att, data = c.attachment(alice, chat) + c.message(alice, chat, '', attachments=[att['id']]) + c.call('GET', base + att['id'] + '/content', 409, actor=bob) + c.message(alice, chat, 'API работает без worker') + wait(lambda: job(att)['published_at'], 'publish while worker stopped') + check(int(compose('exec', '-T', 'broker', 'redis-cli', 'XLEN', 'attachment-jobs-v1')) > 0, 'broker must contain work') + compose('restart', 'api', 'broker') + def ready(): + try: + return c.call('GET', '/health/ready', [200, 503])['status'] == 200 + except Failure: + return False + wait(ready, 'API ready') + compose('start', 'worker') + c.wait_ready(alice, att) + check(c.call('GET', base + att['id'] + '/content', actor=bob)['bytes'] == data, 'restart lost original') + passed('stopped_worker_API_broker_restart', attachment_id=att['id']) + + compose('stop', 'broker') + unavailable = upload(fixture((324, 204))) + check(c.call('GET', '/health/ready')['status'] == 200, 'broker outage must not block API') + check(job(unavailable)['published_at'] is None, 'unsent job must remain in DB') + compose('start', 'broker') + c.wait_ready(alice, unavailable) + passed('broker_unavailable_outbox_recovered') + + # COMMIT is already durable; kill the actual relay before and after XADD. + for fault, code in [('before_publish', 81), ('after_publish', 82)]: + compose('stop', 'dispatcher', 'worker') + att = upload(fixture((321, 201))) + set_fault(att, fault) + check(job(att)['published_at'] is None, 'must have unsent outbox') + compose('start', 'dispatcher') + wait(lambda: exited('dispatcher', code), fault) + check(job(att)['published_at'] is None, 'relay crash must roll back sent flag') + # Dispatcher is stopped; clear through postgres, then restore the same job. + set_fault(att, None) + compose('start', 'dispatcher', 'worker') + c.wait_ready(alice, att) + passed(fault + '_recovered', job_id=job(att)['id']) + + for fault, code in [('after_objects', 83), ('after_commit', 84)]: + compose('stop', 'worker') + att = upload(fixture((322, 202))) + set_fault(att, fault) + compose('start', 'worker') + wait(lambda: exited('worker', code), fault) + before = objects(att) + check(len(before) == 4, 'one original and three objects must exist') + j = job(att) + if fault == 'after_objects': + check(j['state'] == 'processing', 'crash must leave a processing lease') + else: + check(j['state'] == 'ready', 'ready commit must precede ack') + check(pending() > 0, 'crash must leave an unacked delivery') + set_fault(att, None) + compose('start', 'worker') + c.wait_ready(alice, att) + wait(lambda: pending() == 0, 'pending delivery recovery') + check(objects(att) == before, 'redelivery changed/duplicated objects') + stable = job(att) + inject(att); inject(att) + wait(lambda: pending() == 0 and int(compose('exec', '-T', 'broker', 'redis-cli', 'XLEN', 'attachment-jobs-v1')) == 0, 'duplicate ack') + check(job(att)['attempt'] == stable['attempt'], 'ready duplicate must not start a new attempt') + check(objects(att) == before, 'ready duplicate changed output') + passed(fault + '_redelivery_idempotent', attempt=stable['attempt'], objects=len(before)) + + for fault in ['temporary', 'permanent', 'timeout']: + compose('stop', 'worker') + att = upload(fixture((323, 203))) + set_fault(att, fault) + compose('start', 'worker') + failed = wait(lambda: (a if (a := c.call('GET', base + att['id'], actor=alice, schema='Attachment')['json'])['status'] == 'failed' else None), 'terminal failed') + j = job(att) + check(j['cycle_attempt'] == (1 if fault == 'permanent' else 3), 'retry budget/classification') + check(failed['error_code'] == ('invalid_image' if fault == 'permanent' else 'retry_exhausted'), 'safe terminal code') + c.call('GET', base + att['id'] + '/content', 409, actor=bob) + c.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', 409, actor=alice, + body={'text':'', 'attachment_ids':[att['id']]}, headers={'Idempotency-Key':uuid.uuid4().hex}) + registry = json.loads(compose('exec', '-T', 'dispatcher', 'python', '-m', 'src.pipeline.admin', 'failed')) + check(any(x['job_id'] == j['id'] for x in registry), 'failed registry missing') + set_fault(att, None) + compose('exec', '-T', 'dispatcher', 'python', '-m', 'src.pipeline.admin', 'replay', j['id'], '--operator', 'lab5-test', '--reason', 'dependency restored') + c.wait_ready(alice, att) + check(sql("SELECT count(*) FROM job_replays WHERE job_id='" + j['id'] + "';") == '1', 'audit missing') + compose('exec', '-T', 'dispatcher', 'python', '-m', 'src.pipeline.admin', 'replay', j['id'], '--operator', 'lab5-test', '--reason', 'duplicate replay', expected=1) + passed(fault + '_failed_audited_replay', attempts_before_replay=j['attempt']) + + wait(lambda: sql("SELECT count(*) FROM attachments WHERE status='ready' AND staging_key IS NOT NULL;") == '0', 'ready staging cleanup') + passed('staging_cleanup_after_ready') + + compose('stop', 'worker') + retained = upload(fixture((325, 205))) + set_fault(retained, 'permanent') + compose('start', 'worker') + wait(lambda: job(retained)['state'] == 'failed', 'failed retention fixture') + check(sql("SELECT staging_key IS NOT NULL FROM attachments WHERE id='" + retained['id'] + "';") == 't', 'fresh failed staging must remain') + sql("UPDATE attachments SET updated_at=clock_timestamp()-interval '8 days' WHERE id='" + retained['id'] + "';") + wait(lambda: sql("SELECT staging_key IS NULL FROM attachments WHERE id='" + retained['id'] + "';") == 't', 'failed retention cleanup') + compose('exec', '-T', 'dispatcher', 'python', '-m', 'src.pipeline.admin', 'replay', job(retained)['id'], '--operator', 'lab5-test', '--reason', 'expired source', expected=1) + passed('failed_retention_and_expired_replay_rejected') + + compose('stop', 'worker') + active = upload(fixture((326, 206))) + sql("UPDATE attachments SET updated_at=clock_timestamp()-interval '8 days' WHERE id='" + active['id'] + "';") + orphan = 'staging/' + str(uuid.uuid4()) + script = '''import io,sys +from src.core.storage import make_storage +from src.core.settings import settings +make_storage(role='worker').put_object(settings.s3_bucket,sys.argv[1],io.BytesIO(b'orphan'),6) +''' + compose('exec', '-T', 'dispatcher', 'python', '-c', script, orphan) + sql("INSERT INTO upload_intents(object_key,state,created_at) VALUES ('" + orphan + "','pending',clock_timestamp()-interval '8 days');") + wait(lambda: sql("SELECT state FROM upload_intents WHERE object_key='" + orphan + "';") == 'abandoned', 'orphan staging cleanup') + check(sql("SELECT staging_key IS NOT NULL FROM attachments WHERE id='" + active['id'] + "';") == 't', 'cleanup must protect queued work') + compose('start', 'worker') + c.wait_ready(alice, active) + passed('abandoned_staging_cleaned_active_job_protected') + + finally: + # Disable every injection and restore the ordinary configuration even on failure. + try: + sql('UPDATE attachment_jobs SET fault=NULL WHERE fault IS NOT NULL;') + finally: + subprocess.run(['docker','compose','--env-file',os.environ.get('ENV_FILE','.env'),'-p',args.project,'up','-d','--no-deps','--force-recreate','worker','dispatcher'], + cwd=ROOT, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=100, check=True) + override.unlink(missing_ok=True) + override.parent.rmdir() + + +if __name__ == '__main__': + run(main) diff --git a/tests/lab5_limits.py b/tests/lab5_limits.py new file mode 100644 index 0000000..1c55156 --- /dev/null +++ b/tests/lab5_limits.py @@ -0,0 +1,44 @@ +"""Run in Linux worker: actual decoder failures and configured resource ceilings.""" +import io +import json +import os +import subprocess +import sys +from pathlib import Path +from tempfile import TemporaryDirectory + +from PIL import Image + +from src.core.settings import settings + + +def main(): + with TemporaryDirectory() as folder: + root = Path(folder) + source = root / 'input' + def process(data, max_pixels=40000000, memory=768): + source.write_bytes(data) + return subprocess.run([sys.executable,'-m','src.pipeline.images',str(source),str(root), + str(max_pixels),str(memory),'5'],capture_output=True,timeout=10, + env=dict(os.environ, PIPELINE_TEST_IMAGE_DELAY='0')).returncode + buffer = io.BytesIO() + Image.new('RGB',(1000,1000),'red').save(buffer,'PNG') + data = buffer.getvalue() + assert process(b'not an image') == 2 + assert process(data[:33]) == 2 + assert process(data, max_pixels=100) == 2 + assert process(data, memory=1) != 0 + assert process(data) == 0 + # Non-square integer-ratio rounding and independent process are exercised + # by HTTP tests. These values prove the actual stand configuration too. + assert settings.worker_memory_mb == 768 + assert settings.worker_concurrency == 1 + print(json.dumps(dict(result='PASS', invalid_image_rejected=True, + truncated_image_rejected=True, decoded_pixel_limit=True, + address_space_limit=True, subprocess_memory_mb=settings.worker_memory_mb, + wall_timeout_seconds=settings.job_timeout_seconds, + concurrency=settings.worker_concurrency, prefetch_per_consumer=1))) + + +if __name__ == '__main__': + main() diff --git a/tests/lab5_migration.py b/tests/lab5_migration.py new file mode 100644 index 0000000..d173c71 --- /dev/null +++ b/tests/lab5_migration.py @@ -0,0 +1,90 @@ +"""Run inside worker: migrate a real lab4 database and process its existing file.""" +import asyncio +import hashlib +import io +import json +import os +import subprocess +from urllib.parse import urlsplit, urlunsplit +from uuid import uuid4 + +import asyncpg +from PIL import Image +from pydantic import SecretStr +from sqlalchemy import select +from sqlalchemy.ext.asyncio import async_sessionmaker + +from src.core.db.models import Attachment, AttachmentJob +from src.core.db.session import create_engine +from src.core.settings import settings +from src.core.storage import make_storage +from src.pipeline.dispatcher import cleanup +from src.pipeline.worker import Envelope, process_job + + +async def main(): + previous = settings.database_url.get_secret_value() + database = 'migration_' + uuid4().hex + parts = urlsplit(previous) + url = urlunsplit(parts._replace(path='/' + database)) + control = await asyncpg.connect(previous) + storage = make_storage(role='worker') + attachment_id, user_id, chat_id = uuid4(), uuid4(), uuid4() + key = 'originals/' + str(attachment_id) + engine = None + try: + await control.execute('CREATE DATABASE ' + database) + env = dict(os.environ, DATABASE_URL=url) + def migrate(target): + result = subprocess.run(['alembic', 'upgrade', target], env=env, capture_output=True) + assert result.returncode == 0, 'migration failed: ' + result.stderr.decode()[-500:] + await asyncio.to_thread(migrate, '008') + buffer = io.BytesIO() + Image.new('RGB', (320, 200), 'orange').save(buffer, 'PNG') + data = buffer.getvalue() + await asyncio.to_thread(storage.put_object, settings.s3_bucket, key, io.BytesIO(data), len(data), 'image/png') + connection = await asyncpg.connect(url) + try: + await connection.execute("INSERT INTO users (id,username,display_name,password_hash,created_at) VALUES ($1,'migration_user','Migration','unused',now())", user_id) + await connection.execute("INSERT INTO chats (id,title,created_at) VALUES ($1,'Migration',now())", chat_id) + await connection.execute("INSERT INTO chat_members (chat_id,user_id) VALUES ($1,$2)", chat_id, user_id) + await connection.execute("""INSERT INTO attachments + (id,chat_id,owner_id,filename,object_key,content_type,size_bytes,sha256,created_at,updated_at) + VALUES ($1,$2,$3,'legacy.png',$4,'image/png',$5,$6,now(),now())""", + attachment_id, chat_id, user_id, key, len(data), hashlib.sha256(data).hexdigest()) + finally: + await connection.close() + await asyncio.to_thread(migrate, 'head') + settings.database_url = SecretStr(url) + engine = create_engine() + factory = async_sessionmaker(engine, expire_on_commit=False) + async with factory() as session: + attachment = await session.get(Attachment, attachment_id) + job = await session.scalar(select(AttachmentJob).where(AttachmentJob.attachment_id == attachment_id)) + assert attachment.status == 'queued' and attachment.staging_key == key + message = Envelope(schema_version=1,job_id=str(job.id),type='attachment.process.v1', + attachment_id=str(attachment_id),pipeline_version=1,attempt=1,created_at=job.created_at.isoformat()) + assert await process_job(factory, message) + await cleanup(factory) + async with factory() as session: + attachment = await session.get(Attachment, attachment_id) + assert attachment.status == 'ready' and len(attachment.variants) == 3 + assert attachment.staging_key is None + response = storage.get_object(settings.s3_bucket, key) + try: + assert response.read() == data, 'cleanup removed/replaced legacy original' + finally: + response.close(); response.release_conn() + print(json.dumps(dict(result='PASS', migration='008 -> 009', legacy_original_unchanged=True, + variants=3, staging_cleanup_safe=True))) + finally: + settings.database_url = SecretStr(previous) + if engine: + await engine.dispose() + await control.execute('DROP DATABASE IF EXISTS ' + database + ' WITH (FORCE)') + await control.close() + for object_key in [key] + ['variants/' + str(attachment_id) + '/v1/' + name for name in ('thumbnail','crop','optimized')]: + await asyncio.to_thread(storage.remove_object, settings.s3_bucket, object_key) + + +asyncio.run(main()) diff --git a/tests/observability.py b/tests/observability.py new file mode 100644 index 0000000..0cf56c3 --- /dev/null +++ b/tests/observability.py @@ -0,0 +1,207 @@ +"""Short real-backend check; optional bounded incidents for the grade-4 evidence.""" +import json +import os +import subprocess +import time +import urllib.parse +import urllib.request +import uuid +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +from client import Client, Failure, arguments, check, run + +ROOT=Path(__file__).resolve().parents[1] +EVIDENCE=ROOT/'evidence' + + +def get(url): + request=urllib.request.Request(url,headers={'Accept':'application/json'}) + with urllib.request.urlopen(request,timeout=5) as response: + return json.load(response) + + +def wait(fn, label, seconds=35): + deadline=time.monotonic()+seconds + error=None + while time.monotonic()0,'transient retry',25) + compose('unpause','minio');minio_paused=False + wait(lambda:c.call('GET','/health/ready')['status']==200,'API restored') + c.wait_ready(alice,queued) + def retry_chain(): + d=trace_data(retry_trace) + return d if sum(span['name']=='queue.consume' for span in spans(d))>=2 else None + retry_data=wait(retry_chain,'retry chain') + (EVIDENCE/'lab6-retry-trace.json').write_text(json.dumps(retry_data,indent=2)) + record('retry_keeps_upload_trace',trace_id=retry_trace,consumer_spans=sum(span['name']=='queue.consume' for span in spans(retry_data))) + wait(lambda:value('max(queue_depth)')==0 and value('max(ALERTS{alertname="QueueLagGrowing",alertstate="firing"})')==0,'queue alert resolved') + record('incident_worker_resolved') + # Block INSERT while SELECT readiness remains available. This isolates + # DB latency from application CPU/HTTP handler latency. + api=compose('ps','-q','api').split()[0] + code='''import asyncio,asyncpg,sys +from src.core.settings import settings +async def main(): + c=await asyncpg.connect(settings.database_url.get_secret_value()) + await c.execute('BEGIN; LOCK TABLE messages IN SHARE MODE') + print('LOCKED',flush=True) + await asyncio.to_thread(sys.stdin.readline) + await c.execute('ROLLBACK');await c.close() +asyncio.run(main())''' + locker=subprocess.Popen(['docker','exec','-i',api,'python','-u','-c',code],stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.DEVNULL,text=True) + check(locker.stdout.readline().strip()=='LOCKED','DB fault not active') + def failing(_): + return Client(c.base,timeout=12).call('POST','/api/v1/chats/'+chat['id']+'/messages',503, + actor=alice,body={'text':'bounded DB incident'},headers={'Idempotency-Key':uuid.uuid4().hex}) + with ThreadPoolExecutor(max_workers=4) as pool:errors=list(pool.map(failing,range(12))) + wait(lambda:value('max(ALERTS{alertname="ApiHighErrorRate",alertstate="firing"})')==1,'error alert firing',20) + db_trace=wait(lambda: trace_data(errors[0]['headers']['X-Trace-Id']),'DB trace') + failures=[span for span in spans(db_trace) if span['name'].startswith('db.') and span.get('status',{}).get('code') in (2,'STATUS_CODE_ERROR')] + check(failures,'DB error span missing') + db_logs=wait(lambda:logs(event='db.error'),'DB error logs') + (EVIDENCE/'lab6-db-incident-trace.json').write_text(json.dumps(db_trace,indent=2)) + (EVIDENCE/'lab6-db-incident-logs.json').write_text(json.dumps(db_logs,indent=2)) + record('incident_DB_firing',errors=len(errors),db_error_spans=len(failures), + diagnosis='db.INSERT timeout, matching db.error event; API readiness remains available') + locker.stdin.write('release\n');locker.stdin.flush();locker.wait(timeout=10);locker=None + c.message(alice,chat,'DB recovered') + wait(lambda:value('max(ALERTS{alertname="ApiHighErrorRate",alertstate="firing"})')==0,'error alert resolved',40) + record('incident_DB_resolved') + compose('stop','-t','2','tempo');tempo_stopped=True + started=time.monotonic() + for _ in range(15):c.message(alice,chat,'telemetry outage') + other=c.attachment(alice,chat)[0];c.wait_ready(alice,other) + # Bounded retry is 10s; wait only for an observed failure, not a benchmark. + wait(lambda:value('sum(otelcol_exporter_send_failed_spans_total)')>0 or value('sum(otelcol_exporter_send_failed_spans)')>0,'export failure counter',20) + usage=json.loads(compose('exec','-T','api','python','-c','from pathlib import Path;import json;print(json.dumps({"bytes":int(Path("/sys/fs/cgroup/memory.current").read_text()),"limit":int(Path("/sys/fs/cgroup/memory.max").read_text())}))').splitlines()[0]) + check(usage['bytes']