From eafb825657bb82ac91eb90ba4c79b4ddbf07945d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9C=D0=B0=D1=82=D0=B2=D0=B5=D0=B9?= Date: Sat, 7 Nov 2026 13:15:01 +0300 Subject: [PATCH 1/3] add lab6 --- Dockerfile | 23 + REPORT.md | 6 +- compose.yaml | 367 ++++++++ docs/observability.md | 26 + docs/review.md | 36 + docs/slo-runbook.md | 45 + infra/filer.toml | 3 + infra/grafana/dashboards/messenger.json | 543 +++++++++++ .../provisioning/dashboards/dashboards.yaml | 8 + .../provisioning/datasources/datasources.yaml | 25 + infra/haproxy.cfg | 35 + infra/loki/loki.yaml | 17 + infra/otel/collector.yaml | 32 + infra/prometheus/alerts.yaml | 39 + infra/prometheus/prometheus.yaml | 7 + infra/rabbitmq-secret-entrypoint.sh | 12 + infra/tempo/tempo.yaml | 13 + migrations/001_initial.sql | 37 + migrations/002_history_cache_version.sql | 1 + migrations/003_idempotency.sql | 12 + migrations/004_attachments.sql | 18 + migrations/005_async_attachments.sql | 34 + migrations/006_level4_reliability.sql | 17 + migrations/007_job_measurements.sql | 1 + migrations/008_trace_context.sql | 2 + migrations/009_slo_commits.sql | 11 + pyproject.toml | 17 + scripts/audit.py | 70 ++ scripts/init-secrets.py | 39 + scripts/record-environment.py | 44 + scripts/restart.sh | 10 + scripts/run-track1-profiles.py | 31 + scripts/stop-one.sh | 18 + scripts/track1.py | 217 +++++ scripts/verify-persistence.py | 27 + scripts/verify-resilience.py | 48 + src/__init__.py | 1 + src/attachments.py | 141 +++ src/bootstrap_s3.py | 51 + src/broker.py | 90 ++ src/config.py | 125 +++ src/cursor.py | 81 ++ src/errors.py | 12 + src/main.py | 189 ++++ src/migrate.py | 18 + src/models.py | 151 +++ src/outbox.py | 46 + src/pipeline_metrics.py | 37 + src/reconcile.py | 104 ++ src/render.py | 27 + src/replay.py | 24 + src/routes.py | 185 ++++ src/storage/__init__.py | 2 + src/storage/postgres.py | 594 ++++++++++++ src/storage/s3.py | 123 +++ src/telemetry.py | 118 +++ src/worker.py | 265 ++++++ student-tests/alerts.test.yaml | 27 + student-tests/attachment_resilience.py | 120 +++ student-tests/attachments.py | 113 +++ student-tests/burn_rules.test.yaml | 83 ++ student-tests/capture_observability.py | 41 + student-tests/check_dashboard.py | 18 + student-tests/check_retry_trace.py | 31 + student-tests/idempotency_concurrency.py | 32 + student-tests/job_contract.py | 31 + student-tests/pipeline_critical.py | 117 +++ student-tests/pipeline_integration.py | 205 ++++ student-tests/pipeline_invariants.py | 141 +++ student-tests/queue_incident.py | 11 + student-tests/readiness_failover.py | 89 ++ student-tests/readiness_unavailable.py | 16 + student-tests/reconcile.py | 152 +++ student-tests/reconcile_audit.py | 159 ++++ student-tests/reconcile_benchmark.py | 74 ++ student-tests/remaining_regressions.py | 127 +++ student-tests/retry_evidence.py | 30 + student-tests/review_checks.py | 22 + student-tests/review_queries.py | 13 + student-tests/review_regressions.py | 32 + student-tests/review_resilience.py | 20 + student-tests/s3_permissions.py | 54 ++ student-tests/session_expiry.py | 20 + student-tests/slo_analyze.py | 40 + student-tests/slo_commit_integration.py | 45 + student-tests/slo_incidents.py | 123 +++ student-tests/slo_ready_incident.py | 74 ++ student-tests/telemetry_alerts.test.yaml | 31 + student-tests/telemetry_unit.py | 106 +++ student-tests/tempo_outage.py | 60 ++ student-tests/trace_retry.py | 27 + student-tests/upload_memory.py | 106 +++ uv.lock | 885 ++++++++++++++++++ 93 files changed, 7549 insertions(+), 1 deletion(-) create mode 100644 Dockerfile create mode 100644 compose.yaml create mode 100644 docs/observability.md create mode 100644 docs/review.md create mode 100644 docs/slo-runbook.md create mode 100644 infra/filer.toml create mode 100644 infra/grafana/dashboards/messenger.json create mode 100644 infra/grafana/provisioning/dashboards/dashboards.yaml create mode 100644 infra/grafana/provisioning/datasources/datasources.yaml create mode 100644 infra/haproxy.cfg create mode 100644 infra/loki/loki.yaml create mode 100644 infra/otel/collector.yaml create mode 100644 infra/prometheus/alerts.yaml create mode 100644 infra/prometheus/prometheus.yaml create mode 100644 infra/rabbitmq-secret-entrypoint.sh create mode 100644 infra/tempo/tempo.yaml create mode 100644 migrations/001_initial.sql create mode 100644 migrations/002_history_cache_version.sql create mode 100644 migrations/003_idempotency.sql create mode 100644 migrations/004_attachments.sql create mode 100644 migrations/005_async_attachments.sql create mode 100644 migrations/006_level4_reliability.sql create mode 100644 migrations/007_job_measurements.sql create mode 100644 migrations/008_trace_context.sql create mode 100644 migrations/009_slo_commits.sql create mode 100644 pyproject.toml create mode 100644 scripts/audit.py create mode 100644 scripts/init-secrets.py create mode 100644 scripts/record-environment.py create mode 100755 scripts/restart.sh create mode 100644 scripts/run-track1-profiles.py create mode 100755 scripts/stop-one.sh create mode 100755 scripts/track1.py create mode 100644 scripts/verify-persistence.py create mode 100644 scripts/verify-resilience.py create mode 100644 src/__init__.py create mode 100644 src/attachments.py create mode 100644 src/bootstrap_s3.py create mode 100644 src/broker.py create mode 100644 src/config.py create mode 100644 src/cursor.py create mode 100644 src/errors.py create mode 100644 src/main.py create mode 100644 src/migrate.py create mode 100644 src/models.py create mode 100644 src/outbox.py create mode 100644 src/pipeline_metrics.py create mode 100644 src/reconcile.py create mode 100644 src/render.py create mode 100644 src/replay.py create mode 100644 src/routes.py create mode 100644 src/storage/__init__.py create mode 100644 src/storage/postgres.py create mode 100644 src/storage/s3.py create mode 100644 src/telemetry.py create mode 100644 src/worker.py create mode 100644 student-tests/alerts.test.yaml create mode 100644 student-tests/attachment_resilience.py create mode 100644 student-tests/attachments.py create mode 100644 student-tests/burn_rules.test.yaml create mode 100644 student-tests/capture_observability.py create mode 100644 student-tests/check_dashboard.py create mode 100644 student-tests/check_retry_trace.py create mode 100755 student-tests/idempotency_concurrency.py create mode 100644 student-tests/job_contract.py create mode 100644 student-tests/pipeline_critical.py create mode 100644 student-tests/pipeline_integration.py create mode 100644 student-tests/pipeline_invariants.py create mode 100644 student-tests/queue_incident.py create mode 100644 student-tests/readiness_failover.py create mode 100644 student-tests/readiness_unavailable.py create mode 100644 student-tests/reconcile.py create mode 100644 student-tests/reconcile_audit.py create mode 100644 student-tests/reconcile_benchmark.py create mode 100644 student-tests/remaining_regressions.py create mode 100644 student-tests/retry_evidence.py create mode 100644 student-tests/review_checks.py create mode 100644 student-tests/review_queries.py create mode 100644 student-tests/review_regressions.py create mode 100644 student-tests/review_resilience.py create mode 100644 student-tests/s3_permissions.py create mode 100644 student-tests/session_expiry.py create mode 100644 student-tests/slo_analyze.py create mode 100644 student-tests/slo_commit_integration.py create mode 100644 student-tests/slo_incidents.py create mode 100644 student-tests/slo_ready_incident.py create mode 100644 student-tests/telemetry_alerts.test.yaml create mode 100644 student-tests/telemetry_unit.py create mode 100644 student-tests/tempo_outage.py create mode 100644 student-tests/trace_retry.py create mode 100644 student-tests/upload_memory.py create mode 100644 uv.lock diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..ab272d3 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,23 @@ +FROM python:3.12.12-slim + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 + +WORKDIR /app + +RUN pip install --no-cache-dir uv==0.11.28 + +COPY pyproject.toml uv.lock ./ +RUN uv sync --frozen --no-dev --no-install-project + +COPY src ./src +COPY migrations ./migrations + +USER 65534:65534 + +EXPOSE 8080 + +HEALTHCHECK --interval=10s --timeout=3s --start-period=5s --retries=3 \ + CMD ["/app/.venv/bin/python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2).read()"] + +CMD ["/app/.venv/bin/python", "-m", "src.main"] diff --git a/REPORT.md b/REPORT.md index 9801db4..e85e1d1 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,6 +1,6 @@ # Отчёт — лабораторная 7 -- Автор, группа: TODO +- Автор, группа: Kozyrev, BVT2301 - Целевая оценка; выбранный трек 5, если нужен: TODO - Commit/PR: TODO после публикации - Использованные источники, библиотеки и помощь ИИ: TODO @@ -11,6 +11,10 @@ TODO: версии runtime/образов, prerequisites, команды соз ## Архитектура и решения +За основу перенесена реализация lab6 из локального репозитория, commit `7fe48e3`: исходники, миграции, Dockerfile, Compose, инфраструктурные конфигурации, собственные тесты и вспомогательные скрипты. Контракт и публичные проверки lab7 сохранены. Секреты и данные стенда не переносились. + +Перенесённая конфигурация пока относится к lab6: она использует HTTP на ряде соединений и не реализует требования lab7 к TLS/mTLS и OpenBao. Её необходимо переработать до проверки lab7; примеры и тесты lab6 могут содержать адреса HTTP. Работоспособность перенесённого стенда после копирования ещё не подтверждена. + TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab6, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. ## Доказательства критериев diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..4d58c8e --- /dev/null +++ b/compose.yaml @@ -0,0 +1,367 @@ +x-api: &api + build: + context: . + environment: + APP_HOST: 0.0.0.0 + APP_PORT: 8080 + APP_ENV: lab + OTEL_ENABLED: "1" + OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318 + DATABASE_URL_FILE: /run/secrets/database_url + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400} + CURSOR_SECRET_FILE: /run/secrets/cursor_secret + S3_ENDPOINT: http://s3:8333 + S3_BUCKET: messenger-attachments + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + BROKER_URL_FILE: /run/secrets/broker_url + MAX_UPLOAD_BYTES: "10485760" + MAX_IMAGE_PIXELS: "40000000" + UPLOAD_CONCURRENCY: "1" + UPLOAD_TIMEOUT_SECONDS: "30" + UPLOAD_FAULT_AFTER_S3_PUT: ${UPLOAD_FAULT_AFTER_S3_PUT:-0} + OUTBOX_FAULT_AFTER_COMMIT: ${OUTBOX_FAULT_AFTER_COMMIT:-0} + MAX_PENDING_JOBS: ${MAX_PENDING_JOBS:-100} + secrets: + - database_url + - cursor_secret + - s3_access_key_id + - s3_secret_access_key + - broker_url + depends_on: + s3-init: + condition: service_completed_successfully + migrate: + condition: service_completed_successfully + broker: + condition: service_healthy + expose: + - "8080" + stop_grace_period: 10s + healthcheck: + test: ["CMD", "/app/.venv/bin/python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2).read()"] + interval: 5s + timeout: 3s + retries: 12 + start_period: 10s + deploy: + resources: + limits: + cpus: "0.50" + memory: 512M + +services: + broker: + image: rabbitmq:4.1.2-alpine + environment: + RABBITMQ_DEFAULT_USER: messenger + entrypoint: ["/bin/sh", "/usr/local/bin/rabbitmq-secret-entrypoint.sh"] + command: ["rabbitmq-server"] + secrets: + - rabbitmq_password + volumes: + - rabbitmq_data:/var/lib/rabbitmq + - ./infra/rabbitmq-secret-entrypoint.sh:/usr/local/bin/rabbitmq-secret-entrypoint.sh:ro + healthcheck: + test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"] + interval: 5s + timeout: 3s + retries: 20 + start_period: 10s + deploy: + resources: + limits: + cpus: "0.50" + memory: 256M + + s3: + image: chrislusf/seaweedfs:4.17@sha256:186de7ef977a20343ee9a5544073f081976a29e2d29ecf8379891e7bf177fbe9 + command: ["server", "-dir=/data", "-ip=s3", "-ip.bind=0.0.0.0", "-s3", "-s3.config=/run/secrets/s3_config", "-master.volumeSizeLimitMB=128", "-volume.max=64", "-s3.port.iceberg=0", "-filer.exposeDirectoryData=false"] + secrets: + - s3_config + volumes: + - s3_data:/data + - ./infra/filer.toml:/etc/seaweedfs/filer.toml:ro + healthcheck: + test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://localhost:9333/cluster/status"] + interval: 5s + timeout: 3s + retries: 20 + start_period: 15s + deploy: + resources: + limits: + cpus: "1.0" + memory: 512M + + s3-init: + build: + context: . + command: ["/app/.venv/bin/python", "-m", "src.bootstrap_s3"] + environment: + S3_ENDPOINT: http://s3:8333 + S3_BUCKET: messenger-attachments + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_admin_key + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_admin_secret + secrets: + - s3_admin_key + - s3_admin_secret + depends_on: + s3: + condition: service_healthy + restart: "no" + + postgres: + image: postgres:17.6-alpine + command: ["postgres", "-c", "track_commit_timestamp=on"] + environment: + POSTGRES_DB: messenger + POSTGRES_USER: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + secrets: + - postgres_password + volumes: + - postgres_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"] + interval: 5s + timeout: 3s + retries: 12 + start_period: 5s + deploy: + resources: + limits: + cpus: "0.75" + memory: 512M + + migrate: + build: + context: . + command: ["/app/.venv/bin/python", "-m", "src.migrate"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + CURSOR_SECRET_FILE: /run/secrets/cursor_secret + secrets: + - database_url + - cursor_secret + depends_on: + postgres: + condition: service_healthy + restart: "no" + deploy: + resources: + limits: + cpus: "0.25" + memory: 128M + + reconcile: + healthcheck: + disable: true + build: + context: . + entrypoint: ["/app/.venv/bin/python", "-m", "src.reconcile"] + command: ["--apply", "--loop"] + restart: unless-stopped + environment: + DATABASE_URL_FILE: /run/secrets/database_url + CURSOR_SECRET_FILE: /run/secrets/cursor_secret + S3_ENDPOINT: http://s3:8333 + S3_BUCKET: messenger-attachments + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_admin_key + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_admin_secret + STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400} + secrets: + - database_url + - cursor_secret + - s3_admin_key + - s3_admin_secret + depends_on: + postgres: + condition: service_healthy + s3: + condition: service_healthy + deploy: + resources: + limits: + cpus: "0.25" + memory: 128M + + api-a: + <<: *api + + api-b: + <<: *api + + worker: + restart: unless-stopped + healthcheck: + test: ["CMD", "/app/.venv/bin/python", "-c", "import time,pathlib; assert time.time()-float(pathlib.Path('/tmp/worker-heartbeat').read_text()) < 15"] + interval: 5s + timeout: 3s + retries: 3 + deploy: + resources: + limits: + cpus: "2.0" + memory: 4G + build: + context: . + command: ["/app/.venv/bin/python", "-m", "src.worker"] + environment: + APP_ENV: lab + OTEL_ENABLED: "1" + OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318 + DATABASE_URL_FILE: /run/secrets/database_url + CURSOR_SECRET_FILE: /run/secrets/cursor_secret + S3_ENDPOINT: http://s3:8333 + S3_BUCKET: messenger-attachments + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + BROKER_URL_FILE: /run/secrets/broker_url + WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2} + WORKER_FAULT_AFTER_S3_PUT: ${WORKER_FAULT_AFTER_S3_PUT:-0} + RENDER_MEMORY_MB: ${RENDER_MEMORY_MB:-768} + WORKER_PREFETCH: ${WORKER_PREFETCH:-2} + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60} + STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400} + secrets: + - database_url + - cursor_secret + - s3_access_key_id + - s3_secret_access_key + - broker_url + depends_on: + s3-init: + condition: service_completed_successfully + migrate: + condition: service_completed_successfully + broker: + condition: service_healthy + stop_grace_period: 30s + + outbox: + healthcheck: + disable: true + build: + context: . + command: ["/app/.venv/bin/python", "-m", "src.outbox"] + environment: + APP_ENV: lab + OTEL_ENABLED: "1" + OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318 + DATABASE_URL_FILE: /run/secrets/database_url + CURSOR_SECRET_FILE: /run/secrets/cursor_secret + BROKER_URL_FILE: /run/secrets/broker_url + OUTBOX_DISPATCH_INTERVAL_SECONDS: "1" + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + secrets: + - database_url + - cursor_secret + - broker_url + depends_on: + migrate: + condition: service_completed_successfully + broker: + condition: service_healthy + restart: unless-stopped + + load_balancer: + image: haproxy:3.0.11-alpine + ports: + - "127.0.0.1:8080:8080" + volumes: + - ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + depends_on: + api-a: + condition: service_healthy + api-b: + condition: service_healthy + healthcheck: + test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:8080/health/ready"] + interval: 5s + timeout: 3s + retries: 12 + start_period: 5s + deploy: + resources: + limits: + cpus: "0.25" + memory: 128M + + otel-collector: + image: otel/opentelemetry-collector-contrib:0.120.0 + command: ["--config=/etc/otelcol/config.yaml"] + volumes: + - ./infra/otel/collector.yaml:/etc/otelcol/config.yaml:ro + mem_limit: 256m + restart: unless-stopped + prometheus: + image: prom/prometheus:v3.2.1 + command: ["--config.file=/etc/prometheus/prometheus.yaml", "--storage.tsdb.path=/prometheus", "--storage.tsdb.retention.time=7d"] + volumes: + - ./infra/prometheus/prometheus.yaml:/etc/prometheus/prometheus.yaml:ro + - ./infra/prometheus/alerts.yaml:/etc/prometheus/alerts.yaml:ro + - prometheus_data:/prometheus + restart: unless-stopped + loki: + image: grafana/loki:3.4.2 + command: ["-config.file=/etc/loki/loki.yaml"] + volumes: + - ./infra/loki/loki.yaml:/etc/loki/loki.yaml:ro + - loki_data:/loki + restart: unless-stopped + tempo: + image: grafana/tempo:2.7.1 + command: ["-config.file=/etc/tempo/tempo.yaml"] + volumes: + - ./infra/tempo/tempo.yaml:/etc/tempo/tempo.yaml:ro + - tempo_data:/var/tempo + restart: unless-stopped + grafana: + image: grafana/grafana:11.5.2 + ports: ["127.0.0.1:3000:3000"] + environment: + GF_SECURITY_ADMIN_USER: admin + GF_SECURITY_ADMIN_PASSWORD__FILE: /run/secrets/grafana_password + GF_USERS_ALLOW_SIGN_UP: "false" + secrets: [grafana_password] + volumes: + - grafana_data:/var/lib/grafana + - ./infra/grafana/provisioning:/etc/grafana/provisioning:ro + - ./infra/grafana/dashboards:/var/lib/grafana/dashboards:ro + restart: unless-stopped + +secrets: + grafana_password: + file: ./secrets/grafana_password + s3_config: + file: ./secrets/s3.json + s3_admin_key: + file: ./secrets/s3_admin_key + s3_admin_secret: + file: ./secrets/s3_admin_secret + s3_access_key_id: + file: ./secrets/s3_access_key_id + s3_secret_access_key: + file: ./secrets/s3_secret_access_key + postgres_password: + file: ./secrets/postgres_password + database_url: + file: ./secrets/database_url + cursor_secret: + file: ./secrets/cursor_secret + rabbitmq_password: + file: ./secrets/rabbitmq_password + broker_url: + file: ./secrets/broker_url + +volumes: + prometheus_data: + loki_data: + tempo_data: + grafana_data: + s3_data: + postgres_data: + rabbitmq_data: diff --git a/docs/observability.md b/docs/observability.md new file mode 100644 index 0000000..9318302 --- /dev/null +++ b/docs/observability.md @@ -0,0 +1,26 @@ +# Наблюдаемость и проверка lab6 + +## Сигналы и назначение + +API, worker и outbox отправляют OTLP/HTTP в Collector. Prometheus хранит метрики, Loki — логи, Tempo — трассы, Grafana предоставляет запросы и dashboard. Данные хранятся в именованных volumes, retention для Prometheus/Loki/Tempo — 7 дней. Grafana доступна на `http://localhost:3000`, пользователь `admin`, пароль находится в локальном `secrets/grafana_password`. Provisioning импортирует источники и dashboard `Messenger — observability`. + +Сохраняются только выбранные поля логов; трассы не содержат SQL, аргументов запросов, заголовков авторизации, имён файлов и тел. Метрики не размечаются ID пользователей, чатов и вложений. В экспортируемых логах доступен `request_id`, а `trace_id`/`span_id` передаются полями OTLP. В Explore Loki запрос `{service_name="messenger-api"}` находит события API, `{service_name="messenger-worker"}` — worker; в метаданных события доступны ID трассы. По ним можно перейти в Tempo. Локальные stdout-логи содержат JSON с теми же trace/span ID для событий телеметрии. + +При upload текущий W3C trace context сохраняется в `attachment_jobs` одной транзакцией с outbox. Dispatcher извлекает его и публикует сообщение с дочерним producer span и AMQP headers; worker извлекает контекст и создаёт consumer span. Повторная публикация и retry используют сохранённый контекст исходной загрузки. Каждая попытка имеет собственные spans; они являются потомками исходной транзакции загрузки. Baggage не переносится. Контекст — технические данные, не содержимое изображения. + +Dashboard использует счётчики и гистограммы с единицами секунд: p50/p95/p99 HTTP вычисляются через `histogram_quantile` по сумме buckets, не по среднему перцентилей. `messenger_job_ready_duration_seconds` измеряет разность commit публикации ready и commit принятия задания из `attachment_slo` (миграция 009). Оба момента берутся из PostgreSQL; часы worker не участвуют. Гистограмма записывается после успешного commit: аварийное завершение worker между commit и записью метрики может потерять этот histogram sample; DB-когорта SLO при этом сохраняется. Queue depth и age — число и возраст самого старого задания в состоянии queued из PostgreSQL, включая ожидающие retry; это не длина очереди RabbitMQ. Данные обновляет outbox. DB pool отражает пул каждой API-реплики; число ожидающих читается из внутренней очереди asyncpg 0.31.0 и зависит от этой версии. Worker busy/capacity — число занятых executor threads и их лимит; повторная доставка без нового результата имеет outcome `skipped`, успешный commit — `success`, ошибка — `failed`. CPU и RSS — процессы API/worker/outbox, без дочерних процессов render. Queue-метрики имеют label `queue="attachment.process.v1"`. `messenger_queue_sample_time_seconds` показывает время последнего успешного обновления; `MessengerQueueTelemetryStale` сигнализирует об устаревании/отсутствии ряда. + +SLI: доля ответов 5xx среди всех HTTP бизнес-запросов API, исключая health. Целевой SLO: не менее 99% запросов без 5xx за скользящие 24 часа; ошибки балансировщика, которые не дошли до API, SLI не учитывает. Целевое значение выбрано для учебного стенда, а не по измеренным требованиям пользователей. Второй SLO и расчёт burn-rate описаны в [runbook](slo-runbook.md); его DB-счётчики учитывают и невыполненные в срок задания. Alert `MessengerHighErrorRate` срабатывает при >5% за минуту в течение 10 секунд; это оперативный порог, не burn-rate alert. `MessengerQueueLag` срабатывает при возрасте queued >30 секунд в течение 15 секунд. Состояния доступны в Prometheus через Grafana Explore или локальный API Prometheus внутри контейнера. + +Collector использует memory limiter 192 MiB, batch до 1024 записей, очереди экспортёров по 128 batches и retry до 60 секунд; лимит контейнера 256 MiB. SDK использует ограниченные очереди spans/logs по 2048 и timeout экспорта 2 секунды. При долгом отказе хранилища возможна потеря телеметрии; это не должно блокировать бизнес-обработку. Наблюдайте внутренние метрики `otelcol_exporter_queue_size`, `otelcol_exporter_send_failed_spans_total`, `otelcol_exporter_enqueue_failed_spans_total`, `otelcol_exporter_send_failed_log_records_total` и `otelcol_receiver_refused_*`. Очереди не сохраняются после перезапуска Collector. + +## Проверки на локальном учебном стенде + +Приведённые сценарии воспроизведены на локальном стенде; зафиксированные результаты и ограничения — в `evidence/observability.md`. Повторный запуск создаёт новые синтетические данные. + +1. Выполните `python3 scripts/init-secrets.py`, `docker compose up -d --build`, дождитесь readiness и запустите `make test`. В Grafana найдите событие API и worker, раскройте trace_id, проверьте дерево upload → DB/outbox → producer → consumer → S3/DB и убедитесь, что содержимое/credentials не попали в telemetry. Для повторной попытки используйте имеющийся fault injection worker и проверьте новую попытку в том же trace. +2. Очередь: `docker compose stop worker`, затем `python3 student-tests/queue_incident.py`. Смотрите queue age и alert; после перехода в firing выполните `docker compose start worker`, дождитесь уменьшения backlog и resolved. Время и скриншоты/запросы сохраните в evidence. +3. DB: начните `python3 tests/load.py --duration 180 --concurrency 8 --output evidence/db-incident-load.json`; после завершения bootstrap в отдельном терминале выполните `docker compose exec -T postgres psql -U messenger -d messenger -c "BEGIN; LOCK TABLE users IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep(90); COMMIT;"`. Наблюдайте задержку `postgres.user_for_token`, ошибки и alert; после окончания блокировки проверьте восстановление и resolved. Тест работает с синтетическими данными и намеренно мешает запросам. +4. Отказ telemetry backend: `docker compose stop tempo`, затем `make test` и контролируемая нагрузка; сравните `docker stats --no-stream`, queue size и failed exports до/во время отказа. Восстановите `docker compose start tempo` и проверьте восстановление экспорта. Длительный отказ может привести к потере traces; измерьте её, а не предполагайте отсутствие потерь. + +Для каждого сценария фиксируйте UTC-время воздействия, исходный профиль нагрузки, запросы метрик, firing/resolved, trace/log и результат после восстановления. `docker compose down` сохраняет volumes; не используйте `down -v`, если нужны сохранённые наблюдения. diff --git a/docs/review.md b/docs/review.md new file mode 100644 index 0000000..20489b9 --- /dev/null +++ b/docs/review.md @@ -0,0 +1,36 @@ +# Проверка по критериям README + +Проверялись [README](../README.md), [контракт наблюдаемости](../contracts/observability.md), исходный [REPORT](../REPORT.md), код и локальный Docker-стенд. Автоматический smoke не выставляет оценку за наблюдаемость. + +## Итог по уровням + +- **3: подтверждён после исправлений.** [Короткие проверки](../evidence/review-checks.json), [публичные persistence/failover](../evidence/review-resilience.json), [коррелированные логи API/worker и trace](../evidence/review-retry-trace.json). +- **4: подтверждён в проверенном локальном стенде.** [Разбор инцидентов и отказа Tempo](../evidence/review-observability.md), [QueueLag/ready](../evidence/slo-ready-incident.json), [проверка retry-дерева](../evidence/review-retry-summary.json), [сохранность trace после рестарта Tempo](../evidence/review-tempo-persistence.json). +- **5: техническая часть трека 2 проверена, полный зачёт не подтверждён.** [Повторные измерения](../evidence/slo-track.md) и runbook есть; независимый диагноз другого студента не задокументирован. Итоговая подтверждённая оценка этой проверки — **4**. + +## Исправления + +| Уровень | Найденная ошибка | Исправление и проверка | +| --- | --- | --- | +| 3 | Allowlist применялся только к stdout; OTLP LoggingHandler получал произвольные extra и exception | Фильтр до обоих handlers; тест реального SDK log exporter с секретом в extra и exception — [telemetry_unit.py](../student-tests/telemetry_unit.py) | +| 3 | Произвольные HTTP method могли создавать новые metric labels | Стандартные методы и `_OTHER`; шаблон маршрута сохранён — [main.py](../src/main.py), [telemetry.py](../src/telemetry.py) | +| 3 | Повторная доставка без нового результата считалась успешной job | Отдельный исход `skipped`; `success` только после commit ready — [worker.py](../src/worker.py), [pipeline_critical.py](../student-tests/pipeline_critical.py) | +| 4 | Исключение publish оставляло producer span без ERROR | ERROR без экспорта текста исключения; regression test — [broker.py](../src/broker.py), [telemetry_unit.py](../student-tests/telemetry_unit.py) | +| 4 | Histogram ready начинался до durable acceptance и использовал часы worker | Оба конца — commit timestamps PostgreSQL; [миграция 009](../migrations/009_slo_commits.sql), [изолированный интеграционный тест](../student-tests/slo_commit_integration.py) | +| 4 | Queue gauge не имел queue name и мог незаметно устареть | Label queue, timestamp последнего обновления и stale/absent alert; [telemetry.py](../src/telemetry.py), [правила](../infra/prometheus/alerts.yaml) | +| 4 | Потери Tempo приписывались накопленным счётчикам без baseline | Повторяемый опыт с baseline, дельтами, ID Collector и замерами памяти — [tempo_outage.py](../student-tests/tempo_outage.py) | +| 5 | Ready SLO зависел от изменяемых status/updated_at | Отдельная зрелая когорта с однократно зафиксированным bad; retry/replay не меняют результат — [postgres.py](../src/storage/postgres.py) | +| 5 | Старые временные результаты использовались при расхождении UTC/monotonic | Старые отсчёты сохранены в [pre-review](../evidence/pre-review/slo-analysis.json); анализатор выявил расхождение 83,075 с и пометил их непригодными для elapsed-time выводов; причина не установлена | +| 5 | При исключении сценарий мог зависнуть, ожидая бесконечные клиентские потоки | STOP устанавливается до выхода из ThreadPoolExecutor, включая исключения — [slo_incidents.py](../student-tests/slo_incidents.py) | + +Дополнительный прогон выявил дефект теста очереди: требовалось одновременное firing Fast и QueueLag, хотя их окна различаются. Кроме того, выбирался первый metric series старого outbox после пересоздания. [Неудачный прогон](../evidence/pre-review/ready-scenario-failed.json) сохранён; сценарий теперь проверяет наблюдение обоих сигналов и снятие каждого, а абсолютные DB-счётчики читает через max по экземплярам (они представляют одну БД). + +## Ограничения, которые нельзя скрыть оценкой + +- Прохождение runbook другим студентом не подтверждено. Подготовлен [протокол независимой проверки](slo-runbook.md#проверка-другим-студентом); собственная проверка не заменяет внешнюю. +- 99%/24 часа — учебная цель; короткий опыт не доказывает выполнение суточного SLO. Baseline оценивает наблюдавшиеся ложные тревоги только за этот интервал. +- Старые задания до миграции 009 не имеют восстановимых commit timestamps и исключены из новой когорты. Исторические Prometheus-ряды до миграции нельзя смешивать с новой методикой для доказательства SLO. +- PostgreSQL не хранит commit timestamps бессрочно. Outbox сохраняет их в таблицу; если потерял источник до материализации, unknown alert показывает неполноту данных. При перезапуске экспортёра `increase` может не учесть события в разрыве наблюдения. +- Histogram sample может потеряться при crash после commit ready; durable DB-когорта сохраняется. Collector с ограниченной очередью также может терять spans. Это не «нулевые потери». + +Источник границ commit timestamps: [PostgreSQL 17](https://www.postgresql.org/docs/17/functions-info.html#FUNCTIONS-COMMIT-TIMESTAMP). Источник критериев оценки — локальные README и contracts, ссылки выше. diff --git a/docs/slo-runbook.md b/docs/slo-runbook.md new file mode 100644 index 0000000..f6e2c07 --- /dev/null +++ b/docs/slo-runbook.md @@ -0,0 +1,45 @@ +# Runbook: расход бюджета SLO + +Используйте только учебный стенд с синтетическими данными. Grafana: `http://localhost:3000`, имя `admin`, пароль в `secrets/grafana_password`. На dashboard `Messenger — observability` находятся панели API/ready SLI и burn rates. В Explore выберите Prometheus; логи и traces доступны через Loki и Tempo. Правила — [alerts.yaml](../infra/prometheus/alerts.yaml). + +## Что означает сигнал + +Два учебных SLO на скользящие 24 часа: 99% бизнес-запросов API без 5xx и 99% принятых вложений становятся ready не позднее чем через 30 секунд после commit принятия задания. API исключает health и ошибки LB до попадания в приложение; 4xx остаются в знаменателе. Для готовности учтены только задания, достигшие 30-секундного возраста: неготовые, failed и завершённые позднее срока считаются плохими. Счётчики вычисляет outbox по PostgreSQL, поэтому при его недоступности данные не обновляются; события между последним scrape старого процесса и первым scrape нового могут не попасть в расчёт `increase`. Миграция 009 добавляет `attachment_slo`: моменты acceptance и первого ready берутся из commit timestamps PostgreSQL и сохраняются outbox. Результат зрелой когорты фиксируется один раз; retry/replay не меняют историю. Старые задания до миграции исключены, поскольку восстановить точные времена их приёма нельзя. `track_commit_timestamp=on` задан в Compose. PostgreSQL удаляет старые commit timestamps при vacuum; outbox материализует их раз в цикл. Если не успел, растёт `messenger_slo_unknown`, срабатывает `MessengerReadyTimestampUnknown`: неизвестные события нельзя считать доказательством выполнения SLO. Источник: [PostgreSQL 17, commit timestamps](https://www.postgresql.org/docs/17/functions-info.html#FUNCTIONS-COMMIT-TIMESTAMP). Порог 30 секунд — половина 60-секундного бюджета smoke-теста для одного файла при свободном worker; это учебная цель, не измеренное требование пользователей. Цель 99% также выбрана для демонстрации, а не выведена из пользовательского исследования. + +Error budget каждого SLO — `1 − 0,99 = 0,01`. Burn rate — доля плохих событий за окно / 0,01. Приближение расхода бюджета `burn × окно / 24 часа` предполагает стабильную интенсивность событий; при сильно меняющейся нагрузке бюджет, выраженный в доле событий за 24 часа, расходуется иначе. Значение 1 означает, что при сохранении текущей доли плохих событий все допустимые ошибки будут израсходованы за период SLO. + +- Fast: одновременно 1 минута и 20 секунд выше `28,8 × 0,01 = 28,8%`; в коротком окне не менее 5 событий. При постоянной интенсивности это `28,8 × 1 / 1440 = 0,02`, то есть 2% суточного бюджета за минуту. +- Slow: одновременно 5 минут и 1 минута выше `14,4 × 0,01 = 14,4%`; в коротком окне не менее 5 событий. При постоянной интенсивности это `14,4 × 5 / 1440 = 0,05`, то есть 5% бюджета за пять минут. + +В окне недостаточно событий — alert молчит; отсутствие ряда не является доказательством здоровья. Короткие окна и пороги выбраны для воспроизводимого лабораторного опыта, не являются рекомендацией для production. Источник метода multiwindow/multi-burn-rate и ограничений: [Google SRE Workbook](https://sre.google/workbook/alerting-on-slos/). Гистограмма: [Prometheus histograms](https://prometheus.io/docs/practices/histograms/). + +## API burn + +1. Убедитесь в наличии запросов: `sum(rate(messenger_http_requests_total{route!~"/health/.*"}[1m]))`. Отсутствие запросов требует отдельной проверки LB/readiness, а не вывода об отсутствии ошибок. +2. Сравните на dashboard 20s/1m/5m и долю 5xx по маршрутам: `sum by(route) (rate(messenger_http_requests_total{status_class="5xx"}[1m]))`. Если burn high, перейдите в Loki: `{service_name="messenger-api"} | status="503"`. Ограничьте диапазон временем alert. +3. Откройте trace из metadata лога в Tempo. Если `postgres.user_for_token` или другая DB-операция занимает большую часть HTTP span, проверьте подключение к PostgreSQL и блокировки (`pg_locks`, `pg_stat_activity`); не печатайте параметры SQL или credentials. Если вместо этого длителен S3 span — проверьте S3 и сеть. Проверьте также pool used/waiting. +4. Устраните причину в рамках полномочий: в учебном опыте дождитесь окончания контролируемой блокировки. Проверьте уменьшение error ratio, отсутствие активного burn alert и успешный `make test`. Сообщение об устранении возможно после короткого окна; долгая часть условия может ещё некоторое время оставаться выше порога. + +## Ready burn / очередь + +1. Сначала убедитесь, что `MessengerQueueTelemetryStale` и `MessengerReadyTimestampUnknown` не firing. Проверьте `sum(increase(messenger_slo_ready_eligible_total{service_name="messenger-outbox"}[1m]))`, p95 ready и исходы jobs. Счётчик `messenger_slo_ready_bad_total` включает задания, нарушившие срок. Отсутствие событий не делает SLI хорошим: проверьте `messenger_queue_depth{service_name="messenger-outbox"}` и `messenger_queue_age_seconds{service_name="messenger-outbox"}`. +2. В Loki откройте `{service_name="messenger-worker"}`, найдите событие и trace ID. В Tempo проверьте цепочку upload → producer → consumer → S3/DB; посмотрите busy/capacity и состояние broker/worker через `docker compose ps`. +3. В учебном опыте запустите остановленный worker, наблюдайте уменьшение глубины/возраста и новые ready. Проверьте resolved и метаданные вложений. Не удаляйте volumes и не запускайте replay вслепую. + +## Повторение экспериментов + +`python3 student-tests/slo_incidents.py` создаёт синтетические данные, ведёт три клиента и выполняет краткую и длительную блокировку таблицы `users`. Скрипт пишет временные отметки и наблюдения в [evidence/slo-incidents.json](../evidence/slo-incidents.json), не сохраняет tokens; опрашивает alert примерно каждые 5 секунд. `python3 student-tests/slo_analyze.py` строит сводку, включая срабатывания уже после завершения воздействия. Тест намеренно ухудшает локальный API и занимает несколько минут. Первая отметка воздействия — момент, когда скрипт увидел granted `AccessExclusiveLock`; окончание — возврат команды транзакции, поэтому обе оценки имеют задержку наблюдения. Анализатор сравнивает приращения UTC и монотонных часов и сохраняет максимальный фактический интервал опроса. Если `timing_quality.valid_for_elapsed_time_claims=false`, опыт повторяют; причину расхождения часов по этим данным установить нельзя. Аварийное прерывание клиентской команды может оставить server-side `pg_sleep` до окончания заданного ограниченного времени; проверьте состояние блокировки. + +`python3 student-tests/slo_ready_incident.py` проверяет отсутствие незавершённых заданий, останавливает worker, принимает восемь синтетических файлов и восстанавливает worker в `finally`. До запуска убедитесь, что API и worker свободны. Результат — [evidence/slo-ready-incident.json](../evidence/slo-ready-incident.json). Снятие alerts при малом количестве новых событий может означать недостаток событий в коротком окне, а не доказательство выполнения 24-часового SLO. + +Проверка правил без стенда и в ускоренном времени: + +```sh +docker run --rm -v "$PWD:/workspace:ro" --entrypoint promtool prom/prometheus:v3.2.1 test rules /workspace/student-tests/burn_rules.test.yaml +``` + +В runbook зафиксируйте фазу, UTC начала/окончания, первый firing и первое наблюдение после исчезновения alert. Первый и последний интервалы ограничены частотой опроса, а отсутствие ложного firing в одном baseline не оценивает долгосрочный false-positive rate. + +## Проверка другим студентом + +Проверяющий не получает команду инъекции заранее. Автор запускает один из двух ограниченных сценариев; проверяющий получает время и имя alert, затем проходит шаги выше. В отдельный протокол записывают: имя проверяющего, UTC, запрос PromQL/LogQL, trace ID, медленный span или отсутствие consumer, диагноз, действие восстановления и свидетельство resolved. Совпадение диагноза с инъекцией проверяют после записи. Пока такого заполненного протокола нет, независимое прохождение runbook не подтверждено. diff --git a/infra/filer.toml b/infra/filer.toml new file mode 100644 index 0000000..85b2149 --- /dev/null +++ b/infra/filer.toml @@ -0,0 +1,3 @@ +[leveldb2] +enabled = true +dir = "/data/filer" diff --git a/infra/grafana/dashboards/messenger.json b/infra/grafana/dashboards/messenger.json new file mode 100644 index 0000000..57e8548 --- /dev/null +++ b/infra/grafana/dashboards/messenger.json @@ -0,0 +1,543 @@ +{ + "uid": "messenger-lab6", + "title": "Messenger — observability", + "schemaVersion": 39, + "version": 1, + "refresh": "10s", + "time": { + "from": "now-30m", + "to": "now" + }, + "panels": [ + { + "id": 1, + "title": "API RPS", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "fieldConfig": { + "defaults": { + "unit": "reqps" + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "expr": "sum(rate(messenger_http_requests_total{route!~\"/health/.*\"}[1m]))", + "legendFormat": "{{service_name}} {{outcome}}" + } + ] + }, + { + "id": 2, + "title": "API error ratio", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit" + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "expr": "(sum(rate(messenger_http_requests_total{status_class=\"5xx\",route!~\"/health/.*\"}[1m])) or vector(0)) / clamp_min(sum(rate(messenger_http_requests_total{route!~\"/health/.*\"}[1m])),0.001) and sum(increase(messenger_http_requests_total{route!~\"/health/.*\"}[1m])) > 0", + "legendFormat": "{{service_name}} {{outcome}}" + } + ] + }, + { + "id": 3, + "title": "API latency p50 / p95 / p99", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "expr": "histogram_quantile(0.5, sum by (le) (rate(messenger_http_duration_seconds_bucket{route!~\"/health/.*\"}[5m])))", + "legendFormat": "p50" + }, + { + "refId": "B", + "expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_http_duration_seconds_bucket{route!~\"/health/.*\"}[5m])))", + "legendFormat": "p95" + }, + { + "refId": "C", + "expr": "histogram_quantile(0.99, sum by (le) (rate(messenger_http_duration_seconds_bucket{route!~\"/health/.*\"}[5m])))", + "legendFormat": "p99" + } + ] + }, + { + "id": 4, + "title": "Jobs by outcome", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "ops" + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "expr": "sum by (outcome) (rate(messenger_jobs_total[1m]))", + "legendFormat": "{{service_name}} {{outcome}}" + } + ] + }, + { + "id": 5, + "title": "Job duration p95", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_job_duration_seconds_bucket[5m])))", + "legendFormat": "{{service_name}} {{outcome}}" + } + ] + }, + { + "id": 6, + "title": "Acceptance to ready p95", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_job_ready_duration_seconds_bucket[5m])))", + "legendFormat": "{{service_name}} {{outcome}}" + } + ] + }, + { + "id": 7, + "title": "Queued jobs", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "expr": "max(messenger_queue_depth{service_name=\"messenger-outbox\"})", + "legendFormat": "{{service_name}} {{outcome}}" + } + ] + }, + { + "id": 8, + "title": "Oldest queued age", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "expr": "max(messenger_queue_age_seconds{service_name=\"messenger-outbox\"})", + "legendFormat": "{{service_name}} {{outcome}}" + } + ] + }, + { + "id": 9, + "title": "DB pool used / capacity / waiting", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 32 + }, + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "expr": "messenger_pool_used{service_name=\"messenger-api\"}", + "legendFormat": "used" + }, + { + "refId": "B", + "expr": "messenger_pool_capacity{service_name=\"messenger-api\"}", + "legendFormat": "capacity" + }, + { + "refId": "C", + "expr": "messenger_pool_waiting{service_name=\"messenger-api\"}", + "legendFormat": "waiting" + } + ] + }, + { + "id": 10, + "title": "Worker busy / capacity", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 32 + }, + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "expr": "messenger_worker_busy{service_name=\"messenger-worker\"}", + "legendFormat": "busy" + }, + { + "refId": "B", + "expr": "messenger_worker_capacity{service_name=\"messenger-worker\"}", + "legendFormat": "capacity" + } + ] + }, + { + "id": 11, + "title": "Process CPU", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 40 + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit" + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "expr": "sum by (service_name, service_instance_id) (rate(messenger_process_cpu_seconds_total[1m]))", + "legendFormat": "{{service_name}} {{outcome}}" + } + ] + }, + { + "id": 12, + "title": "Process resident memory", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 40 + }, + "fieldConfig": { + "defaults": { + "unit": "bytes" + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "expr": "messenger_process_memory_bytes", + "legendFormat": "{{service_name}} {{outcome}}" + } + ] + }, + { + "id": 13, + "title": "Collector failed / dropped spans per second", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 48 + }, + "fieldConfig": { + "defaults": { + "unit": "ops" + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "expr": "sum by (exporter) (rate(otelcol_exporter_send_failed_spans_total[1m]))", + "legendFormat": "send failed {{exporter}}" + }, + { + "refId": "B", + "expr": "sum by (exporter) (rate(otelcol_exporter_enqueue_failed_spans_total[1m]))", + "legendFormat": "queue full {{exporter}}" + } + ] + }, + { + "id": 14, + "title": "API SLI: successful ratio over 24h", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 48 + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "min": 0, + "max": 1 + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "expr": "1 - (sum(increase(messenger_http_requests_total{status_class=\"5xx\",route!~\"/health/.*\"}[24h])) or vector(0)) / clamp_min(sum(increase(messenger_http_requests_total{route!~\"/health/.*\"}[24h])),1) and sum(increase(messenger_http_requests_total{route!~\"/health/.*\"}[24h])) > 0", + "legendFormat": "SLI" + } + ] + }, + { + "id": 15, + "title": "API SLO burn rate (budget 1%)", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 56 + }, + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "expr": "(((sum(increase(messenger_http_requests_total{status_class=\"5xx\",route!~\"/health/.*\"}[20s])) or vector(0)) / clamp_min(sum(increase(messenger_http_requests_total{route!~\"/health/.*\"}[20s])), 1)) / 0.01) and sum(increase(messenger_http_requests_total{route!~\"/health/.*\"}[20s])) > 0", + "legendFormat": "burn 20s" + }, + { + "refId": "B", + "expr": "(((sum(increase(messenger_http_requests_total{status_class=\"5xx\",route!~\"/health/.*\"}[1m])) or vector(0)) / clamp_min(sum(increase(messenger_http_requests_total{route!~\"/health/.*\"}[1m])), 1)) / 0.01) and sum(increase(messenger_http_requests_total{route!~\"/health/.*\"}[1m])) > 0", + "legendFormat": "burn 1m" + }, + { + "refId": "C", + "expr": "(((sum(increase(messenger_http_requests_total{status_class=\"5xx\",route!~\"/health/.*\"}[5m])) or vector(0)) / clamp_min(sum(increase(messenger_http_requests_total{route!~\"/health/.*\"}[5m])), 1)) / 0.01) and sum(increase(messenger_http_requests_total{route!~\"/health/.*\"}[5m])) > 0", + "legendFormat": "burn 5m" + } + ] + }, + { + "id": 16, + "title": "READY SLO burn rate (budget 1%)", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 56 + }, + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "expr": "(((sum(increase(messenger_slo_ready_bad_total{service_name=\"messenger-outbox\"}[20s])) or vector(0)) / clamp_min(sum(increase(messenger_slo_ready_eligible_total{service_name=\"messenger-outbox\"}[20s])), 1)) / 0.01) and sum(increase(messenger_slo_ready_eligible_total{service_name=\"messenger-outbox\"}[20s])) > 0", + "legendFormat": "burn 20s" + }, + { + "refId": "B", + "expr": "(((sum(increase(messenger_slo_ready_bad_total{service_name=\"messenger-outbox\"}[1m])) or vector(0)) / clamp_min(sum(increase(messenger_slo_ready_eligible_total{service_name=\"messenger-outbox\"}[1m])), 1)) / 0.01) and sum(increase(messenger_slo_ready_eligible_total{service_name=\"messenger-outbox\"}[1m])) > 0", + "legendFormat": "burn 1m" + }, + { + "refId": "C", + "expr": "(((sum(increase(messenger_slo_ready_bad_total{service_name=\"messenger-outbox\"}[5m])) or vector(0)) / clamp_min(sum(increase(messenger_slo_ready_eligible_total{service_name=\"messenger-outbox\"}[5m])), 1)) / 0.01) and sum(increase(messenger_slo_ready_eligible_total{service_name=\"messenger-outbox\"}[5m])) > 0", + "legendFormat": "burn 5m" + } + ] + }, + { + "id": 17, + "title": "Accepted jobs ready within 30 s — 24h (matured cohort)", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 64 + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "min": 0, + "max": 1 + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "expr": "1 - ((sum(increase(messenger_slo_ready_bad_total{service_name=\"messenger-outbox\"}[24h])) or vector(0)) / clamp_min(sum(increase(messenger_slo_ready_eligible_total{service_name=\"messenger-outbox\"}[24h])), 1)) and sum(increase(messenger_slo_ready_eligible_total{service_name=\"messenger-outbox\"}[24h])) > 0", + "legendFormat": "ready SLI" + } + ] + } + ] +} diff --git a/infra/grafana/provisioning/dashboards/dashboards.yaml b/infra/grafana/provisioning/dashboards/dashboards.yaml new file mode 100644 index 0000000..c278fc1 --- /dev/null +++ b/infra/grafana/provisioning/dashboards/dashboards.yaml @@ -0,0 +1,8 @@ +apiVersion: 1 +providers: + - name: messenger + orgId: 1 + folder: Messenger + type: file + disableDeletion: false + options: {path: /var/lib/grafana/dashboards} diff --git a/infra/grafana/provisioning/datasources/datasources.yaml b/infra/grafana/provisioning/datasources/datasources.yaml new file mode 100644 index 0000000..2264ab3 --- /dev/null +++ b/infra/grafana/provisioning/datasources/datasources.yaml @@ -0,0 +1,25 @@ +apiVersion: 1 +datasources: + - name: Prometheus + uid: prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + - name: Loki + uid: loki + type: loki + access: proxy + url: http://loki:3100 + jsonData: + derivedFields: + - name: TraceID + matcherType: label + matcherRegex: '^trace_?id$' + url: '$${__value.raw}' + datasourceUid: tempo + - name: Tempo + uid: tempo + type: tempo + access: proxy + url: http://tempo:3200 diff --git a/infra/haproxy.cfg b/infra/haproxy.cfg new file mode 100644 index 0000000..00f0b6c --- /dev/null +++ b/infra/haproxy.cfg @@ -0,0 +1,35 @@ +global + log stdout format raw local0 + +defaults + mode http + log global + option httplog + option forwardfor + timeout connect 1s + timeout client 30s + timeout server 10s + timeout check 2s + retries 2 + +resolvers docker + nameserver dns 127.0.0.11:53 + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + hold valid 2s + hold obsolete 2s + +frontend http + bind :8080 + default_backend api + +backend api + balance roundrobin + option httpchk + http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host localhost + http-check expect status 200 + default-server check inter 2s fall 1 rise 2 resolvers docker resolve-prefer ipv4 init-addr libc,none + server api-a api-a:8080 + server api-b api-b:8080 + diff --git a/infra/loki/loki.yaml b/infra/loki/loki.yaml new file mode 100644 index 0000000..ec83888 --- /dev/null +++ b/infra/loki/loki.yaml @@ -0,0 +1,17 @@ +auth_enabled: false +server: {http_listen_port: 3100} +common: + path_prefix: /loki + replication_factor: 1 + ring: {kvstore: {store: inmemory}} +schema_config: + configs: + - from: '2024-01-01' + store: tsdb + object_store: filesystem + schema: v13 + index: {prefix: index_, period: 24h} +storage_config: + filesystem: {directory: /loki/chunks} +limits_config: {allow_structured_metadata: true, retention_period: 168h} +compactor: {working_directory: /loki/compactor, retention_enabled: true, delete_request_store: filesystem} diff --git a/infra/otel/collector.yaml b/infra/otel/collector.yaml new file mode 100644 index 0000000..d29d6fb --- /dev/null +++ b/infra/otel/collector.yaml @@ -0,0 +1,32 @@ +receivers: + otlp: + protocols: + grpc: {endpoint: "0.0.0.0:4317"} + http: {endpoint: "0.0.0.0:4318"} +processors: + memory_limiter: {check_interval: 1s, limit_mib: 192, spike_limit_mib: 48} + batch: {timeout: 2s, send_batch_size: 512, send_batch_max_size: 1024} +exporters: + prometheus: + endpoint: "0.0.0.0:8889" + resource_to_telemetry_conversion: {enabled: true} + otlp/tempo: + endpoint: tempo:4317 + tls: {insecure: true} + sending_queue: {enabled: true, queue_size: 128, num_consumers: 2} + retry_on_failure: {enabled: true, initial_interval: 1s, max_interval: 10s, max_elapsed_time: 60s} + otlphttp/loki: + endpoint: http://loki:3100/otlp + sending_queue: {enabled: true, queue_size: 128, num_consumers: 2} + retry_on_failure: {enabled: true, initial_interval: 1s, max_interval: 10s, max_elapsed_time: 60s} +service: + telemetry: + metrics: + readers: + - pull: + exporter: + prometheus: {host: 0.0.0.0, port: 8888} + pipelines: + metrics: {receivers: [otlp], processors: [memory_limiter, batch], exporters: [prometheus]} + traces: {receivers: [otlp], processors: [memory_limiter, batch], exporters: [otlp/tempo]} + logs: {receivers: [otlp], processors: [memory_limiter, batch], exporters: [otlphttp/loki]} diff --git a/infra/prometheus/alerts.yaml b/infra/prometheus/alerts.yaml new file mode 100644 index 0000000..af62567 --- /dev/null +++ b/infra/prometheus/alerts.yaml @@ -0,0 +1,39 @@ +groups: + - name: messenger + rules: + - alert: MessengerQueueTelemetryStale + expr: time() - max(messenger_queue_sample_time_seconds{service_name="messenger-outbox"}) > 20 or sum(absent(messenger_queue_sample_time_seconds{service_name="messenger-outbox"})) + for: 15s + labels: {severity: warning} + annotations: {summary: 'Queue and ready SLO telemetry is stale', runbook: 'See docs/slo-runbook.md; check outbox and DB before interpreting ready SLO.'} + - alert: MessengerReadyTimestampUnknown + expr: max(messenger_slo_unknown{service_name="messenger-outbox"}) > 0 + for: 15s + labels: {severity: warning} + annotations: {summary: 'Ready SLO commit timestamps unavailable', runbook: 'Check track_commit_timestamp and outbox; unknown events are not proof of SLO compliance.'} + - alert: MessengerHighErrorRate + expr: (sum(rate(messenger_http_requests_total{status_class="5xx",route!~"/health/.*"}[1m])) or vector(0)) / clamp_min(sum(rate(messenger_http_requests_total{route!~"/health/.*"}[1m])), 0.001) > 0.05 + for: 10s + labels: {severity: warning} + annotations: {summary: 'API 5xx ratio exceeds 5%', runbook: 'Inspect API latency, traces and dependency spans; check DB readiness.'} + - alert: MessengerQueueLag + expr: max(messenger_queue_age_seconds{service_name="messenger-outbox"}) > 30 + for: 15s + labels: {severity: warning} + annotations: {summary: 'Oldest queued job exceeds 30 seconds', runbook: 'Check worker health, busy/capacity, broker and consumer traces.'} + - alert: MessengerApiBurnFast + expr: '((sum(increase(messenger_http_requests_total{status_class="5xx",route!~"/health/.*"}[1m])) or vector(0)) / clamp_min(sum(increase(messenger_http_requests_total{route!~"/health/.*"}[1m])), 1)) > 28.8 * 0.01 and ((sum(increase(messenger_http_requests_total{status_class="5xx",route!~"/health/.*"}[20s])) or vector(0)) / clamp_min(sum(increase(messenger_http_requests_total{route!~"/health/.*"}[20s])), 1)) > 28.8 * 0.01 and sum(increase(messenger_http_requests_total{route!~"/health/.*"}[20s])) >= 5' + labels: {severity: critical, slo: api, burn: fast} + annotations: {summary: 'api SLO fast burn', runbook: 'See docs/slo-runbook.md'} + - alert: MessengerApiBurnSlow + expr: '((sum(increase(messenger_http_requests_total{status_class="5xx",route!~"/health/.*"}[5m])) or vector(0)) / clamp_min(sum(increase(messenger_http_requests_total{route!~"/health/.*"}[5m])), 1)) > 14.4 * 0.01 and ((sum(increase(messenger_http_requests_total{status_class="5xx",route!~"/health/.*"}[1m])) or vector(0)) / clamp_min(sum(increase(messenger_http_requests_total{route!~"/health/.*"}[1m])), 1)) > 14.4 * 0.01 and sum(increase(messenger_http_requests_total{route!~"/health/.*"}[1m])) >= 5' + labels: {severity: warning, slo: api, burn: slow} + annotations: {summary: 'api SLO slow burn', runbook: 'See docs/slo-runbook.md'} + - alert: MessengerReadyBurnFast + expr: '((sum(increase(messenger_slo_ready_bad_total{service_name="messenger-outbox"}[1m])) or vector(0)) / clamp_min(sum(increase(messenger_slo_ready_eligible_total{service_name="messenger-outbox"}[1m])), 1)) > 28.8 * 0.01 and ((sum(increase(messenger_slo_ready_bad_total{service_name="messenger-outbox"}[20s])) or vector(0)) / clamp_min(sum(increase(messenger_slo_ready_eligible_total{service_name="messenger-outbox"}[20s])), 1)) > 28.8 * 0.01 and sum(increase(messenger_slo_ready_eligible_total{service_name="messenger-outbox"}[20s])) >= 5' + labels: {severity: critical, slo: ready, burn: fast} + annotations: {summary: 'ready SLO fast burn', runbook: 'See docs/slo-runbook.md'} + - alert: MessengerReadyBurnSlow + expr: '((sum(increase(messenger_slo_ready_bad_total{service_name="messenger-outbox"}[5m])) or vector(0)) / clamp_min(sum(increase(messenger_slo_ready_eligible_total{service_name="messenger-outbox"}[5m])), 1)) > 14.4 * 0.01 and ((sum(increase(messenger_slo_ready_bad_total{service_name="messenger-outbox"}[1m])) or vector(0)) / clamp_min(sum(increase(messenger_slo_ready_eligible_total{service_name="messenger-outbox"}[1m])), 1)) > 14.4 * 0.01 and sum(increase(messenger_slo_ready_eligible_total{service_name="messenger-outbox"}[1m])) >= 5' + labels: {severity: warning, slo: ready, burn: slow} + annotations: {summary: 'ready SLO slow burn', runbook: 'See docs/slo-runbook.md'} diff --git a/infra/prometheus/prometheus.yaml b/infra/prometheus/prometheus.yaml new file mode 100644 index 0000000..4db25df --- /dev/null +++ b/infra/prometheus/prometheus.yaml @@ -0,0 +1,7 @@ +global: {scrape_interval: 5s, evaluation_interval: 5s} +rule_files: ['/etc/prometheus/alerts.yaml'] +scrape_configs: + - job_name: collector + static_configs: [{targets: ['otel-collector:8889']}] + - job_name: collector-internal + static_configs: [{targets: ['otel-collector:8888']}] diff --git a/infra/rabbitmq-secret-entrypoint.sh b/infra/rabbitmq-secret-entrypoint.sh new file mode 100644 index 0000000..8ed866f --- /dev/null +++ b/infra/rabbitmq-secret-entrypoint.sh @@ -0,0 +1,12 @@ +#!/bin/sh +set -eu + +# RabbitMQ 4 no longer supports *_FILE. Keep the password in a Docker +# secret and export it only inside the running container, never in Compose. +RABBITMQ_DEFAULT_PASS="$(cat /run/secrets/rabbitmq_password)" +if [ -z "$RABBITMQ_DEFAULT_PASS" ]; then + echo "RabbitMQ password secret must not be empty" >&2 + exit 1 +fi +export RABBITMQ_DEFAULT_PASS +exec /usr/local/bin/docker-entrypoint.sh "$@" diff --git a/infra/tempo/tempo.yaml b/infra/tempo/tempo.yaml new file mode 100644 index 0000000..2e501ea --- /dev/null +++ b/infra/tempo/tempo.yaml @@ -0,0 +1,13 @@ +server: {http_listen_port: 3200} +distributor: + receivers: + otlp: + protocols: + grpc: {endpoint: "0.0.0.0:4317"} +ingester: {trace_idle_period: 10s} +compactor: {compaction: {block_retention: 168h}} +storage: + trace: + backend: local + wal: {path: /var/tempo/wal} + local: {path: /var/tempo/blocks} diff --git a/migrations/001_initial.sql b/migrations/001_initial.sql new file mode 100644 index 0000000..c03d5fd --- /dev/null +++ b/migrations/001_initial.sql @@ -0,0 +1,37 @@ +CREATE TABLE users ( + id uuid PRIMARY KEY, + username varchar(32) NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'), + display_name varchar(100) NOT NULL CHECK (char_length(btrim(display_name)) > 0), + password_hash text NOT NULL, + created_at timestamptz NOT NULL DEFAULT now() +); + +CREATE TABLE chats ( + id uuid PRIMARY KEY, + title varchar(100) NOT NULL CHECK (char_length(btrim(title)) > 0), + created_at timestamptz NOT NULL DEFAULT now() +); + +CREATE TABLE chat_members ( + chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + user_id uuid NOT NULL REFERENCES users(id) ON DELETE RESTRICT, + PRIMARY KEY (chat_id, user_id) +); +CREATE INDEX chat_members_user_chat_idx ON chat_members (user_id, chat_id); + +CREATE TABLE messages ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + sender_id uuid NOT NULL REFERENCES users(id) ON DELETE RESTRICT, + text varchar(4000) NOT NULL CHECK (char_length(btrim(text)) > 0), + created_at timestamptz NOT NULL DEFAULT now() +); +CREATE INDEX messages_chat_created_id_idx ON messages (chat_id, created_at, id); + +CREATE TABLE sessions ( + token_hash char(64) PRIMARY KEY, + user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE, + expires_at timestamptz NOT NULL, + created_at timestamptz NOT NULL DEFAULT now() +); +CREATE INDEX sessions_expiry_idx ON sessions (expires_at); diff --git a/migrations/002_history_cache_version.sql b/migrations/002_history_cache_version.sql new file mode 100644 index 0000000..7b7972c --- /dev/null +++ b/migrations/002_history_cache_version.sql @@ -0,0 +1 @@ +ALTER TABLE chats ADD COLUMN history_version bigint NOT NULL DEFAULT 0; diff --git a/migrations/003_idempotency.sql b/migrations/003_idempotency.sql new file mode 100644 index 0000000..7fcf665 --- /dev/null +++ b/migrations/003_idempotency.sql @@ -0,0 +1,12 @@ +CREATE TABLE idempotency_records ( + user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE, + method varchar(8) NOT NULL, + path text NOT NULL, + idempotency_key varchar(128) NOT NULL, + request_hash char(64) NOT NULL, + message_id uuid REFERENCES messages(id) ON DELETE RESTRICT, + created_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (user_id, method, path, idempotency_key) +); + +CREATE INDEX idempotency_records_created_at_idx ON idempotency_records (created_at); diff --git a/migrations/004_attachments.sql b/migrations/004_attachments.sql new file mode 100644 index 0000000..2fe0f9e --- /dev/null +++ b/migrations/004_attachments.sql @@ -0,0 +1,18 @@ +CREATE TABLE attachments ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats(id), + owner_id uuid NOT NULL REFERENCES users(id), + filename varchar(255) NOT NULL, + content_type text NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg')), + size_bytes integer NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760), + sha256 char(64) NOT NULL, + object_key text NOT NULL UNIQUE, + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + UNIQUE (id, chat_id) +); +ALTER TABLE messages DROP CONSTRAINT messages_text_check; +ALTER TABLE messages ADD COLUMN attachment_ids uuid[] NOT NULL DEFAULT '{}'; +ALTER TABLE messages ADD CONSTRAINT messages_content_check + CHECK (char_length(btrim(text)) > 0 OR cardinality(attachment_ids) > 0); +CREATE INDEX attachments_chat_idx ON attachments(chat_id); diff --git a/migrations/005_async_attachments.sql b/migrations/005_async_attachments.sql new file mode 100644 index 0000000..fa3bcca --- /dev/null +++ b/migrations/005_async_attachments.sql @@ -0,0 +1,34 @@ +ALTER TABLE attachments + RENAME COLUMN object_key TO staging_key; +ALTER TABLE attachments + ADD COLUMN original_key text, + ADD COLUMN status text NOT NULL DEFAULT 'queued' + CHECK (status IN ('queued', 'processing', 'ready', 'failed')), + ADD COLUMN error_code text, + ADD COLUMN pipeline_version integer NOT NULL DEFAULT 1, + ADD COLUMN processing_started_at timestamptz; + +CREATE TABLE attachment_variants ( + attachment_id uuid NOT NULL REFERENCES attachments(id) ON DELETE CASCADE, + pipeline_version integer NOT NULL, + name text NOT NULL CHECK (name IN ('thumbnail', 'crop', 'optimized')), + object_key text NOT NULL UNIQUE, + content_type text NOT NULL, + size_bytes integer NOT NULL CHECK (size_bytes > 0), + width integer NOT NULL CHECK (width > 0), + height integer NOT NULL CHECK (height > 0), + sha256 char(64) NOT NULL, + PRIMARY KEY (attachment_id, pipeline_version, name) +); + +CREATE TABLE attachment_jobs ( + job_id uuid PRIMARY KEY, + attachment_id uuid NOT NULL UNIQUE REFERENCES attachments(id) ON DELETE CASCADE, + pipeline_version integer NOT NULL DEFAULT 1, + attempt integer NOT NULL DEFAULT 1 CHECK (attempt >= 1), + status text NOT NULL DEFAULT 'queued' CHECK (status IN ('queued', 'processing', 'ready', 'failed')), + last_error text, + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now() +); +CREATE INDEX attachment_jobs_status_idx ON attachment_jobs(status, updated_at); diff --git a/migrations/006_level4_reliability.sql b/migrations/006_level4_reliability.sql new file mode 100644 index 0000000..3be954c --- /dev/null +++ b/migrations/006_level4_reliability.sql @@ -0,0 +1,17 @@ +ALTER TABLE attachment_jobs + ADD COLUMN next_attempt_at timestamptz NOT NULL DEFAULT now(), + ADD COLUMN lease_until timestamptz, + ADD COLUMN replayed_at timestamptz; +CREATE INDEX attachment_jobs_due_idx ON attachment_jobs(status, next_attempt_at); +CREATE INDEX attachment_jobs_lease_idx ON attachment_jobs(status, lease_until); + +CREATE TABLE attachment_outbox ( + id uuid PRIMARY KEY, + job_id uuid NOT NULL REFERENCES attachment_jobs(job_id) ON DELETE CASCADE, + payload jsonb NOT NULL, + created_at timestamptz NOT NULL DEFAULT now(), + published_at timestamptz, + attempts integer NOT NULL DEFAULT 0, + UNIQUE (job_id, published_at) +); +CREATE INDEX attachment_outbox_pending_idx ON attachment_outbox(created_at) WHERE published_at IS NULL; diff --git a/migrations/007_job_measurements.sql b/migrations/007_job_measurements.sql new file mode 100644 index 0000000..12fc193 --- /dev/null +++ b/migrations/007_job_measurements.sql @@ -0,0 +1 @@ +ALTER TABLE attachment_jobs ADD COLUMN first_started_at timestamptz; diff --git a/migrations/008_trace_context.sql b/migrations/008_trace_context.sql new file mode 100644 index 0000000..4eee3fc --- /dev/null +++ b/migrations/008_trace_context.sql @@ -0,0 +1,2 @@ +-- Context is technical metadata, never user content. Retained across retry/outbox recreation. +ALTER TABLE attachment_jobs ADD COLUMN trace_context jsonb NOT NULL DEFAULT '{}'::jsonb; diff --git a/migrations/009_slo_commits.sql b/migrations/009_slo_commits.sql new file mode 100644 index 0000000..dd5f37c --- /dev/null +++ b/migrations/009_slo_commits.sql @@ -0,0 +1,11 @@ +-- A separate immutable cohort: replay and job cleanup must not rewrite SLO history. +-- Existing jobs have no recoverable acceptance commit and are deliberately not backfilled. +CREATE TABLE attachment_slo ( + attachment_id uuid PRIMARY KEY, + accepted_xid xid NOT NULL DEFAULT pg_current_xact_id()::xid, + ready_xid xid, + accepted_at timestamptz, + ready_at timestamptz, + bad boolean +); +CREATE INDEX attachment_slo_unresolved_idx ON attachment_slo(accepted_at) WHERE bad IS NULL; diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..df9aa92 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,17 @@ +[project] +name = "mtusi2026lab6" +version = "1.6.0" +description = "MTUSI high-load messenger, laboratory 6" +readme = "README.md" +requires-python = ">=3.12" +dependencies = [ + "argon2-cffi==25.1.0", + "asyncpg==0.31.0", + "boto3==1.43.115", + "fastapi==0.141.1", + "pillow==13.0.0", + "pika==1.3.2", + "uvicorn==0.40.0", + "opentelemetry-sdk==1.36.0", + "opentelemetry-exporter-otlp-proto-http==1.36.0", +] diff --git a/scripts/audit.py b/scripts/audit.py new file mode 100644 index 0000000..5c15de2 --- /dev/null +++ b/scripts/audit.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 +"""Sequential lab3-5 verification; capture actual output and exit status, fail fast.""" +from datetime import datetime, timezone +import argparse +import json +from pathlib import Path +import subprocess +import sys +import time + +ROOT = Path(__file__).resolve().parents[1] +DESTINATION = ROOT / "evidence/audit" +CHECKS = [ + ("contracts", ["make", "check", "validate", "PYTHON=.venv/bin/python"]), + ("compose", ["docker", "compose", "config", "--quiet"]), + ("smoke", ["make", "test"]), + ("attachments", ["make", "test-attachments"]), + ("idempotency", [sys.executable, "student-tests/idempotency_concurrency.py"]), + ("regressions", [sys.executable, "student-tests/review_regressions.py"]), + ("private-bucket", ["make", "test-private"]), + ("s3-permissions", ["make", "test-s3-permissions"]), + ("bootstrap", ["docker", "compose", "run", "--rm", "--no-deps", "s3-init"]), + ("upload-failover", ["make", "test-upload-failover"]), + ("public-failover", ["make", "test-failover", "ACTION_SCRIPT=scripts/stop-one.sh"]), + ("restore-replicas", ["docker", "compose", "up", "-d", "--wait", "api-a", "api-b", "load_balancer"]), + ("s3-outage", ["make", "test-s3-outage"]), + ("persistence", [sys.executable, "scripts/verify-persistence.py"]), + ("reconcile", ["make", "test-reconcile"]), + ("reconcile-audit", ["make", "test-reconcile-audit"]), + ("upload-memory", ["make", "test-upload-memory"]), + ("reconcile-benchmark", ["make", "test-reconcile-benchmark"]), + ("final-smoke", ["make", "test"]), + ("environment", [sys.executable, "scripts/record-environment.py"]), +] + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--resume-from', choices=[name for name, _ in CHECKS]) + args = parser.parse_args() + DESTINATION.mkdir(parents=True, exist_ok=True) + results = [] + checks = CHECKS + if args.resume_from: + offset = [name for name, _ in CHECKS].index(args.resume_from) + previous = json.loads((DESTINATION / 'results.json').read_text())['checks'] + results = previous[:offset] + if [item['name'] for item in results] != [name for name, _ in CHECKS[:offset]] or any(item['exit_code'] for item in results): + parser.error('resume requires successful evidence for every earlier step') + checks = CHECKS[offset:] + for name, command in checks: + print(f"RUN {name}", flush=True) + started = time.monotonic() + with (DESTINATION / f"{name}.txt").open("w") as stream: + result = subprocess.run(command, cwd=ROOT, stdout=stream, stderr=subprocess.STDOUT) + results.append({"name": name, "command": command, "exit_code": result.returncode, + "seconds": round(time.monotonic() - started, 3)}) + (DESTINATION / "results.json").write_text(json.dumps({ + "recorded_at_utc": datetime.now(timezone.utc).isoformat(), + "checks": results, "complete": len(results) == len(CHECKS), + }, indent=2) + "\n") + print(f"{'PASS' if result.returncode == 0 else 'FAIL'} {name}: exit={result.returncode}", flush=True) + if result.returncode: + print((DESTINATION / f"{name}.txt").read_text()[-8000:], flush=True) + return result.returncode + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/init-secrets.py b/scripts/init-secrets.py new file mode 100644 index 0000000..616933d --- /dev/null +++ b/scripts/init-secrets.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 +"""Create local credentials without printing or overwriting existing values.""" +import json +from pathlib import Path +import secrets + +root = Path(__file__).resolve().parents[1] / "secrets" +root.mkdir(mode=0o700, exist_ok=True) + + +def write(name, value): + path = root / name + if path.exists(): + return path.read_text().strip() + with path.open("x") as stream: + path.chmod(0o600) + stream.write(value + "\n") + return value + + +write("grafana_password", secrets.token_urlsafe(24)) +password = write("postgres_password", secrets.token_hex(32)) +write("database_url", f"postgresql://messenger:{password}@postgres:5432/messenger") +write("cursor_secret", secrets.token_hex(32)) +admin_key = write("s3_admin_key", secrets.token_hex(16)) +admin_secret = write("s3_admin_secret", secrets.token_hex(32)) +app_key = write("s3_access_key_id", secrets.token_hex(16)) +app_secret = write("s3_secret_access_key", secrets.token_hex(32)) +rabbitmq_password = write("rabbitmq_password", secrets.token_urlsafe(32)) +write("broker_url", f"amqp://messenger:{rabbitmq_password}@broker:5672/%2F") +config_path = root / "s3.json" +config_path.write_text(json.dumps({"identities": [ + {"name": "bootstrap", "credentials": [{"accessKey": admin_key, "secretKey": admin_secret}], + "actions": ["Admin", "Read", "Write", "List", "Tagging"]}, + {"name": "api", "credentials": [{"accessKey": app_key, "secretKey": app_secret}], + "actions": ["Read:messenger-attachments", "Write:messenger-attachments", "List:messenger-attachments"]}, +]}, indent=2) + "\n") +config_path.chmod(0o600) +print("Local secrets ready; existing values preserved.") diff --git a/scripts/record-environment.py b/scripts/record-environment.py new file mode 100644 index 0000000..961dc2f --- /dev/null +++ b/scripts/record-environment.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +"""Sanitized environment and source fingerprint for local test evidence.""" +from datetime import datetime, timezone +from hashlib import sha256 +import json +from pathlib import Path +import platform +import subprocess + +root = Path(__file__).resolve().parents[1] + + +def command(*args): + return subprocess.check_output(args, cwd=root, text=True).strip() + + +paths = [root / name for name in ('Dockerfile', 'compose.yaml', 'pyproject.toml', 'uv.lock', 'Makefile')] +for directory in ('src', 'migrations', 'infra', 'scripts', 'student-tests'): + paths.extend(path for path in (root / directory).rglob('*') + if path.is_file() and '__pycache__' not in path.parts) +report = { + 'recorded_at_utc': datetime.now(timezone.utc).isoformat(), + 'base_commit': command('git', 'rev-parse', 'HEAD'), + 'branch': command('git', 'branch', '--show-current'), + 'note': 'Results apply to this working tree, not only the base commit. Per-profile load settings are recorded in real-c*-p*-s*.json.', + 'host': platform.platform(), + 'docker': json.loads(command('docker', 'info', '--format', + '{"server":"{{.ServerVersion}}","cpus":{{.NCPU}},"memory_bytes":{{.MemTotal}}}')), + 'compose_version': command('docker', 'compose', 'version', '--short'), + 'sources_sha256': {str(path.relative_to(root)): sha256(path.read_bytes()).hexdigest() for path in sorted(paths)}, + 'services': {}, +} +for service in ('postgres', 's3', 'broker', 'worker', 'outbox', 'reconcile', 'api-a', 'api-b', 'load_balancer'): + container = command('docker', 'compose', 'ps', '-q', service) + info = json.loads(command('docker', 'inspect', container))[0] + report['services'][service] = { + 'image': info['Config']['Image'], 'image_id': info['Image'], + 'state': info['State']['Status'], 'health': info['State'].get('Health', {}).get('Status'), + 'memory_limit_bytes': info['HostConfig']['Memory'], + 'nano_cpus_limit': info['HostConfig']['NanoCpus'], + 'volumes': [mount['Name'] for mount in info['Mounts'] if mount['Type'] == 'volume'], + } +(root / 'evidence/environment.json').write_text(json.dumps(report, ensure_ascii=False, indent=2) + '\n') +print('PASS: sanitized environment and source hashes saved') diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 0000000..958b700 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,10 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." + +# Deliberately preserves named volumes and test data. +docker compose restart postgres s3 broker +docker compose up -d --wait postgres s3 broker +docker compose run --rm --no-deps s3-init +docker compose restart worker outbox reconcile api-a api-b load_balancer +docker compose up -d --wait worker outbox reconcile api-a api-b load_balancer diff --git a/scripts/run-track1-profiles.py b/scripts/run-track1-profiles.py new file mode 100644 index 0000000..ba3f140 --- /dev/null +++ b/scripts/run-track1-profiles.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python3 +"""Run real, sequential Docker profiles; restore normal settings on exit.""" +import os +from pathlib import Path +import subprocess +import sys + +ROOT = Path(__file__).resolve().parents[1] + + +def main(): + try: + for workers in (1, 2, 4): + env = dict(os.environ, WORKER_CONCURRENCY=str(workers), + WORKER_PREFETCH=str(workers), MAX_PENDING_JOBS='32') + subprocess.run(['docker', 'compose', 'up', '-d', '--no-deps', '--wait', + 'worker', 'api-a', 'api-b'], cwd=ROOT, env=env, check=True) + for size in (512, 2400): + subprocess.run([sys.executable, 'scripts/track1.py', + '--run-name', f'real-c{workers}-p{workers}-s{size}', + '--workers', str(workers), '--prefetch', str(workers), + '--max-pending', '32', '--images', '160', + '--concurrency', '16', '--size', str(size), + '--require-backpressure'], cwd=ROOT, check=True) + finally: + subprocess.run(['docker', 'compose', 'up', '-d', '--no-deps', '--wait', + 'worker', 'api-a', 'api-b'], cwd=ROOT, check=True) + + +if __name__ == '__main__': + main() diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 0000000..befd6cf --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,18 @@ +#!/bin/sh +set -eu + +: "${TARGET_INSTANCE_ID:?test provides target API instance alias}" +cd "$(dirname "$0")/.." + +for service in api-a api-b; do + container="$(docker compose ps -q "$service")" + [ -n "$container" ] || continue + hostname="$(docker inspect --format '{{.Config.Hostname}}' "$container")" + if [ "$hostname" = "$TARGET_INSTANCE_ID" ]; then + docker compose stop "$service" + exit 0 + fi +done + +printf '%s\n' 'TARGET_INSTANCE_ID does not match a running API replica' >&2 +exit 2 diff --git a/scripts/track1.py b/scripts/track1.py new file mode 100755 index 0000000..d72240d --- /dev/null +++ b/scripts/track1.py @@ -0,0 +1,217 @@ +#!/usr/bin/env python3 +"""Closed-loop upload burst with concurrent readiness observations and drain proof.""" +from concurrent.futures import ThreadPoolExecutor, as_completed +from io import BytesIO +import json +import math +from pathlib import Path +import subprocess +import sys +import threading +import time + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, Failure, arguments, check, run + + +def percentile(values, point): + values = sorted(values) + return round(values[max(0, math.ceil(len(values) * point) - 1)] * 1000, 3) if values else None + + +def command_json(*args): + result = subprocess.run(args, cwd=ROOT, text=True, capture_output=True, timeout=20) + if result.returncode: + raise Failure("metrics command failed: " + args[0]) + try: + return json.loads(result.stdout) + except ValueError: + raise Failure("metrics command returned invalid JSON") from None + + +def metrics(chat_id): + return command_json("docker", "compose", "exec", "-T", "worker", + "/app/.venv/bin/python", "-m", "src.pipeline_metrics", chat_id) + + +def broker_depth(): + rows = command_json("docker", "compose", "exec", "-T", "broker", + "rabbitmqctl", "list_queues", "name", "messages_ready", + "messages_unacknowledged", "--formatter", "json") + return next(row for row in rows if row["name"] == "attachment.process.v1") + + +def container_info(service): + result = subprocess.run(['docker', 'compose', 'ps', '-q', service], cwd=ROOT, + capture_output=True, text=True, timeout=10, check=True) + ids = result.stdout.split() + check(len(ids) == 1, 'expected exactly one container: ' + service) + return command_json('docker', 'inspect', ids[0])[0] + + +def main(): + def configure(p): + p.add_argument("--run-name", required=True) + p.add_argument("--workers", type=int, required=True) + p.add_argument("--prefetch", type=int, required=True) + p.add_argument("--max-pending", type=int, required=True) + p.add_argument("--images", type=int, default=40) + p.add_argument("--concurrency", type=int, default=8) + p.add_argument("--size", type=int, default=1600) + p.add_argument("--max-ready-seconds", type=float, default=180) + p.add_argument("--require-backpressure", action="store_true") + p.add_argument("--output") + args = arguments(__doc__, configure) + check(1 <= args.images <= 1000 and 1 <= args.concurrency <= 128, "invalid workload") + check(1 <= args.size <= 6000 and 0 < args.max_ready_seconds <= 3600, "invalid size/deadline") + check(args.workers > 0 and args.prefetch > 0 and args.max_pending > 0, "invalid profile") + from PIL import Image + # Reproducible, nonuniform synthetic image; no real personal data. + row = bytes(v for x in range(args.size) for v in ((x * 13) % 256, (x * 37) % 256, (x * 71) % 256)) + image = Image.frombytes("RGB", (args.size, args.size), row * args.size) + stream = BytesIO() + image.save(stream, "PNG") + data = stream.getvalue() + client = Client(args.base_url, args.ca_file, timeout=30) + alice, bob, _, chat = client.bootstrap() + baseline = metrics(chat["id"]) + check(baseline["concurrency"] == args.workers and baseline["prefetch"] == args.prefetch, + "worker settings do not match requested profile") + check(baseline["pending"] == 0, "use an idle isolated stack") + for service in ("api-a", "api-b"): + env = dict(item.split('=', 1) for item in container_info(service)['Config']['Env']) + check(int(env['MAX_PENDING_JOBS']) == args.max_pending, + 'running API MAX_PENDING_JOBS does not match profile') + resources = container_info('worker')['HostConfig'] + path = "/api/v1/chats/" + chat["id"] + "/attachments?filename=track1.png" + pending, records, samples, errors = {}, [], [], [] + lock, finished = threading.Lock(), threading.Event() + started = time.monotonic() + def observe(): + local = Client(args.base_url, args.ca_file, timeout=5) + while not finished.wait(.1): + with lock: + batch = list(pending.items()) + for key, record in batch: + if finished.is_set(): + break + try: + att = local.call("GET", "/api/v1/attachments/" + key, actor=bob, schema="Attachment")["json"] + if att["status"] in ("ready", "failed"): + with lock: + record["terminal"] = att["status"] + record["terminal_at"] = time.monotonic() + pending.pop(key, None) + except Failure: + continue + def sample(): + while not finished.is_set(): + try: + sample = metrics(chat["id"]) + sample["queue"] = broker_depth() + sample["seconds"] = time.monotonic() - started + with lock: + samples.append(sample) + except (Failure, OSError, subprocess.TimeoutExpired) as exc: + with lock: + errors.append(type(exc).__name__) + if finished.wait(1): + break + observer = threading.Thread(target=observe, daemon=True) + sampler = threading.Thread(target=sample, daemon=True) + observer.start() + sampler.start() + def upload(index): + local = Client(args.base_url, args.ca_file, timeout=30) + before = time.monotonic() + try: + response = local.call("POST", path, (202, 429), actor=alice, raw=data, + headers={"Content-Type": "image/png"}, schema="Attachment") + received = time.monotonic() + record = {"index": index, "status": response["status"], "upload_seconds": received - before} + if response["status"] == 202: + record.update(id=response["json"]["id"], accepted_at=received) + with lock: + records.append(record) + if response["status"] == 202: + pending[record["id"]] = record + except Failure: + with lock: + records.append({"index": index, "status": "transport_or_http_error"}) + try: + with ThreadPoolExecutor(max_workers=args.concurrency) as pool: + list(pool.map(upload, range(args.images))) + upload_finished = time.monotonic() + deadline = upload_finished + args.max_ready_seconds + while time.monotonic() < deadline: + with lock: + empty = not pending + if empty: + break + time.sleep(.1) + finally: + measured_end = time.monotonic() + finished.set() + observer.join(timeout=10) + sampler.join(timeout=45) + elapsed = measured_end - started + final = metrics(chat["id"]) + queue = broker_depth() + for _ in range(20): + if queue["messages_ready"] == 0 and queue["messages_unacknowledged"] == 0: + break + time.sleep(.25) + queue = broker_depth() + accepted = [r for r in records if r["status"] == 202] + ready = [r for r in accepted if r.get("terminal") == "ready"] + latencies = [r["terminal_at"] - r["accepted_at"] for r in ready] + waits = [r["queue_wait_seconds"] for r in final["jobs"] if r["queue_wait_seconds"] is not None] + drained = (bool(accepted) and len(ready) == len(accepted) and final["pending"] == 0 + and queue["messages_ready"] == 0 and queue["messages_unacknowledged"] == 0) + recovery = False + if drained: + try: + att, _ = client.attachment(alice, chat) + client.wait_ready(bob, att, wait_seconds=args.max_ready_seconds) + recovery = True + except Failure: + pass + rejected = sum(r["status"] == 429 for r in records) + report = { + "run_name": args.run_name, + "settings": {"workers": baseline["concurrency"], "prefetch": baseline["prefetch"], + "max_pending": args.max_pending, + "worker_limits": {key: resources[key] for key in ('Memory', 'NanoCpus')}}, + "workload": {"images": args.images, "concurrency": args.concurrency, + "size": args.size, "bytes": len(data), "type": "closed-loop burst"}, + "results": {"accepted": len(accepted), "rejected_429": rejected, "ready": len(ready), + "failed": sum(r.get("terminal") == "failed" for r in accepted), + "timed_out": len(pending), "elapsed_seconds": elapsed, + "ready_per_second": len(ready) / elapsed, + "ready_latency_ms": {str(p): percentile(latencies, p) for p in (.5, .95, .99)}, + "queue_wait_ms": {str(p): percentile(waits, p) for p in (.5, .95, .99)}, + "max_sampled_rss_bytes": max((s["rss_bytes"] for s in samples), default=None), + "max_sampled_pending": max((s["pending"] for s in samples), default=None), + "sample_count": len(samples), + "drained": drained, "recovery_upload_ready": recovery}, + "records": [{k: v for k, v in r.items() if k not in ("accepted_at", "terminal_at")} | + ({"ready_seconds": r["terminal_at"] - r["accepted_at"]} if "terminal_at" in r else {}) + for r in records], + "samples": samples, "metric_errors": errors, + "limitations": "Closed-loop; ready time is polling upper bound. RSS is sum of worker/container processes, shared pages may be counted twice. Sampling may miss peaks. Queue wait is first claim minus durable job creation.", + } + target = Path(args.output or ROOT / "evidence" / (args.run_name + ".json")) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(json.dumps(report, indent=2) + "\n") + print(json.dumps(report["results"], indent=2)) + check(not errors and bool(samples), "incomplete metrics; result saved but not valid evidence") + check(all(s['pending'] <= args.max_pending for s in samples), "admission limit exceeded") + check(all(r["status"] in (202, 429) for r in records), "unexpected HTTP/transport failures") + check(drained and recovery, "pipeline did not drain/recover; see saved result") + if args.require_backpressure: + check(rejected > 0, "backpressure was not observed") + + +if __name__ == "__main__": + run(main) diff --git a/scripts/verify-persistence.py b/scripts/verify-persistence.py new file mode 100644 index 0000000..5bfc5a4 --- /dev/null +++ b/scripts/verify-persistence.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python3 +"""Public persistence test plus proof of actual container restarts.""" +import json +from pathlib import Path +import subprocess +import sys + +ROOT = Path(__file__).resolve().parents[1] + + +def snapshot(): + result = {} + for service in ('postgres', 's3', 'api-a', 'api-b', 'load_balancer'): + container = subprocess.check_output(['docker', 'compose', 'ps', '-q', service], cwd=ROOT, text=True).strip() + started = subprocess.check_output(['docker', 'inspect', '--format', '{{.State.StartedAt}}', container], text=True).strip() + result[service] = {'container': container[:12], 'started_at': started} + return result + + +before = snapshot() +print('Before:', json.dumps(before, sort_keys=True), flush=True) +subprocess.run([sys.executable, 'tests/resilience.py', '--mode', 'persistence', + '--action-script', 'scripts/restart.sh'], cwd=ROOT, check=True) +after = snapshot() +print('After:', json.dumps(after, sort_keys=True), flush=True) +assert all(before[key]['started_at'] != after[key]['started_at'] for key in before) +print('PASS: PostgreSQL, S3, both API replicas and LB actually restarted; volumes preserved') diff --git a/scripts/verify-resilience.py b/scripts/verify-resilience.py new file mode 100644 index 0000000..02ccd00 --- /dev/null +++ b/scripts/verify-resilience.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +"""Sequential public tests with safe container-state evidence; no credentials.""" +import json +from pathlib import Path +import subprocess +import sys + +ROOT = Path(__file__).resolve().parents[1] + + +def states(): + ids = subprocess.check_output(['docker', 'compose', 'ps', '-aq'], text=True).split() + rows = json.loads(subprocess.check_output(['docker', 'inspect', *ids], text=True)) + return {r['Config']['Labels']['com.docker.compose.service']: + {'id': r['Id'][:12], 'started_at': r['State']['StartedAt'], 'status': r['State']['Status']} + for r in rows} + + +def main(): + results, stopped = [], set() + try: + for mode in ['persistence'] + ['failover'] * 6: + before = states() + hook = 'restart' if mode == 'persistence' else 'stop-one' + result = subprocess.run([sys.executable, 'tests/resilience.py', '--mode', mode, + '--action-script', 'scripts/' + hook + '.sh'], + text=True, capture_output=True, timeout=200) + after = states() + results.append(dict(mode=mode, returncode=result.returncode, + output=result.stdout + result.stderr, before=before, after=after)) + (ROOT / 'evidence/resilience.json').write_text(json.dumps(results, indent=2) + '\n') + print(result.stdout + result.stderr, flush=True) + assert result.returncode == 0, 'public resilience failed' + if mode == 'persistence': + for service in ('postgres', 's3', 'broker', 'worker', 'outbox', 'api-a', 'api-b'): + assert before[service]['started_at'] != after[service]['started_at'], service + else: + stopped.update(s for s in ('api-a', 'api-b') if after[s]['status'] == 'exited') + subprocess.run(['docker', 'compose', 'up', '-d', '--no-deps', '--wait', 'api-a', 'api-b'], check=True) + if len(stopped) == 2: + break + assert len(stopped) == 2, 'did not exercise both API replicas' + finally: + subprocess.run(['docker', 'compose', 'up', '-d', '--no-deps', '--wait', 'api-a', 'api-b'], check=True) + + +if __name__ == '__main__': + main() diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/src/__init__.py @@ -0,0 +1 @@ + diff --git a/src/attachments.py b/src/attachments.py new file mode 100644 index 0000000..ea496ee --- /dev/null +++ b/src/attachments.py @@ -0,0 +1,141 @@ +import asyncio +from hashlib import sha256 +import os +from tempfile import TemporaryFile +import unicodedata +from uuid import UUID, uuid4 +import warnings + +from fastapi import APIRouter, Depends, Query, Request, Response +from PIL import Image, UnidentifiedImageError +from starlette.concurrency import run_in_threadpool +from starlette.requests import ClientDisconnect + +from .errors import ApiError +from .models import Attachment +from .routes import get_current_user, get_store +from .storage.postgres import PostgresStore + +router = APIRouter() +Image.MAX_IMAGE_PIXELS = 40000000 +warnings.filterwarnings("error", category=Image.DecompressionBombWarning) + + +def inspect_image(file, content_type, max_pixels): + try: + file.seek(0) + with Image.open(file, formats=("PNG", "JPEG")) as image: + if image.width * image.height > max_pixels: + raise ApiError(413, "payload_too_large", "Слишком много пикселей") + actual = {"PNG": "image/png", "JPEG": "image/jpeg"}[image.format] + if actual != content_type or getattr(image, "n_frames", 1) != 1: + raise ApiError(415, "unsupported_media_type", "Нужен обычный PNG или JPEG") + image.verify() + # verify checks PNG CRC; load also catches truncated pixel streams/JPEGs. + file.seek(0) + with Image.open(file, formats=("PNG", "JPEG")) as image: + image.load() + except (Image.DecompressionBombError, Image.DecompressionBombWarning) as error: + raise ApiError(413, "payload_too_large", "Слишком много пикселей") from error + except (UnidentifiedImageError, OSError, ValueError, SyntaxError) as error: + raise ApiError(415, "unsupported_media_type", "Повреждённое изображение") from error + file.seek(0) + + +@router.post("/api/v1/chats/{chat_id}/attachments", response_model=Attachment, status_code=202) +async def upload(chat_id: UUID, request: Request, filename: str = Query(...), + user: dict = Depends(get_current_user), store: PostgresStore = Depends(get_store)): + await store.get_chat_for_member(chat_id, user["id"]) + if (not 1 <= len(filename) <= 255 or not filename.strip() or + any(c in "/\\" or unicodedata.category(c) in {"Cc", "Cs"} for c in filename)): + raise ApiError(400, "invalid_request", "Некорректное имя файла") + content_type = request.headers.get("content-type", "").lower() + if content_type not in {"image/png", "image/jpeg"}: + raise ApiError(415, "unsupported_media_type", "Нужен PNG или JPEG") + s3 = request.app.state.s3 + limit = s3.settings.max_bytes + if "content-length" in request.headers: + try: + length = int(request.headers["content-length"]) + if length < 0: + raise ValueError() + except ValueError: + raise ApiError(400, "invalid_request", "Некорректная длина") from None + if length > limit: + raise ApiError(413, "payload_too_large", "Файл больше лимита") + try: + timeout = int(os.environ.get("UPLOAD_TIMEOUT_SECONDS", "30")) + if not 1 <= timeout <= 120: + raise ApiError(503, "unavailable", "Некорректный лимит времени загрузки") + async with asyncio.timeout(timeout): + async with request.app.state.upload_slot: + # Temporary scratch space only; deleted on every exit, never used for downloads. + with TemporaryFile() as file: + size = 0 + digest = sha256() + async for chunk in request.stream(): + size += len(chunk) + if size > limit: + raise ApiError(413, "payload_too_large", "Файл больше лимита") + digest.update(chunk) + await run_in_threadpool(file.write, chunk) + await run_in_threadpool(inspect_image, file, content_type, s3.settings.max_pixels) + attachment_id, job_id = uuid4(), uuid4() + key = f"staging/{attachment_id}/original" + async with store.attachment_transaction() as connection: + try: + max_pending = int(os.environ.get("MAX_PENDING_JOBS", "100")) + except ValueError as error: + raise ApiError(503, "unavailable", "Некорректный лимит очереди") from error + if max_pending < 1: + raise ApiError(503, "unavailable", "Некорректный лимит очереди") + await store.ensure_attachment_queue_capacity(connection, max_pending) + await s3.put(key, file, size, content_type) + if os.environ.get("UPLOAD_FAULT_AFTER_S3_PUT") == "1": + raise ApiError(503, "unavailable", "Инъецирован отказ после S3 PUT") + attachment = await store.create_attachment( + attachment_id, chat_id, user["id"], filename, content_type, + size, digest.hexdigest(), key, job_id, connection=connection, + ) + # Fault-injection for level-4 proof: durable outbox must recover this commit. + if os.environ.get("OUTBOX_FAULT_AFTER_COMMIT") == "1": + os._exit(70) + attachment["variants"] = [] + return attachment + except ClientDisconnect: + raise ApiError(400, "invalid_request", "Загрузка прервана") from None + except TimeoutError: + raise ApiError(503, "unavailable", "Время загрузки истекло") from None + + +@router.get("/api/v1/attachments/{attachment_id}", response_model=Attachment) +async def metadata(attachment_id: UUID, user: dict = Depends(get_current_user), + store: PostgresStore = Depends(get_store)): + return await store.attachment_for_member(attachment_id, user["id"]) + + +@router.get("/api/v1/attachments/{attachment_id}/content") +async def download(attachment_id: UUID, request: Request, + variant: str = Query("original", pattern="^(original|thumbnail|crop|optimized)$"), + user: dict = Depends(get_current_user), store: PostgresStore = Depends(get_store)): + attachment = await store.attachment_for_member(attachment_id, user["id"]) + if attachment["status"] != "ready": + raise ApiError(409, "conflict", "Вложение ещё не готово") + if variant == "original": + data = await request.app.state.s3.download(attachment) + content_type = attachment["content_type"] + else: + matching = [item for item in attachment["variants"] if item["name"] == variant] + if not matching: + raise ApiError(404, "not_found", "Вариант не найден") + item = matching[0] + object_key = await store.pool.fetchval( + 'SELECT object_key FROM attachment_variants WHERE attachment_id=$1 AND pipeline_version=$2 AND name=$3', + attachment_id, attachment['pipeline_version'], variant, + ) + data = await request.app.state.s3.download_key( + object_key, item["size_bytes"], item["sha256"] + ) + content_type = item["content_type"] + return Response(data, media_type=content_type, + headers={"X-Content-Type-Options": "nosniff", "Cache-Control": "private, no-store"}) diff --git a/src/bootstrap_s3.py b/src/bootstrap_s3.py new file mode 100644 index 0000000..28f2c8b --- /dev/null +++ b/src/bootstrap_s3.py @@ -0,0 +1,51 @@ +"""Idempotent private bucket bootstrap; executed with separate admin credentials.""" +import time +from uuid import uuid4 +from botocore.exceptions import BotoCoreError, ClientError +from .storage.s3 import S3Settings, S3Store + + +def main(): + store = S3Store(S3Settings.load()) + try: + for attempt in range(60): + try: + store.client.list_buckets() + break + except (BotoCoreError, ClientError): + if attempt == 59: + raise RuntimeError("S3 startup timed out") from None + time.sleep(1) + bucket = store.settings.bucket + try: + store.client.head_bucket(Bucket=bucket) + except ClientError as error: + if str(error.response["Error"]["Code"]) not in {"404", "NoSuchBucket", "NotFound"}: + raise + store.client.create_bucket(Bucket=bucket) + # No anonymous identity; revoke any previously configured public grants/policy. + store.client.put_bucket_acl(Bucket=bucket, ACL="private") + try: + store.client.delete_bucket_policy(Bucket=bucket) + except ClientError as error: + if error.response["Error"]["Code"] != "NoSuchBucketPolicy": + raise + # The master health endpoint can become available before a volume accepts + # writes after a restart. Verify the actual data path before APIs start. + probe_key = f"_bootstrap/write-probe-{uuid4()}" + for attempt in range(30): + try: + store.client.put_object(Bucket=bucket, Key=probe_key, Body=b"ready") + store.client.delete_object(Bucket=bucket, Key=probe_key) + break + except (BotoCoreError, ClientError): + if attempt == 29: + raise RuntimeError("S3 write path startup timed out") from None + time.sleep(1) + print("PASS: private S3 bucket ready") + finally: + store.client.close() + + +if __name__ == "__main__": + main() diff --git a/src/broker.py b/src/broker.py new file mode 100644 index 0000000..ed306b5 --- /dev/null +++ b/src/broker.py @@ -0,0 +1,90 @@ +"""Small durable RabbitMQ adapter; broker messages contain no attachment bytes.""" +import json +import os +from pathlib import Path + +import pika +from starlette.concurrency import run_in_threadpool + +from .errors import ApiError +from . import telemetry as tel + +QUEUE = "attachment.process.v1" +DLQ = "attachment.process.v1.dlq" + + +def broker_url() -> str: + path = os.environ.get("BROKER_URL_FILE") + value = Path(path).read_text(encoding="utf-8").strip() if path else os.environ.get("BROKER_URL", "") + if not value.startswith(("amqp://", "amqps://")): + raise ValueError("BROKER_URL или BROKER_URL_FILE должен содержать AMQP URL") + return value + + +def _connection() -> pika.BlockingConnection: + connection = pika.BlockingConnection(pika.URLParameters(broker_url())) + channel = connection.channel() + channel.queue_declare(queue=QUEUE, durable=True) + channel.queue_declare(queue=DLQ, durable=True) + return connection + + +def _publish(payload: dict, headers: dict | None = None) -> None: + connection = _connection() + try: + channel = connection.channel() + channel.confirm_delivery() + channel.basic_publish( + exchange="", routing_key=QUEUE, + body=json.dumps(payload, separators=(",", ":")).encode("utf-8"), + properties=pika.BasicProperties(delivery_mode=pika.DeliveryMode.Persistent, content_type="application/json", headers=headers), + mandatory=True, + ) + finally: + connection.close() + + +async def publish(payload: dict, context=None) -> None: + try: + with tel.tracer.start_as_current_span("attachment.publish", context=tel.extract(context), kind=tel.SpanKind.PRODUCER, record_exception=False, set_status_on_exception=False) as span: + span.set_attribute("messaging.retry.attempt", int(payload.get("attempt", 1))) + try: + await run_in_threadpool(_publish, payload, tel.carrier()) + except Exception: + span.set_status(tel.Status(tel.StatusCode.ERROR)) + raise + except (pika.exceptions.AMQPError, OSError, ValueError) as error: + raise ApiError(503, "unavailable", "Брокер недоступен") from error + + +async def publish_dlq(payload: dict) -> None: + payload = {**payload, "failure": payload.get('failure', 'retry_exhausted')} + try: + await run_in_threadpool(_publish_to, DLQ, payload) + except (pika.exceptions.AMQPError, OSError, ValueError) as error: + raise RuntimeError("DLQ is unavailable") from error + + +def _publish_to(queue: str, payload: dict) -> None: + connection = _connection() + try: + channel = connection.channel() + channel.confirm_delivery() + channel.basic_publish( + exchange="", routing_key=queue, body=json.dumps(payload, separators=(",", ":")).encode("utf-8"), + properties=pika.BasicProperties(delivery_mode=pika.DeliveryMode.Persistent, content_type="application/json"), + mandatory=True, + ) + finally: + connection.close() + + +async def ready() -> None: + connection = None + try: + connection = await run_in_threadpool(_connection) + except (pika.exceptions.AMQPError, OSError, ValueError) as error: + raise ApiError(503, "unavailable", "Брокер недоступен") from error + finally: + if connection is not None: + await run_in_threadpool(connection.close) diff --git a/src/config.py b/src/config.py new file mode 100644 index 0000000..320063a --- /dev/null +++ b/src/config.py @@ -0,0 +1,125 @@ +from dataclasses import dataclass +import os +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +_CONFIGURATION_KEYS = { + "APP_HOST", + "APP_PORT", + "APP_ENV", + "DATABASE_URL", + "DATABASE_URL_FILE", + "SESSION_TTL_SECONDS", + "IDEMPOTENCY_TTL_SECONDS", + "INSTANCE_ID", + "CURSOR_SECRET", + "CURSOR_SECRET_FILE", + "S3_ENDPOINT", "S3_REGION", "S3_BUCKET", + "S3_ACCESS_KEY_ID", "S3_ACCESS_KEY_ID_FILE", + "S3_SECRET_ACCESS_KEY", "S3_SECRET_ACCESS_KEY_FILE", + "BROKER_URL", "BROKER_URL_FILE", + "MAX_UPLOAD_BYTES", "MAX_IMAGE_PIXELS", + "UPLOAD_CONCURRENCY", "UPLOAD_TIMEOUT_SECONDS", "UPLOAD_FAULT_AFTER_S3_PUT", + "WORKER_CONCURRENCY", "JOB_MAX_ATTEMPTS", "JOB_TIMEOUT_SECONDS", "STAGING_RETENTION_SECONDS", + "WORKER_PREFETCH", "MAX_PENDING_JOBS", "RENDER_MEMORY_MB", +} + + +@dataclass(frozen=True, slots=True) +class Settings: + """Проверенная конфигурация запуска HTTP-сервера.""" + + host: str + port: int + environment: str + database_url: str + session_ttl_seconds: int + idempotency_ttl_seconds: int + instance_id: str + cursor_secret: bytes + + +def load_settings(env_file: Path = ROOT / ".env") -> Settings: + """Читает разрешённые ключи .env, не заменяя явно переданное окружение.""" + + if env_file.is_file(): + for raw_line in env_file.read_text(encoding="utf-8").splitlines(): + line = raw_line.strip() + if not line or line.startswith("#"): + continue + key, separator, value = line.partition("=") + if not separator or key not in _CONFIGURATION_KEYS: + continue + value = value.strip() + if len(value) >= 2 and value[0] == value[-1] and value[0] in {"'", '"'}: + value = value[1:-1] + os.environ.setdefault(key, value) + + def value(name: str, default: str = "") -> str: + return os.environ.get(name, default).strip() + + def number(name: str, default: int) -> int: + try: + result = int(value(name, str(default))) + except ValueError as error: + raise ValueError(f"{name} должен быть целым числом") from error + if result < 1: + raise ValueError(f"{name} должен быть положительным") + return result + + def url(name: str, file_name: str, prefixes: tuple[str, ...]) -> str: + result = value(name) + path = value(file_name) + if path: + try: + result = Path(path).read_text(encoding="utf-8").strip() + except OSError as error: + raise ValueError(f"{file_name} недоступен") from error + if result and not result.startswith(prefixes): + raise ValueError(f"{name} содержит неверный URL") + return result + + def secret(name: str, file_name: str) -> bytes: + result = value(name) + path = value(file_name) + if path: + try: + result = Path(path).read_text(encoding="utf-8").strip() + except OSError as error: + raise ValueError(f"{file_name} недоступен") from error + encoded = result.encode("utf-8") + if len(encoded) < 32: + raise ValueError(f"{name} должен содержать не менее 32 байт") + return encoded + + host = value("APP_HOST", "0.0.0.0") + environment = value("APP_ENV", "lab") + if not host: + raise ValueError("APP_HOST не может быть пустым") + if not environment: + raise ValueError("APP_ENV не может быть пустым") + port = number("APP_PORT", 8080) + if port > 65535: + raise ValueError("APP_PORT должен находиться в диапазоне 1..65535") + database_url = url( + "DATABASE_URL", "DATABASE_URL_FILE", ("postgresql://", "postgres://") + ) + if not database_url: + raise ValueError("Нужен PostgreSQL DATABASE_URL или DATABASE_URL_FILE") + instance_id = value("INSTANCE_ID") or os.uname().nodename + if not instance_id: + raise ValueError("INSTANCE_ID не может быть пустым") + idempotency_ttl_seconds = number("IDEMPOTENCY_TTL_SECONDS", 86400) + if idempotency_ttl_seconds < 86400: + raise ValueError("IDEMPOTENCY_TTL_SECONDS должен быть не менее 86400") + return Settings( + host=host, + port=port, + environment=environment, + database_url=database_url, + session_ttl_seconds=number("SESSION_TTL_SECONDS", 3600), + idempotency_ttl_seconds=idempotency_ttl_seconds, + instance_id=instance_id, + cursor_secret=secret("CURSOR_SECRET", "CURSOR_SECRET_FILE"), + ) diff --git a/src/cursor.py b/src/cursor.py new file mode 100644 index 0000000..5dd8eab --- /dev/null +++ b/src/cursor.py @@ -0,0 +1,81 @@ +import base64 +from datetime import UTC, datetime +import hashlib +import hmac +import json +from uuid import UUID + + +_secret: bytes | None = None + + +class CursorError(ValueError): + """Недопустимый, повреждённый или применённый не к тому ресурсу курсор.""" + + +def configure_secret(secret: bytes) -> None: + """Устанавливает общий для всех реплик ключ подписи курсоров.""" + global _secret + if len(secret) < 32: + raise ValueError("cursor secret must contain at least 32 bytes") + _secret = secret + + +def signing_secret() -> bytes: + if _secret is None: + raise RuntimeError("cursor secret has not been configured") + return _secret + + +def _encode_bytes(value: bytes) -> str: + return base64.urlsafe_b64encode(value).decode().rstrip("=") + + +def _decode_bytes(value: str) -> bytes: + try: + return base64.urlsafe_b64decode(value + "=" * (-len(value) % 4)) + except (ValueError, UnicodeEncodeError) as error: + raise CursorError from error + + +def encode_cursor(scope: str, created_at: datetime, item_id: str) -> str: + payload = json.dumps( + { + "scope": scope, + "created_at": created_at.astimezone(UTC) + .isoformat() + .replace("+00:00", "Z"), + "id": item_id, + }, + separators=(",", ":"), + sort_keys=True, + ).encode() + signature = hmac.new(signing_secret(), payload, hashlib.sha256).digest() + return _encode_bytes(payload) + "." + _encode_bytes(signature) + + +def decode_cursor(cursor: str, expected_scope: str) -> tuple[datetime, str]: + try: + encoded_payload, encoded_signature = cursor.split(".") + payload = _decode_bytes(encoded_payload) + signature = _decode_bytes(encoded_signature) + except ValueError as error: + raise CursorError from error + + expected_signature = hmac.new(signing_secret(), payload, hashlib.sha256).digest() + if not hmac.compare_digest(signature, expected_signature): + raise CursorError + + try: + data = json.loads(payload) + if set(data) != {"scope", "created_at", "id"}: + raise CursorError + if data["scope"] != expected_scope: + raise CursorError + item_id = str(UUID(data["id"])) + created_at = datetime.fromisoformat(data["created_at"].replace("Z", "+00:00")) + if created_at.tzinfo is None: + raise CursorError + return created_at.astimezone(UTC), item_id + except (TypeError, ValueError, KeyError) as error: + raise CursorError from error diff --git a/src/errors.py b/src/errors.py new file mode 100644 index 0000000..258d699 --- /dev/null +++ b/src/errors.py @@ -0,0 +1,12 @@ +"""Ошибки, которые преобразуются в контрактные HTTP-ответы.""" + + +class ApiError(Exception): + + def __init__( + self, status_code: int, code: str, message: str, headers: dict[str, str] | None = None + ) -> None: + self.status_code = status_code + self.code = code + self.message = message + self.headers = headers or {} diff --git a/src/main.py b/src/main.py new file mode 100644 index 0000000..004b3e6 --- /dev/null +++ b/src/main.py @@ -0,0 +1,189 @@ +import asyncio +import logging +import os +import sys +import time +from contextlib import asynccontextmanager +from uuid import uuid4 + +from fastapi import FastAPI, Request +from fastapi.exceptions import RequestValidationError +import uvicorn +from starlette.responses import JSONResponse, Response +from starlette.exceptions import HTTPException + +from .errors import ApiError +from .cursor import configure_secret +from .models import ErrorDetails, ErrorResponse +from .config import load_settings +from .routes import router +from .attachments import router as attachment_router +from .storage.s3 import S3Settings, S3Store +from .storage.postgres import PostgresStore +from . import telemetry as tel + +tel.setup("messenger-api") + + +def logger(name: str) -> logging.Logger: + result = logging.getLogger(name) + if result.handlers: + return result + handler = logging.StreamHandler(sys.stdout) + handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(name)s %(message)s")) + result.addHandler(handler) + result.setLevel(logging.INFO) + result.propagate = False + return result + + +access_logger = logger("messenger.access") + + +@asynccontextmanager +async def lifespan(application: FastAPI): + settings = load_settings() + configure_secret(settings.cursor_secret) + store = PostgresStore(settings.database_url) + await store.connect() + application.state.store = store + application.state.settings = settings + application.state.s3 = S3Store(S3Settings.load()) + concurrency = int(os.environ.get("UPLOAD_CONCURRENCY", "1")) + if not 1 <= concurrency <= 8: + raise ValueError("UPLOAD_CONCURRENCY must be 1..8") + application.state.upload_slot = asyncio.Semaphore(concurrency) + application.state.accepting_requests = True + async def sample_pool(): + while True: + pool = store.pool + tel.state.update(pool_used=pool.get_size()-pool.get_idle_size(), pool_capacity=pool.get_max_size(), pool_waiting=len(pool._queue._getters)) + await asyncio.sleep(1) + sampler = asyncio.create_task(sample_pool()) + try: + yield + finally: + sampler.cancel() + try: + await sampler + except asyncio.CancelledError: + pass + application.state.accepting_requests = False + application.state.s3.client.close() + await store.close() + + +app = FastAPI( + title="MTUSI Messenger", + version="1.6.0", + redoc_url=None, + lifespan=lifespan, +) + + +@app.middleware("http") +async def add_request_id(request: Request, call_next) -> Response: + """Добавляет ID запроса и пишет безопасную строку access-лога.""" + + request.state.request_id = str(uuid4()) + started_at = time.monotonic() + status_code = 500 + with tel.tracer.start_as_current_span("HTTP request", context=tel.extract(dict(request.headers)), kind=tel.SpanKind.SERVER, record_exception=False, set_status_on_exception=False) as span: + try: + response = await call_next(request) + status_code = response.status_code + response.headers["X-Request-Id"] = request.state.request_id + response.headers["X-Instance-Id"] = request.app.state.settings.instance_id + return response + finally: + route = getattr(request.scope.get("route"), "path", "unmatched") + method = tel.http_method(request.method) + attrs = {"method": method, "route": route, "status_class": f"{status_code // 100}xx"} + span.update_name(f"{method} {route}") + span.set_attributes({"http.request.method": method, "http.route": route, "http.response.status_code": status_code}) + if status_code >= 500: + span.set_status(tel.Status(tel.StatusCode.ERROR)) + tel.http_requests.add(1, attrs) + tel.http_duration.record(time.monotonic() - started_at, attrs) + tel.log.info("http.request", extra={"request_id": request.state.request_id, "method": method, "route": route, "status": status_code}) + + +def error_response( + request: Request, status_code: int, code: str, message: str, headers: dict[str, str] | None = None +) -> JSONResponse: + request_id = request.state.request_id + body = ErrorResponse( + error=ErrorDetails(code=code, message=message, request_id=request_id) + ) + # Неожиданная ошибка обрабатывается за пределами HTTP-middleware. + # Поэтому её ответ тоже должен сам получить заголовок из контракта. + return JSONResponse( + status_code=status_code, + content=body.model_dump(), + headers={ + "X-Request-Id": request_id, + "X-Instance-Id": request.app.state.settings.instance_id, + **(headers or {}), + }, + ) + + +@app.exception_handler(ApiError) +async def handle_api_error(request: Request, error: ApiError) -> JSONResponse: + """Преобразует контролируемую ошибку API в стандартный JSON-ответ.""" + + return error_response(request, error.status_code, error.code, error.message, error.headers) + + +@app.exception_handler(RequestValidationError) +async def handle_validation_error( + request: Request, error: RequestValidationError +) -> JSONResponse: + """Преобразует ошибки JSON и Pydantic в обязательный ответ 400.""" + + return error_response(request, 400, "invalid_request", "Некорректный запрос") + + +@app.exception_handler(HTTPException) +async def handle_http_error(request: Request, error: HTTPException) -> JSONResponse: + """Включает ошибки декодирования тела, возникающие до Pydantic.""" + codes = { + 400: "invalid_request", + 401: "unauthorized", + 403: "forbidden", + 404: "not_found", + 405: "method_not_allowed", + 503: "unavailable", + } + code = codes.get(error.status_code, "internal_error") + return error_response( + request, error.status_code, code, + "Некорректный запрос" if error.status_code < 500 else "Ошибка сервера", + error.headers, + ) + + +@app.exception_handler(Exception) +async def handle_unexpected_error(request: Request, error: Exception) -> JSONResponse: + """Не раскрывает внутренние детали при непредвиденной ошибке приложения.""" + + return error_response(request, 500, "internal_error", "Внутренняя ошибка сервера") + + +app.include_router(router) +app.include_router(attachment_router) + + +def main() -> None: + settings = load_settings() + uvicorn.run( + app, + host=settings.host, + port=settings.port, + access_log=False, + timeout_graceful_shutdown=8, + ) + + +if __name__ == "__main__": + main() diff --git a/src/migrate.py b/src/migrate.py new file mode 100644 index 0000000..4376e8d --- /dev/null +++ b/src/migrate.py @@ -0,0 +1,18 @@ +import asyncio + +from .config import load_settings +from .storage.postgres import PostgresStore + + +async def run() -> None: + settings = load_settings() + store = PostgresStore(settings.database_url) + await store.connect() + try: + await store.migrate() + finally: + await store.close() + + +if __name__ == "__main__": + asyncio.run(run()) diff --git a/src/models.py b/src/models.py new file mode 100644 index 0000000..15f3810 --- /dev/null +++ b/src/models.py @@ -0,0 +1,151 @@ +from datetime import datetime +from uuid import UUID + +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator + +class CreateUserRequest(BaseModel): + model_config = ConfigDict(extra="forbid", strict=True) + + username: str = Field(min_length=3, max_length=32, pattern=r"^[a-z0-9_]{3,32}$") + display_name: str = Field(min_length=1, max_length=100) + password: str = Field(min_length=12, max_length=128) + + @field_validator("display_name") + @classmethod + def display_name_must_not_be_whitespace(cls, value: str) -> str: + """Не допускает обязательное имя, состоящее только из пробелов.""" + + if not value.strip(): + raise ValueError("display_name не может состоять только из пробелов") + return value + + +class CreateChatRequest(BaseModel): + model_config = ConfigDict(extra="forbid") + + title: str = Field(min_length=1, max_length=100) + member_ids: list[UUID] = Field(min_length=1, max_length=100) + + @field_validator("title") + @classmethod + def title_must_not_be_whitespace(cls, value: str) -> str: + """Не допускает обязательное название, состоящее только из пробелов.""" + + if not value.strip(): + raise ValueError("title не может состоять только из пробелов") + return value + + +class ChatPageResponse(BaseModel): + model_config = ConfigDict(extra="forbid") + + items: list["Chat"] + next_cursor: str | None + + +class CreateMessageRequest(BaseModel): + model_config = ConfigDict(extra="forbid", strict=True) + + text: str = Field(max_length=4000) + attachment_ids: list[str] = Field(default_factory=list, max_length=10) + + @field_validator("attachment_ids") + @classmethod + def valid_ids(cls, values: list[str]) -> list[str]: + normalized = [str(UUID(value)) for value in values] + if len(normalized) != len(set(normalized)): + raise ValueError("Повторяющиеся attachment_ids") + return normalized + + @model_validator(mode="after") + def has_content(self): + if not self.text.strip() and not self.attachment_ids: + raise ValueError("Сообщение без содержимого") + return self + + +class MessagePageResponse(BaseModel): + model_config = ConfigDict(extra="forbid") + + items: list["Message"] + next_cursor: str | None + + +class ErrorDetails(BaseModel): + model_config = ConfigDict(extra="forbid") + + code: str + message: str + request_id: str + + +class ErrorResponse(BaseModel): + model_config = ConfigDict(extra="forbid") + + error: ErrorDetails + + +class User(BaseModel): + model_config = ConfigDict(extra="forbid") + + id: UUID + username: str + display_name: str + created_at: datetime + + +class Chat(BaseModel): + model_config = ConfigDict(extra="forbid") + + id: UUID + title: str + member_ids: list[UUID] + created_at: datetime + + +class Message(BaseModel): + model_config = ConfigDict(extra="forbid") + + id: UUID + chat_id: UUID + sender_id: UUID + text: str + attachment_ids: list[UUID] = Field(default_factory=list) + created_at: datetime + + +class Session(BaseModel): + model_config = ConfigDict(extra="forbid") + + token: str = Field(min_length=32) + token_type: str = "Bearer" + expires_at: datetime + user: User + + +class Attachment(BaseModel): + model_config = ConfigDict(extra="ignore") + + id: UUID + chat_id: UUID + owner_id: UUID + filename: str + content_type: str + size_bytes: int + sha256: str + status: str + variants: list["Variant"] = Field(default_factory=list) + error_code: str | None = None + created_at: datetime + updated_at: datetime + + +class Variant(BaseModel): + model_config = ConfigDict(extra="forbid") + + name: str + content_type: str + size_bytes: int + width: int + height: int + sha256: str diff --git a/src/outbox.py b/src/outbox.py new file mode 100644 index 0000000..a8ac14d --- /dev/null +++ b/src/outbox.py @@ -0,0 +1,46 @@ +"""Transactional-outbox dispatcher; safe to run as one or more replicas.""" +import asyncio +import logging +import os +import json +import time + +from .broker import publish +from .config import load_settings +from .storage.postgres import PostgresStore +from . import telemetry as tel + +logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s outbox %(message)s") + + +async def run() -> None: + tel.setup("messenger-outbox") + settings = load_settings() + store = PostgresStore(settings.database_url) + await store.connect() + interval = max(0.1, float(os.environ.get("OUTBOX_DISPATCH_INTERVAL_SECONDS", "1"))) + try: + while True: + try: + await store.schedule_due_jobs(int(os.environ.get('JOB_MAX_ATTEMPTS', '5'))) + for event in await store.due_outbox_events(): + context = event['trace_context'] + await publish(event["payload"], json.loads(context) if isinstance(context, str) else context) + await store.mark_outbox_published(event["id"]) + except Exception: + logging.exception("outbox dispatch failed; durable event remains pending") + try: + queue = await store.pool.fetchrow("SELECT count(*) AS depth, COALESCE(EXTRACT(EPOCH FROM now()-min(created_at)), 0)::float8 AS age FROM attachment_jobs WHERE status='queued'") + slo = await store.sample_ready_slo() + tel.state.update(queue_depth=queue['depth'], queue_age=queue['age'], + slo_ready_eligible=slo['eligible'], slo_ready_bad=slo['bad'], + slo_unknown=slo['unknown'], queue_sample_time=time.time()) + except Exception: + logging.warning("queue telemetry unavailable") + await asyncio.sleep(interval) + finally: + await store.close() + + +if __name__ == "__main__": + asyncio.run(run()) diff --git a/src/pipeline_metrics.py b/src/pipeline_metrics.py new file mode 100644 index 0000000..93cde0b --- /dev/null +++ b/src/pipeline_metrics.py @@ -0,0 +1,37 @@ +"""Operator-only metrics for one synthetic chat; never exposes credentials.""" +import asyncio +import json +import os +from pathlib import Path +import sys +from uuid import UUID +from .config import load_settings +from .storage.postgres import PostgresStore + + +async def main(chat): + store = PostgresStore(load_settings().database_url) + await store.connect() + try: + rows = await store.pool.fetch('SELECT a.id, a.status, extract(epoch from (j.first_started_at-j.created_at))::float8 AS queue_wait_seconds FROM attachments a JOIN attachment_jobs j ON j.attachment_id=a.id WHERE a.chat_id=$1', chat) + jobs = [dict(row, id=str(row['id'])) for row in rows] + pending = await store.pool.fetchval("SELECT count(*) FROM attachment_jobs WHERE status IN ('queued','processing')") + finally: + await store.close() + rss = 0 + for path in Path('/proc').glob('[0-9]*/status'): + try: + if int(path.parent.name) == os.getpid(): + continue + for line in path.read_text().splitlines(): + if line.startswith('VmRSS:'): + rss += int(line.split()[1]) * 1024 + except (OSError, ValueError): + pass + print(json.dumps({'jobs': jobs, 'pending': pending, 'rss_bytes': rss, + 'concurrency': int(os.environ['WORKER_CONCURRENCY']), + 'prefetch': int(os.environ['WORKER_PREFETCH'])})) + + +if __name__ == '__main__': + asyncio.run(main(UUID(sys.argv[1]))) diff --git a/src/reconcile.py b/src/reconcile.py new file mode 100644 index 0000000..1b60bb8 --- /dev/null +++ b/src/reconcile.py @@ -0,0 +1,104 @@ +"""Safely reconcile attachment metadata and original S3 objects. + +The command is dry-run by default. It only deletes unreferenced objects older +than a configured grace period; missing objects are reported and never cause a +metadata delete. That keeps a delayed PostgreSQL commit and active attachment +records safe from a maintenance run. +""" +import argparse +import asyncio +from datetime import datetime, timedelta, timezone +import json +import os +from time import perf_counter + +from .config import load_settings +from .storage.postgres import PostgresStore +from .storage.s3 import S3Settings, S3Store + + +async def run(args) -> dict: + settings = load_settings() + db = PostgresStore(settings.database_url) + s3 = S3Store(S3Settings.load()) + try: + await db.connect() + async with db.attachment_transaction(maintenance=True): + return await scan(args, db, s3) + finally: + s3.client.close() + await db.close() + + +async def scan(args, db, s3) -> dict: + # Age is measured at scan start, not after a potentially slow inventory. + cutoff = datetime.now(timezone.utc) - timedelta(seconds=args.grace_seconds) + started = perf_counter() + objects, s3_pages = {}, 0 + for prefix in ('staging/', 'originals/', 'variants/'): + entries, pages = await s3.list_with_pages(prefix) + objects.update(entries) + s3_pages += pages + listed_at = perf_counter() + metadata = await db.attachment_keys() + metadata_at = perf_counter() + orphan = sorted(key for key, modified in objects.items() + if key not in metadata and modified <= cutoff) + fresh = sorted(key for key, modified in objects.items() + if key not in metadata and modified > cutoff) + missing = sorted({str(attachment_id): key for key, attachment_id in metadata.items() + if key not in objects}.items()) + deleted = [] + selected = set(args.object_key) if args.object_key else None + if args.apply: + for key in orphan: + if selected is not None and key not in selected: + continue + await s3.delete(key) + deleted.append(key) + finished = perf_counter() + return { + "mode": "apply" if args.apply else "dry-run", + "grace_seconds": args.grace_seconds, + "s3_objects": len(objects), + "metadata_rows": len(metadata), + "orphan_candidates": orphan, + "fresh_unreferenced": fresh, + "missing_objects": [{"attachment_id": item[0], "object_key": item[1]} for item in missing], + "deleted": deleted, + "selected_keys": sorted(selected) if selected is not None else None, + "scan_metrics": { + "s3_list_pages": s3_pages, + "s3_list_seconds": round(listed_at - started, 6), + "postgres_metadata_seconds": round(metadata_at - listed_at, 6), + "total_seconds": round(finished - started, 6), + }, + } + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--apply", action="store_true", help="delete only aged orphan objects") + parser.add_argument("--grace-seconds", type=int, default=int(os.environ.get('STAGING_RETENTION_SECONDS', '86400'))) + parser.add_argument('--loop', action='store_true') + parser.add_argument("--object-key", action="append", default=[], + help="restrict apply to these exact keys; repeatable, inventory stays complete") + args = parser.parse_args() + if not 0 <= args.grace_seconds <= 7 * 24 * 3600: + parser.error("--grace-seconds must be 0..604800") + if any(not key.startswith(('originals/', 'staging/', 'variants/')) for key in args.object_key): + parser.error('unsupported object prefix') + if args.loop: + import time + while True: + try: + print(json.dumps(asyncio.run(run(args)), ensure_ascii=False, sort_keys=True)) + except Exception: + print('cleanup deferred: dependency unavailable', flush=True) + time.sleep(60) + else: + print(json.dumps(asyncio.run(run(args)), ensure_ascii=False, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/src/render.py b/src/render.py new file mode 100644 index 0000000..dbb1d1d --- /dev/null +++ b/src/render.py @@ -0,0 +1,27 @@ +"""Isolated image renderer. Parent enforces wall time and kills this process.""" +import os +import pickle +import sys + + +def main(): + if sys.platform == 'linux': + import resource + limit = int(os.environ.get('RENDER_MEMORY_MB', '768')) * 1024 * 1024 + resource.setrlimit(resource.RLIMIT_AS, (limit, limit)) + from .worker import variants + from PIL import UnidentifiedImageError, Image + try: + data = sys.stdin.buffer.read(10485761) + if len(data) > 10485760: + raise ValueError('bytes limit') + result = variants(data, int(os.environ.get('MAX_IMAGE_PIXELS', '40000000'))) + pickle.dump(result, sys.stdout.buffer) + except MemoryError: + sys.exit(21) + except (ValueError, OSError, SyntaxError, UnidentifiedImageError, Image.DecompressionBombError): + sys.exit(20) + + +if __name__ == '__main__': + main() diff --git a/src/replay.py b/src/replay.py new file mode 100644 index 0000000..2360cb5 --- /dev/null +++ b/src/replay.py @@ -0,0 +1,24 @@ +"""Operator-only safe replay for a failed attachment job.""" +import argparse +import asyncio +from uuid import UUID + +from .config import load_settings +from .storage.postgres import PostgresStore + + +async def run(attachment_id: UUID) -> None: + store = PostgresStore(load_settings().database_url) + await store.connect() + try: + if not await store.replay_failed_attachment(attachment_id): + raise SystemExit("attachment is not in failed state") + finally: + await store.close() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("attachment_id", type=UUID) + args = parser.parse_args() + asyncio.run(run(args.attachment_id)) diff --git a/src/routes.py b/src/routes.py new file mode 100644 index 0000000..fcd2e74 --- /dev/null +++ b/src/routes.py @@ -0,0 +1,185 @@ +import base64 +import re +from datetime import datetime +from uuid import UUID + +from fastapi import APIRouter, Depends, Header, Query, Request, Response + +from .cursor import CursorError, decode_cursor, encode_cursor +from .errors import ApiError +from .storage.postgres import PostgresStore +from .broker import ready as broker_ready +from .models import ( + Chat, + ChatPageResponse, + CreateChatRequest, + CreateMessageRequest, + CreateUserRequest, + Message, + MessagePageResponse, + Session, + User, +) + +router = APIRouter() +BEARER_CHALLENGE = {"WWW-Authenticate": 'Bearer realm="messenger"'} +BASIC_CHALLENGE = {"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'} +IDEMPOTENCY_KEY = re.compile(r"^[A-Za-z0-9._:-]{1,128}$") + +def get_store(request: Request) -> PostgresStore: + return request.app.state.store + + +async def get_current_user( + authorization: str | None = Header(default=None), + store: PostgresStore = Depends(get_store), +) -> dict: + if authorization is None or not authorization.startswith("Bearer "): + raise ApiError(401, "unauthorized", "Требуется Bearer-сессия", BEARER_CHALLENGE) + token = authorization[7:] + if not token or not token.isascii() or token.strip() != token: + raise ApiError(401, "unauthorized", "Требуется Bearer-сессия", BEARER_CHALLENGE) + user = await store.user_for_token(token) + if user is None: + raise ApiError(401, "unauthorized", "Недействительная сессия", BEARER_CHALLENGE) + return user + +def parse_basic(authorization: str | None) -> tuple[str, str]: + if authorization is None or not authorization.startswith("Basic "): + raise ApiError(401, "unauthorized", "Неверные учётные данные", BASIC_CHALLENGE) + try: + decoded = base64.b64decode(authorization[6:], validate=True) + username_bytes, password_bytes = decoded.split(b":", 1) + username = username_bytes.decode("ascii") + password = password_bytes.decode("utf-8") + except (UnicodeDecodeError, ValueError) as error: + raise ApiError(401, "unauthorized", "Неверные учётные данные", BASIC_CHALLENGE) from error + if not username or not (12 <= len(password) <= 128): + raise ApiError(401, "unauthorized", "Неверные учётные данные", BASIC_CHALLENGE) + return username, password + + +def cursor_position(cursor: str | None, scope: str) -> tuple[datetime, UUID] | None: + if cursor is None: + return None + try: + created_at, item_id = decode_cursor(cursor, scope) + return created_at, UUID(item_id) + except (CursorError, ValueError) as error: + raise ApiError(400, "invalid_request", "Некорректный cursor") from error + + +def next_page_cursor(rows: list[dict], limit: int, scope: str) -> str | None: + if len(rows) <= limit: + return None + last = rows[limit - 1] + return encode_cursor(scope, last["created_at"], str(last["id"])) + +@router.get("/health/live") +async def liveness() -> dict[str, str]: + return {"status": "ok"} + +@router.get("/health/ready") +async def readiness(request: Request, store: PostgresStore = Depends(get_store)) -> dict[str, str]: + if not request.app.state.accepting_requests: + raise ApiError(503, "unavailable", "API завершает работу") + await store.ready() + await request.app.state.s3.ready() + await broker_ready() + return {"status": "ok"} + +@router.post("/api/v1/users", response_model=User, status_code=201) +async def register_user( + payload: CreateUserRequest, store: PostgresStore = Depends(get_store) +) -> dict: + return await store.create_user(payload.username, payload.display_name, payload.password) + +@router.post("/api/v1/auth/sessions", response_model=Session, status_code=201) +async def create_session( + request: Request, + authorization: str | None = Header(default=None), + store: PostgresStore = Depends(get_store), +) -> dict: + if await request.body(): + raise ApiError(400, "invalid_request", "Тело запроса не требуется") + username, password = parse_basic(authorization) + user = await store.authenticate(username, password) + if user is None: + raise ApiError(401, "unauthorized", "Неверные учётные данные", BASIC_CHALLENGE) + return await store.create_session(user, request.app.state.settings.session_ttl_seconds) + +@router.delete("/api/v1/auth/sessions/current", status_code=204) +async def revoke_session( + authorization: str | None = Header(default=None), + store: PostgresStore = Depends(get_store), + user: dict = Depends(get_current_user), +) -> Response: + del user + await store.revoke_session(authorization[7:]) + return Response(status_code=204) + +@router.get("/api/v1/users/me", response_model=User) +async def current_user(user: dict = Depends(get_current_user)) -> dict: + return user + +@router.post("/api/v1/chats", response_model=Chat, status_code=201) +async def create_chat( + payload: CreateChatRequest, + user: dict = Depends(get_current_user), + store: PostgresStore = Depends(get_store), +) -> dict: + return await store.create_chat(user["id"], payload.title, payload.member_ids) + +@router.get("/api/v1/chats", response_model=ChatPageResponse) +async def list_chats( + user: dict = Depends(get_current_user), + store: PostgresStore = Depends(get_store), + limit: int = Query(default=50, ge=1, le=100), + cursor: str | None = Query(default=None, min_length=1), +) -> ChatPageResponse: + scope = "chats:" + str(user["id"]) + rows = await store.list_chats_for_member(user["id"], cursor_position(cursor, scope), limit) + page = rows[:limit] + return ChatPageResponse(items=page, next_cursor=next_page_cursor(rows, limit, scope)) + +@router.get("/api/v1/chats/{chat_id}", response_model=Chat) +async def get_chat( + chat_id: UUID, + user: dict = Depends(get_current_user), + store: PostgresStore = Depends(get_store), +) -> dict: + return await store.get_chat_for_member(chat_id, user["id"]) + +@router.post("/api/v1/chats/{chat_id}/messages", response_model=Message, status_code=201) +async def create_message( + chat_id: UUID, + payload: CreateMessageRequest, + request: Request, + idempotency_key: str | None = Header(default=None, alias="Idempotency-Key"), + user: dict = Depends(get_current_user), + store: PostgresStore = Depends(get_store), +) -> dict: + if idempotency_key is None or IDEMPOTENCY_KEY.fullmatch(idempotency_key) is None: + raise ApiError(400, "invalid_request", "Некорректный Idempotency-Key") + return await store.create_message_idempotent( + chat_id, + user["id"], + payload.text, + idempotency_key, + request.app.state.settings.idempotency_ttl_seconds, + [UUID(value) for value in payload.attachment_ids], + ) + +@router.get("/api/v1/chats/{chat_id}/messages", response_model=MessagePageResponse) +async def list_messages( + chat_id: UUID, + user: dict = Depends(get_current_user), + store: PostgresStore = Depends(get_store), + limit: int = Query(default=50, ge=1, le=100), + cursor: str | None = Query(default=None, min_length=1), +) -> MessagePageResponse: + scope = "messages:" + str(chat_id) + after = cursor_position(cursor, scope) + rows = await store.list_messages_for_member(chat_id, user["id"], after, limit) + page = rows[:limit] + return MessagePageResponse(items=page, next_cursor=next_page_cursor(rows, limit, scope)) diff --git a/src/storage/__init__.py b/src/storage/__init__.py new file mode 100644 index 0000000..e2e0419 --- /dev/null +++ b/src/storage/__init__.py @@ -0,0 +1,2 @@ +"""Слой in-memory хранения для приложения первой лабораторной работы.""" +"""Persistent storage adapters.""" diff --git a/src/storage/postgres.py b/src/storage/postgres.py new file mode 100644 index 0000000..9e6edfe --- /dev/null +++ b/src/storage/postgres.py @@ -0,0 +1,594 @@ +from datetime import datetime, timedelta, timezone +from contextlib import asynccontextmanager +from hashlib import sha256 +import json +from pathlib import Path +from secrets import token_urlsafe +from uuid import UUID, uuid4 + +import asyncpg +from argon2 import PasswordHasher +from argon2.exceptions import InvalidHashError, VerificationError + +from ..errors import ApiError +from .. import telemetry as tel + + +ROOT = Path(__file__).resolve().parents[2] +MIGRATIONS = ROOT / "migrations" +PASSWORD_HASHER = PasswordHasher() +DB_ERRORS = (asyncpg.PostgresError, asyncpg.InterfaceError, OSError, TimeoutError) + + +class PostgresStore: + def __init__(self, database_url: str) -> None: + self._database_url = database_url + self._pool: asyncpg.Pool | None = None + + async def connect(self) -> None: + try: + self._pool = await asyncpg.create_pool( + self._database_url, min_size=1, max_size=10, command_timeout=5 + ) + except DB_ERRORS as error: + raise RuntimeError("PostgreSQL is unavailable") from error + + async def close(self) -> None: + if self._pool is not None: + await self._pool.close() + + @property + def pool(self) -> asyncpg.Pool: + if self._pool is None: + raise RuntimeError("PostgreSQL pool has not been initialized") + return self._pool + + async def migrate(self) -> None: + async with self.pool.acquire() as connection: + await connection.execute( + "CREATE TABLE IF NOT EXISTS schema_migrations (version text PRIMARY KEY, applied_at timestamptz NOT NULL DEFAULT now())" + ) + for migration in sorted(MIGRATIONS.glob("*.sql")): + applied = await connection.fetchval( + "SELECT 1 FROM schema_migrations WHERE version = $1", migration.name + ) + if applied: + continue + async with connection.transaction(): + await connection.execute(migration.read_text(encoding="utf-8")) + await connection.execute( + "INSERT INTO schema_migrations (version) VALUES ($1)", migration.name + ) + + @staticmethod + def _user(row: asyncpg.Record) -> dict: + return { + "id": row["id"], + "username": row["username"], + "display_name": row["display_name"], + "created_at": row["created_at"], + } + + @staticmethod + def _chat(row: asyncpg.Record) -> dict: + return { + "id": row["id"], + "title": row["title"], + "member_ids": row["member_ids"], + "created_at": row["created_at"], + } + + @staticmethod + def _message(row: asyncpg.Record) -> dict: + return { + "id": row["id"], + "chat_id": row["chat_id"], + "sender_id": row["sender_id"], + "text": row["text"], + "attachment_ids": row["attachment_ids"], + "created_at": row["created_at"], + } + + async def ready(self) -> None: + try: + await self.pool.fetchval("SELECT 1") + except DB_ERRORS as error: + raise ApiError(503, "unavailable", "PostgreSQL недоступен") from error + + async def create_user(self, username: str, display_name: str, password: str) -> dict: + try: + row = await self.pool.fetchrow( + "INSERT INTO users (id, username, display_name, password_hash) VALUES ($1, $2, $3, $4) RETURNING id, username, display_name, created_at", + uuid4(), + username, + display_name, + PASSWORD_HASHER.hash(password), + ) + except asyncpg.UniqueViolationError as error: + raise ApiError(409, "conflict", "Username уже занят") from error + except DB_ERRORS as error: + raise ApiError(503, "unavailable", "PostgreSQL недоступен") from error + return self._user(row) + + async def authenticate(self, username: str, password: str) -> dict | None: + try: + row = await self.pool.fetchrow( + "SELECT id, username, display_name, created_at, password_hash FROM users WHERE username = $1", + username, + ) + except DB_ERRORS as error: + raise ApiError(503, "unavailable", "PostgreSQL недоступен") from error + if row is None: + return None + try: + valid = PASSWORD_HASHER.verify(row["password_hash"], password) + except (InvalidHashError, VerificationError): + valid = False + return self._user(row) if valid else None + + async def create_session(self, user: dict, ttl_seconds: int) -> dict: + token = token_urlsafe(32) + token_hash = sha256(token.encode("ascii")).hexdigest() + try: + row = await self.pool.fetchrow( + "INSERT INTO sessions (token_hash, user_id, expires_at) VALUES ($1, $2, now() + $3 * interval '1 second') RETURNING expires_at", + token_hash, + user["id"], + ttl_seconds, + ) + except DB_ERRORS as error: + raise ApiError(503, "unavailable", "PostgreSQL недоступен") from error + return {"token": token, "token_type": "Bearer", "expires_at": row["expires_at"], "user": user} + + async def user_for_token(self, token: str) -> dict | None: + if not token.isascii(): + return None + token_hash = sha256(token.encode("ascii")).hexdigest() + try: + row = await self.pool.fetchrow( + "SELECT u.id, u.username, u.display_name, u.created_at FROM sessions s JOIN users u ON u.id = s.user_id WHERE s.token_hash = $1 AND s.expires_at > now()", + token_hash, + ) + except DB_ERRORS as error: + raise ApiError(503, "unavailable", "PostgreSQL недоступен") from error + return self._user(row) if row else None + + async def revoke_session(self, token: str) -> None: + try: + await self.pool.execute( + "DELETE FROM sessions WHERE token_hash = $1", sha256(token.encode("ascii")).hexdigest() + ) + except DB_ERRORS as error: + raise ApiError(503, "unavailable", "PostgreSQL недоступен") from error + + async def create_chat(self, creator_id: UUID, title: str, member_ids: list[UUID]) -> dict: + if len(member_ids) != len(set(member_ids)) or creator_id not in member_ids: + raise ApiError(400, "invalid_request", "Некорректный список участников") + try: + async with self.pool.acquire() as connection, connection.transaction(): + existing = await connection.fetchval( + "SELECT count(*) FROM users WHERE id = ANY($1::uuid[])", member_ids + ) + if existing != len(member_ids): + raise ApiError(400, "invalid_request", "Один или несколько участников не существуют") + row = await connection.fetchrow( + "INSERT INTO chats (id, title) VALUES ($1, $2) RETURNING id, title, created_at", + uuid4(), + title, + ) + await connection.executemany( + "INSERT INTO chat_members (chat_id, user_id) VALUES ($1, $2)", + [(row["id"], member_id) for member_id in member_ids], + ) + except DB_ERRORS as error: + raise ApiError(503, "unavailable", "PostgreSQL недоступен") from error + return { + "id": row["id"], + "title": row["title"], + "member_ids": member_ids, + "created_at": row["created_at"], + } + + async def get_chat_for_member(self, chat_id: UUID, user_id: UUID) -> dict: + try: + row = await self.pool.fetchrow( + "SELECT c.id, c.title, c.created_at, array_agg(cm_all.user_id ORDER BY cm_all.user_id) AS member_ids FROM chats c JOIN chat_members mine ON mine.chat_id = c.id AND mine.user_id = $2 JOIN chat_members cm_all ON cm_all.chat_id = c.id WHERE c.id = $1 GROUP BY c.id", + chat_id, + user_id, + ) + except DB_ERRORS as error: + raise ApiError(503, "unavailable", "PostgreSQL недоступен") from error + if row is None: + raise ApiError(404, "not_found", "Чат не найден") + return self._chat(row) + + async def list_chats_for_member( + self, user_id: UUID, after: tuple[datetime, UUID] | None, limit: int + ) -> list[dict]: + query = "SELECT c.id, c.title, c.created_at, array_agg(cm_all.user_id ORDER BY cm_all.user_id) AS member_ids FROM chats c JOIN chat_members mine ON mine.chat_id = c.id AND mine.user_id = $1 JOIN chat_members cm_all ON cm_all.chat_id = c.id" + args: list = [user_id] + if after is not None: + query += " WHERE (c.created_at, c.id) > ($2, $3)" + args.extend(after) + query += " GROUP BY c.id ORDER BY c.created_at, c.id LIMIT $" + str(len(args) + 1) + args.append(limit + 1) + try: + rows = await self.pool.fetch(query, *args) + except DB_ERRORS as error: + raise ApiError(503, "unavailable", "PostgreSQL недоступен") from error + return [self._chat(row) for row in rows] + + async def create_message_idempotent( + self, chat_id: UUID, sender_id: UUID, text: str, idempotency_key: str, ttl_seconds: int, attachment_ids: list[UUID] | None = None + ) -> dict: + """Создаёт сообщение и фиксирует результат retry в одной транзакции.""" + path = f"/api/v1/chats/{chat_id}/messages" + attachment_ids = attachment_ids or [] + # Preserve hashes of text-only requests already accepted by lab3. + canonical = {"text": text} + if attachment_ids: + canonical["attachment_ids"] = [str(value) for value in attachment_ids] + request_hash = sha256( + json.dumps(canonical, ensure_ascii=False, separators=(",", ":"), sort_keys=True).encode("utf-8") + ).hexdigest() + try: + async with self.pool.acquire() as connection, connection.transaction(): + member = await connection.fetchval( + "SELECT 1 FROM chat_members WHERE chat_id = $1 AND user_id = $2", chat_id, sender_id + ) + if member is None: + raise ApiError(404, "not_found", "Чат не найден") + await connection.execute( + "DELETE FROM idempotency_records WHERE created_at <= now() - $1 * interval '1 second'", ttl_seconds + ) + if attachment_ids: + count = await connection.fetchval( + "SELECT count(*) FROM attachments WHERE chat_id = $1 AND id = ANY($2::uuid[])", + chat_id, attachment_ids, + ) + if count != len(attachment_ids): + raise ApiError(404, "not_found", "Вложение не найдено") + failed = await connection.fetchval( + "SELECT 1 FROM attachments WHERE id=ANY($1::uuid[]) AND status='failed' LIMIT 1", attachment_ids + ) + if failed: + raise ApiError(409, 'conflict', 'Вложение завершилось ошибкой') + claimed = await connection.fetchrow( + "INSERT INTO idempotency_records (user_id, method, path, idempotency_key, request_hash) " + "VALUES ($1, 'POST', $2, $3, $4) ON CONFLICT DO NOTHING RETURNING message_id", + sender_id, path, idempotency_key, request_hash, + ) + if claimed is None: + recorded = await connection.fetchrow( + "SELECT request_hash, message_id FROM idempotency_records " + "WHERE user_id = $1 AND method = 'POST' AND path = $2 AND idempotency_key = $3 FOR UPDATE", + sender_id, path, idempotency_key, + ) + if recorded is None: + raise ApiError(503, "unavailable", "Не удалось получить результат идемпотентного запроса") + if recorded["request_hash"] != request_hash: + raise ApiError(409, "conflict", "Idempotency-Key использован с другим телом") + if recorded["message_id"] is None: + raise ApiError(503, "unavailable", "Не удалось получить результат идемпотентного запроса") + row = await connection.fetchrow( + "SELECT id, chat_id, sender_id, text, attachment_ids, created_at FROM messages WHERE id = $1", recorded["message_id"] + ) + if row is None: + raise ApiError(503, "unavailable", "Не удалось получить сохранённое сообщение") + return self._message(row) + row = await connection.fetchrow( + "INSERT INTO messages (id, chat_id, sender_id, text, attachment_ids) VALUES ($1, $2, $3, $4, $5) " + "RETURNING id, chat_id, sender_id, text, attachment_ids, created_at", + uuid4(), chat_id, sender_id, text, attachment_ids, + ) + await connection.execute( + "UPDATE idempotency_records SET message_id = $1 WHERE user_id = $2 AND method = 'POST' " + "AND path = $3 AND idempotency_key = $4", + row["id"], sender_id, path, idempotency_key, + ) + except DB_ERRORS as error: + raise ApiError(503, "unavailable", "PostgreSQL недоступен") from error + return self._message(row) + + async def list_messages_for_member( + self, chat_id: UUID, user_id: UUID, after: tuple[datetime, UUID] | None, limit: int + ) -> list[dict]: + await self.get_chat_for_member(chat_id, user_id) + query = "SELECT id, chat_id, sender_id, text, attachment_ids, created_at FROM messages WHERE chat_id = $1" + args: list = [chat_id] + if after is not None: + query += " AND (created_at, id) > ($2, $3)" + args.extend(after) + query += " ORDER BY created_at, id LIMIT $" + str(len(args) + 1) + args.append(limit + 1) + try: + rows = await self.pool.fetch(query, *args) + except DB_ERRORS as error: + raise ApiError(503, "unavailable", "PostgreSQL недоступен") from error + return [self._message(row) for row in rows] + + + @asynccontextmanager + async def attachment_transaction(self, *, maintenance=False): + """Shared writer / exclusive reconciler lock across all API replicas. + + Writers hold the transaction from before PUT through metadata COMMIT. + Maintenance holds the exclusive lock through inventory and deletion. + All attachment writers must participate in this protocol. + """ + try: + with tel.tracer.start_as_current_span("postgres.attachment_transaction", kind=tel.SpanKind.CLIENT, record_exception=False, set_status_on_exception=False) as span: + try: + async with self.pool.acquire() as connection, connection.transaction(): + await connection.execute("SET LOCAL lock_timeout = '5s'") + lock = "pg_advisory_xact_lock" if maintenance else "pg_advisory_xact_lock_shared" + await connection.execute(f"SELECT {lock}(41004, 1)") + yield connection + except Exception: + span.set_status(tel.Status(tel.StatusCode.ERROR)) + raise + except DB_ERRORS as error: + raise ApiError(503, "unavailable", "PostgreSQL недоступен") from error + + async def create_attachment(self, attachment_id, chat_id, owner_id, filename, + content_type, size_bytes, digest, staging_key, job_id, *, connection=None) -> dict: + try: + row = await (connection or self.pool).fetchrow( + "INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, staging_key) " + "VALUES ($1,$2,$3,$4,$5,$6,$7,$8) RETURNING *", + attachment_id, chat_id, owner_id, filename, content_type, size_bytes, digest, staging_key, + ) + await (connection or self.pool).execute( + "INSERT INTO attachment_jobs (job_id, attachment_id, trace_context) VALUES ($1, $2, $3::jsonb)", job_id, attachment_id, json.dumps(tel.carrier()) + ) + await (connection or self.pool).execute( + "INSERT INTO attachment_slo (attachment_id) VALUES ($1)", attachment_id + ) + payload = { + "schema_version": 1, "job_id": str(job_id), "type": "attachment.process.v1", + "attachment_id": str(attachment_id), "pipeline_version": 1, "attempt": 1, + "created_at": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"), + } + await (connection or self.pool).execute( + "INSERT INTO attachment_outbox (id, job_id, payload) VALUES ($1, $2, $3::jsonb)", + uuid4(), job_id, json.dumps(payload, separators=(",", ":")), + ) + await (connection or self.pool).execute( + "UPDATE attachment_jobs SET next_attempt_at = 'infinity' WHERE job_id = $1", job_id + ) + except DB_ERRORS as error: + raise ApiError(503, "unavailable", "PostgreSQL недоступен") from error + return dict(row) + + async def ensure_attachment_queue_capacity(self, connection: asyncpg.Connection, maximum: int) -> None: + """Globally serialize admission so replicas cannot over-accept queued work.""" + try: + await connection.execute("SELECT pg_advisory_xact_lock(41005, 1)") + pending = await connection.fetchval( + "SELECT count(*) FROM attachment_jobs WHERE status IN ('queued', 'processing')" + ) + except DB_ERRORS as error: + raise ApiError(503, "unavailable", "PostgreSQL недоступен") from error + if pending >= maximum: + raise ApiError(429, "rate_limited", "Очередь обработки заполнена") + + async def attachment_for_member(self, attachment_id: UUID, user_id: UUID) -> dict: + try: + row = await self.pool.fetchrow( + "SELECT a.* FROM attachments a JOIN chat_members m ON m.chat_id = a.chat_id " + "WHERE a.id = $1 AND m.user_id = $2", attachment_id, user_id, + ) + except DB_ERRORS as error: + raise ApiError(503, "unavailable", "PostgreSQL недоступен") from error + if row is None: + raise ApiError(404, "not_found", "Вложение не найдено") + return await self._attachment_with_variants(row) + + async def _attachment_with_variants(self, row: asyncpg.Record | dict) -> dict: + result = dict(row) + variants = await self.pool.fetch( + "SELECT name, content_type, size_bytes, width, height, sha256 FROM attachment_variants " + "WHERE attachment_id = $1 AND pipeline_version = $2 ORDER BY name", + result["id"], result["pipeline_version"], + ) + result["variants"] = [dict(item) for item in variants] + return result + + async def claim_attachment_job(self, attachment_id: UUID, lease_seconds: int) -> dict | None: + try: + async with self.pool.acquire() as connection, connection.transaction(): + row = await connection.fetchrow( + "SELECT a.*, j.job_id, j.attempt, j.lease_until, " + "(j.next_attempt_at <> 'infinity'::timestamptz AND j.next_attempt_at > now()) AS retry_waiting " + "FROM attachments a JOIN attachment_jobs j ON j.attachment_id = a.id " + "WHERE a.id = $1 FOR UPDATE", attachment_id + ) + # Only the dispatcher recovers an expired lease and increments + # its attempt. Redelivery must not renew it or bypass backoff. + if row is None or row["status"] != "queued" or row["retry_waiting"]: + return None + await connection.execute( + "UPDATE attachments SET status = 'processing', processing_started_at = now(), updated_at = now() WHERE id = $1", + attachment_id, + ) + await connection.execute( + "UPDATE attachment_jobs SET status = 'processing', first_started_at=COALESCE(first_started_at, now()), lease_until = now() + $2 * interval '1 second', updated_at = now() WHERE attachment_id = $1", + attachment_id, lease_seconds, + ) + result = dict(row) + result['claim_time'] = await connection.fetchval('SELECT processing_started_at FROM attachments WHERE id=$1', attachment_id) + return result + except DB_ERRORS as error: + raise RuntimeError("PostgreSQL is unavailable") from error + + async def complete_attachment_job(self, attachment_id: UUID, original_key: str, variants: list[dict], claim_time) -> bool: + try: + async with self.pool.acquire() as connection, connection.transaction(): + valid = await connection.fetchval( + "SELECT 1 FROM attachments a JOIN attachment_jobs j ON j.attachment_id=a.id WHERE a.id=$1 AND a.status='processing' AND a.processing_started_at=$2 AND j.lease_until > now() FOR UPDATE", attachment_id, claim_time + ) + if not valid: + return False + for variant in variants: + await connection.execute( + "INSERT INTO attachment_variants (attachment_id, pipeline_version, name, object_key, content_type, size_bytes, width, height, sha256) " + "VALUES ($1, 1, $2, $3, $4, $5, $6, $7, $8) " + "ON CONFLICT (attachment_id, pipeline_version, name) DO UPDATE SET object_key = EXCLUDED.object_key, content_type = EXCLUDED.content_type, size_bytes = EXCLUDED.size_bytes, width = EXCLUDED.width, height = EXCLUDED.height, sha256 = EXCLUDED.sha256", + attachment_id, variant["name"], variant["object_key"], variant["content_type"], variant["size_bytes"], variant["width"], variant["height"], variant["sha256"], + ) + await connection.execute( + "UPDATE attachments SET status = 'ready', original_key = $2, error_code = NULL, updated_at = now() WHERE id = $1", + attachment_id, original_key, + ) + await connection.execute("UPDATE attachment_jobs SET status = 'ready', lease_until = NULL, updated_at = now() WHERE attachment_id = $1", attachment_id) + await connection.execute( + "UPDATE attachment_slo SET ready_xid=pg_current_xact_id()::xid " + "WHERE attachment_id=$1 AND ready_xid IS NULL", attachment_id + ) + return True + except DB_ERRORS as error: + raise RuntimeError("PostgreSQL is unavailable") from error + + async def ready_seconds(self, attachment_id: UUID): + # Both ends are DB commit timestamps, not transaction start or a worker's clock. + return await self.pool.fetchval(""" + SELECT EXTRACT(EPOCH FROM ( + COALESCE(ready_at, pg_xact_commit_timestamp(ready_xid)) - + COALESCE(accepted_at, pg_xact_commit_timestamp(accepted_xid))))::float8 + FROM attachment_slo WHERE attachment_id=$1 + """, attachment_id) + + async def sample_ready_slo(self): + # Materialize promptly: PostgreSQL does not retain commit timestamps forever. + async with self.pool.acquire() as connection, connection.transaction(): + await connection.execute(""" + UPDATE attachment_slo SET + accepted_at=COALESCE(accepted_at, pg_xact_commit_timestamp(accepted_xid)), + ready_at=COALESCE(ready_at, pg_xact_commit_timestamp(ready_xid)) + WHERE accepted_at IS NULL OR (ready_xid IS NOT NULL AND ready_at IS NULL) + """) + await connection.execute(""" + UPDATE attachment_slo SET bad=(ready_at IS NULL OR ready_at>accepted_at+interval '30 seconds') + WHERE bad IS NULL AND accepted_at<=now()-interval '30 seconds' + AND (ready_xid IS NULL OR ready_at IS NOT NULL) + """) + return await connection.fetchrow(""" + SELECT count(bad) AS eligible, count(*) FILTER (WHERE bad) AS bad, + count(*) FILTER (WHERE accepted_at IS NULL OR + (ready_xid IS NOT NULL AND ready_at IS NULL)) AS unknown + FROM attachment_slo + """) + + async def retry_attachment_job(self, attachment_id: UUID, max_attempts: int, delay_seconds: float, claim_time, permanent_code=None) -> bool: + """Return true only when a bounded at-least-once retry was recorded.""" + try: + async with self.pool.acquire() as connection, connection.transaction(): + job = await connection.fetchrow( + "SELECT j.attempt FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id WHERE a.id=$1 AND a.status='processing' AND a.processing_started_at=$2 FOR UPDATE", attachment_id, claim_time + ) + if job is None: + return False + if permanent_code is None and job["attempt"] < max_attempts: + await connection.execute( + "UPDATE attachment_jobs SET attempt = attempt + 1, status = 'queued', last_error = 'processing_failed', lease_until = NULL, next_attempt_at = now() + $2 * interval '1 second', updated_at = now() WHERE attachment_id = $1", + attachment_id, delay_seconds, + ) + await connection.execute( + "UPDATE attachments SET status = 'queued', updated_at = now() WHERE id = $1", attachment_id + ) + return True + code = permanent_code or 'retry_exhausted' + await connection.execute("UPDATE attachments SET status='failed', error_code=$2, updated_at=now() WHERE id=$1", attachment_id, code) + await connection.execute("UPDATE attachment_jobs SET status='failed', last_error=$2, lease_until=NULL, updated_at=now() WHERE attachment_id=$1", attachment_id, code) + return False + except DB_ERRORS as error: + raise RuntimeError("PostgreSQL is unavailable") from error + + async def due_outbox_events(self, limit: int = 20) -> list[dict]: + try: + return [{**dict(row), 'payload': json.loads(row['payload']) if isinstance(row['payload'], str) else row['payload']} for row in await self.pool.fetch( + "SELECT o.id, o.payload, j.trace_context FROM attachment_outbox o JOIN attachment_jobs j ON j.job_id=o.job_id WHERE o.published_at IS NULL ORDER BY o.created_at LIMIT $1", limit + )] + except DB_ERRORS as error: + raise RuntimeError("PostgreSQL is unavailable") from error + + async def mark_outbox_published(self, event_id: UUID) -> None: + await self.pool.execute("UPDATE attachment_outbox SET published_at = now(), attempts = attempts + 1 WHERE id = $1", event_id) + + async def schedule_due_jobs(self, max_attempts: int = 5) -> None: + """Outbox dispatcher also restores expired leases and scheduled retries.""" + try: + async with self.pool.acquire() as connection, connection.transaction(): + rows = await connection.fetch( + "SELECT job_id, attachment_id, pipeline_version, attempt, status FROM attachment_jobs " + "WHERE (status = 'queued' AND (next_attempt_at <= now() OR " + "(next_attempt_at = 'infinity'::timestamptz AND updated_at < now() - interval '60 seconds'))) " + "OR (status = 'processing' AND lease_until < now()) " + "FOR UPDATE SKIP LOCKED" + ) + for row in rows: + row = dict(row) + if row["status"] == "processing": + if row['attempt'] >= max_attempts: + await connection.execute("UPDATE attachment_jobs SET status='failed', lease_until=NULL, last_error='retry_exhausted', updated_at=now() WHERE job_id=$1", row['job_id']) + await connection.execute("UPDATE attachments SET status='failed', error_code='retry_exhausted', updated_at=now() WHERE id=$1", row['attachment_id']) + continue + row['attempt'] += 1 + await connection.execute( + "UPDATE attachment_jobs SET status = 'queued', attempt=$2, lease_until = NULL, next_attempt_at = now(), last_error = 'lease_expired', updated_at = now() WHERE job_id = $1", row["job_id"], row['attempt'] + ) + await connection.execute("UPDATE attachments SET status = 'queued', updated_at = now() WHERE id = $1", row["attachment_id"]) + payload = {"schema_version": 1, "job_id": str(row["job_id"]), "type": "attachment.process.v1", "attachment_id": str(row["attachment_id"]), "pipeline_version": row["pipeline_version"], "attempt": row["attempt"], "created_at": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")} + await connection.execute("INSERT INTO attachment_outbox (id, job_id, payload) VALUES ($1, $2, $3::jsonb)", uuid4(), row["job_id"], json.dumps(payload, separators=(",", ":"))) + # Refresh the watchdog on publication intent, not on claim. + # Unclaimed jobs are republished after 60s. At-least-once + # duplicates are safe, and a published-but-lost delivery + # can no longer strand durable accepted work forever. + await connection.execute("UPDATE attachment_jobs SET next_attempt_at = 'infinity', updated_at=now() WHERE job_id = $1", row["job_id"]) + except DB_ERRORS as error: + raise RuntimeError("PostgreSQL is unavailable") from error + + async def replay_failed_attachment(self, attachment_id: UUID) -> bool: + """Explicit operator-only replay; never requeues an already successful job.""" + try: + async with self.pool.acquire() as connection, connection.transaction(): + job = await connection.fetchrow( + "SELECT job_id, status FROM attachment_jobs WHERE attachment_id = $1 FOR UPDATE", attachment_id + ) + if job is None or job["status"] != "failed": + return False + await connection.execute( + "UPDATE attachment_jobs SET status = 'queued', attempt = 1, next_attempt_at = now(), lease_until = NULL, replayed_at = now(), last_error = NULL, updated_at = now() WHERE attachment_id = $1", attachment_id + ) + await connection.execute("UPDATE attachments SET status = 'queued', error_code = NULL, updated_at = now() WHERE id = $1", attachment_id) + return True + except DB_ERRORS as error: + raise RuntimeError("PostgreSQL is unavailable") from error + + async def attachment_keys(self) -> dict[str, UUID]: + """Inventory used only by the maintenance reconciler.""" + try: + rows = await self.pool.fetch("SELECT staging_key AS object_key, id FROM attachments WHERE status <> 'ready' UNION ALL SELECT original_key, id FROM attachments WHERE original_key IS NOT NULL UNION ALL SELECT object_key, attachment_id AS id FROM attachment_variants") + except DB_ERRORS as error: + raise RuntimeError("PostgreSQL is unavailable") from error + return {row["object_key"]: row["id"] for row in rows} + +# Span names describe operations only; never export SQL, parameters, credentials or IDs. +def _trace_db(method): + from functools import wraps + @wraps(method) + async def wrapped(*args, **kwargs): + with tel.tracer.start_as_current_span('postgres.' + method.__name__, kind=tel.SpanKind.CLIENT, + record_exception=False, set_status_on_exception=False) as span: + try: + return await method(*args, **kwargs) + except Exception: + span.set_status(tel.Status(tel.StatusCode.ERROR)) + raise + return wrapped + +for _name in ('ready', 'create_user', 'authenticate', 'create_session', 'user_for_token', 'revoke_session', + 'create_chat', 'get_chat_for_member', 'list_chats_for_member', 'create_message_idempotent', + 'list_messages_for_member', 'create_attachment', 'attachment_for_member', 'claim_attachment_job', + 'complete_attachment_job', 'retry_attachment_job'): + setattr(PostgresStore, _name, _trace_db(getattr(PostgresStore, _name))) diff --git a/src/storage/s3.py b/src/storage/s3.py new file mode 100644 index 0000000..7188fb2 --- /dev/null +++ b/src/storage/s3.py @@ -0,0 +1,123 @@ +"""Bounded S3 operations; synchronous SDK calls run outside the event loop.""" +from dataclasses import dataclass +from hashlib import sha256 +import os +from pathlib import Path + +import boto3 +from botocore.config import Config +from botocore.exceptions import BotoCoreError, ClientError +from starlette.concurrency import run_in_threadpool + +from ..errors import ApiError +from .. import telemetry as tel + + +@dataclass(frozen=True) +class S3Settings: + endpoint: str + region: str + bucket: str + access_key: str + secret_key: str + max_bytes: int + max_pixels: int + + @classmethod + def load(cls): + def secret(name): + path = os.environ.get(name + "_FILE") + result = Path(path).read_text().strip() if path else os.environ.get(name, "") + if not result: + raise ValueError(f"Missing {name}") + return result + + max_bytes = int(os.environ.get("MAX_UPLOAD_BYTES", "10485760")) + max_pixels = int(os.environ.get("MAX_IMAGE_PIXELS", "40000000")) + if not 1 <= max_bytes <= 10485760 or not 1 <= max_pixels <= 40000000: + raise ValueError("Image limits exceed contract") + return cls(os.environ.get("S3_ENDPOINT", "http://s3:8333"), + os.environ.get("S3_REGION", "us-east-1"), + os.environ.get("S3_BUCKET", "messenger-attachments"), + secret("S3_ACCESS_KEY_ID"), secret("S3_SECRET_ACCESS_KEY"), + max_bytes, max_pixels) + + +class S3Store: + def __init__(self, settings): + self.settings = settings + self.client = boto3.client( + "s3", endpoint_url=settings.endpoint, region_name=settings.region, + aws_access_key_id=settings.access_key, aws_secret_access_key=settings.secret_key, + config=Config(signature_version="s3v4", connect_timeout=2, read_timeout=5, + retries={"total_max_attempts": 1}, + s3={"addressing_style": "path"}, + request_checksum_calculation="when_required", + response_checksum_validation="when_required"), + ) + + async def call(self, method, **kwargs): + with tel.tracer.start_as_current_span("s3." + method, kind=tel.SpanKind.CLIENT, record_exception=False, set_status_on_exception=False) as span: + try: + return await run_in_threadpool(getattr(self.client, method), **kwargs) + except (BotoCoreError, ClientError, OSError) as error: + span.set_status(tel.Status(tel.StatusCode.ERROR)) + raise ApiError(503, "unavailable", "S3 недоступно") from error + + async def ready(self): + await self.call("head_bucket", Bucket=self.settings.bucket) + + async def put(self, key, file, size, content_type): + file.seek(0) + await self.call("put_object", Bucket=self.settings.bucket, Key=key, + Body=file, ContentLength=size, ContentType=content_type) + + async def copy(self, source_key: str, destination_key: str, content_type: str) -> None: + await self.call( + "copy_object", Bucket=self.settings.bucket, Key=destination_key, + CopySource={"Bucket": self.settings.bucket, "Key": source_key}, + ContentType=content_type, MetadataDirective="REPLACE", + ) + + async def list(self, prefix: str) -> dict[str, object]: + """Return every object under a controlled prefix, handling S3 pagination.""" + objects, _ = await self.list_with_pages(prefix) + return objects + + async def list_with_pages(self, prefix: str) -> tuple[dict[str, object], int]: + """Return inventory and number of S3 ListObjectsV2 pages consumed.""" + result: dict[str, object] = {} + token = None + pages = 0 + while True: + args = {"Bucket": self.settings.bucket, "Prefix": prefix} + if token: + args["ContinuationToken"] = token + page = await self.call("list_objects_v2", **args) + pages += 1 + for item in page.get("Contents", []): + result[item["Key"]] = item["LastModified"] + if not page.get("IsTruncated"): + return result, pages + token = page["NextContinuationToken"] + + async def delete(self, key: str) -> None: + await self.call("delete_object", Bucket=self.settings.bucket, Key=key) + + async def download(self, attachment): + return await self.download_key(attachment["original_key"], attachment["size_bytes"], attachment["sha256"]) + + async def download_key(self, key: str, expected_size: int | None = None, + expected_sha256: str | None = None) -> bytes: + response = await self.call("get_object", Bucket=self.settings.bucket, Key=key) + body = response["Body"] + try: + data = await run_in_threadpool(body.read, self.settings.max_bytes + 1) + except (BotoCoreError, OSError) as error: + raise ApiError(503, "unavailable", "S3 недоступно") from error + finally: + body.close() + if ((expected_size is not None and len(data) != expected_size) or + (expected_sha256 is not None and sha256(data).hexdigest() != expected_sha256)): + raise ApiError(503, "unavailable", "Оригинал недоступен") + return data diff --git a/src/telemetry.py b/src/telemetry.py new file mode 100644 index 0000000..082661e --- /dev/null +++ b/src/telemetry.py @@ -0,0 +1,118 @@ +"""Bounded OTLP telemetry; only explicitly selected, non-sensitive fields leave the process.""" +import json +import logging +import os +import socket +import time +import resource +from datetime import datetime, timezone + +from opentelemetry import metrics, trace +from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator +from opentelemetry.trace import SpanKind, Status, StatusCode + +tracer = trace.get_tracer("messenger") +meter = metrics.get_meter("messenger") +trace_context = TraceContextTextMapPropagator() +http_requests = meter.create_counter("messenger.http.requests", unit="{request}") +http_duration = meter.create_histogram("messenger.http.duration", unit="s") +jobs = meter.create_counter("messenger.jobs", unit="{job}") +job_duration = meter.create_histogram("messenger.job.duration", unit="s") +ready_duration = meter.create_histogram("messenger.job.ready.duration", unit="s") +busy = meter.create_up_down_counter("messenger.worker.busy", unit="{worker}") +capacity = meter.create_up_down_counter("messenger.worker.capacity", unit="{worker}") +state = {"queue_depth": 0, "queue_age": 0, "pool_used": 0, "pool_capacity": 0, "pool_waiting": 0, + "slo_ready_eligible": 0, "slo_ready_bad": 0, "slo_unknown": 0, "queue_sample_time": 0} +for name, unit in (("queue_sample_time", "s"), ("slo_unknown", "{job}"), ("queue_depth", "{job}"), ("queue_age", "s"), ("pool_used", "{connection}"), ("pool_capacity", "{connection}"), ("pool_waiting", "{request}")): + meter.create_observable_gauge("messenger." + name, callbacks=[lambda options, key=name: [metrics.Observation(state[key], {"queue": "attachment.process.v1"} if key.startswith("queue_") else {})]], unit=unit) +for name in ("slo_ready_eligible", "slo_ready_bad"): + meter.create_observable_counter("messenger." + name, callbacks=[lambda options, key=name: [metrics.Observation(state[key])]], unit="{job}") + +def process_cpu(options): + usage = resource.getrusage(resource.RUSAGE_SELF) + return [metrics.Observation(usage.ru_utime + usage.ru_stime)] + +def process_rss(options): + try: + with open('/proc/self/statm', encoding='ascii') as stream: + return [metrics.Observation(int(stream.read().split()[1]) * os.sysconf('SC_PAGE_SIZE'))] + except (OSError, ValueError, IndexError): + return [] + +meter.create_observable_counter("messenger.process.cpu", callbacks=[process_cpu], unit="s") +meter.create_observable_gauge("messenger.process.memory", callbacks=[process_rss], unit="By") + +log = logging.getLogger("messenger.telemetry") +log.setLevel(logging.INFO) +log.propagate = False + +LOG_FIELDS = ("request_id", "method", "route", "status", "outcome") +LOG_EVENTS = {"http.request", "job.processed", "telemetry.unavailable"} +STANDARD_LOG_FIELDS = frozenset(logging.makeLogRecord({}).__dict__) + + +def http_method(value): + return value if value in {"GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS", "CONNECT", "TRACE"} else "_OTHER" + + +class SafeLogFilter(logging.Filter): + """Sanitize the shared record BEFORE both stdout and OTLP handlers.""" + def filter(self, record): + for key in list(record.__dict__): + if key not in STANDARD_LOG_FIELDS and key not in LOG_FIELDS: + del record.__dict__[key] + record.msg = record.msg if isinstance(record.msg, str) and record.msg in LOG_EVENTS else "telemetry.unavailable" + record.args = () + record.exc_info = record.exc_text = record.stack_info = None + return True + + +log.addFilter(SafeLogFilter()) + +class JsonFormatter(logging.Formatter): + def format(self, record): + ctx = trace.get_current_span().get_span_context() + obj = {"timestamp": datetime.now(timezone.utc).isoformat(), "severity": record.levelname, + "service.name": os.environ.get("OTEL_SERVICE_NAME", "messenger"), "event": record.getMessage()} + if ctx.is_valid: + obj.update(trace_id=f"{ctx.trace_id:032x}", span_id=f"{ctx.span_id:016x}") + for field in LOG_FIELDS: + if hasattr(record, field): obj[field] = getattr(record, field) + return json.dumps(obj, ensure_ascii=False) + +def setup(service): + if os.environ.get("OTEL_ENABLED", "0") != "1": + return + os.environ["OTEL_SERVICE_NAME"] = service + from opentelemetry.sdk.resources import Resource + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.export import BatchSpanProcessor + from opentelemetry.sdk.metrics import MeterProvider + from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader + from opentelemetry.sdk.metrics.view import View, ExplicitBucketHistogramAggregation + from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler + from opentelemetry.sdk._logs.export import BatchLogRecordProcessor + from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter + from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter + from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter + resource = Resource.create({"service.name": service, "service.version": "1.6.0", "service.instance.id": socket.gethostname(), "deployment.environment.name": "lab"}) + provider = TracerProvider(resource=resource) + provider.add_span_processor(BatchSpanProcessor(OTLPSpanExporter(timeout=2), max_queue_size=2048, max_export_batch_size=256, schedule_delay_millis=1000)) + trace.set_tracer_provider(provider) + buckets = [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 120, 300, 600] + views = [View(instrument_name=name, aggregation=ExplicitBucketHistogramAggregation(boundaries=buckets)) for name in ("messenger.http.duration", "messenger.job.duration", "messenger.job.ready.duration")] + metrics.set_meter_provider(MeterProvider(resource=resource, views=views, metric_readers=[PeriodicExportingMetricReader(OTLPMetricExporter(timeout=2), export_interval_millis=5000)])) + logs = LoggerProvider(resource=resource) + logs.add_log_record_processor(BatchLogRecordProcessor(OTLPLogExporter(timeout=2), max_queue_size=2048, max_export_batch_size=256, schedule_delay_millis=1000)) + log.addHandler(LoggingHandler(level=logging.INFO, logger_provider=logs)) + stream = logging.StreamHandler() + stream.setFormatter(JsonFormatter()) + log.addHandler(stream) + +def carrier(): + result = {} + trace_context.inject(result) + return result + +def extract(value): + return trace_context.extract(value or {}) diff --git a/src/worker.py b/src/worker.py new file mode 100644 index 0000000..6aa4f8b --- /dev/null +++ b/src/worker.py @@ -0,0 +1,265 @@ +"""RabbitMQ consumer for the version-1 attachment pipeline.""" +import asyncio +from hashlib import sha256 +from io import BytesIO +import json +import logging +import os +import random +import pickle +import subprocess +import sys +import time +from pathlib import Path +from concurrent.futures import ThreadPoolExecutor +from uuid import UUID, uuid4 + +from PIL import Image, ImageOps + +from .broker import QUEUE, _connection, publish_dlq +from .config import load_settings +from .storage.postgres import PostgresStore +from .storage.s3 import S3Settings, S3Store +from . import telemetry as tel +from datetime import datetime, timezone + +logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s worker %(message)s") + + +class PermanentFailure(Exception): + pass + + +def fit(image, bound): + longest = max(image.size) + if longest <= bound: + return image.copy() + return image.resize((max(1, image.width * bound // longest), + max(1, image.height * bound // longest)), Image.Resampling.LANCZOS) + + +def render_isolated(data, timeout): + try: + result = subprocess.run([sys.executable, '-m', 'src.render'], input=data, + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, timeout=timeout) + except subprocess.TimeoutExpired as error: + raise TimeoutError('render deadline exceeded') from error + if result.returncode in (20, 21): + raise PermanentFailure('invalid_image' if result.returncode == 20 else 'image_resource_limit') + if result.returncode: + raise RuntimeError('renderer exited') + # Only our own child process produces this payload; no external pickle input. + return pickle.loads(result.stdout) + + +def thumbnail(data: bytes) -> tuple[bytes, int, int]: + with Image.open(BytesIO(data), formats=("PNG", "JPEG")) as source: + image = ImageOps.exif_transpose(source) + if image.mode in {"RGBA", "LA"} or (image.mode == "P" and "transparency" in image.info): + background = Image.new("RGBA", image.size, "white") + background.alpha_composite(image.convert("RGBA")) + image = background.convert("RGB") + else: + image = image.convert("RGB") + image = fit(image, 256) + output = BytesIO() + image.save(output, format="JPEG", quality=85, optimize=True) + return output.getvalue(), image.width, image.height + + +def variants(data: bytes, max_pixels: int = 40_000_000) -> list[tuple[str, bytes, str, int, int]]: + """Render required variants without preserving EXIF or other input metadata.""" + with Image.open(BytesIO(data), formats=("PNG", "JPEG")) as source: + if source.width * source.height > max_pixels: + raise ValueError("image exceeds configured pixel limit") + image = ImageOps.exif_transpose(source).convert("RGBA") + white = Image.new("RGBA", image.size, "white") + white.alpha_composite(image) + rgb = white.convert("RGB") + + thumb = fit(rgb, 256) + thumbnail_out = BytesIO(); thumb.save(thumbnail_out, "JPEG", quality=85, optimize=True) + + side = min(rgb.width, rgb.height) + left, top = (rgb.width - side) // 2, (rgb.height - side) // 2 + crop = rgb.crop((left, top, left + side, top + side)).resize((128, 128), Image.Resampling.LANCZOS) + crop_out = BytesIO(); crop.save(crop_out, "JPEG", quality=85, optimize=True) + + optimized = fit(rgb, 1280) + optimized_out = BytesIO(); optimized.save(optimized_out, "WEBP", quality=80, method=4) + return [ + ("thumbnail", thumbnail_out.getvalue(), "image/jpeg", thumb.width, thumb.height), + ("crop", crop_out.getvalue(), "image/jpeg", 128, 128), + ("optimized", optimized_out.getvalue(), "image/webp", optimized.width, optimized.height), + ] + + +async def process(store: PostgresStore, s3: S3Store, payload: dict, lease_seconds: int) -> str: + # Excludes cleanup for the whole lifetime of S3 effects, including stale attempts. + async with store.attachment_transaction(): + return await process_locked(store, s3, payload, lease_seconds) + + +async def process_locked(store: PostgresStore, s3: S3Store, payload: dict, lease_seconds: int) -> str: + attachment_id = UUID(payload["attachment_id"]) + attachment = await store.claim_attachment_job(attachment_id, lease_seconds) + if attachment is None: + return "skipped" + payload['_claim_time'] = attachment['claim_time'] + payload['attempt'] = attachment.get('attempt', payload.get('attempt', 1)) + attempt_key = uuid4().hex + original_key = f"originals/{attachment_id}/{attempt_key}" + await s3.copy(attachment["staging_key"], original_key, attachment["content_type"]) + original = await s3.download_key(attachment["staging_key"], attachment["size_bytes"], attachment["sha256"]) + rendered = await asyncio.to_thread(render_isolated, original, lease_seconds) + metadata = [] + for name, content, content_type, width, height in rendered: + extension = "webp" if name == "optimized" else "jpg" + object_key = f"variants/{attachment_id}/{attempt_key}/{name}.{extension}" + await s3.put(object_key, BytesIO(content), len(content), content_type) + metadata.append({"name": name, "object_key": object_key, "content_type": content_type, + "size_bytes": len(content), "width": width, "height": height, + "sha256": sha256(content).hexdigest()}) + # Explicit, opt-in integration fault. Marker survives container restart, + # so one crash exercises lease recovery without a permanent crash loop. + if os.environ.get('WORKER_FAULT_AFTER_S3_PUT') == '1': + marker = Path('/tmp/worker-fault-after-s3') + try: + with marker.open('x') as stream: + stream.write(str(attachment_id)) + except FileExistsError: + pass + else: + os._exit(70) + committed = await store.complete_attachment_job(attachment_id, original_key, metadata, attachment['claim_time']) + if committed: + try: + duration = await store.ready_seconds(attachment_id) + if duration is not None: + tel.ready_duration.record(duration, {"job_type": "attachment"}) + except Exception: + tel.log.warning("telemetry.unavailable") + try: + await s3.delete(attachment["staging_key"]) + except Exception: + logging.warning('staging cleanup deferred attachment_id=%s', attachment_id) + + return "success" if committed else "skipped" + +def settle_delivery(channel, delivery_tag, future): + """Only a successful durable disposition permits acknowledgement.""" + try: + success = future.result() is True + except Exception: + success = False + if channel.is_open: + if success: + channel.basic_ack(delivery_tag) + else: + channel.basic_nack(delivery_tag, requeue=True) + + +def main() -> None: + tel.setup("messenger-worker") + settings = load_settings() + connection = _connection() + channel = connection.channel() + workers = max(1, int(os.environ.get("WORKER_CONCURRENCY", "2"))) + tel.capacity.add(workers) + channel.basic_qos(prefetch_count=max(1, int(os.environ.get("WORKER_PREFETCH", str(workers))))) + + async def handle_async(payload): + """Each executor thread owns its event loop and its DB/S3 clients.""" + store = PostgresStore(settings.database_url) + s3 = S3Store(S3Settings.load()) + await store.connect() + try: + return await process(store, s3, payload, max(1, int(os.environ.get("JOB_TIMEOUT_SECONDS", "60")))) + finally: + s3.client.close() + await store.close() + + async def schedule_retry(attachment_id, delay, claim_time, permanent_code=None): + retry_store = PostgresStore(settings.database_url) + await retry_store.connect() + try: + return await retry_store.retry_attachment_job( + attachment_id, max(1, int(os.environ.get("JOB_MAX_ATTEMPTS", "5"))), delay, claim_time, permanent_code + ) + finally: + await retry_store.close() + + def handle_job(body): + payload = None + try: + payload = json.loads(body) + if not isinstance(payload, dict): + raise ValueError('job must be an object') + if payload["type"] != "attachment.process.v1" or payload["schema_version"] != 1: + raise ValueError("unexpected broker payload") + UUID(payload['attachment_id']) + UUID(payload['job_id']) + tel.trace.get_current_span().set_attribute("messaging.retry.attempt", int(payload.get('attempt', 1))) + except (ValueError, KeyError, TypeError): + try: + asyncio.run(publish_dlq({'failure': 'invalid_job'})) + return True, "failed" + except Exception: + return False, "failed" + try: + outcome = asyncio.run(handle_async(payload)) + except Exception as error: + logging.warning("job failed attachment_id=%s", payload['attachment_id']) + if '_claim_time' not in payload: + return False, "failed" + try: + attachment_id = UUID(payload["attachment_id"]) + attempt = int(payload.get("attempt", 1)) + delay = min(60.0, float(2 ** min(attempt, 6))) * random.uniform(0.5, 1.5) + asyncio.run(schedule_retry(attachment_id, delay, payload['_claim_time'], + str(error) if isinstance(error, PermanentFailure) else None)) + except Exception: + return False, "failed" + return True, "failed" + return True, outcome + + def handle(body, headers): + started = time.monotonic() + tel.busy.add(1) + with tel.tracer.start_as_current_span("attachment.consume", context=tel.extract(headers), kind=tel.SpanKind.CONSUMER, record_exception=False, set_status_on_exception=False) as span: + try: + acknowledged, outcome = handle_job(body) + return acknowledged + except Exception: + outcome = "failed" + raise + finally: + if 'outcome' not in locals(): outcome = "failed" + if outcome == "failed": span.set_status(tel.Status(tel.StatusCode.ERROR)) + attrs = {"job_type": "attachment", "outcome": outcome} + tel.jobs.add(1, attrs) + tel.job_duration.record(time.monotonic() - started, attrs) + tel.log.info("job.processed", extra={"outcome": outcome}) + tel.busy.add(-1) + + def consume(ch, method, properties, body): + future = executor.submit(handle, body, properties.headers or {}) + def acknowledged(_future): + if connection.is_open: + connection.add_callback_threadsafe(lambda: settle_delivery(ch, method.delivery_tag, _future)) + future.add_done_callback(acknowledged) + + try: + def heartbeat(): + Path('/tmp/worker-heartbeat').write_text(str(time.time())) + connection.call_later(2, heartbeat) + heartbeat() + with ThreadPoolExecutor(max_workers=workers, thread_name_prefix="attachment-worker") as executor: + channel.basic_consume(queue=QUEUE, on_message_callback=consume, auto_ack=False) + channel.start_consuming() + finally: + connection.close() + + +if __name__ == "__main__": + main() diff --git a/student-tests/alerts.test.yaml b/student-tests/alerts.test.yaml new file mode 100644 index 0000000..30c3b72 --- /dev/null +++ b/student-tests/alerts.test.yaml @@ -0,0 +1,27 @@ +rule_files: + - /workspace/infra/prometheus/alerts.yaml +evaluation_interval: 5s +tests: + - interval: 5s + input_series: + - series: 'messenger_http_requests_total{method="POST",route="/api/v1/chats/{chat_id}/messages",status_class="5xx"}' + values: '0+50x30' + - series: 'messenger_http_requests_total{method="POST",route="/api/v1/chats/{chat_id}/messages",status_class="2xx"}' + values: '0+450x30' + - series: 'messenger_queue_age_seconds{service_name="messenger-outbox"}' + values: '60x30' + alert_rule_test: + - eval_time: 2m + alertname: MessengerHighErrorRate + exp_alerts: + - exp_labels: {severity: warning} + exp_annotations: + summary: 'API 5xx ratio exceeds 5%' + runbook: 'Inspect API latency, traces and dependency spans; check DB readiness.' + - eval_time: 2m + alertname: MessengerQueueLag + exp_alerts: + - exp_labels: {severity: warning} + exp_annotations: + summary: 'Oldest queued job exceeds 30 seconds' + runbook: 'Check worker health, busy/capacity, broker and consumer traces.' diff --git a/student-tests/attachment_resilience.py b/student-tests/attachment_resilience.py new file mode 100644 index 0000000..4f72d9f --- /dev/null +++ b/student-tests/attachment_resilience.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 +"""Explicit local Docker tests: private bucket, uploader failover, S3 outage.""" +import json +from pathlib import Path +import subprocess +import sys +import time + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client, Failure, arguments, check, run + + +def compose(*args, input=None): + return subprocess.check_output(['docker', 'compose', *args], cwd=ROOT, input=input, + text=True, stderr=subprocess.DEVNULL) + + +def wait_ready(c): + deadline = time.monotonic() + 90 + while True: + try: + c.call('GET', '/health/ready', schema='Health') + return + except Failure: + check(time.monotonic() < deadline, 'stack did not recover') + time.sleep(1) + + +def main(): + def configure(parser): + parser.add_argument('--mode', required=True, choices=['private', 'failover', 's3-outage']) + args = arguments(__doc__, configure) + c = Client(args.base_url, args.ca_file) + alice, bob, eve, chat = c.bootstrap() + data = (ROOT / 'tests/fixtures/sample.png').read_bytes() + uploaded = c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=sample.png', + 201, actor=alice, raw=data, headers={'Content-Type': 'image/png'}, schema='Attachment') + att = uploaded['json'] + content_path = '/api/v1/attachments/' + att['id'] + '/content' + if args.mode == 'private': + script = '''import sys, urllib.request, urllib.error +url = "http://s3:8333/messenger-attachments/originals/" + sys.stdin.read().strip() +try: + urllib.request.urlopen(url, timeout=5) +except urllib.error.HTTPError as error: + assert error.code == 403, error.code + print("PASS: anonymous S3 GET returns 403 for an existing object") +else: + raise AssertionError("private object leaked") +''' + print(compose('exec', '-T', 'api-a', '/app/.venv/bin/python', '-c', script, input=att['id']).strip()) + elif args.mode == 'failover': + target = uploaded['headers']['X-Instance-Id'] + service = None + for candidate in ('api-a', 'api-b'): + container = compose('ps', '-q', candidate).strip() + hostname = subprocess.check_output(['docker', 'inspect', '--format', '{{.Config.Hostname}}', container], text=True).strip() + if hostname == target: + service = candidate + check(service is not None, 'uploading replica not found') + try: + compose('stop', service) + deadline = time.monotonic() + 15 + while True: + try: + download = c.call('GET', content_path, actor=bob) + break + except Failure: + check(time.monotonic() < deadline, 'download did not recover') + time.sleep(0.5) + check(download['headers']['X-Instance-Id'] != target, 'download used stopped uploader') + check(download['bytes'] == data, 'original changed after stopping uploader') + print('PASS: uploaded via ' + service + '; stopped that container; original downloaded through another replica') + finally: + compose('start', service) + wait_ready(c) + else: + try: + compose('stop', 's3') + # Direct API request inside the network: LB excludes unready backends. + script = '''import json, sys, urllib.request, urllib.error +payload = json.load(sys.stdin) +for method, path, body, headers in [ + ("GET", "/health/ready", None, {}), + ("GET", payload["path"], None, {"Authorization": "Bearer " + payload["token"]}), + ("POST", payload["upload_path"], bytes.fromhex(payload["bytes"]), + {"Authorization": "Bearer " + payload["token"], "Content-Type": "image/png"}), +]: + req = urllib.request.Request("http://localhost:8080" + path, data=body, headers=headers, method=method) + try: + urllib.request.urlopen(req, timeout=15) + except urllib.error.HTTPError as error: + assert error.code == 503, error.code + document = json.load(error) + assert document["error"]["code"] == "unavailable" + assert document["error"]["request_id"] == error.headers["X-Request-Id"] + else: + raise AssertionError("S3 outage was not reported") +assert urllib.request.urlopen("http://localhost:8080/health/live").status == 200 +print("PASS: S3 outage gives 503 for readiness, download and upload; liveness stays 200") +''' + payload = {'path': content_path, 'token': bob['token'], 'bytes': data.hex(), + 'upload_path': '/api/v1/chats/' + chat['id'] + '/attachments?filename=outage.png'} + print(compose('exec', '-T', 'api-a', '/app/.venv/bin/python', '-c', script, + input=json.dumps(payload)).strip()) + finally: + compose('start', 's3') + # HeadBucket/read recovery alone does not prove the write path is ready. + # Use the same write-probing bootstrap as the documented restart hook. + compose('run', '--rm', '--no-deps', 's3-init') + wait_ready(c) + check(c.call('GET', content_path, actor=bob)['bytes'] == data, 'original lost after S3 outage') + c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=after-recovery.png', + 201, actor=alice, raw=data, headers={'Content-Type': 'image/png'}, schema='Attachment') + print('PASS: original available and new upload accepted after S3 restart/bootstrap') + + +if __name__ == '__main__': + run(main) diff --git a/student-tests/attachments.py b/student-tests/attachments.py new file mode 100644 index 0000000..f9f7531 --- /dev/null +++ b/student-tests/attachments.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +"""Attachment contract checks against the running lab5 stack.""" +import hashlib +import http.client +from io import BytesIO +from pathlib import Path +import struct +import sys +import uuid +import zlib +from urllib.parse import quote, urlsplit + +from PIL import Image + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests")) +from client import Client, arguments, check, run + + +def main(): + args = arguments(__doc__) + c = Client(args.base_url, args.ca_file, timeout=30) + alice, bob, eve, chat = c.bootstrap() + path = '/api/v1/chats/' + chat['id'] + '/attachments?filename=' + png = (Path(__file__).resolve().parents[1] / 'tests/fixtures/sample.png').read_bytes() + + def upload(data, mime='image/png', filename='photo.png', status=202, actor=alice): + return c.call('POST', path + quote(filename, safe=''), status, actor=actor, raw=data, + headers={'Content-Type': mime}, schema='Attachment' if status == 202 else None) + + for fmt, mime in [('PNG', 'image/png'), ('JPEG', 'image/jpeg')]: + buf = BytesIO() + Image.new('RGB', (32, 20), 'blue').save(buf, fmt) + data = buf.getvalue() + att = upload(data, mime)['json'] + check(att['variants'] == [] and att['status'] == 'queued', 'wrong attachment state') + att = c.wait_ready(bob, att) + check(att['sha256'] == hashlib.sha256(data).hexdigest(), 'wrong checksum') + content_path = '/api/v1/attachments/' + att['id'] + check(c.call('GET', content_path, actor=bob, schema='Attachment')['json'] == att, + 'metadata changed') + check(c.call('GET', content_path + '/content', actor=bob)['bytes'] == data, 'bytes changed') + c.call('GET', content_path, 404, actor=eve) + c.call('GET', content_path + '/content', 404, actor=eve) + c.call('GET', content_path + '/content', 401) + c.call('GET', content_path + '/content?variant=thumbnail', actor=bob) + message = c.message(bob, chat, '', attachments=[att['id']])['json'] + history = c.call('GET', '/api/v1/chats/' + chat['id'] + '/messages', + actor=alice, schema='MessagePage')['json']['items'] + check(message in history, 'attachment message missing in history') + print('PASS: PNG/JPEG originals, metadata, attachment-only messages and ACL') + + for name in ('../x.png', 'x/y.png', 'x\\y.png', 'x\x00.png', 'x\n.png', 'x\x7f.png', ' ', 'x' * 256): + upload(png, filename=name, status=400) + upload(png, mime='image/jpeg', status=415) + upload(b'not an image', status=415) + upload(png[:30], status=415) + upload(png, mime='application/octet-stream', status=415) + upload(png, status=404, actor=eve) + upload(b'x' * (10485760 + 1), status=413) + + # Valid PNG header with an oversized canvas: reject before attempting pixel decoding. + def chunk(kind, data): + return struct.pack('>I', len(data)) + kind + data + struct.pack('>I', zlib.crc32(kind + data)) + huge = b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('>IIBBBBB', 8001, 5000, 8, 2, 0, 0, 0)) + huge += chunk(b'IDAT', zlib.compress(b'')) + chunk(b'IEND', b'') + upload(huge, status=413) + + # No Content-Length: limits must also work for chunked requests. + url = urlsplit(c.base) + connection = http.client.HTTPConnection(url.hostname, url.port, timeout=30) + connection.request('POST', path + 'chunked.png', body=iter([b'x' * 65536] * 161), + headers={'Authorization': 'Bearer ' + alice['token'], 'Content-Type': 'image/png'}, + encode_chunked=True) + response = connection.getresponse() + check(response.status == 413, 'chunked upload bypassed byte limit') + response.read() + connection.close() + print('PASS: filename, MIME, malformed image, byte/pixel and chunked limits') + + att = upload(png)['json'] + other_chat = c.call('POST', '/api/v1/chats', 201, actor=alice, + body={'title': 'other', 'member_ids': [alice['id']]}, schema='Chat')['json'] + for target, ids, code in ((chat, [str(uuid.uuid4())], 404), + (other_chat, [att['id']], 404), + (chat, [att['id'], att['id']], 400)): + c.call('POST', '/api/v1/chats/' + target['id'] + '/messages', code, actor=alice, + body={'text': '', 'attachment_ids': ids}, headers={'Idempotency-Key': uuid.uuid4().hex}) + key = uuid.uuid4().hex + first = c.message(alice, chat, 'same', key=key)['json'] + check(c.message(alice, chat, 'same', key=key, attachments=[])['json'] == first, + 'omitted and empty attachment_ids differ') + key = uuid.uuid4().hex + first = c.message(alice, chat, '', key=key, attachments=[att['id']])['json'] + check(c.message(alice, chat, '', key=key, attachments=[att['id']])['json'] == first, 'replay differs') + c.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', 409, actor=alice, + body={'text': 'changed', 'attachment_ids': [att['id']]}, headers={'Idempotency-Key': key}) + print('PASS: attachment scope, duplicates and idempotency normalization') + + # Exact limits are accepted; original bytes (including PNG trailing data) survive. + exact = png + b'\0' * (10485760 - len(png)) + att = upload(exact)['json'] + c.wait_ready(bob, att) + check(c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=bob)['bytes'] == exact, + '10 MiB original changed') + buf = BytesIO() + Image.new('RGB', (8000, 5000)).save(buf, 'PNG') + att = upload(buf.getvalue())['json'] + check(att['size_bytes'] == len(buf.getvalue()), '40M pixel upload failed') + print('PASS: inclusive 10 MiB and 40 million pixel boundaries') + + +if __name__ == '__main__': + run(main) diff --git a/student-tests/burn_rules.test.yaml b/student-tests/burn_rules.test.yaml new file mode 100644 index 0000000..7b25538 --- /dev/null +++ b/student-tests/burn_rules.test.yaml @@ -0,0 +1,83 @@ +rule_files: + - /workspace/infra/prometheus/alerts.yaml +evaluation_interval: 5s +tests: + - name: healthy + interval: 5s + input_series: + - series: 'messenger_http_requests_total{route="/api/v1/users",status_class="2xx"}' + values: '0+100x80' + - series: 'messenger_http_requests_total{route="/api/v1/users",status_class="5xx"}' + values: '0+0x80' + - series: 'messenger_slo_ready_eligible_total{service_name="messenger-outbox"}' + values: '0+100x80' + - series: 'messenger_slo_ready_bad_total{service_name="messenger-outbox"}' + values: '0+0x80' + alert_rule_test: + - {eval_time: 5m, alertname: MessengerApiBurnFast, exp_alerts: []} + - {eval_time: 5m, alertname: MessengerApiBurnSlow, exp_alerts: []} + - {eval_time: 5m, alertname: MessengerReadyBurnFast, exp_alerts: []} + - {eval_time: 5m, alertname: MessengerReadyBurnSlow, exp_alerts: []} + - name: brief severe spike and recovery + interval: 5s + input_series: + - series: 'messenger_http_requests_total{route="/api/v1/users",status_class="2xx"}' + values: '0+100x12 1200+0x8 1200+100x60' + - series: 'messenger_http_requests_total{route="/api/v1/users",status_class="5xx"}' + values: '0+0x12 0+100x8 800+0x60' + alert_rule_test: + - eval_time: 1m30s + alertname: MessengerApiBurnFast + exp_alerts: + - exp_labels: {severity: critical, slo: api, burn: fast} + exp_annotations: {summary: 'api SLO fast burn', runbook: 'See docs/slo-runbook.md'} + - {eval_time: 3m, alertname: MessengerApiBurnFast, exp_alerts: []} + - name: sustained moderate degradation + interval: 5s + input_series: + - series: 'messenger_http_requests_total{route="/api/v1/users",status_class="2xx"}' + values: '0+80x80' + - series: 'messenger_http_requests_total{route="/api/v1/users",status_class="5xx"}' + values: '0+20x80' + - series: 'messenger_slo_ready_eligible_total{service_name="messenger-outbox"}' + values: '0+100x80' + - series: 'messenger_slo_ready_bad_total{service_name="messenger-outbox"}' + values: '0+20x80' + alert_rule_test: + - {eval_time: 5m, alertname: MessengerApiBurnFast, exp_alerts: []} + - {eval_time: 5m, alertname: MessengerReadyBurnFast, exp_alerts: []} + - eval_time: 5m + alertname: MessengerApiBurnSlow + exp_alerts: + - exp_labels: {severity: warning, slo: api, burn: slow} + exp_annotations: {summary: 'api SLO slow burn', runbook: 'See docs/slo-runbook.md'} + - eval_time: 5m + alertname: MessengerReadyBurnSlow + exp_alerts: + - exp_labels: {severity: warning, slo: ready, burn: slow} + exp_annotations: {summary: 'ready SLO slow burn', runbook: 'See docs/slo-runbook.md'} + - name: low-volume guard + interval: 5s + input_series: + - series: 'messenger_http_requests_total{route="/api/v1/users",status_class="5xx"}' + values: '0+1x80' + - series: 'messenger_slo_ready_eligible_total{service_name="messenger-outbox"}' + values: '0+1x80' + - series: 'messenger_slo_ready_bad_total{service_name="messenger-outbox"}' + values: '0+1x80' + alert_rule_test: + - {eval_time: 5m, alertname: MessengerApiBurnFast, exp_alerts: []} + - {eval_time: 5m, alertname: MessengerReadyBurnFast, exp_alerts: []} + - name: late ready events fast burn + interval: 5s + input_series: + - series: 'messenger_slo_ready_eligible_total{service_name="messenger-outbox"}' + values: '0+100x30' + - series: 'messenger_slo_ready_bad_total{service_name="messenger-outbox"}' + values: '0+100x30' + alert_rule_test: + - eval_time: 1m + alertname: MessengerReadyBurnFast + exp_alerts: + - exp_labels: {severity: critical, slo: ready, burn: fast} + exp_annotations: {summary: 'ready SLO fast burn', runbook: 'See docs/slo-runbook.md'} diff --git a/student-tests/capture_observability.py b/student-tests/capture_observability.py new file mode 100644 index 0000000..f0d066c --- /dev/null +++ b/student-tests/capture_observability.py @@ -0,0 +1,41 @@ +"""Export synthetic trace/log evidence through Grafana; credentials never enter artifacts.""" +import argparse +import base64 +from datetime import datetime, timezone +import json +from pathlib import Path +import urllib.parse +import urllib.request + +ROOT = Path(__file__).resolve().parents[1] +auth = 'Basic ' + base64.b64encode(('admin:' + (ROOT/'secrets/grafana_password').read_text().strip()).encode()).decode() +def get(source, path, params=None): + url = f'http://127.0.0.1:3000/api/datasources/proxy/uid/{source}/' + path + if params: url += '?' + urllib.parse.urlencode(params) + with urllib.request.urlopen(urllib.request.Request(url,headers={'Authorization':auth}),timeout=15) as r: + return json.load(r) + +def capture(trace_id=None, query=None): + query = query or '{service_name="messenger-api"} | status="503"' + if trace_id: + query = '{service_name=~"messenger-api|messenger-worker"} | trace_id="' + trace_id + '"' + logs = get('loki','loki/api/v1/query_range',{'query':query,'since':'30m','limit':20,'direction':'backward'}) + if not trace_id: + for stream in logs['data']['result']: + for value in stream['values']: + metadata = dict(stream['stream']) + if len(value)>2: metadata.update(value[2]) + trace_id = metadata.get('trace_id') + if trace_id: break + if trace_id: break + if not trace_id: + raise RuntimeError('No correlated log/trace found') + trace = get('tempo','api/traces/'+trace_id) + return {'captured_at':datetime.now(timezone.utc).isoformat(),'query':query,'trace_id':trace_id,'logs':logs,'trace':trace} + +if __name__=='__main__': + parser=argparse.ArgumentParser();parser.add_argument('--trace-id');parser.add_argument('--output',required=True) + args=parser.parse_args() + result=capture(args.trace_id) + (ROOT/args.output).write_text(json.dumps(result,ensure_ascii=False,indent=2)+'\n') + print('Saved correlated evidence:',args.output,'trace_id:',result['trace_id']) diff --git a/student-tests/check_dashboard.py b/student-tests/check_dashboard.py new file mode 100644 index 0000000..cef28a8 --- /dev/null +++ b/student-tests/check_dashboard.py @@ -0,0 +1,18 @@ +"""Execute every provisioned PromQL target against the running Grafana datasource.""" +import importlib.util +import json +from pathlib import Path +ROOT=Path(__file__).resolve().parents[1] +spec=importlib.util.spec_from_file_location('capture',ROOT/'student-tests/capture_observability.py') +capture=importlib.util.module_from_spec(spec);spec.loader.exec_module(capture) +dashboard=json.loads((ROOT/'infra/grafana/dashboards/messenger.json').read_text()) +results=[] +for panel in dashboard['panels']: + for target in panel.get('targets',[]): + if 'expr' not in target:continue + query=target['expr'].replace('$__rate_interval','1m').replace('$__interval','5s') + result=capture.get('prometheus','api/v1/query',{'query':query}) + assert result['status']=='success',result + results.append({'panel':panel['title'],'query':query,'series':len(result['data']['result'])}) +(ROOT/'evidence/review-dashboard.json').write_text(json.dumps(results,ensure_ascii=False,indent=2)+'\n') +print('PASS:',len(results),'dashboard queries; empty targets:',sum(r['series']==0 for r in results)) diff --git a/student-tests/check_retry_trace.py b/student-tests/check_retry_trace.py new file mode 100644 index 0000000..1f75474 --- /dev/null +++ b/student-tests/check_retry_trace.py @@ -0,0 +1,31 @@ +"""Verify exported parent relationships and the completed retry, not just trace existence.""" +import base64 +import json +from collections import Counter +from pathlib import Path +ROOT=Path(__file__).resolve().parents[1] +data=json.loads((ROOT/'evidence/review-retry-trace.json').read_text()) +spans=[s for batch in data['trace']['batches'] for scope in batch.get('scopeSpans',[]) for s in scope.get('spans',[])] +by_id={s['spanId']:s for s in spans} +counts=Counter(s['name'] for s in spans) +producers=[s for s in spans if s['kind']=='SPAN_KIND_PRODUCER'] +consumers=[s for s in spans if s['kind']=='SPAN_KIND_CONSUMER'] +assert len(producers)>=2 and consumers,counts +for producer in producers: + ancestor = producer + for _ in range(len(spans)): + ancestor = by_id[ancestor['parentSpanId']] + if ancestor['kind']=='SPAN_KIND_SERVER': break + else: raise AssertionError('producer has no API ancestor') +for consumer in consumers: + assert by_id[consumer['parentSpanId']]['kind']=='SPAN_KIND_PRODUCER' +assert counts['postgres.complete_attachment_job']>=1,counts +assert any(s['name'].startswith('s3.') for s in spans),counts +assert all(base64.b64decode(s['traceId']).hex()==data['trace_id'] for s in spans) +services={stream['stream']['service_name'] for stream in data['logs']['data']['result']} +assert {'messenger-api','messenger-worker'}<=services,services +result={'trace_id':data['trace_id'],'span_count':len(spans),'span_names':dict(counts), + 'producer_count':len(producers),'consumer_count':len(consumers),'log_services':sorted(services), + 'parent_relationships':'PASS','note':'Abrupt os._exit may lose unfinished spans of the crashed attempt; redelivery and committed retry remain visible.'} +(ROOT/'evidence/review-retry-summary.json').write_text(json.dumps(result,indent=2)+'\n') +print('PASS:',json.dumps(result)) diff --git a/student-tests/idempotency_concurrency.py b/student-tests/idempotency_concurrency.py new file mode 100755 index 0000000..fcf958f --- /dev/null +++ b/student-tests/idempotency_concurrency.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 +"""Один ключ одновременно через LB создаёт ровно одно сообщение.""" +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +import sys +import uuid + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests")) +from client import Client, check, run # noqa: E402 + + +def main() -> None: + client = Client() + alice, _bob, _eve, chat = client.bootstrap() + key = uuid.uuid4().hex + text = "concurrent idempotency " + key + + def send(_number: int): + return Client().message(alice, chat, text, key=key)["json"] + + with ThreadPoolExecutor(max_workers=16) as pool: + replies = list(pool.map(send, range(16))) + check(all(reply == replies[0] for reply in replies), "replays returned different messages") + + path = "/api/v1/chats/" + chat["id"] + "/messages?limit=100" + history = client.call("GET", path, actor=alice, schema="MessagePage")["json"]["items"] + check(sum(message["id"] == replies[0]["id"] for message in history) == 1, "concurrent retries created duplicates") + print("PASS: concurrent idempotency through load balancer") + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/job_contract.py b/student-tests/job_contract.py new file mode 100644 index 0000000..961c0f9 --- /dev/null +++ b/student-tests/job_contract.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python3 +"""Check real producer payloads and redeliver a complete persisted message.""" +import json +import time +from jsonschema import Draft202012Validator, FormatChecker +from pipeline_integration import ROOT, Client, remote, duplicate, snapshot, wait, check + + +def main(): + c = Client('http://localhost:8080', timeout=30) + actor, _, _, chat = c.bootstrap() + att, _ = c.attachment(actor, chat) + wait(c, actor, att) + payloads = remote('''rows = await db.pool.fetch('SELECT payload FROM attachment_outbox') +print(json.dumps([json.loads(row['payload']) if isinstance(row['payload'], str) else row['payload'] for row in rows]))''') + validator = Draft202012Validator(json.loads((ROOT / 'contracts/jobs.schema.json').read_text()), format_checker=FormatChecker()) + for payload in payloads: + validator.validate(payload) + before = snapshot(att) + duplicate(att) + duplicate(att) + time.sleep(3) + check(snapshot(att) == before, 'persisted redelivery changed result') + result = dict(status='PASS', validated_outbox_payloads=len(payloads), + extra_payload_fields=False, persisted_redelivery_noop=True) + (ROOT / 'evidence/job-contract.json').write_text(json.dumps(result, indent=2) + '\n') + print(json.dumps(result)) + + +if __name__ == '__main__': + main() diff --git a/student-tests/pipeline_critical.py b/student-tests/pipeline_critical.py new file mode 100644 index 0000000..3750ba9 --- /dev/null +++ b/student-tests/pipeline_critical.py @@ -0,0 +1,117 @@ +"""Offline regression checks for delivery and terminal-state safety.""" +import asyncio +from contextlib import asynccontextmanager +from datetime import datetime, timezone +from io import BytesIO +import json +from pathlib import Path +import sys +from types import SimpleNamespace +import unittest +from unittest.mock import AsyncMock, MagicMock, patch +from uuid import uuid4 + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from src.storage.postgres import PostgresStore +from src.worker import process_locked as process, settle_delivery +from concurrent.futures import Future +from src.broker import _publish_to + + +class CriticalTests(unittest.IsolatedAsyncioTestCase): + def test_failed_or_unrecorded_work_is_requeued(self): + for result in (False, None, RuntimeError('database unavailable')): + future, channel = Future(), MagicMock() + if isinstance(result, Exception): + future.set_exception(result) + else: + future.set_result(result) + settle_delivery(channel, 7, future) + channel.basic_ack.assert_not_called() + channel.basic_nack.assert_called_once_with(7, requeue=True) + + def test_durable_success_is_acked(self): + future, channel = Future(), MagicMock() + future.set_result(True) + settle_delivery(channel, 7, future) + channel.basic_ack.assert_called_once_with(7) + channel.basic_nack.assert_not_called() + + async def test_outbox_decodes_jsonb(self): + store = PostgresStore('unused') + store._pool = AsyncMock() + payload = {'type': 'attachment.process.v1', 'attachment_id': str(uuid4())} + store.pool.fetch.return_value = [{'id': uuid4(), 'payload': json.dumps(payload)}] + event, = await store.due_outbox_events() + self.assertEqual(json.loads(json.dumps(event['payload'])), payload) + + async def run_processing(self, committed): + store, s3 = AsyncMock(), AsyncMock() + stamp = datetime.now(timezone.utc) + store.claim_attachment_job.return_value = { + 'claim_time': stamp, 'staging_key': 'staging/test', + 'content_type': 'image/png', 'size_bytes': 1, 'sha256': 'unused', + } + store.complete_attachment_job.return_value = committed + store.ready_seconds.return_value = 0.25 + s3.settings = SimpleNamespace(max_pixels=40000000) + s3.download_key.return_value = b'fixture' + s3.delete.side_effect = RuntimeError('storage temporarily unavailable') + payload = {'attachment_id': str(uuid4())} + with patch('src.worker.render_isolated', return_value=[('thumbnail', b'jpeg', 'image/jpeg', 1, 1)]): + outcome = await process(store, s3, payload, 60) + self.assertEqual(outcome, "success" if committed else "skipped") + self.assertEqual(store.complete_attachment_job.call_args.args[-1], stamp) + return store, s3 + + async def test_cleanup_error_does_not_retry_ready(self): + store, s3 = await self.run_processing(True) + s3.delete.assert_awaited_once() + store.retry_attachment_job.assert_not_awaited() + + async def test_stale_completion_keeps_staging(self): + _, s3 = await self.run_processing(False) + s3.delete.assert_not_awaited() + + async def test_duplicate_is_skipped_not_successful(self): + store, s3 = AsyncMock(), AsyncMock() + store.claim_attachment_job.return_value = None + result = await process(store, s3, {'attachment_id': str(uuid4())}, 60) + self.assertEqual(result, 'skipped') + store.complete_attachment_job.assert_not_awaited() + s3.copy.assert_not_awaited() + + async def test_stale_claim_cannot_publish_metadata(self): + store = PostgresStore('unused') + connection = AsyncMock() + connection.fetchval.return_value = None + @asynccontextmanager + async def context(): + yield connection + connection.transaction = context + store._pool = SimpleNamespace(acquire=context) + self.assertFalse(await store.complete_attachment_job(uuid4(), 'original', [], datetime.now(timezone.utc))) + connection.execute.assert_not_awaited() + + async def test_retry_of_terminal_or_stale_attempt_is_noop(self): + store = PostgresStore('unused') + connection = AsyncMock() + connection.fetchrow.return_value = None + @asynccontextmanager + async def context(): + yield connection + connection.transaction = context + store._pool = SimpleNamespace(acquire=context) + self.assertFalse(await store.retry_attachment_job(uuid4(), 5, 1, datetime.now(timezone.utc))) + connection.execute.assert_not_awaited() + + def test_dlq_waits_for_publisher_confirm(self): + connection = MagicMock() + with patch('src.broker._connection', return_value=connection): + _publish_to('dlq', {'failure': 'invalid_job'}) + connection.channel.return_value.confirm_delivery.assert_called_once() + self.assertTrue(connection.channel.return_value.basic_publish.call_args.kwargs['mandatory']) + + +if __name__ == '__main__': + unittest.main() diff --git a/student-tests/pipeline_integration.py b/student-tests/pipeline_integration.py new file mode 100644 index 0000000..c230440 --- /dev/null +++ b/student-tests/pipeline_integration.py @@ -0,0 +1,205 @@ +#!/usr/bin/env python3 +"""Destructive-to-availability tests on the local lab stack; volumes preserved. + +Only synthetic attachments created here are mutated. Tokens stay in memory. +Run sequentially, never alongside load or another fault test. +""" +import json +import os +from pathlib import Path +import subprocess +import sys +import time + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client, Failure, check + +RESULTS = [] + + +def compose(*args, env=None, source=None): + result = subprocess.run(['docker', 'compose', *args], cwd=ROOT, + env=dict(os.environ, **(env or {})), input=source, + text=True, capture_output=True, timeout=150) + check(result.returncode == 0, 'compose failed: ' + ' '.join(args[:4]) + '\n' + result.stderr[-1000:]) + return result.stdout.strip() + + +def remote(body, **values): + source = '''import asyncio, json, os +from uuid import UUID +from src.config import load_settings +from src.storage.postgres import PostgresStore +from src.storage.s3 import S3Store, S3Settings +from src.broker import publish +V = json.loads(%r) +async def main(): + db = PostgresStore(load_settings().database_url) + await db.connect() + s3 = S3Store(S3Settings.load()) + try: +%s + finally: + s3.client.close() + await db.close() +asyncio.run(main()) +''' % (json.dumps(values), '\n'.join(' ' + line for line in body.splitlines())) + return json.loads(compose('exec', '-T', 'api-a', '/app/.venv/bin/python', '-', source=source)) + + +def record(name, **facts): + RESULTS.append(dict(test=name, status='PASS', **facts)) + print('PASS: ' + name, flush=True) + (ROOT / 'evidence/pipeline-integration.json').write_text(json.dumps(RESULTS, indent=2) + '\n') + + +def ready(c): + deadline = time.monotonic() + 90 + successes = 0 + while time.monotonic() < deadline: + try: + c.call('GET', '/health/ready') + successes += 1 + if successes >= 5: + return + time.sleep(.5) + except Failure: + successes = 0 + time.sleep(.5) + raise Failure('API did not recover') + + +def wait(c, actor, att, status='ready', timeout=120): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + try: + value = c.call('GET', '/api/v1/attachments/' + att['id'], actor=actor, schema='Attachment')['json'] + except Failure: + time.sleep(.25) + continue + if value['status'] == status: + return value + check(value['status'] != 'failed' or status == 'failed', 'unexpected failed attachment') + time.sleep(.25) + raise Failure('attachment did not reach ' + status) + + +def snapshot(att): + return remote('''row = await db.pool.fetchrow("SELECT a.status, j.attempt, j.job_id, a.original_key, a.staging_key FROM attachments a JOIN attachment_jobs j ON j.attachment_id=a.id WHERE a.id=$1", UUID(V['id'])) +objects = sorted(list(await s3.list('originals/' + V['id'] + '/')) + list(await s3.list('variants/' + V['id'] + '/'))) +print(json.dumps(dict(row) | {'objects': objects}, default=str))''', id=att['id']) + + +def duplicate(att): + return remote('''payload = await db.pool.fetchval("SELECT o.payload FROM attachment_outbox o JOIN attachment_jobs j ON j.job_id=o.job_id WHERE j.attachment_id=$1 ORDER BY o.created_at LIMIT 1", UUID(V['id'])) +await publish(json.loads(payload) if isinstance(payload, str) else payload) +print('true')''', id=att['id']) + + +def main(): + c = Client('http://localhost:8080', timeout=30) + alice, bob, _, chat = c.bootstrap() + + # No worker: HTTP admission and message creation must remain asynchronous. + compose('stop', '-t', '2', 'worker') + att, data = c.attachment(alice, chat) + check(att['status'] == 'queued', 'upload is not queued with stopped worker') + c.message(alice, chat, 'queued attachment', attachments=[att['id']]) + time.sleep(2) + compose('restart', 'broker') + compose('up', '-d', '--no-deps', '--wait', 'broker') + compose('restart', 'api-a', 'api-b') + ready(c) + compose('start', 'worker') + completed = wait(c, bob, att) + check(c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=bob)['bytes'] == data, 'original changed') + check({v['name'] for v in completed['variants']} == {'thumbnail', 'crop', 'optimized'}, 'variants missing') + record('stopped worker, queued message, broker/API restart, ready', attachment_id=att['id']) + + before = snapshot(att) + duplicate(att) + duplicate(att) + time.sleep(3) + check(snapshot(att) == before, 'duplicate changed result or created S3 objects') + refused = remote("print(json.dumps(await db.replay_failed_attachment(UUID(V['id']))))", id=att['id']) + check(refused is False, 'ready replay accepted') + record('duplicate delivery and replay of ready are no-ops', object_count=len(before['objects'])) + + # Pause publication so the API crash is demonstrably before any publish. + compose('stop', '-t', '2', 'outbox') + compose('up', '-d', '--no-deps', '--wait', 'api-a', 'api-b', env={'OUTBOX_FAULT_AFTER_COMMIT': '1'}) + ready(c) + fault_actor, _, _, fault_chat = c.bootstrap() + try: + c.attachment(fault_actor, fault_chat) + raise AssertionError('crash hook did not interrupt upload') + except Failure: + pass + compose('up', '-d', '--no-deps', '--wait', 'api-a', 'api-b') + ready(c) + row = remote('''rows = await db.pool.fetch("SELECT a.id, o.published_at FROM attachments a JOIN attachment_jobs j ON j.attachment_id=a.id JOIN attachment_outbox o ON o.job_id=j.job_id WHERE a.chat_id=$1", UUID(V['chat'])) +assert len(rows) == 1 and rows[0]['published_at'] is None +print(json.dumps({'id': str(rows[0]['id'])}))''', chat=fault_chat['id']) + compose('start', 'outbox') + wait(c, fault_actor, row) + record('crash after COMMIT before publish recovered from existing outbox', attachment_id=row['id']) + + compose('up', '-d', '--no-deps', '--wait', 'worker', env={'WORKER_FAULT_AFTER_S3_PUT': '1', 'JOB_TIMEOUT_SECONDS': '10'}) + att, _ = c.attachment(alice, chat) + wait(c, alice, att) + snap = snapshot(att) + check(snap['attempt'] >= 2 and len(snap['objects']) >= 8, 'side effect crash/lease retry not observed') + marker = compose('exec', '-T', 'worker', 'cat', '/tmp/worker-fault-after-s3') + check(marker == att['id'], 'wrong fault target') + record('worker crash after S3 before commit/ack, lease recovery', attempt=snap['attempt'], objects_before_cleanup=len(snap['objects'])) + compose('up', '-d', '--no-deps', '--wait', 'worker') + + # Select only this test's unreferenced attempt keys for cleanup. + cleaned = remote('''from argparse import Namespace +from src.reconcile import run +keys = V['keys'] +result = await run(Namespace(apply=True, grace_seconds=0, object_key=keys)) +print(json.dumps(result['deleted']))''', keys=snap['objects']) + check(len(cleaned) == 4, 'stale attempt objects not cleaned safely') + check(len(snapshot(att)['objects']) == 4, 'live attempt affected by cleanup') + wait(c, alice, att) + record('targeted orphan cleanup preserves committed attempt', deleted=len(cleaned)) + + # Lose only this synthetic staging object to exercise transient retry exhaustion. + compose('stop', '-t', '2', 'worker') + att, data = c.attachment(alice, chat) + remote("await s3.delete('staging/' + V['id'] + '/original')\nprint('true')", id=att['id']) + compose('up', '-d', '--no-deps', '--wait', 'worker', env={'JOB_MAX_ATTEMPTS': '2'}) + failed = wait(c, alice, att, 'failed') + snap = snapshot(att) + check(snap['attempt'] == 2 and failed['error_code'] == 'retry_exhausted', 'unbounded/wrong retry') + # Restore exactly the bytes originally accepted, then explicit failed-only replay. + import base64 + replayed = remote('''from io import BytesIO +import base64 +content = base64.b64decode(V['data']) +await s3.put('staging/' + V['id'] + '/original', BytesIO(content), len(content), 'image/png') +print(json.dumps(await db.replay_failed_attachment(UUID(V['id']))))''', id=att['id'], data=base64.b64encode(data).decode()) + check(replayed, 'failed replay refused') + wait(c, alice, att) + record('bounded transient retry, failed registry and explicit replay', attempts_before_replay=snap['attempt']) + compose('up', '-d', '--no-deps', '--wait', 'worker') + + # Real Linux child with deliberately insufficient address space. + memory = remote('''import subprocess +from io import BytesIO +from PIL import Image +buf = BytesIO(); Image.new('RGB', (2400, 2400)).save(buf, 'PNG') +env = dict(os.environ, RENDER_MEMORY_MB='120') +p = subprocess.run(['/app/.venv/bin/python', '-m', 'src.render'], input=buf.getvalue(), capture_output=True, env=env, timeout=30) +print(json.dumps({'returncode': p.returncode}))''') + check(memory['returncode'] == 21, 'Linux memory limit did not produce bounded resource failure') + record('Linux renderer memory limit', **memory) + + +if __name__ == '__main__': + try: + main() + finally: + compose('up', '-d', '--no-deps', '--wait', 'api-a', 'api-b', 'worker', 'outbox') diff --git a/student-tests/pipeline_invariants.py b/student-tests/pipeline_invariants.py new file mode 100644 index 0000000..e571551 --- /dev/null +++ b/student-tests/pipeline_invariants.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python3 +"""Real PostgreSQL/S3/RabbitMQ invariant and security checks, synthetic data only.""" +import json +from pathlib import Path +import time +from pipeline_integration import Client, check, compose, remote, wait, ROOT + + +def main(): + results = [] + c = Client('http://localhost:8080', timeout=30) + alice, bob, eve, chat = c.bootstrap() + att, _ = c.attachment(alice, chat) + wait(c, bob, att) + from io import BytesIO + from PIL import Image + for name, fmt, bound in (('thumbnail', 'JPEG', 256), ('crop', 'JPEG', 128), ('optimized', 'WEBP', 1280)): + content = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=' + name, actor=bob)['bytes'] + with Image.open(BytesIO(content)) as image: + check(image.format == fmt and max(image.size) <= bound and not image.getexif(), 'variant format/size/EXIF') + if name == 'crop': + check(image.size == (128, 128), 'crop dimensions') + results.append('actual thumbnail/crop/WebP formats, dimensions and removed EXIF') + # Verify actual private object key, not a nonexistent legacy key. + private = remote('''import urllib.request, urllib.error +key = await db.pool.fetchval('SELECT original_key FROM attachments WHERE id=$1', UUID(V['id'])) +try: + urllib.request.urlopen('http://s3:8333/' + s3.settings.bucket + '/' + key, timeout=5) +except urllib.error.HTTPError as error: + assert error.code == 403, error.code +else: + raise AssertionError('anonymous object leaked') +print('true')''', id=att['id']) + check(private, 'private S3 check failed') + for variant in ('thumbnail', 'crop', 'optimized'): + c.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=' + variant, 404, actor=eve) + results.append('anonymous S3 403 on existing object; variant ACL 404') + + lock = remote('''import time +entered = asyncio.Event() +async def writer(): + async with db.attachment_transaction(): + entered.set() + await asyncio.sleep(1) +task = asyncio.create_task(writer()) +await entered.wait() +started = time.monotonic() +async with db.attachment_transaction(maintenance=True): + waited = time.monotonic() - started +await task +assert waited >= .9, waited +print(json.dumps({'maintenance_wait_seconds': waited}))''') + results.append(lock) + + # Poison staging only for a newly created synthetic attachment and update its + # digest so invalid-image classification (not transport checksum) is exercised. + compose('stop', '-t', '2', 'worker') + guarded, _ = c.attachment(alice, chat) + fence = remote('''key = UUID(V['id']) +job_id = await db.pool.fetchval('SELECT job_id FROM attachment_jobs WHERE attachment_id=$1', key) +before = await db.pool.fetchval('SELECT count(*) FROM attachment_outbox WHERE job_id=$1', job_id) +await db.pool.execute("UPDATE attachment_jobs SET updated_at=now()-interval '61 seconds' WHERE attachment_id=$1", key) +await db.schedule_due_jobs() +after = await db.pool.fetchval('SELECT count(*) FROM attachment_outbox WHERE job_id=$1', job_id) +assert after == before + 1, 'unclaimed published job stranded' +assert await db.pool.fetchval('SELECT attempt FROM attachment_jobs WHERE attachment_id=$1', key) == 1 +await db.schedule_due_jobs() +assert await db.pool.fetchval('SELECT count(*) FROM attachment_outbox WHERE job_id=$1', job_id) == after, 'watchdog floods outbox' +first = await db.claim_attachment_job(key, 30) +assert first is not None +assert await db.retry_attachment_job(key, 5, 30, first['claim_time']) +assert await db.claim_attachment_job(key, 30) is None, 'duplicate bypassed retry delay' +await db.pool.execute("UPDATE attachment_jobs SET next_attempt_at=now() WHERE attachment_id=$1", key) +await db.schedule_due_jobs() +second = await db.claim_attachment_job(key, 30) +assert second is not None and second['attempt'] == 2 +assert not await db.complete_attachment_job(key, 'invalid-stale-key', [], first['claim_time']) +assert not await db.retry_attachment_job(key, 5, 0, first['claim_time']) +await db.pool.execute("UPDATE attachment_jobs SET lease_until=now()-interval '1 second' WHERE attachment_id=$1", key) +assert await db.claim_attachment_job(key, 30) is None, 'duplicate bypassed lease attempt accounting' +await db.schedule_due_jobs() +third = await db.claim_attachment_job(key, 30) +assert third['attempt'] == 3 +assert await db.retry_attachment_job(key, 5, 0, third['claim_time']) +print('true')''', id=guarded['id']) + check(fence, 'SQL fencing/backoff invariant') + results.append('real SQL: queued watchdog republishes without attempt increment/flood; duplicate respects backoff; expired lease counted; stale completion/retry fenced') + att, _ = c.attachment(alice, chat) + remote('''from io import BytesIO +from hashlib import sha256 +data = b'invalid image payload' +await s3.put('staging/' + V['id'] + '/original', BytesIO(data), len(data), 'image/png') +await db.pool.execute('UPDATE attachments SET size_bytes=$2, sha256=$3 WHERE id=$1', UUID(V['id']), len(data), sha256(data).hexdigest()) +print('true')''', id=att['id']) + compose('start', 'worker') + wait(c, alice, guarded) + failed = wait(c, alice, att, 'failed') + check(failed['error_code'] == 'invalid_image', 'permanent classification failed') + attempt = remote("print(await db.pool.fetchval('SELECT attempt FROM attachment_jobs WHERE attachment_id=$1', UUID(V['id'])))", id=att['id']) + check(attempt == 1, 'permanent error retried') + results.append('permanent invalid_image reaches failed on attempt 1') + + # DLQ uses confirmed persistent publication; count change avoids consuming + # messages belonging to a different test. + dlq = remote('''from src.broker import _connection, _publish, DLQ +connection = _connection(); channel = connection.channel() +before = channel.queue_declare(queue=DLQ, durable=True).method.message_count +connection.close() +_publish({'malformed': True}) +print(json.dumps({'before': before}))''') + deadline = time.monotonic() + 20 + while time.monotonic() < deadline: + count = remote('''from src.broker import _connection, DLQ +connection = _connection() +print(connection.channel().queue_declare(queue=DLQ, durable=True).method.message_count) +connection.close()''') + if count > dlq['before']: + break + time.sleep(.5) + check(count == dlq['before'] + 1, 'invalid job not delivered to DLQ') + results.append('malformed broker message delivered to DLQ') + + # Actual Linux child timeout: no renderer can finish within this deadline. + timeout = remote('''from src.worker import render_isolated +try: + render_isolated(b'x', .000001) +except TimeoutError: + print('true') +else: + raise AssertionError('timeout not enforced')''') + check(timeout, 'timeout missing') + results.append('Linux render subprocess killed/reaped on deadline') + (ROOT / 'evidence/pipeline-invariants.json').write_text(json.dumps(results, indent=2) + '\n') + print(json.dumps(results, indent=2), flush=True) + + +if __name__ == '__main__': + try: + main() + finally: + compose('up', '-d', '--no-deps', '--wait', 'worker') diff --git a/student-tests/queue_incident.py b/student-tests/queue_incident.py new file mode 100644 index 0000000..6e07a5c --- /dev/null +++ b/student-tests/queue_incident.py @@ -0,0 +1,11 @@ +"""Submit synthetic attachments without waiting for worker; no credentials printed.""" +import sys +from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests')) +from client import Client + +client = Client() +actor, _, _, chat = client.bootstrap() +for _ in range(3): + attachment, _ = client.attachment(actor, chat) + print('accepted attachment:', attachment['id']) diff --git a/student-tests/readiness_failover.py b/student-tests/readiness_failover.py new file mode 100644 index 0000000..cdb6c0a --- /dev/null +++ b/student-tests/readiness_failover.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +"""Запрет БД только для api-a через pg_hba; восстановление в finally.""" +import ipaddress +import json +from pathlib import Path +import subprocess +import sys +import time +import uuid + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, check, run + + +def compose(*args): + return subprocess.check_output(["docker", "compose", *args], cwd=ROOT, text=True).strip() + + +def sql(statement): + return compose("exec", "-T", "postgres", "psql", "-U", "messenger", "-d", + "messenger", "-v", "ON_ERROR_STOP=1", "-Atc", statement) + + +def ready(service): + code = """import urllib.request, urllib.error +try: + print(urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=7).status) +except urllib.error.HTTPError as error: + print(error.code) +""" + return compose("exec", "-T", service, "/app/.venv/bin/python", "-c", code) + + +def main(): + client = Client() + container = json.loads(subprocess.check_output( + ["docker", "inspect", compose("ps", "-q", "api-a")], text=True))[0] + addresses = list(container["NetworkSettings"]["Networks"].values()) + check(len(addresses) == 1, "test expects one API network") + address = str(ipaddress.IPv4Address(addresses[0]["IPAddress"])) + bad_id = container["Config"]["Hostname"] + before = {client.call("GET", "/health/live")["headers"]["X-Instance-Id"] for _ in range(40)} + check(len(before) == 2 and bad_id in before, "both API replicas must be reachable") + check(ready("api-a") == ready("api-b") == "200", "both APIs must be ready") + # Сохраняем исходный pg_hba; чужую оставшуюся резервную копию не перезаписываем. + compose("exec", "-T", "postgres", "sh", "-ec", + 'test ! -e "$PGDATA/pg_hba.conf.readiness-backup"; ' + 'cp -p "$PGDATA/pg_hba.conf" "$PGDATA/pg_hba.conf.readiness-backup"') + try: + compose("exec", "-T", "postgres", "sh", "-ec", + '{ printf "host all all %s/32 reject\\n" "$1"; ' + 'cat "$PGDATA/pg_hba.conf.readiness-backup"; } > "$PGDATA/pg_hba.conf"', + "sh", address) + sql("SELECT pg_reload_conf()") + time.sleep(0.5) + sql(f"SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE client_addr = '{address}'::inet") + check(ready("api-a") == "503", "api-a must be unready") + check(ready("api-b") == "200", "api-b must remain ready") + # 2s interval + 2s check timeout, с запасом на планировщик Docker Desktop. + time.sleep(6) + for _ in range(40): + result = client.call("GET", "/health/live", 200) + check(result["headers"]["X-Instance-Id"] != bad_id, "LB selected unready api-a") + for _ in range(20): + result = client.call("POST", "/api/v1/users", 201, body={ + "username": "r_" + uuid.uuid4().hex[:16], + "display_name": "Readiness regression", + "password": "Test-password-" + uuid.uuid4().hex, + }, schema="User") + check(result["headers"]["X-Instance-Id"] != bad_id, "registration reached api-a") + print("PASS: api-a ready=503, api-b ready=200; 40/40 live and 20/20 registrations use api-b") + finally: + compose("exec", "-T", "postgres", "sh", "-ec", + 'mv "$PGDATA/pg_hba.conf.readiness-backup" "$PGDATA/pg_hba.conf"') + sql("SELECT pg_reload_conf()") + deadline = time.monotonic() + 20 + while time.monotonic() < deadline: + observed = {client.call("GET", "/health/live")["headers"]["X-Instance-Id"] for _ in range(10)} + if observed == before: + check(ready("api-a") == "200", "restored api-a must be ready") + print("PASS: database access restored; both API replicas returned to LB") + return + time.sleep(0.5) + raise AssertionError("api-a did not return to LB") + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/readiness_unavailable.py b/student-tests/readiness_unavailable.py new file mode 100644 index 0000000..cf4bffa --- /dev/null +++ b/student-tests/readiness_unavailable.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python3 +"""Проверка 503 readiness при остановленном PostgreSQL.""" +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests")) +from client import Client, run # noqa: E402 + + +def main() -> None: + Client().call("GET", "/health/ready", 503) + print("PASS: readiness rejects unavailable PostgreSQL") + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/reconcile.py b/student-tests/reconcile.py new file mode 100644 index 0000000..04a8fd7 --- /dev/null +++ b/student-tests/reconcile.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python3 +"""Inject S3/DB disagreement and prove dry-run, grace period and safe repair.""" +import json +import base64 +from pathlib import Path +import subprocess +import sys +import time +import uuid + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, Failure, arguments, check, run + + +def compose(*args, input=None, env=None) -> str: + return subprocess.check_output(["docker", "compose", *args], cwd=ROOT, text=True, + input=input, env=env) + + +def reconcile(*args) -> dict: + output = compose("--profile", "maintenance", "run", "--rm", "--no-deps", "reconcile", *args) + return json.loads(next(line for line in reversed(output.splitlines()) if line.startswith("{"))) + + +def s3_code(action: str, key: str, data: bytes | None = None) -> None: + code = """from src.storage.s3 import S3Settings, S3Store +import base64, json, sys +s3 = S3Store(S3Settings.load()) +payload = json.load(sys.stdin) +key = payload['key'] +try: + if sys.argv[1] == 'put': + s3.client.put_object(Bucket=s3.settings.bucket, Key=key, + Body=base64.b64decode(payload.get('data', 'b3JwaGFu'))) + else: + s3.client.delete_object(Bucket=s3.settings.bucket, Key=key) +finally: + s3.client.close() +""" + payload = {"key": key} + if data is not None: + payload["data"] = base64.b64encode(data).decode() + compose("run", "--rm", "--no-deps", "s3-init", "/app/.venv/bin/python", "-c", code, action, + input=json.dumps(payload)) + + +def wait_ready(client: Client, env=None) -> None: + subprocess.run(["docker", "compose", "up", "-d", "--wait", "--no-deps", "api-a"], + cwd=ROOT, env=env, check=True, stdout=subprocess.DEVNULL) + deadline = time.monotonic() + 15 + while True: + try: + client.call("GET", "/health/ready", schema="Health") + return + except Failure: + check(time.monotonic() < deadline, "stack did not recover") + time.sleep(.5) + + +def main(): + args = arguments(__doc__) + client = Client(args.base_url, args.ca_file, timeout=30) + alice, bob, _eve, chat = client.bootstrap() + image = (ROOT / "tests/fixtures/sample.png").read_bytes() + path = "/api/v1/chats/" + chat["id"] + "/attachments?filename=" + + def upload(name, expected=201): + return client.call("POST", path + name, expected, actor=alice, raw=image, + headers={"Content-Type": "image/png"}, + schema="Attachment" if expected == 201 else None) + + # A reference object must survive an apply sweep, even with zero grace. + protected = upload("protected.png")["json"] + message = client.message(alice, chat, "protected", attachments=[protected["id"]])["json"] + orphan = "originals/test-orphan-" + uuid.uuid4().hex + s3_code("put", orphan) + + report = reconcile("--grace-seconds", "3600") + check(orphan in report["fresh_unreferenced"], "fresh orphan missing from dry-run") + check(not report["deleted"], "dry-run deleted an object") + report = reconcile("--apply", "--grace-seconds", "3600", "--object-key", orphan) + check(orphan in report["fresh_unreferenced"] and not report["deleted"], "grace period ignored") + report = reconcile("--apply", "--grace-seconds", "0", "--object-key", orphan) + check(report["deleted"] == [orphan], "aged orphan was not deleted safely") + report = reconcile("--apply", "--grace-seconds", "0", "--object-key", orphan) + check(orphan not in report["deleted"] and orphan not in report["orphan_candidates"], + "second repair run was not idempotent") + check(client.call("GET", "/api/v1/attachments/" + protected["id"] + "/content", actor=bob)["bytes"] == image, + "reconciler deleted a referenced object") + + # This models loss after metadata has committed: report it, do not delete metadata. + missing = upload("missing.png")["json"] + missing_key = "originals/" + missing["id"] + s3_code("delete", missing_key) + try: + report = reconcile("--apply", "--grace-seconds", "0", "--object-key", missing_key) + expected = {"attachment_id": missing["id"], "object_key": missing_key} + check(expected in report["missing_objects"], "missing metadata/object pair not reported") + client.call("GET", "/api/v1/attachments/" + missing["id"], actor=bob, schema="Attachment") + client.call("GET", "/api/v1/attachments/" + missing["id"] + "/content", 503, actor=bob) + finally: + s3_code("put", missing_key, image) + check(client.call("GET", "/api/v1/attachments/" + missing["id"] + "/content", actor=bob)["bytes"] == image, + "test object was not restored") + report = reconcile("--grace-seconds", "0") + check(expected not in report["missing_objects"], "restored object remained missing on second scan") + print("PASS: S3-side missing object, dry-run, grace period and idempotent orphan repair", flush=True) + + # Inject a process-equivalent failure after PUT directly into the configured + # replica. The public load balancer may correctly select the healthy peer, + # which would not exercise this boundary. + environment = dict(__import__("os").environ, UPLOAD_FAULT_AFTER_S3_PUT="1") + before_fault = set(reconcile("--grace-seconds", "0")["orphan_candidates"]) + try: + subprocess.run(["docker", "compose", "up", "-d", "--force-recreate", "--no-deps", "api-a"], + cwd=ROOT, env=environment, check=True, stdout=subprocess.DEVNULL) + wait_ready(client, environment) + fault = """import base64, json, sys, urllib.error, urllib.request +payload = json.load(sys.stdin) +request = urllib.request.Request('http://127.0.0.1:8080/api/v1/chats/' + payload['chat_id'] + + '/attachments?filename=fault-after-put.png', data=base64.b64decode(payload['image']), + method='POST', headers={'Authorization': 'Bearer ' + payload['token'], 'Content-Type': 'image/png'}) +try: + urllib.request.urlopen(request, timeout=30) +except urllib.error.HTTPError as error: + assert error.code == 503, error.code + assert json.load(error)['error']['code'] == 'unavailable' +else: + raise AssertionError('fault injection unexpectedly returned success') +""" + compose("exec", "-T", "api-a", "/app/.venv/bin/python", "-c", fault, + input=json.dumps({"chat_id": chat["id"], "token": alice["token"], + "image": base64.b64encode(image).decode()})) + report = reconcile("--grace-seconds", "0") + injected = sorted(set(report["orphan_candidates"]) - before_fault) + check(len(injected) == 1, "fault must create exactly one new orphan; run without other writers") + report = reconcile("--apply", "--grace-seconds", "0", "--object-key", injected[0]) + check(set(injected).issubset(report["deleted"]), "reconciler did not clean injected orphan") + history = client.call("GET", "/api/v1/chats/" + chat["id"] + "/messages", actor=bob, schema="MessagePage")["json"] + check(message in history["items"], "message reference was lost") + check(client.call("GET", "/api/v1/attachments/" + protected["id"] + "/content", actor=bob)["bytes"] == image, + "used original was lost") + print("PASS: injected pre-INSERT failure repaired; message and used original survive", flush=True) + finally: + environment["UPLOAD_FAULT_AFTER_S3_PUT"] = "0" + subprocess.run(["docker", "compose", "up", "-d", "--wait", "--force-recreate", "--no-deps", "api-a"], + cwd=ROOT, env=environment, check=True, stdout=subprocess.DEVNULL) + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/reconcile_audit.py b/student-tests/reconcile_audit.py new file mode 100644 index 0000000..8c1e046 --- /dev/null +++ b/student-tests/reconcile_audit.py @@ -0,0 +1,159 @@ +#!/usr/bin/env python3 +"""Real database rejection, concurrent commit, interrupted upload and pagination.""" +import base64 +import json +from pathlib import Path +import subprocess +import sys + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, arguments, run + +PROBE = r''' +import asyncio, base64, hashlib, http.client, json, sys, urllib.request, urllib.error +from types import SimpleNamespace +from uuid import UUID, uuid4 +from src.config import load_settings +from src.storage.postgres import PostgresStore +from src.storage.s3 import S3Store, S3Settings +from src.reconcile import run as reconcile, scan + +payload = json.load(sys.stdin) +image = base64.b64decode(payload['image']) +chat_id, user_id = UUID(payload['chat_id']), UUID(payload['user_id']) + +def upload(filename): + req = urllib.request.Request('http://api-a:8080/api/v1/chats/' + str(chat_id) + + '/attachments?filename=' + filename, data=image, method='POST', + headers={'Authorization': 'Bearer ' + payload['token'], 'Content-Type': 'image/png'}) + try: + with urllib.request.urlopen(req, timeout=30) as response: + return response.status, json.load(response) + except urllib.error.HTTPError as error: + return error.code, json.load(error) + +async def main(): + db = PostgresStore(load_settings().database_url) + s3 = S3Store(S3Settings.load()) + await db.connect() + def args(key): + return SimpleNamespace(apply=True, grace_seconds=0, object_key=[key]) + try: + # A failed metadata inventory must not authorize any S3 deletion. + key = 'originals/audit-inventory-' + uuid4().hex + await s3.call('put_object', Bucket=s3.settings.bucket, Key=key, Body=image) + class BrokenInventory: + async def attachment_keys(self): + # Real SQL failure, not an empty inventory on error. + return await db.pool.fetch('SELECT * FROM audit_missing_table_' + uuid4().hex) + try: + async with db.attachment_transaction(maintenance=True): + try: + await scan(args(key), BrokenInventory(), s3) + except Exception: + pass + else: + raise AssertionError('failed inventory did not abort') + assert key in await s3.list('originals/') + finally: + await s3.delete(key) + print('PASS: failed PostgreSQL inventory aborts apply without deleting the orphan', flush=True) + + # An actual PostgreSQL trigger rejects only this test chat's INSERT. + suffix = uuid4().hex + name = 'audit_reject_' + suffix + before = await s3.list('originals/') + await db.pool.execute(f"""CREATE FUNCTION {name}() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN IF NEW.chat_id = '{chat_id}'::uuid THEN + RAISE EXCEPTION 'synthetic audit rejection' USING ERRCODE = '23514'; + END IF; RETURN NEW; END $$""") + try: + await db.pool.execute(f'CREATE TRIGGER {name} BEFORE INSERT ON attachments FOR EACH ROW EXECUTE FUNCTION {name}()') + status, body = await asyncio.to_thread(upload, 'db-failure.png') + assert status == 503 and body['error']['code'] == 'unavailable', (status, body) + assert await db.pool.fetchval('SELECT count(*) FROM attachments WHERE chat_id=$1', chat_id) == 0 + finally: + await db.pool.execute(f'DROP TRIGGER IF EXISTS {name} ON attachments') + await db.pool.execute(f'DROP FUNCTION {name}()') + created = set(await s3.list('originals/')) - set(before) + assert len(created) == 1, created + key = created.pop() + report = await reconcile(SimpleNamespace(apply=False, grace_seconds=0, object_key=[key])) + assert key in report['orphan_candidates'] and not report['deleted'] + assert (await reconcile(args(key)))['deleted'] == [key] + assert not (await reconcile(args(key)))['deleted'] + print('PASS: actual PostgreSQL INSERT rejection -> 503, dry-run detection, scoped repair, repeat no-op', flush=True) + + # Writer holds a shared lock after PUT; cleanup must wait for COMMIT. + identifier = uuid4() + key = 'originals/' + str(identifier) + async with db.attachment_transaction() as connection: + await s3.call('put_object', Bucket=s3.settings.bucket, Key=key, Body=image) + sweep = asyncio.create_task(reconcile(args(key))) + for _ in range(40): + waiting = await db.pool.fetchval("SELECT count(*) FROM pg_locks WHERE locktype='advisory' AND classid=41004 AND objid=1 AND NOT granted") + if waiting: + break + await asyncio.sleep(.05) + assert waiting and not sweep.done(), 'maintenance failed to wait for active writer' + await db.create_attachment(identifier, chat_id, user_id, 'concurrent.png', 'image/png', + len(image), hashlib.sha256(image).hexdigest(), key, connection=connection) + report = await sweep + assert key not in report['deleted'] and key not in report['orphan_candidates'] + row = await db.attachment_for_member(identifier, user_id) + assert await s3.download(row) == image + print('PASS: apply with zero grace waits for in-flight PUT/COMMIT; committed original survives', flush=True) + + # Abrupt client close before all bytes arrive must not persist a file. + before = await s3.list('originals/') + rows = await db.attachment_keys() + def interrupt(): + conn = http.client.HTTPConnection('api-a', 8080, timeout=10) + conn.putrequest('POST', '/api/v1/chats/' + str(chat_id) + '/attachments?filename=interrupted.png') + conn.putheader('Authorization', 'Bearer ' + payload['token']) + conn.putheader('Content-Type', 'image/png') + conn.putheader('Content-Length', str(len(image) + 100000)) + conn.endheaders() + conn.send(image[:32]) + conn.close() + await asyncio.to_thread(interrupt) + await asyncio.sleep(1) + assert await db.attachment_keys() == rows + assert await s3.list('originals/') == before + status, body = await asyncio.to_thread(upload, 'after-disconnect.png') + assert status == 201, (status, body) + print('PASS: interrupted upload leaves no metadata/object and next upload succeeds', flush=True) + + # Exercise pagination on real S3 without adding thousands of fixtures. + original_call = s3.call + async def small_pages(method, **kwargs): + if method == 'list_objects_v2': + kwargs['MaxKeys'] = 7 + return await original_call(method, **kwargs) + expected = await s3.list('originals/') + s3.call = small_pages + actual, pages = await s3.list_with_pages('originals/') + assert actual == expected and pages > 1 + print(f'PASS: S3 pagination: {len(actual)} keys across {pages} pages, no missing/duplicate keys', flush=True) + finally: + s3.client.close() + await db.close() + +asyncio.run(main()) +''' + + +def main(): + args = arguments(__doc__) + client = Client(args.base_url, args.ca_file, timeout=30) + alice, _, _, chat = client.bootstrap() + payload = {"chat_id": chat["id"], "user_id": alice["id"], "token": alice["token"], + "image": base64.b64encode((ROOT / "tests/fixtures/sample.png").read_bytes()).decode()} + subprocess.run(["docker", "compose", "--profile", "maintenance", "run", "--rm", "--no-deps", + "--entrypoint", "/app/.venv/bin/python", "reconcile", "-c", PROBE], + cwd=ROOT, input=json.dumps(payload), text=True, check=True) + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/reconcile_benchmark.py b/student-tests/reconcile_benchmark.py new file mode 100644 index 0000000..358ba40 --- /dev/null +++ b/student-tests/reconcile_benchmark.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +"""Measure repeatable reconciler inventory cost for two controlled S3 sizes.""" +import json +from pathlib import Path +import statistics +import subprocess +import sys +import uuid + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import arguments, check, run + + +def compose(*args, input=None) -> str: + return subprocess.check_output(["docker", "compose", *args], cwd=ROOT, text=True, input=input) + + +def reconcile() -> dict: + output = compose("--profile", "maintenance", "run", "--rm", "--no-deps", "reconcile", + "--grace-seconds", "3600") + return json.loads(next(line for line in reversed(output.splitlines()) if line.startswith("{"))) + + +def mutate(action: str, keys: list[str]) -> None: + code = """from src.storage.s3 import S3Settings, S3Store +import json, sys +s3 = S3Store(S3Settings.load()) +try: + for key in json.load(sys.stdin): + if sys.argv[1] == 'put': + s3.client.put_object(Bucket=s3.settings.bucket, Key=key, Body=b'track-2-benchmark') + else: + s3.client.delete_object(Bucket=s3.settings.bucket, Key=key) +finally: + s3.client.close() +""" + compose("run", "--rm", "--no-deps", "s3-init", "/app/.venv/bin/python", "-c", code, action, + input=json.dumps(keys)) + + +def main() -> None: + arguments(__doc__) # Keeps the common --base-url/--ca-file command interface. + results = [] + for requested in (32, 128): + prefix = f"originals/track2-benchmark-{uuid.uuid4().hex}-" + keys = [prefix + str(index) for index in range(requested)] + try: + mutate("put", keys) + reports = [reconcile() for _ in range(3)] + for report in reports: + check(set(keys).issubset(report["fresh_unreferenced"]), + "benchmark objects were not included in the inventory") + check(not set(keys) & set(report["orphan_candidates"]), + "grace period classified fresh benchmark objects as deletable") + seconds = [report["scan_metrics"]["total_seconds"] for report in reports] + results.append({ + "added_unreferenced_objects": requested, + "runs": 3, + "s3_objects_scanned": reports[-1]["s3_objects"], + "metadata_rows_scanned": reports[-1]["metadata_rows"], + "s3_list_pages": reports[-1]["scan_metrics"]["s3_list_pages"], + "seconds": seconds, + "median_seconds": round(statistics.median(seconds), 6), + }) + finally: + mutate("delete", keys) + print(json.dumps({"method": "three dry-run inventories per controlled size", "results": results}, + sort_keys=True)) + print("PASS: reconciler cost measured for 32 and 128 added S3 objects; fixtures removed") + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/remaining_regressions.py b/student-tests/remaining_regressions.py new file mode 100644 index 0000000..af79f76 --- /dev/null +++ b/student-tests/remaining_regressions.py @@ -0,0 +1,127 @@ +"""Contract, isolated-render and benchmark regressions without Docker.""" +import asyncio +from contextlib import asynccontextmanager +from datetime import datetime, timedelta, timezone +import importlib.util +from io import BytesIO +import json +from pathlib import Path +import sys +import tempfile +import threading +import time +from types import SimpleNamespace +import unittest +from unittest.mock import AsyncMock, patch +from uuid import uuid4 + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) +from PIL import Image +from src.worker import variants, render_isolated, PermanentFailure +from src.storage.postgres import PostgresStore +from src.reconcile import scan +from src.errors import ApiError + + +def png(width, height): + buffer = BytesIO() + Image.new('RGBA', (width, height), (255, 0, 0, 0)).save(buffer, 'PNG') + return buffer.getvalue() + + +class Regressions(unittest.IsolatedAsyncioTestCase): + async def test_failed_attachment_is_rejected(self): + store, connection = PostgresStore('unused'), AsyncMock() + connection.fetchval.side_effect = [1, 1, 1] + @asynccontextmanager + async def context(): + yield connection + connection.transaction = context + store._pool = SimpleNamespace(acquire=context) + with self.assertRaises(ApiError) as error: + await store.create_message_idempotent(uuid4(), uuid4(), '', 'key', 86400, [uuid4()]) + self.assertEqual(error.exception.status_code, 409) + connection.fetchrow.assert_not_awaited() + + def test_floor_and_small_images(self): + self.assertEqual(variants(png(300, 200))[0][3:], (256, 170)) + self.assertEqual(variants(png(200, 300))[0][3:], (170, 256)) + self.assertEqual(variants(png(10, 8))[0][3:], (10, 8)) + for name, data, mime, width, height in variants(png(300, 200)): + with Image.open(BytesIO(data)) as image: + self.assertEqual(image.size, (width, height)) + self.assertTrue(all(value > 240 for value in image.convert('RGB').getpixel((0, 0)))) + + def test_renderer_success_and_permanent_error(self): + self.assertEqual(render_isolated(png(300, 200), 10)[0][3:], (256, 170)) + with self.assertRaises(PermanentFailure): + render_isolated(b'not an image', 10) + + def test_renderer_is_killed_on_deadline(self): + started = time.monotonic() + with self.assertRaises(TimeoutError): + render_isolated(png(300, 200), .0001) + self.assertLess(time.monotonic() - started, 2) + + async def test_cleanup_retains_active_and_failed_staging(self): + old = datetime.now(timezone.utc) - timedelta(days=2) + db, s3 = AsyncMock(), AsyncMock() + db.attachment_keys.return_value = {'staging/active': uuid4(), 'staging/failed': uuid4(), 'originals/ready': uuid4()} + s3.list_with_pages.side_effect = [({'staging/active': old, 'staging/failed': old, 'staging/orphan': old}, 1), + ({'originals/ready': old}, 1), ({'variants/orphan': old}, 1)] + report = await scan(SimpleNamespace(grace_seconds=60, apply=True, object_key=[]), db, s3) + self.assertEqual(report['deleted'], ['staging/orphan', 'variants/orphan']) + + async def test_permanent_failure_is_atomic_and_not_retried(self): + store = PostgresStore('unused') + connection = AsyncMock() + connection.fetchrow.return_value = {'attempt': 1} + @asynccontextmanager + async def context(): + yield connection + connection.transaction = context + store._pool = SimpleNamespace(acquire=context) + result = await store.retry_attachment_job(uuid4(), 5, 1, datetime.now(timezone.utc), 'invalid_image') + self.assertFalse(result) + self.assertEqual(connection.execute.await_count, 2) + for call in connection.execute.call_args_list: + self.assertIn("status='failed'", call.args[0]) + self.assertEqual(call.args[2], 'invalid_image') + + def test_benchmark_partial_batch_and_recovery(self): + spec = importlib.util.spec_from_file_location('track1', ROOT / 'scripts/track1.py') + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + jobs, lock = {}, threading.Lock() + class FakeClient: + def __init__(self, *args, **kwargs): pass + def bootstrap(self): return {}, {}, {}, {'id': str(uuid4())} + def call(self, method, path, *args, **kwargs): + if method == 'POST': + key = str(uuid4()) + with lock: jobs[key] = {'id': key, 'status': 'ready', 'queue_wait_seconds': .01} + return {'status': 202, 'json': {'id': key}} + return {'json': {'status': 'ready'}} + def attachment(self, *args): return {}, b'' + def wait_ready(self, *args, **kwargs): return {} + def metrics(_): + with lock: + return {'jobs': list(jobs.values()), 'pending': 0, 'rss_bytes': 1000, 'concurrency': 2, 'prefetch': 2} + with tempfile.TemporaryDirectory() as directory: + destination = str(Path(directory) / 'result.json') + args = SimpleNamespace(run_name='test', workers=2, prefetch=2, max_pending=20, images=10, + concurrency=8, size=8, max_ready_seconds=2, require_backpressure=False, + output=destination, base_url=None, ca_file=None) + info = {'Config': {'Env': ['MAX_PENDING_JOBS=20']}, 'HostConfig': {'Memory': 4096, 'NanoCpus': 2}} + with patch.object(module, 'arguments', return_value=args), patch.object(module, 'Client', FakeClient), \ + patch.object(module, 'metrics', side_effect=metrics), patch.object(module, 'container_info', return_value=info), \ + patch.object(module, 'broker_depth', return_value={'messages_ready': 0, 'messages_unacknowledged': 0}): + module.main() + report = json.loads(Path(destination).read_text()) + self.assertEqual(report['results']['ready'], 10) + self.assertTrue(report['results']['recovery_upload_ready']) + + +if __name__ == '__main__': + unittest.main() diff --git a/student-tests/retry_evidence.py b/student-tests/retry_evidence.py new file mode 100644 index 0000000..f3b5b01 --- /dev/null +++ b/student-tests/retry_evidence.py @@ -0,0 +1,30 @@ +"""Crash one idle lab worker after S3 writes; restore normal configuration in finally.""" +import json +import os +from pathlib import Path +import subprocess +import sys +import time +ROOT=Path(__file__).resolve().parents[1] +def compose(*args, env=None): + return subprocess.run(['docker','compose',*args],cwd=ROOT,env=env,check=True,capture_output=True,text=True).stdout +count=compose('exec','-T','postgres','psql','-U','messenger','-d','messenger','-Atc',"SELECT count(*) FROM attachment_jobs WHERE status IN ('queued','processing')") +if int(count.strip()):raise RuntimeError('Pending work; refusing crash injection') +try: + compose('stop','-t','5','worker') + compose('up','-d','--no-deps','worker',env={**os.environ,'WORKER_FAULT_AFTER_S3_PUT':'1'}) + subprocess.run([sys.executable,'student-tests/trace_retry.py'],cwd=ROOT,check=True,timeout=180) + out=json.loads((ROOT/'evidence/review-retry-job.json').read_text()) + from uuid import UUID + identity=str(UUID(out['attachment_id'])) + out['db_attempt']=int(compose('exec','-T','postgres','psql','-U','messenger','-d','messenger','-Atc',f"SELECT attempt FROM attachment_jobs WHERE attachment_id='{identity}'").strip()) + cid=compose('ps','-q','worker').strip() + out['restart_count']=int(subprocess.run(['docker','inspect','--format','{{.RestartCount}}',cid],check=True,capture_output=True,text=True).stdout.strip()) + assert out['db_attempt']==2 and out['restart_count']==1,out + (ROOT/'evidence/review-retry-job.json').write_text(json.dumps(out,indent=2)+'\n') + time.sleep(10) + subprocess.run([sys.executable,'student-tests/capture_observability.py','--trace-id',out['trace_id'],'--output','evidence/review-retry-trace.json'],cwd=ROOT,check=True) +finally: + compose('stop','-t','5','worker') + compose('up','-d','--no-deps','worker',env={**os.environ,'WORKER_FAULT_AFTER_S3_PUT':'0'}) +print('PASS: attempt 2, one restart, normal worker restored') diff --git a/student-tests/review_checks.py b/student-tests/review_checks.py new file mode 100644 index 0000000..a22b16c --- /dev/null +++ b/student-tests/review_checks.py @@ -0,0 +1,22 @@ +"""Repeat short review checks; experiments are separate and intentionally disruptive.""" +from datetime import datetime, timezone +import json +from pathlib import Path +import subprocess +ROOT=Path(__file__).resolve().parents[1] +checks=[['make','check'],['make','validate','PYTHON=.venv/bin/python'], + ['docker','compose','config','--quiet'], + ['docker','run','--rm','-v',f'{ROOT}:/workspace:ro','--entrypoint','promtool','prom/prometheus:v3.2.1','test','rules', + '/workspace/student-tests/alerts.test.yaml','/workspace/student-tests/burn_rules.test.yaml','/workspace/student-tests/telemetry_alerts.test.yaml'], + ['make','test'],['python3','student-tests/check_dashboard.py']] +results=[] +for command in checks: + proc=subprocess.run(command,cwd=ROOT,capture_output=True,text=True,timeout=180) + results.append({'command':command,'exit_code':proc.returncode,'output':proc.stdout+proc.stderr}) +for filename in ('telemetry_unit.py','pipeline_critical.py','slo_commit_integration.py'): + command=['docker','compose','exec','-T','api-a','/app/.venv/bin/python','-'] + proc=subprocess.run(command,input=(ROOT/'student-tests'/filename).read_text(),cwd=ROOT,capture_output=True,text=True,timeout=60) + results.append({'command':command,'stdin':f'student-tests/{filename}','exit_code':proc.returncode,'output':proc.stdout+proc.stderr}) +(ROOT/'evidence/review-checks.json').write_text(json.dumps({'utc':datetime.now(timezone.utc).isoformat(),'checks':results},ensure_ascii=False,indent=2)+'\n') +for r in results:print('PASS' if r['exit_code']==0 else 'FAIL',r.get('stdin',r['command'])) +raise SystemExit(any(r['exit_code'] for r in results)) diff --git a/student-tests/review_queries.py b/student-tests/review_queries.py new file mode 100644 index 0000000..1c4d9ae --- /dev/null +++ b/student-tests/review_queries.py @@ -0,0 +1,13 @@ +"""Read current health of telemetry, without printing access credentials.""" +import importlib.util +import json +from pathlib import Path +ROOT=Path(__file__).resolve().parents[1] +spec=importlib.util.spec_from_file_location('capture',ROOT/'student-tests/capture_observability.py') +c=importlib.util.module_from_spec(spec);spec.loader.exec_module(c) +queries=['messenger_slo_unknown{service_name="messenger-outbox"}', + 'time()-max(messenger_queue_sample_time_seconds{service_name="messenger-outbox"})', + 'messenger_queue_depth{service_name="messenger-outbox"}', + 'messenger_job_ready_duration_seconds_count{service_name="messenger-worker"}'] +for q in queries: + print(q,json.dumps(c.get('prometheus','api/v1/query',{'query':q})['data']['result'])) diff --git a/student-tests/review_regressions.py b/student-tests/review_regressions.py new file mode 100644 index 0000000..7e2bba7 --- /dev/null +++ b/student-tests/review_regressions.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 +"""Регрессии ревью: не-ASCII Bearer и невалидная кодировка JSON через LB.""" +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests")) +from client import Client, arguments, check, run + + +def main(): + args = arguments(__doc__) + client = Client(args.base_url, args.ca_file) + for path in ("/api/v1/users/me", "/api/v1/auth/sessions/current"): + method = "DELETE" if path.endswith("/current") else "GET" + result = client.call(method, path, 401, + headers={"Authorization": "Bearer é"}, schema="Error") + check(result["headers"].get("WWW-Authenticate", "").startswith("Bearer"), + "missing Bearer challenge") + check(result["json"]["error"]["code"] == "unauthorized", "wrong error code") + check(result["json"]["error"]["request_id"] == result["headers"]["X-Request-Id"], + "request IDs differ") + for raw in (b"\xff", b"{", b'{"username":'): + result = client.call("POST", "/api/v1/users", 400, raw=raw, + headers={"Content-Type": "application/json"}, schema="Error") + check(result["json"]["error"]["code"] == "invalid_request", "wrong JSON error code") + check(result["json"]["error"]["request_id"] == result["headers"]["X-Request-Id"], + "request IDs differ") + print("PASS: non-ASCII Bearer returns 401/challenge; malformed JSON returns contract Error/400") + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/review_resilience.py b/student-tests/review_resilience.py new file mode 100644 index 0000000..cfc49e1 --- /dev/null +++ b/student-tests/review_resilience.py @@ -0,0 +1,20 @@ +"""Run inherited public durability/failover checks and restore both API replicas.""" +from datetime import datetime, timezone +import json +from pathlib import Path +import subprocess +ROOT=Path(__file__).resolve().parents[1] +results=[] +try: + for command in [ + ['make','test-persistence','ACTION_SCRIPT=scripts/restart.sh'], + ['make','test-failover','ACTION_SCRIPT=scripts/stop-one.sh']]: + result=subprocess.run(command,cwd=ROOT,capture_output=True,text=True,timeout=240) + results.append({'command':command,'exit_code':result.returncode,'output':result.stdout+result.stderr}) + print('PASS' if result.returncode==0 else 'FAIL',command,flush=True) + if result.returncode:break +finally: + restore=subprocess.run(['docker','compose','up','-d','--no-deps','--wait','api-a','api-b','worker','outbox'],cwd=ROOT,capture_output=True,text=True,timeout=90) + results.append({'restore_exit_code':restore.returncode,'output':restore.stdout+restore.stderr}) + (ROOT/'evidence/review-resilience.json').write_text(json.dumps({'utc':datetime.now(timezone.utc).isoformat(),'checks':results},ensure_ascii=False,indent=2)+'\n') +raise SystemExit(any(r.get('exit_code',r.get('restore_exit_code')) for r in results)) diff --git a/student-tests/s3_permissions.py b/student-tests/s3_permissions.py new file mode 100644 index 0000000..d67e7be --- /dev/null +++ b/student-tests/s3_permissions.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +"""Prove that the API identity is bucket-scoped and lacks admin operations.""" +import subprocess +from pathlib import Path +import sys +import uuid + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, arguments, check, run + + +def compose(*args, input=None): + return subprocess.check_output(["docker", "compose", *args], cwd=ROOT, text=True, + input=input, stderr=subprocess.DEVNULL) + + +def main(): + args = arguments(__doc__) + client = Client(args.base_url, args.ca_file) + alice, _bob, _eve, chat = client.bootstrap() + data = (ROOT / "tests/fixtures/sample.png").read_bytes() + client.call("POST", "/api/v1/chats/" + chat["id"] + "/attachments?filename=permission.png", 202, + actor=alice, raw=data, headers={"Content-Type": "image/png"}, schema="Attachment") + bucket = "permission-probe-" + uuid.uuid4().hex + code = """from botocore.exceptions import ClientError +from src.storage.s3 import S3Settings, S3Store +import sys +s3 = S3Store(S3Settings.load()) +try: + try: + s3.client.create_bucket(Bucket=sys.stdin.read().strip()) + except ClientError as error: + assert error.response['Error']['Code'] in {'AccessDenied', 'Unauthorized'}, error.response + else: + raise AssertionError('API S3 identity unexpectedly has admin bucket creation') +finally: + s3.client.close() +""" + compose("exec", "-T", "api-a", "/app/.venv/bin/python", "-c", code, input=bucket) + admin = """from src.storage.s3 import S3Settings, S3Store +import sys +s3 = S3Store(S3Settings.load()) +try: + s3.client.list_buckets() +finally: + s3.client.close() +""" + compose("run", "--rm", "--no-deps", "s3-init", "/app/.venv/bin/python", "-c", admin) + print("PASS: API identity cannot create buckets; maintenance identity has separate admin access") + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/session_expiry.py b/student-tests/session_expiry.py new file mode 100644 index 0000000..37e29fb --- /dev/null +++ b/student-tests/session_expiry.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 +"""Проверка expiry; запускается при SESSION_TTL_SECONDS=2.""" +from pathlib import Path +import sys +import time + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests")) +from client import Client, check, run # noqa: E402 + + +def main() -> None: + client = Client() + actor = client.user("expiry") + time.sleep(3) + client.call("GET", "/api/v1/users/me", 401, actor=actor) + print("PASS: expired Bearer session was rejected") + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/slo_analyze.py b/student-tests/slo_analyze.py new file mode 100644 index 0000000..d01f1d4 --- /dev/null +++ b/student-tests/slo_analyze.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 +"""Summarize observed alert transitions; sampling bounds detection precision.""" +import json +from datetime import datetime +from pathlib import Path + +root = Path(__file__).resolve().parents[1] +data = json.loads((root / 'evidence/slo-incidents.json').read_text()) +samples = sorted((s for s in data['samples'] if 'firing' in s), key=lambda s: s['monotonic']) +alerts = ['MessengerApiBurnFast', 'MessengerApiBurnSlow'] +output = {'sampling_interval_seconds': 5, 'monitor_errors': sum('monitor_error' in s for s in data['samples']), 'phases': {}} +drifts = [abs((datetime.fromisoformat(b['time']) - datetime.fromisoformat(a['time'])).total_seconds() + - (b['monotonic'] - a['monotonic'])) for a, b in zip(samples, samples[1:])] +gaps = [b['monotonic'] - a['monotonic'] for a, b in zip(samples, samples[1:])] +output['timing_quality'] = {'max_wall_monotonic_disagreement_seconds': round(max(drifts, default=0), 3), + 'max_observed_sampling_gap_seconds': round(max(gaps, default=0), 3), + 'valid_for_elapsed_time_claims': max(drifts, default=0) <= 1, + 'note': 'Disagreement does not identify its cause; rerun if invalid. Precision is bounded by actual gaps, not the configured sleep.'} +base = [s for s in samples if s['phase'] == 'baseline'] +output['baseline'] = {'samples': len(base), 'samples_with_burn_alert': sum(bool(s['firing']) for s in base), 'scope': 'one observed healthy baseline, not a long-term false-positive rate'} +for name, period in data['periods'].items(): + start, end = period['lock_observed']['monotonic'], period['command_completed']['monotonic'] + active = [s for s in samples if start <= s['monotonic'] <= end] + recovery = [s for s in samples if s['monotonic'] > end and s['phase'] == name + '_recovery'] + previous = [s for s in samples if s['monotonic'] < start] + row = {'observed_lock_seconds': round(end-start, 3), 'alerts': {}} + for alert in alerts: + before = bool(previous and alert in previous[-1]['firing']) + hits = [s for s in active + recovery if alert in s['firing']] + clear = [s for s in recovery if hits and s['monotonic'] > hits[0]['monotonic'] and alert not in s['firing']] + row['alerts'][alert] = {'active_before_lock': before, + 'first_firing': hits[0]['time'] if hits else None, + 'first_firing_after_incident': bool(hits and hits[0]['monotonic'] > end), + 'detection_seconds': round(hits[0]['monotonic']-start, 3) if hits and not before else None, + 'first_clear': clear[0]['time'] if hits and clear else None, + 'recovery_seconds': round(clear[0]['monotonic']-end, 3) if hits and clear else None} + output['phases'][name] = row +path = root / 'evidence/slo-analysis.json' +path.write_text(json.dumps(output, ensure_ascii=False, indent=2) + '\n') +print(json.dumps(output, ensure_ascii=False, indent=2)) diff --git a/student-tests/slo_commit_integration.py b/student-tests/slo_commit_integration.py new file mode 100644 index 0000000..acc1dd0 --- /dev/null +++ b/student-tests/slo_commit_integration.py @@ -0,0 +1,45 @@ +"""Run inside api-a; session-local table shadows real SLO data (no business mutations).""" +import asyncio +from datetime import datetime, timezone +from uuid import uuid4 +import asyncpg +from src.config import load_settings +from src.storage.postgres import PostgresStore + +async def main(): + store = PostgresStore(load_settings().database_url) + store._pool = await asyncpg.create_pool(load_settings().database_url, min_size=1, max_size=1) + try: + async with store.pool.acquire() as conn: + await conn.execute('CREATE TEMP TABLE attachment_slo (LIKE public.attachment_slo INCLUDING DEFAULTS)') + identity = uuid4() + async with conn.transaction(): + await conn.execute('INSERT INTO attachment_slo (attachment_id) VALUES ($1)', identity) + await asyncio.sleep(.2) + before_accept_commit = await conn.fetchval('SELECT clock_timestamp()') + async with conn.transaction(): + await conn.execute('UPDATE attachment_slo SET ready_xid=pg_current_xact_id()::xid') + await asyncio.sleep(.2) + before_ready_commit = await conn.fetchval('SELECT clock_timestamp()') + await store.sample_ready_slo() + row = await store.pool.fetchrow('SELECT * FROM attachment_slo') + assert row['accepted_at'] >= before_accept_commit + assert row['ready_at'] >= before_ready_commit + elapsed = await store.ready_seconds(identity) + assert elapsed >= .2, elapsed + # Mature a successful synthetic cohort, then emulate replay metadata changes. + await store.pool.execute("UPDATE attachment_slo SET accepted_at=now()-interval '32 seconds',ready_at=now()-interval '31 seconds'") + first = dict(await store.sample_ready_slo()) + assert first == {'eligible': 1, 'bad': 0, 'unknown': 0}, first + await store.pool.execute("UPDATE attachment_slo SET ready_at=now()+interval '1 minute'") + assert dict(await store.sample_ready_slo()) == first + await store.pool.execute("INSERT INTO attachment_slo (attachment_id,accepted_at) VALUES ($1,now()-interval '31 seconds')", uuid4()) + late = dict(await store.sample_ready_slo()) + assert late == {'eligible': 2, 'bad': 1, 'unknown': 0}, late + await store.pool.execute('UPDATE attachment_slo SET ready_at=now(),ready_xid=pg_current_xact_id()::xid WHERE ready_at IS NULL') + assert dict(await store.sample_ready_slo()) == late + print('PASS: commit boundaries; immutable good/bad cohorts; late completion remains bad') + finally: + await store.close() + +asyncio.run(main()) diff --git a/student-tests/slo_incidents.py b/student-tests/slo_incidents.py new file mode 100644 index 0000000..09c2bcc --- /dev/null +++ b/student-tests/slo_incidents.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 +"""Run two bounded DB-lock incidents against the local Compose project. + +Creates synthetic data, records only aggregate counts/alert states, and releases +bounded locks; interrupted server-side sleeps may persist until their deadline. Run on the isolated lab6 stack, never a shared database. +""" +import base64 +from concurrent.futures import ThreadPoolExecutor +from datetime import datetime, timezone +import json +from pathlib import Path +import subprocess +import sys +import threading +import time +import urllib.parse +import urllib.request + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client, Failure + +OUTPUT = ROOT / 'evidence' / 'slo-incidents.json' +STOP = threading.Event() +LOCK = threading.Lock() +COUNTS = {'ok': 0, 'failed': 0} +SCENARIO = {'phase': 'bootstrap'} +SAMPLES = [] +AUTH = 'Basic ' + base64.b64encode(('admin:' + (ROOT / 'secrets/grafana_password').read_text().strip()).encode()).decode() + +def utc(): return datetime.now(timezone.utc).isoformat() + +def get(path, params=None): + url = 'http://127.0.0.1:3000/api/datasources/proxy/uid/prometheus/api/v1/' + path + if params: url += '?' + urllib.parse.urlencode(params) + with urllib.request.urlopen(urllib.request.Request(url, headers={'Authorization': AUTH}), timeout=5) as response: + result = json.load(response) + if result.get('status') != 'success': raise RuntimeError('Prometheus query failed') + return result['data'] + +def monitor(): + while not STOP.is_set(): + try: + alerts = get('alerts')['alerts'] + firing = sorted(a['labels']['alertname'] for a in alerts if a['state'] == 'firing' and 'Burn' in a['labels']['alertname']) + with LOCK: sample = {'time': utc(), 'monotonic': time.monotonic(), 'phase': SCENARIO['phase'], 'counts': dict(COUNTS), 'firing': firing, 'all_firing': sorted(a['labels']['alertname'] for a in alerts if a['state'] == 'firing')} + SAMPLES.append(sample) + except Exception as error: + SAMPLES.append({'time': utc(), 'monotonic': time.monotonic(), 'phase': SCENARIO['phase'], 'monitor_error': type(error).__name__}) + STOP.wait(5) + +def worker(actor, chat): + client = Client(timeout=9) + while not STOP.is_set(): + try: + client.message(actor, chat, 'synthetic SLO experiment') + outcome = 'ok' + except Failure: + outcome = 'failed' + with LOCK: COUNTS[outcome] += 1 + STOP.wait(1) + +def phase(name): + with LOCK: SCENARIO['phase'] = name + print(utc(), name, flush=True) + +def db_is_locked(): + result = subprocess.run(['docker','compose','exec','-T','postgres','psql','-U','messenger','-d','messenger','-Atc', + "SELECT EXISTS (SELECT 1 FROM pg_locks WHERE relation='users'::regclass AND mode='AccessExclusiveLock' AND granted)"], + cwd=ROOT, capture_output=True, text=True, timeout=10, check=True) + return result.stdout.strip() == 't' + +def incident(name, seconds): + cmd = ['docker','compose','exec','-T','postgres','psql','-U','messenger','-d','messenger','-c', + f'BEGIN; LOCK TABLE users IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep({seconds}); COMMIT;'] + start_command = {'time': utc(), 'monotonic': time.monotonic()} + proc = subprocess.Popen(cmd, cwd=ROOT, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE) + try: + deadline = time.monotonic() + 15 + while time.monotonic() < deadline: + if proc.poll() is not None: raise RuntimeError('DB lock command ended early') + if db_is_locked(): break + time.sleep(.5) + else: raise RuntimeError('DB lock was not observed') + phase(name) + start = {'time': utc(), 'monotonic': time.monotonic(), 'command_started': start_command} + code = proc.wait(timeout=seconds+20) + if code: raise RuntimeError('DB lock command failed') + end = {'time': utc(), 'monotonic': time.monotonic()} + phase(name + '_recovery') + return {'lock_observed': start, 'command_completed': end} + finally: + if proc.poll() is None: + proc.terminate() + try: proc.wait(timeout=5) + except subprocess.TimeoutExpired: proc.kill(); proc.wait() + +def main(): + client = Client(timeout=9) + actor, _, _, chat = client.bootstrap() + result = {'description': 'controlled DB table lock; three synthetic clients; 5s alert sampling', 'periods': {}, 'samples': SAMPLES} + monitor_thread = threading.Thread(target=monitor, daemon=True); monitor_thread.start() + try: + with ThreadPoolExecutor(max_workers=3) as pool: + futures = [pool.submit(worker, actor, chat) for _ in range(3)] + try: + phase('baseline'); time.sleep(60) + result['periods']['burst'] = incident('burst', 75) + time.sleep(100) + result['periods']['sustained'] = incident('sustained', 180) + time.sleep(100) + phase('complete') + finally: + STOP.set() + for future in futures: future.result(timeout=15) + finally: + STOP.set(); monitor_thread.join(timeout=10) + result['end_time'] = utc() + result['final_counts'] = COUNTS + OUTPUT.write_text(json.dumps(result, ensure_ascii=False, indent=2) + '\n') + print('Aggregate evidence:', OUTPUT, flush=True) + +if __name__ == '__main__': main() diff --git a/student-tests/slo_ready_incident.py b/student-tests/slo_ready_incident.py new file mode 100644 index 0000000..f745af7 --- /dev/null +++ b/student-tests/slo_ready_incident.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +"""Stop only this project's idle worker, submit 8 jobs, verify ready SLO.""" +import base64 +from datetime import datetime, timezone +import json +from pathlib import Path +import subprocess +import sys +import time +import urllib.parse +import urllib.request + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client + +secret = (ROOT / 'secrets/grafana_password').read_text().strip() +auth = 'Basic ' + base64.b64encode(('admin:' + secret).encode()).decode() +def utc(): return datetime.now(timezone.utc).isoformat() +def prom(path, query=None): + url = 'http://127.0.0.1:3000/api/datasources/proxy/uid/prometheus/api/v1/' + path + if query is not None: url += '?' + urllib.parse.urlencode({'query': query}) + with urllib.request.urlopen(urllib.request.Request(url, headers={'Authorization': auth}), timeout=5) as r: return json.load(r)['data'] +def metric(name): + result = prom('query', 'max(' + name + '{service_name="messenger-outbox"})')['result'] + return float(result[0]['value'][1]) if result else None +def sample(phase): + alerts = prom('alerts')['alerts'] + return {'time': utc(), 'monotonic': time.monotonic(), 'phase': phase, + 'firing': sorted(a['labels']['alertname'] for a in alerts if a['state']=='firing' and a['labels']['alertname'].startswith('MessengerReadyBurn')), + 'all_firing': sorted(a['labels']['alertname'] for a in alerts if a['state']=='firing'), + 'queue_depth': metric('messenger_queue_depth'), 'queue_age': metric('messenger_queue_age_seconds'), + 'eligible': metric('messenger_slo_ready_eligible_total'), 'bad': metric('messenger_slo_ready_bad_total')} +def docker(*args): subprocess.run(['docker','compose',*args], cwd=ROOT, check=True, stdout=subprocess.DEVNULL) +def run(): + status = subprocess.run(['docker','compose','exec','-T','postgres','psql','-U','messenger','-d','messenger','-Atc', + "SELECT count(*) FROM attachment_jobs WHERE status IN ('queued','processing')"], cwd=ROOT, check=True, capture_output=True, text=True) + if int(status.stdout.strip()): raise RuntimeError('Pending jobs exist; refusing to stop worker') + client=Client(); actor,_,_,chat=client.bootstrap(); samples=[]; out={'samples':samples,'jobs':8} + worker_stopped=False + try: + samples.append(sample('baseline')) + docker('stop','-t','5','worker'); worker_stopped=True + accepted=[] + for _ in range(8): accepted.append(client.attachment(actor,chat)[0]) + out['accepted_at']={'time':utc(),'monotonic':time.monotonic()} + print('8 synthetic jobs accepted; waiting for burn alert', flush=True) + seen=set() + deadline=time.monotonic()+100 + while time.monotonic()=0 for value in out['counter_deltas'].values()),'counter reset' + out['load']=json.loads((ROOT/'evidence/review-tempo-load.json').read_text()) + assert out['load']['failed_requests']==0,out['load'] + out['result']='PASS' + finally: + if load is not None and load.poll() is None: + load.terminate();load.wait(timeout=15) + if stopped: docker('up','-d','--no-deps','tempo') + (ROOT/'evidence/review-tempo-outage.json').write_text(json.dumps(out,indent=2)+'\n') + print('PASS: Tempo outage; baseline/deltas/memory in evidence/review-tempo-outage.json') +if __name__=='__main__': main() diff --git a/student-tests/trace_retry.py b/student-tests/trace_retry.py new file mode 100644 index 0000000..056143b --- /dev/null +++ b/student-tests/trace_retry.py @@ -0,0 +1,27 @@ +"""Synthetic upload with a known W3C trace ID; wait through a worker restart.""" +import secrets +import json +from datetime import datetime, timezone +import sys +import time +from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests')) +from client import Client + +client = Client() +actor, _, _, chat = client.bootstrap() +trace_id = secrets.token_hex(16) +parent_id = secrets.token_hex(8) +content = (Path(__file__).resolve().parents[1] / 'tests/fixtures/sample.png').read_bytes() +attachment = client.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=sample.png', 202, + actor=actor, raw=content, headers={'Content-Type':'image/png', + 'traceparent':f'00-{trace_id}-{parent_id}-01'}, schema='Attachment')['json'] +print('trace_id:', trace_id, 'attachment_id:', attachment['id'], flush=True) +started = time.monotonic() +result = client.wait_ready(actor, attachment, wait_seconds=150) +print('status:', result['status'], 'elapsed_seconds:', round(time.monotonic()-started, 3), flush=True) + +(Path(__file__).resolve().parents[1] / 'evidence/review-retry-job.json').write_text(json.dumps({ + 'utc': datetime.now(timezone.utc).isoformat(), 'trace_id': trace_id, + 'attachment_id': attachment['id'], 'status': result['status'], + 'elapsed_seconds': round(time.monotonic()-started, 3)}, indent=2)+'\n') diff --git a/student-tests/upload_memory.py b/student-tests/upload_memory.py new file mode 100644 index 0000000..1075a4e --- /dev/null +++ b/student-tests/upload_memory.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 +"""Ten concurrent bounded uploads, with API-container peak memory sampling.""" +from concurrent.futures import ThreadPoolExecutor +import json +import http.client +from pathlib import Path +import subprocess +import sys +import threading +import time +from urllib.parse import quote +from urllib.parse import urlsplit + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, arguments, check, run + + +def api_memory() -> dict[str, int]: + ids = subprocess.check_output(["docker", "compose", "ps", "-q", "api-a", "api-b"], cwd=ROOT, text=True).split() + return {container[:12]: int(subprocess.check_output( + ["docker", "exec", container, "cat", "/sys/fs/cgroup/memory.current"], text=True)) + for container in ids} + + +def main(): + def configure(parser): + parser.add_argument("--output", default="evidence/upload-memory.json") + parser.add_argument("--size-mib", type=int, default=5) + args = arguments(__doc__, configure) + check(1 <= args.size_mib <= 10, "--size-mib must be 1..10") + client = Client(args.base_url, args.ca_file, timeout=60) + alice, _bob, _eve, chat = client.bootstrap() + seed = (ROOT / "tests/fixtures/sample.png").read_bytes() + payload = seed + b"\0" * (args.size_mib * 1024 * 1024 - len(seed)) + baseline, peak, samples, stop = api_memory(), {}, [], threading.Event() + check(len(baseline) == 2, "memory experiment requires two running APIs") + sampling_errors = [] + timestamps = [] + barrier = threading.Barrier(10) + + def sampler(): + while not stop.is_set(): + try: + values = api_memory() + check(set(values) == set(baseline), "API containers changed during measurement") + except Exception as error: + sampling_errors.append(type(error).__name__) + return + timestamps.append(time.monotonic()) + samples.append(values) + for name, value in values.items(): + peak[name] = max(peak.get(name, 0), value) + time.sleep(.1) + + def upload(index): + barrier.wait(timeout=15) + parsed = urlsplit(client.base) + connection = http.client.HTTPConnection(parsed.hostname, parsed.port, timeout=60) + + def chunks(): + for offset in range(0, len(payload), 512 * 1024): + yield payload[offset:offset + 512 * 1024] + # Keep all ten client streams open long enough for multiple memory samples. + time.sleep(.5) + + connection.request("POST", "/api/v1/chats/" + chat["id"] + "/attachments?filename=" + quote(f"load-{index}.png"), + body=chunks(), encode_chunked=True, + headers={"Authorization": "Bearer " + alice["token"], "Content-Type": "image/png"}) + response = connection.getresponse() + body = response.read() + connection.close() + check(response.status == 201, "upload %s returned %s: %s" % (index, response.status, body[:200])) + return json.loads(body) + + started = time.monotonic() + thread = threading.Thread(target=sampler, daemon=True) + thread.start() + try: + with ThreadPoolExecutor(max_workers=10) as pool: + attachments = list(pool.map(upload, range(10))) + finally: + stop.set() + thread.join(timeout=5) + elapsed = time.monotonic() - started + check(len({item["id"] for item in attachments}) == 10, "uploads were not distinct") + check(not sampling_errors and len(samples) >= 2 and not thread.is_alive(), + "memory sampler failed or did not collect enough samples: " + repr(sampling_errors)) + report = { + "uploads": 10, "file_bytes": len(payload), "concurrency": 10, + "api_upload_concurrency_limit": 1, "elapsed_seconds": round(elapsed, 3), + "baseline_memory_bytes": baseline, "peak_memory_bytes": peak, + "peak_growth_bytes": {name: peak.get(name, 0) - baseline.get(name, 0) for name in baseline}, + "samples": len(samples), + "sample_intervals_seconds": [round(b-a, 3) for a, b in zip(timestamps, timestamps[1:])], + "method": "cgroup memory.current via Docker; 0.1s pause AFTER each poll, actual intervals include Docker latency; ten clients start at a barrier and send chunked 512 KiB blocks every 0.5s. Peak is sampled, not continuous.", + } + destination = ROOT / args.output + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n") + print(json.dumps(report, ensure_ascii=False)) + check(all(value <= 512 * 1024 * 1024 for value in peak.values()), "API exceeded configured memory limit") + + +if __name__ == "__main__": + run(main) diff --git a/uv.lock b/uv.lock new file mode 100644 index 0000000..33119d4 --- /dev/null +++ b/uv.lock @@ -0,0 +1,885 @@ +version = 1 +revision = 3 +requires-python = ">=3.12" +resolution-markers = [ + "python_full_version >= '3.14'", + "python_full_version < '3.14'", +] + +[[package]] +name = "annotated-doc" +version = "0.0.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5a/8e/38aa427ed5402449e226975b649c5dc73ccadfefeb95e6aecb8f8ea4b6b6/annotated_doc-0.0.5.tar.gz", hash = "sha256:c7e58ce09192557605d8bbd92836d7e1d520ac9580096042c0bfd197efacf1bb", size = 10758, upload-time = "2026-07-28T13:50:58.129Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3e/30/e900b21425a860e195f32e37657aa1f7c7f2b1bfb26f03ca209b90933c06/annotated_doc-0.0.5-py3-none-any.whl", hash = "sha256:117bac03a25ede5df5440e855b32d556049ca169ead221505badf432fed4b101", size = 5302, upload-time = "2026-07-28T13:50:57.239Z" }, +] + +[[package]] +name = "annotated-types" +version = "0.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5f/56/a8120250d128bed162cd73c76d45f6ef9991f3e068f62a8ee060afa3104a/annotated_types-0.8.0.tar.gz", hash = "sha256:13b2beaad985e05e2d6407ee4c4f35590b11f8d693a258a561055cac8f64cab7", size = 15893, upload-time = "2026-07-23T20:16:13.995Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/99/91/8acff4f5e50511b911bbccb72b8628a49c68ce14148cd9f6431094859a90/annotated_types-0.8.0-py3-none-any.whl", hash = "sha256:f072f4d804ea359e4eaf198b1af7a8b0943881a87f31bb764f8bf219bb9419e0", size = 13427, upload-time = "2026-07-23T20:16:12.938Z" }, +] + +[[package]] +name = "anyio" +version = "4.15.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "idna" }, + { name = "typing-extensions", marker = "python_full_version < '3.15'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a9/d2/f4d173e22df740bc37b1db102b386ba719b66e95b0f0d751f556b387e6d2/anyio-4.15.1.tar.gz", hash = "sha256:9f28306018cbd6d329e64a36d58256edff76dd996fe423bc957326e578b82a94", size = 276966, upload-time = "2026-09-05T10:42:39.44Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/12/b8/4bd346e22b28902df4d651910f5242c28d84e4a5c2435ca5c3f797ed7e2e/anyio-4.15.1-py3-none-any.whl", hash = "sha256:6152fdbbf9a77fdec97731721bebf7c4c44f7c29b424b0065826173efc7ed101", size = 132079, upload-time = "2026-09-05T10:42:37.923Z" }, +] + +[[package]] +name = "argon2-cffi" +version = "25.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "argon2-cffi-bindings" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0e/89/ce5af8a7d472a67cc819d5d998aa8c82c5d860608c4db9f46f1162d7dab9/argon2_cffi-25.1.0.tar.gz", hash = "sha256:694ae5cc8a42f4c4e2bf2ca0e64e51e23a040c6a517a85074683d3959e1346c1", size = 45706, upload-time = "2025-06-03T06:55:32.073Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4f/d3/a8b22fa575b297cd6e3e3b0155c7e25db170edf1c74783d6a31a2490b8d9/argon2_cffi-25.1.0-py3-none-any.whl", hash = "sha256:fdc8b074db390fccb6eb4a3604ae7231f219aa669a2652e0f20e16ba513d5741", size = 14657, upload-time = "2025-06-03T06:55:30.804Z" }, +] + +[[package]] +name = "argon2-cffi-bindings" +version = "26.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0b/43/bb8b6e8708d49a5ab36781333af092d9f483b198a2710d01281204640055/argon2_cffi_bindings-26.1.0.tar.gz", hash = "sha256:63505c71542a44b68b1e38060450fb006404170da375feb31af153e7f9c6205d", size = 1790807, upload-time = "2026-08-20T07:44:22.492Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e7/d2/0ae991f1b2181e5be49007c574710a800ad36c2978683addb3e67c474e55/argon2_cffi_bindings-26.1.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:21ca0396fe5ec995dd54431c32698189666f9224810acfa752e50d2bd94d9df2", size = 25521, upload-time = "2026-08-20T07:32:43.019Z" }, + { url = "https://files.pythonhosted.org/packages/7e/e4/ad91d8297638aa2258aad4501c306aca99480dfe76ccd638173fa3702db9/argon2_cffi_bindings-26.1.0-cp310-abi3-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:78de2d65e0b9ea7ce9d1b1c3e87297b2d7305a02c266ee2a2d6910daddd7ee69", size = 27177, upload-time = "2026-08-20T07:32:44.158Z" }, + { url = "https://files.pythonhosted.org/packages/6f/86/5363df11b86d02cf3662208e7406496327649cc90eb365bf6f4e8a54a41f/argon2_cffi_bindings-26.1.0-cp310-abi3-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:27f1821903e2ceadcb88ec2b45ef190897b7682449c772f4d9b53e42c520cf29", size = 26597, upload-time = "2026-08-20T07:32:45.172Z" }, + { url = "https://files.pythonhosted.org/packages/f4/b5/a14dcc592652347dad23ee93b278a4da5d2a25c9ed3ebd10d68eea823a4f/argon2_cffi_bindings-26.1.0-cp310-abi3-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d88e5f7e60f28ae0b0cc6b2f16c43e87cd642a196a86f85e0d8bb6fe016fc16d", size = 27403, upload-time = "2026-08-20T07:32:46.13Z" }, + { url = "https://files.pythonhosted.org/packages/b3/81/b4a20d4902af7f796390bf9245ff83c5217dfa7367efa1d14986956c482b/argon2_cffi_bindings-26.1.0-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:34b7d9c24a4165a2c61cc8ae11d44d48c9ce2830fb536cb7914e11fdd9962728", size = 27132, upload-time = "2026-08-20T07:32:47.13Z" }, + { url = "https://files.pythonhosted.org/packages/7e/1b/c8de358af07b1c490e0fcb863ef98e46ddb486e45567aca5a60bd68d9daa/argon2_cffi_bindings-26.1.0-cp310-abi3-musllinux_1_2_riscv64.whl", hash = "sha256:224865cbbcb7a2bd1356741dff12b0134df726b6d44bb7b500df8e303cbd9e81", size = 27588, upload-time = "2026-08-20T07:32:48.087Z" }, + { url = "https://files.pythonhosted.org/packages/48/2f/7ee62a6e79f9309f9d9982d301b22a00010adb580c05c8109b94d7b33de0/argon2_cffi_bindings-26.1.0-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:ffff613aaa9ce6236766e2fc6dc560bb5abde7a2e2416e3db1f9ae395a2b4dd4", size = 26785, upload-time = "2026-08-20T07:32:48.977Z" }, + { url = "https://files.pythonhosted.org/packages/e9/10/960d0ee93d4897741bcaf4799c697dae2d81499f66fd1ed042a7dd54c1f4/argon2_cffi_bindings-26.1.0-cp310-abi3-win32.whl", hash = "sha256:a86c069c91a747a2c4e5c51473590aeb48172fff9b2130d23729a42d98665ecb", size = 23898, upload-time = "2026-08-20T07:32:50.114Z" }, + { url = "https://files.pythonhosted.org/packages/6d/3a/0cc14a05810e6add9bce5e87693334baa2222de5f647fa31781885b6573f/argon2_cffi_bindings-26.1.0-cp310-abi3-win_amd64.whl", hash = "sha256:2c36ff87b5dfaa477d0bd51e9d7f6abdae7c8955d2983c97419085d842154b3e", size = 25730, upload-time = "2026-08-20T07:32:51.091Z" }, + { url = "https://files.pythonhosted.org/packages/4e/db/d83cf2af140547f0b9cdaece05b2dc2dcbf991be4667331d073eff771435/argon2_cffi_bindings-26.1.0-cp310-abi3-win_arm64.whl", hash = "sha256:f9c4420a7a864fe1b86ce35befc95b8e39fb852493b81cf798671ddc265de638", size = 24478, upload-time = "2026-08-20T07:32:52.111Z" }, + { url = "https://files.pythonhosted.org/packages/bb/5f/f652055e18d2627e2eed94c7f31a792127cfe38df786635395d742321674/argon2_cffi_bindings-26.1.0-cp313-cp313-pyemscripten_2025_0_wasm32.whl", hash = "sha256:af11ac37a7c53dc16cb7950a6190851b0870fe218b6c60c0bb7ac355234e3083", size = 15434, upload-time = "2026-08-20T07:32:53.143Z" }, + { url = "https://files.pythonhosted.org/packages/76/38/de696045960f5b846d428c0fb6c130ed3da87aac2af209b05c193815404c/argon2_cffi_bindings-26.1.0-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:db0fcd827ca61622a01b220aadfbece01939acf53888f2cb98cd93e9b1e2c97e", size = 15449, upload-time = "2026-08-20T07:32:54.075Z" }, + { url = "https://files.pythonhosted.org/packages/91/0a/c25af768f6b75a5a71e31207f87c540656b2808c015260444a22763221ad/argon2_cffi_bindings-26.1.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:28524438cd3e723f25412f63d4fd516ff5bae9ae5aa56acbe2a1404398a0cf31", size = 25683, upload-time = "2026-08-20T07:32:55.05Z" }, + { url = "https://files.pythonhosted.org/packages/a8/7e/be212c751ab0bcea7f646615f933bf262e8e50b3f7bef32f861d0a2d066b/argon2_cffi_bindings-26.1.0-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ac82fc756a446b6ccd7139ce70efa9d8bbe541e7ad579a12dcb52764b7175c5f", size = 27311, upload-time = "2026-08-20T07:32:56.166Z" }, + { url = "https://files.pythonhosted.org/packages/a6/ee/f84b28e4afd13d3cac36c1d8fa8c239d2dc2c51cd978d02ee5d5ad98d9bb/argon2_cffi_bindings-26.1.0-cp314-cp314t-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6a4e68eed961a8de6928d1c17ff3dc2a547e0e923c17f8f1cd79fb7bc9502f98", size = 26771, upload-time = "2026-08-20T07:32:57.206Z" }, + { url = "https://files.pythonhosted.org/packages/21/c3/95c07a023691ecd529da9cb6a8f0779e13ebc1bdfaa86d145fdc1c6e7e79/argon2_cffi_bindings-26.1.0-cp314-cp314t-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:151dfaad9de753f4af2a7854e707e4784f2acc434340ade64239c5b104b2d605", size = 27568, upload-time = "2026-08-20T07:32:58.361Z" }, + { url = "https://files.pythonhosted.org/packages/e6/31/3a18e31406d8694b4d6a31573c3e572fff6bed318bb744453eb653766d22/argon2_cffi_bindings-26.1.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:061a6919145bbf282ebf1f9c59d3135d4833c25313c8595c0d68cf7712ddfce2", size = 27280, upload-time = "2026-08-20T07:32:59.343Z" }, + { url = "https://files.pythonhosted.org/packages/0b/39/d4be4577e178b2397aa5b5575c8a309bf0da2afe05fe0c72c8f398662d63/argon2_cffi_bindings-26.1.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:62ff20cd130c956c7c9144d5fe35228f98b51c579b2439e988b27ef93e16c02a", size = 27776, upload-time = "2026-08-20T07:33:00.325Z" }, + { url = "https://files.pythonhosted.org/packages/71/47/78f4dd96f7411339f723b96fe24039c1bd5835102b8a5ba71ac4ec712ac7/argon2_cffi_bindings-26.1.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:19423e5d7ac1cc354baab59eaabf18db2ec04ef6593b5abe5a34f323c4a8f87a", size = 26932, upload-time = "2026-08-20T07:33:01.272Z" }, + { url = "https://files.pythonhosted.org/packages/3b/cd/96bfd37434cc0a848a9066c291d84b28846c4c9ea289ed9866b1164d622b/argon2_cffi_bindings-26.1.0-cp314-cp314t-win32.whl", hash = "sha256:4f84cdd868978d7b7350a566c254042d44216d9e37f241f3a6d3b1dfebeede35", size = 24878, upload-time = "2026-08-20T07:33:02.189Z" }, + { url = "https://files.pythonhosted.org/packages/f1/42/d8b6810abd9b1bd2f47ebbccf460da59c9f32e94888bea4f7b137d998797/argon2_cffi_bindings-26.1.0-cp314-cp314t-win_amd64.whl", hash = "sha256:2b741888c93147444fdfc851abd81cc207f37f7f7da42062a00deb3888e57da8", size = 26656, upload-time = "2026-08-20T07:33:03.222Z" }, + { url = "https://files.pythonhosted.org/packages/a9/d1/095d95eaf2ed1d9f77268cf3291bde148c6cd56121f8db2c74c1ba618a0e/argon2_cffi_bindings-26.1.0-cp314-cp314t-win_arm64.whl", hash = "sha256:6ab674f668d5962a3a4136ae0812519b0f1586874263723a32181d60d64137e1", size = 25378, upload-time = "2026-08-20T07:33:04.332Z" }, + { url = "https://files.pythonhosted.org/packages/66/cb/214092c39c4dbcb72cf98b12234ddac2221f8fe2c0acf29c6a70fa83be53/argon2_cffi_bindings-26.1.0-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:1d98e33bd8bd67d7206c124e200bf2229c4cfa8c9c19f7b44a897f0fc71837eb", size = 25683, upload-time = "2026-08-20T07:33:05.337Z" }, + { url = "https://files.pythonhosted.org/packages/83/e5/02015b83e9b05ccb85ff2ced424cf6e83a12d3810bc7f66d679a92b69ffb/argon2_cffi_bindings-26.1.0-cp315-cp315t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ccaf0a46cbb380f1fd102a874e32aa629fd3cb0c0e94f4943fa1f6d5edc5dac6", size = 27310, upload-time = "2026-08-20T07:33:06.344Z" }, + { url = "https://files.pythonhosted.org/packages/c3/4a/85e612787d0796878b3b4f6bd53dcd5484b6fe7b64cc6fc7b6e6a04cf835/argon2_cffi_bindings-26.1.0-cp315-cp315t-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f0c3103fcff20183e593459cfea6e012281c0e76ae3ed8b5565ad1b92eac3990", size = 26771, upload-time = "2026-08-20T07:33:07.429Z" }, + { url = "https://files.pythonhosted.org/packages/f6/84/ccb003b6f9969820e87656398f4d49c857def71a85ca1588a0e809afd7ce/argon2_cffi_bindings-26.1.0-cp315-cp315t-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c49e853a3bef9dd10329f31f702e7fa9b5c58229ff9c2ff6d069efaf09177c08", size = 27569, upload-time = "2026-08-20T07:33:08.598Z" }, + { url = "https://files.pythonhosted.org/packages/88/07/c26b76debf0998ee08fbe947ab2058ac5de37d4b9d46b06c17abaa6c4ce9/argon2_cffi_bindings-26.1.0-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:6376d4b3aca039375ca8bf92f770da0ec424a1ce3a37077a8d3c557411aa56ca", size = 27279, upload-time = "2026-08-20T07:33:09.518Z" }, + { url = "https://files.pythonhosted.org/packages/ee/0d/ead6ddc029f91bc9b9390686dad3c808ab08100d348f6266b5f93f8970ee/argon2_cffi_bindings-26.1.0-cp315-cp315t-musllinux_1_2_riscv64.whl", hash = "sha256:9bacedc04b0402837586a17f0919e3dfdd95291f441f1f56bd80ec274c2840a1", size = 27774, upload-time = "2026-08-20T07:33:10.728Z" }, + { url = "https://files.pythonhosted.org/packages/7d/47/c108530d9eb86036b78d3af4de28b83b4a2d9a70512bd10ff8e59966aab4/argon2_cffi_bindings-26.1.0-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:76ae29acace5d33355344612844d588e19deaaba4639d8bb01601e4b1418ef36", size = 26933, upload-time = "2026-08-20T07:33:11.661Z" }, + { url = "https://files.pythonhosted.org/packages/a9/02/0bfc59e781c89acf64c31c388aade9d9d1c1ea38aa1ba1292fe07f607fe9/argon2_cffi_bindings-26.1.0-cp315-cp315t-win32.whl", hash = "sha256:df612391feca41c44d20118f3b88d1b86419465cd1f5496859f715ca60ec2210", size = 24875, upload-time = "2026-08-20T07:33:12.616Z" }, + { url = "https://files.pythonhosted.org/packages/61/c7/c3e46068cddffccecb8ad94d71135e9bf62bbc789589e7dfadc7c6f59214/argon2_cffi_bindings-26.1.0-cp315-cp315t-win_amd64.whl", hash = "sha256:1a0a29ed86960e44eaace7e081bdfab4f08b012fd96ec8edba71e2ad020939e4", size = 26655, upload-time = "2026-08-20T07:33:13.521Z" }, + { url = "https://files.pythonhosted.org/packages/f4/ca/18b9c8c45fecf34b9100ec6d7946057f14a158f2eaa20ea123a3e82351cb/argon2_cffi_bindings-26.1.0-cp315-cp315t-win_arm64.whl", hash = "sha256:d157ddfab1e8b21f2f1dedda9c09645d98b5ed0b667b0626be600a345d426440", size = 25376, upload-time = "2026-08-20T07:33:14.491Z" }, +] + +[[package]] +name = "asyncpg" +version = "0.31.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fe/cc/d18065ce2380d80b1bcce927c24a2642efd38918e33fd724bc4bca904877/asyncpg-0.31.0.tar.gz", hash = "sha256:c989386c83940bfbd787180f2b1519415e2d3d6277a70d9d0f0145ac73500735", size = 993667, upload-time = "2025-11-24T23:27:00.812Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/a6/59d0a146e61d20e18db7396583242e32e0f120693b67a8de43f1557033e2/asyncpg-0.31.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b44c31e1efc1c15188ef183f287c728e2046abb1d26af4d20858215d50d91fad", size = 662042, upload-time = "2025-11-24T23:25:49.578Z" }, + { url = "https://files.pythonhosted.org/packages/36/01/ffaa189dcb63a2471720615e60185c3f6327716fdc0fc04334436fbb7c65/asyncpg-0.31.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0c89ccf741c067614c9b5fc7f1fc6f3b61ab05ae4aaa966e6fd6b93097c7d20d", size = 638504, upload-time = "2025-11-24T23:25:51.501Z" }, + { url = "https://files.pythonhosted.org/packages/9f/62/3f699ba45d8bd24c5d65392190d19656d74ff0185f42e19d0bbd973bb371/asyncpg-0.31.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:12b3b2e39dc5470abd5e98c8d3373e4b1d1234d9fbdedf538798b2c13c64460a", size = 3426241, upload-time = "2025-11-24T23:25:53.278Z" }, + { url = "https://files.pythonhosted.org/packages/8c/d1/a867c2150f9c6e7af6462637f613ba67f78a314b00db220cd26ff559d532/asyncpg-0.31.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:aad7a33913fb8bcb5454313377cc330fbb19a0cd5faa7272407d8a0c4257b671", size = 3520321, upload-time = "2025-11-24T23:25:54.982Z" }, + { url = "https://files.pythonhosted.org/packages/7a/1a/cce4c3f246805ecd285a3591222a2611141f1669d002163abef999b60f98/asyncpg-0.31.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3df118d94f46d85b2e434fd62c84cb66d5834d5a890725fe625f498e72e4d5ec", size = 3316685, upload-time = "2025-11-24T23:25:57.43Z" }, + { url = "https://files.pythonhosted.org/packages/40/ae/0fc961179e78cc579e138fad6eb580448ecae64908f95b8cb8ee2f241f67/asyncpg-0.31.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:bd5b6efff3c17c3202d4b37189969acf8927438a238c6257f66be3c426beba20", size = 3471858, upload-time = "2025-11-24T23:25:59.636Z" }, + { url = "https://files.pythonhosted.org/packages/52/b2/b20e09670be031afa4cbfabd645caece7f85ec62d69c312239de568e058e/asyncpg-0.31.0-cp312-cp312-win32.whl", hash = "sha256:027eaa61361ec735926566f995d959ade4796f6a49d3bde17e5134b9964f9ba8", size = 527852, upload-time = "2025-11-24T23:26:01.084Z" }, + { url = "https://files.pythonhosted.org/packages/b5/f0/f2ed1de154e15b107dc692262395b3c17fc34eafe2a78fc2115931561730/asyncpg-0.31.0-cp312-cp312-win_amd64.whl", hash = "sha256:72d6bdcbc93d608a1158f17932de2321f68b1a967a13e014998db87a72ed3186", size = 597175, upload-time = "2025-11-24T23:26:02.564Z" }, + { url = "https://files.pythonhosted.org/packages/95/11/97b5c2af72a5d0b9bc3fa30cd4b9ce22284a9a943a150fdc768763caf035/asyncpg-0.31.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c204fab1b91e08b0f47e90a75d1b3c62174dab21f670ad6c5d0f243a228f015b", size = 661111, upload-time = "2025-11-24T23:26:04.467Z" }, + { url = "https://files.pythonhosted.org/packages/1b/71/157d611c791a5e2d0423f09f027bd499935f0906e0c2a416ce712ba51ef3/asyncpg-0.31.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:54a64f91839ba59008eccf7aad2e93d6e3de688d796f35803235ea1c4898ae1e", size = 636928, upload-time = "2025-11-24T23:26:05.944Z" }, + { url = "https://files.pythonhosted.org/packages/2e/fc/9e3486fb2bbe69d4a867c0b76d68542650a7ff1574ca40e84c3111bb0c6e/asyncpg-0.31.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c0e0822b1038dc7253b337b0f3f676cadc4ac31b126c5d42691c39691962e403", size = 3424067, upload-time = "2025-11-24T23:26:07.957Z" }, + { url = "https://files.pythonhosted.org/packages/12/c6/8c9d076f73f07f995013c791e018a1cd5f31823c2a3187fc8581706aa00f/asyncpg-0.31.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bef056aa502ee34204c161c72ca1f3c274917596877f825968368b2c33f585f4", size = 3518156, upload-time = "2025-11-24T23:26:09.591Z" }, + { url = "https://files.pythonhosted.org/packages/ae/3b/60683a0baf50fbc546499cfb53132cb6835b92b529a05f6a81471ab60d0c/asyncpg-0.31.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:0bfbcc5b7ffcd9b75ab1558f00db2ae07db9c80637ad1b2469c43df79d7a5ae2", size = 3319636, upload-time = "2025-11-24T23:26:11.168Z" }, + { url = "https://files.pythonhosted.org/packages/50/dc/8487df0f69bd398a61e1792b3cba0e47477f214eff085ba0efa7eac9ce87/asyncpg-0.31.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:22bc525ebbdc24d1261ecbf6f504998244d4e3be1721784b5f64664d61fbe602", size = 3472079, upload-time = "2025-11-24T23:26:13.164Z" }, + { url = "https://files.pythonhosted.org/packages/13/a1/c5bbeeb8531c05c89135cb8b28575ac2fac618bcb60119ee9696c3faf71c/asyncpg-0.31.0-cp313-cp313-win32.whl", hash = "sha256:f890de5e1e4f7e14023619399a471ce4b71f5418cd67a51853b9910fdfa73696", size = 527606, upload-time = "2025-11-24T23:26:14.78Z" }, + { url = "https://files.pythonhosted.org/packages/91/66/b25ccb84a246b470eb943b0107c07edcae51804912b824054b3413995a10/asyncpg-0.31.0-cp313-cp313-win_amd64.whl", hash = "sha256:dc5f2fa9916f292e5c5c8b2ac2813763bcd7f58e130055b4ad8a0531314201ab", size = 596569, upload-time = "2025-11-24T23:26:16.189Z" }, + { url = "https://files.pythonhosted.org/packages/3c/36/e9450d62e84a13aea6580c83a47a437f26c7ca6fa0f0fd40b6670793ea30/asyncpg-0.31.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:f6b56b91bb0ffc328c4e3ed113136cddd9deefdf5f79ab448598b9772831df44", size = 660867, upload-time = "2025-11-24T23:26:17.631Z" }, + { url = "https://files.pythonhosted.org/packages/82/4b/1d0a2b33b3102d210439338e1beea616a6122267c0df459ff0265cd5807a/asyncpg-0.31.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:334dec28cf20d7f5bb9e45b39546ddf247f8042a690bff9b9573d00086e69cb5", size = 638349, upload-time = "2025-11-24T23:26:19.689Z" }, + { url = "https://files.pythonhosted.org/packages/41/aa/e7f7ac9a7974f08eff9183e392b2d62516f90412686532d27e196c0f0eeb/asyncpg-0.31.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:98cc158c53f46de7bb677fd20c417e264fc02b36d901cc2a43bd6cb0dc6dbfd2", size = 3410428, upload-time = "2025-11-24T23:26:21.275Z" }, + { url = "https://files.pythonhosted.org/packages/6f/de/bf1b60de3dede5c2731e6788617a512bc0ebd9693eac297ee74086f101d7/asyncpg-0.31.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9322b563e2661a52e3cdbc93eed3be7748b289f792e0011cb2720d278b366ce2", size = 3471678, upload-time = "2025-11-24T23:26:23.627Z" }, + { url = "https://files.pythonhosted.org/packages/46/78/fc3ade003e22d8bd53aaf8f75f4be48f0b460fa73738f0391b9c856a9147/asyncpg-0.31.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:19857a358fc811d82227449b7ca40afb46e75b33eb8897240c3839dd8b744218", size = 3313505, upload-time = "2025-11-24T23:26:25.235Z" }, + { url = "https://files.pythonhosted.org/packages/bf/e9/73eb8a6789e927816f4705291be21f2225687bfa97321e40cd23055e903a/asyncpg-0.31.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ba5f8886e850882ff2c2ace5732300e99193823e8107e2c53ef01c1ebfa1e85d", size = 3434744, upload-time = "2025-11-24T23:26:26.944Z" }, + { url = "https://files.pythonhosted.org/packages/08/4b/f10b880534413c65c5b5862f79b8e81553a8f364e5238832ad4c0af71b7f/asyncpg-0.31.0-cp314-cp314-win32.whl", hash = "sha256:cea3a0b2a14f95834cee29432e4ddc399b95700eb1d51bbc5bfee8f31fa07b2b", size = 532251, upload-time = "2025-11-24T23:26:28.404Z" }, + { url = "https://files.pythonhosted.org/packages/d3/2d/7aa40750b7a19efa5d66e67fc06008ca0f27ba1bd082e457ad82f59aba49/asyncpg-0.31.0-cp314-cp314-win_amd64.whl", hash = "sha256:04d19392716af6b029411a0264d92093b6e5e8285ae97a39957b9a9c14ea72be", size = 604901, upload-time = "2025-11-24T23:26:30.34Z" }, + { url = "https://files.pythonhosted.org/packages/ce/fe/b9dfe349b83b9dee28cc42360d2c86b2cdce4cb551a2c2d27e156bcac84d/asyncpg-0.31.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:bdb957706da132e982cc6856bb2f7b740603472b54c3ebc77fe60ea3e57e1bd2", size = 702280, upload-time = "2025-11-24T23:26:32Z" }, + { url = "https://files.pythonhosted.org/packages/6a/81/e6be6e37e560bd91e6c23ea8a6138a04fd057b08cf63d3c5055c98e81c1d/asyncpg-0.31.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6d11b198111a72f47154fa03b85799f9be63701e068b43f84ac25da0bda9cb31", size = 682931, upload-time = "2025-11-24T23:26:33.572Z" }, + { url = "https://files.pythonhosted.org/packages/a6/45/6009040da85a1648dd5bc75b3b0a062081c483e75a1a29041ae63a0bf0dc/asyncpg-0.31.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:18c83b03bc0d1b23e6230f5bf8d4f217dc9bc08644ce0502a9d91dc9e634a9c7", size = 3581608, upload-time = "2025-11-24T23:26:35.638Z" }, + { url = "https://files.pythonhosted.org/packages/7e/06/2e3d4d7608b0b2b3adbee0d0bd6a2d29ca0fc4d8a78f8277df04e2d1fd7b/asyncpg-0.31.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e009abc333464ff18b8f6fd146addffd9aaf63e79aa3bb40ab7a4c332d0c5e9e", size = 3498738, upload-time = "2025-11-24T23:26:37.275Z" }, + { url = "https://files.pythonhosted.org/packages/7d/aa/7d75ede780033141c51d83577ea23236ba7d3a23593929b32b49db8ed36e/asyncpg-0.31.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:3b1fbcb0e396a5ca435a8826a87e5c2c2cc0c8c68eb6fadf82168056b0e53a8c", size = 3401026, upload-time = "2025-11-24T23:26:39.423Z" }, + { url = "https://files.pythonhosted.org/packages/ba/7a/15e37d45e7f7c94facc1e9148c0e455e8f33c08f0b8a0b1deb2c5171771b/asyncpg-0.31.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:8df714dba348efcc162d2adf02d213e5fab1bd9f557e1305633e851a61814a7a", size = 3429426, upload-time = "2025-11-24T23:26:41.032Z" }, + { url = "https://files.pythonhosted.org/packages/13/d5/71437c5f6ae5f307828710efbe62163974e71237d5d46ebd2869ea052d10/asyncpg-0.31.0-cp314-cp314t-win32.whl", hash = "sha256:1b41f1afb1033f2b44f3234993b15096ddc9cd71b21a42dbd87fc6a57b43d65d", size = 614495, upload-time = "2025-11-24T23:26:42.659Z" }, + { url = "https://files.pythonhosted.org/packages/3c/d7/8fb3044eaef08a310acfe23dae9a8e2e07d305edc29a53497e52bc76eca7/asyncpg-0.31.0-cp314-cp314t-win_amd64.whl", hash = "sha256:bd4107bb7cdd0e9e65fae66a62afd3a249663b844fa34d479f6d5b3bef9c04c3", size = 706062, upload-time = "2025-11-24T23:26:44.086Z" }, +] + +[[package]] +name = "boto3" +version = "1.43.115" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore" }, + { name = "jmespath" }, + { name = "s3transfer" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0e/4a/b0ce191492a1724be73d8aacaf70d508bc8cbb9674c046d2ff7181dc51fb/boto3-1.43.115.tar.gz", hash = "sha256:c0f35e5f9cdf0edb365195e306d5586d3970b224660a11a87a4ec361ec3d7168", size = 112677, upload-time = "2026-10-19T19:19:53.093Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/be/bc/40dffae3238ef8d3415393132751fe2c88e8663ef84a31f0950e439ed2d5/boto3-1.43.115-py3-none-any.whl", hash = "sha256:6667a0c21d48dc125dbba88444bc8a601ce881a99341a0e6169f9333898e620d", size = 140044, upload-time = "2026-10-19T19:19:51.444Z" }, +] + +[[package]] +name = "botocore" +version = "1.43.115" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jmespath" }, + { name = "python-dateutil" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6f/10/ca525e88db50f5153ddad2a3d55908b5ace915d10f443d5e5135ba808d8d/botocore-1.43.115.tar.gz", hash = "sha256:aa4d9c4e7ef99988b7347e0cd276ba84e1eb1625510557b0e36cd57ecb19bbf2", size = 16383796, upload-time = "2026-10-19T19:19:45.382Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/77/2d/4b6b6f5704d1ec9b412c1dd9700bda0c315619dbac4a21418180a81a2780/botocore-1.43.115-py3-none-any.whl", hash = "sha256:9ceda640c868bc15f5399129e8cfdef8cbb4b350e8a99547c9b5f477cfb9619a", size = 16085101, upload-time = "2026-10-19T19:19:42.203Z" }, +] + +[[package]] +name = "certifi" +version = "2026.10.23" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fd/11/b524cbd9bbdaca58608e4cf4053fd463417549e1959fa8d3aa18319f9555/certifi-2026.10.23.tar.gz", hash = "sha256:a9895e6e2758b371e0545e321a3a8fd1fcfd9829ada8824a936dfb16aaaf36cd", size = 136225, upload-time = "2026-10-23T11:36:16.478Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/32/d1/a3652fe2c19134d0d15922177821d863cbd7e842d53a21bbe7e8e3a1b5ac/certifi-2026.10.23-py3-none-any.whl", hash = "sha256:e9b54de76e530bb6cf0ccd036787103acb58fd42ba930ea8bf948ddbaa2f61d1", size = 135082, upload-time = "2026-10-23T11:36:14.763Z" }, +] + +[[package]] +name = "cffi" +version = "2.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pycparser", marker = "implementation_name != 'PyPy'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9e/ef/008a1939e372c06329a3fce4279c02f328488f3526744906eeec3da7ad5f/cffi-2.1.1.tar.gz", hash = "sha256:dd31f52ea1086513bb9df30f8fcee9b8918323ae067a3d5b78bc826a000712be", size = 530807, upload-time = "2026-08-03T21:21:18.939Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/10/69/43965eccfdead3b9220015fd1320e117be8c6ed01a62ffab76eeb752f5d5/cffi-2.1.1-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:c8c69575568085ba0b1b10c0249d779a214aea6f6522e949a0fc9fb0fcb449d0", size = 184821, upload-time = "2026-08-03T21:19:44.887Z" }, + { url = "https://files.pythonhosted.org/packages/54/7d/16e5a096677b5e313ca80cd5e5170efa3ea44624a82bb111925522da64b1/cffi-2.1.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f81b3b8f3d4e343550fa4baa0e479bba9f2d29ce9c2e9b51d1ce1718d7442fcf", size = 184719, upload-time = "2026-08-03T21:19:46.129Z" }, + { url = "https://files.pythonhosted.org/packages/56/e6/8941622732edec876dd17d0453dce07317ae96db34f2ec1436c9d3785986/cffi-2.1.1-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:811bd1e21d32de12efca32393a0ab3f5133b54fce9bd44b8bd77ab07da14bf6a", size = 214799, upload-time = "2026-08-03T21:19:47.218Z" }, + { url = "https://files.pythonhosted.org/packages/44/de/f98430906df1545ffde0d543dd124a7a439bc2cd32b36b9c53f805df7333/cffi-2.1.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:68e62fe11f30d5ca8289242866f0a5291402d8529ca2178ab8afc5c9694ae890", size = 222389, upload-time = "2026-08-03T21:19:48.331Z" }, + { url = "https://files.pythonhosted.org/packages/6a/5b/717f1526b9957b34456313c31645c5b82b8fb5c3fe9e4752999be7128bfc/cffi-2.1.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:4a7c934f7360e8cd64fe9efadcbd10c7c6364f531e432b9a4bf5ccbc9e0e8b50", size = 210249, upload-time = "2026-08-03T21:19:49.543Z" }, + { url = "https://files.pythonhosted.org/packages/64/b3/f8aa4f3e34986c7e4ec45072d1b1b9dd295b6b18007b45518d79726dd725/cffi-2.1.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:3143d81e29e1e20a9ce10901ec369012947876596f75a222235965f2b7ae832e", size = 208775, upload-time = "2026-08-03T21:19:50.918Z" }, + { url = "https://files.pythonhosted.org/packages/b1/db/dceb9dd5b231e1da801793f8acc9f3c52a7e1afe40bb1aae37e02b0faad5/cffi-2.1.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c1453022f490d2459a11819d83ad1d586e9ff65a12ac3e705ffebd46d3685dcf", size = 221822, upload-time = "2026-08-03T21:19:52.054Z" }, + { url = "https://files.pythonhosted.org/packages/a0/d2/6cd24ae3be000a634109c247d1475d62e5616d0dc78c82770942ec384248/cffi-2.1.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:208f941bb9d18e768138677f0a6d2ce01f590df56043dda1df1535ac57c88517", size = 225232, upload-time = "2026-08-03T21:19:53.109Z" }, + { url = "https://files.pythonhosted.org/packages/cb/52/3fa190537004dd7f0ab860a6dc7c0175b8667f68d1e618a46f5498d30250/cffi-2.1.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:210019b6c7cf07f081b4c54635c8cf744377001350e29cc0f81c4377b4797735", size = 223597, upload-time = "2026-08-03T21:19:54.515Z" }, + { url = "https://files.pythonhosted.org/packages/80/fb/0bb75b7039588c074b37ae99f40d9bfddf990ecb2fbc346ebccd2e56b9be/cffi-2.1.1-cp312-cp312-win32.whl", hash = "sha256:046bfc24911b37851ee1b51aab8bffe713d89c68c6a057b09484ce9fd5f69b4e", size = 175292, upload-time = "2026-08-03T21:19:55.566Z" }, + { url = "https://files.pythonhosted.org/packages/d9/79/615cc094e2fb508cade7de88d3b4f6c4ec2bab695c97bce9153dc65aadf5/cffi-2.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:f53e442b08449d42821fa4a4fba000095af9f62742a500f978a9f557ec44339a", size = 185919, upload-time = "2026-08-03T21:19:56.89Z" }, + { url = "https://files.pythonhosted.org/packages/70/c6/d0ea84713fe46b243a436a18fcd47d639732747e21635c8a27191b06dc30/cffi-2.1.1-cp312-cp312-win_arm64.whl", hash = "sha256:7bde5e4cc5c10140859842b9d383af292b22639a4dffb725314baf45968cef80", size = 180093, upload-time = "2026-08-03T21:19:58.155Z" }, + { url = "https://files.pythonhosted.org/packages/9d/f4/035513d4117049066b4779dc3b7c0c0fdad175fa13731c9f4003f1cd1478/cffi-2.1.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:b5bdfd1c873d4e093aabc0ca84c4ca6dbc4f752afb5c86f146d9742580c9da2e", size = 194248, upload-time = "2026-08-03T21:19:59.399Z" }, + { url = "https://files.pythonhosted.org/packages/76/af/2aeb4dbb5fc41a04161ae9ff1518de7cec08e164f44a8ce6a4cf7fd2cd1d/cffi-2.1.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:31348097ff5bbe827ccc41795d4dd099d9f0625e7def00ee653c137a490c2a6c", size = 196908, upload-time = "2026-08-03T21:20:00.746Z" }, + { url = "https://files.pythonhosted.org/packages/a7/46/2e5fdde8555706dd98139a910ca11be02809f3f605ce956f655d0214e100/cffi-2.1.1-cp313-cp313-macosx_10_15_x86_64.whl", hash = "sha256:9d2055050ea716bd38b7f7f1579c275386646b4894c155a3e2f3cd62ed41b7c6", size = 184805, upload-time = "2026-08-03T21:20:02.02Z" }, + { url = "https://files.pythonhosted.org/packages/55/41/4c7042f317b9217502988f0873af87e16ad606dc20f84e546e3e6ce9764c/cffi-2.1.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:19ee6127ee34de7d83ce3d371ebc5ed91addbdcc39f9ab15ce4eb35a4e534971", size = 184764, upload-time = "2026-08-03T21:20:03.141Z" }, + { url = "https://files.pythonhosted.org/packages/43/1f/1c3d90d91811c8f86ced9ed637956c54bfe5b79ca98fe976d7f8c8979f6b/cffi-2.1.1-cp313-cp313-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:6a8dddef476fab96d066d578fc88526767b836ab5ab21754e1d5bf3879c31c7c", size = 214722, upload-time = "2026-08-03T21:20:04.377Z" }, + { url = "https://files.pythonhosted.org/packages/37/6f/3b5ce4c3b2192d250f04908f2bfd91ef34552ec8f7716a5d4abdb8d67bb2/cffi-2.1.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f16c709686a78c727bbbf059f92b0bf41c6fc60deec706d2dc19f529175a6125", size = 222369, upload-time = "2026-08-03T21:20:05.544Z" }, + { url = "https://files.pythonhosted.org/packages/02/10/4b3c75dde3d9663c9e02ba05c2668b954f671d4bbe346413ca8c696b295a/cffi-2.1.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:fcd22650c908d7b7da162bbfaab594a1227a15d1643a98c68b122ac642fa2264", size = 210175, upload-time = "2026-08-03T21:20:06.75Z" }, + { url = "https://files.pythonhosted.org/packages/df/62/14f74b9543e605d17701dc797b815958b8bb70b7624ce1b832ddad48ed6c/cffi-2.1.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:aa9511c62d14da7aacc9b4bf51f3f697a621e83b2d6919008243c3aad168eea3", size = 208670, upload-time = "2026-08-03T21:20:08.04Z" }, + { url = "https://files.pythonhosted.org/packages/95/95/86342356ff5953b3fb06f7ef7c5bee212d45e770abc7218d451b9148313c/cffi-2.1.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a931079504ecc49efed7744c476a5c343a92fabf66dec2db95edb1b2fdc770e2", size = 221824, upload-time = "2026-08-03T21:20:09.274Z" }, + { url = "https://files.pythonhosted.org/packages/eb/ff/7b3429ff53aafe931ed8a5fc69f481bbef7ba6de87ddcbb63d08f483f613/cffi-2.1.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a2d7755bef5a12ed488f4ef1f1b69ee9191d7396083b755a5d2295f6edb4768b", size = 225148, upload-time = "2026-08-03T21:20:10.7Z" }, + { url = "https://files.pythonhosted.org/packages/34/34/a95870b9221e09cf4f2ce3178b1a210abdfe63a1bd357da940418d7b8d15/cffi-2.1.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e0bcb7e0f677f543555d2adff3bf19c05f66cdb4796e5ff602442ab2fe3c4ef7", size = 223564, upload-time = "2026-08-03T21:20:12.165Z" }, + { url = "https://files.pythonhosted.org/packages/70/ea/839b50531021a647fb5e929f72cf97bc1ff702b5472166164b5b6e76b851/cffi-2.1.1-cp313-cp313-win32.whl", hash = "sha256:334644fbac4eff73d985a17a91226df55d0f394160c4cfb880e084c8f7161cac", size = 175263, upload-time = "2026-08-03T21:20:13.559Z" }, + { url = "https://files.pythonhosted.org/packages/60/a6/8b149b2c3f2e11aaa1618ef64500b45f50f22c57a977a4dff1aff1f91042/cffi-2.1.1-cp313-cp313-win_amd64.whl", hash = "sha256:1aa5645c30469b09530c4ebca77ebf8f17618293c58f8549cb1a543a50236e7d", size = 185688, upload-time = "2026-08-03T21:20:14.69Z" }, + { url = "https://files.pythonhosted.org/packages/01/9a/11f687cb39d6a3504060d5242f04f48c735afb4d3d533958a20594890cb2/cffi-2.1.1-cp313-cp313-win_arm64.whl", hash = "sha256:63bbfd5ded17c4840ac07cd8f1c21ba9d9708141f840b324f422f41b207e3973", size = 180078, upload-time = "2026-08-03T21:20:15.917Z" }, + { url = "https://files.pythonhosted.org/packages/d3/7b/d6bbf82b8b96e7391438898c42f5bd96dd02030fd5b64937d248220003e2/cffi-2.1.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:7dbb61fe3a7699468030f71bbe5f8a0e326a151daa91beb11a6fc1f980c55e1c", size = 194064, upload-time = "2026-08-03T21:20:17.148Z" }, + { url = "https://files.pythonhosted.org/packages/94/e6/bcc91b283be94735e268487a054004f0aa19947b6348fa367db53230abc8/cffi-2.1.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:f24fb43132a4c6b4cb4eb029492919b2db645be6808d738f244fd146c03c32cb", size = 196720, upload-time = "2026-08-03T21:20:18.268Z" }, + { url = "https://files.pythonhosted.org/packages/d9/99/c4b0c17cacdc9c3b8f280026286a9826d6a208c0f047591a3c3ce99b91fd/cffi-2.1.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:d28630f5854ab07ab1fd4aba756de52326c82e6be15d414b12793f1975048b54", size = 184964, upload-time = "2026-08-03T21:20:19.708Z" }, + { url = "https://files.pythonhosted.org/packages/b3/a9/9db617d05d7367c1ad0ab00b3aa6e6f9281edd689b4ee9ea0e5a84e89c97/cffi-2.1.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:661c298b4821edebead0c91edd2b00374d67ad7c5a1f7a91d4442633b79d6a72", size = 184962, upload-time = "2026-08-03T21:20:20.833Z" }, + { url = "https://files.pythonhosted.org/packages/67/b8/b42132ca113dc567d37684437b46ca1dafc885902b02a110a02d5b511857/cffi-2.1.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:58acb8ab8e295e6c5ea12f888cbb13cf21511ef2a3303a23f4325c29d17fe5c1", size = 222328, upload-time = "2026-08-03T21:20:22.118Z" }, + { url = "https://files.pythonhosted.org/packages/80/10/c5c0cbf0a657aecf59ef511409734230bf556f05a0d6c9eed7aa5c0a0166/cffi-2.1.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:456a61fa52d579ebf9df2e9552ead5129855dbaff6c1e5a9b1bc408809bdc062", size = 209985, upload-time = "2026-08-03T21:20:23.401Z" }, + { url = "https://files.pythonhosted.org/packages/d5/6c/bfa0b87b03b9238148beca990292843c9396ba069b54496596594173de7b/cffi-2.1.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:a4f00aa42f75d6e4595e8866e748cc1705adc0cddfeb2ca86d0d03993d63ba03", size = 208530, upload-time = "2026-08-03T21:20:24.628Z" }, + { url = "https://files.pythonhosted.org/packages/e9/02/4e7d553a7ac4b4238b38b3c1b80d486e9d4436f8d2acbf87a0997fe3f402/cffi-2.1.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b0431303acaea1089ad4b3e9ce4e6518193def1118d4073ca848635ee4ea2e96", size = 221525, upload-time = "2026-08-03T21:20:25.758Z" }, + { url = "https://files.pythonhosted.org/packages/82/1d/a4aaf9babd75acb4d5f223bff71533bee748dd770a382619a798960ee9ba/cffi-2.1.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:64faea20f4e2613363a1a9b9c7dd73058f3ecd00133a511e72ad7c511658f527", size = 225053, upload-time = "2026-08-03T21:20:26.985Z" }, + { url = "https://files.pythonhosted.org/packages/81/10/5dc0e7bdd18e22107054288283380fc97a06ae3f1656a106908d666a3c88/cffi-2.1.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5c58fe613dc5e5336357eff555824a314d8e43282600435c8d1cb6a7a2fedd13", size = 223213, upload-time = "2026-08-03T21:20:28.277Z" }, + { url = "https://files.pythonhosted.org/packages/0b/e9/d0061c364cde06ee43168a0d076ac1da512cbc380d44767b844ba34fe2b6/cffi-2.1.1-cp314-cp314-win32.whl", hash = "sha256:1a18a57b58cfb21fc28d72e876acf10eaed67a1ed96226f92af4df681d571c4c", size = 177682, upload-time = "2026-08-03T21:20:44.288Z" }, + { url = "https://files.pythonhosted.org/packages/a7/06/1c3e01e3ba14c39f6d10bfbac52753b7e22259e38088e5cfe1d704918690/cffi-2.1.1-cp314-cp314-win_amd64.whl", hash = "sha256:3222ba5d678f80a030e6afbcc33dc1ae5cb45facabb61cee2c7016b8432fde48", size = 187949, upload-time = "2026-08-03T21:20:45.623Z" }, + { url = "https://files.pythonhosted.org/packages/87/5b/da4e39efe18eeb89cf580ea9cfc66b6a7c3eadb808fc0cc1d3a295cb5a5d/cffi-2.1.1-cp314-cp314-win_arm64.whl", hash = "sha256:ab36d55f9ed2d067327667c2fea18dda018eb628dd6347aa01dda6cf1f5d3836", size = 182947, upload-time = "2026-08-03T21:20:46.955Z" }, + { url = "https://files.pythonhosted.org/packages/23/59/40338bf421c5accea1d45158170c87006ef1cd371b05c077e76476949728/cffi-2.1.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:7750c6449dff7864bb9bb27ddfb0267756189201a3afc911d82b3caacd70dfc3", size = 188504, upload-time = "2026-08-03T21:20:29.495Z" }, + { url = "https://files.pythonhosted.org/packages/7d/47/5ecf1023850036e674c77ec4de86182d309ae344e39e7cba984b7df5d647/cffi-2.1.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:0beceaabe56af686895136a2de78db54ecd8e4046b236b8fd6d6cb61389e9bf2", size = 188259, upload-time = "2026-08-03T21:20:31.291Z" }, + { url = "https://files.pythonhosted.org/packages/2a/9c/92934c3bea9f785b23eba304538c0b4d37a2a96d2431eb3a1bc87a11aa19/cffi-2.1.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:49cbc70e6542d4ccccb936558d1064a8012541e78f821f955cff24e357776c94", size = 223864, upload-time = "2026-08-03T21:20:32.571Z" }, + { url = "https://files.pythonhosted.org/packages/4d/45/ba4c93527bc38616a8bd36488acb69a2212d60486794f0c1f318949bbb76/cffi-2.1.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:e2d65b31f36619cda3999b78b2aa9632e76b78448e7a56fc4240824200e7c4fc", size = 211538, upload-time = "2026-08-03T21:20:33.808Z" }, + { url = "https://files.pythonhosted.org/packages/80/e9/b6ef565e452acb932fb0cb5443f44a78efbd1233e566f02b5a83855e9115/cffi-2.1.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:28907ab9bfb6aa13184cfc17c6b8e1023c5ab6fd7076d8c20a35e59fe04f8f29", size = 210688, upload-time = "2026-08-03T21:20:34.974Z" }, + { url = "https://files.pythonhosted.org/packages/9a/95/eff5f0cee78d2eabc7eebffec40d3fc1876b5f3c95582e018bb4b99601f2/cffi-2.1.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:51b31d1c98274844cfd7838ce00bfc27c7423a4dc00fc0772fc3331c2cc90676", size = 223803, upload-time = "2026-08-03T21:20:36.564Z" }, + { url = "https://files.pythonhosted.org/packages/fa/01/579d39fb8bef00a335a23d83757b44feb24cd6345a2c451b64cb67b9c362/cffi-2.1.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:5e7cecbaadb83884793e05828cee59b210b24583b9c7425d0ba6a754fe22eb4e", size = 226763, upload-time = "2026-08-03T21:20:37.816Z" }, + { url = "https://files.pythonhosted.org/packages/8d/b0/0b44f47c60b01b57b6e2bbd92343f13a85a1d93bc46ccf6e47e244acd99c/cffi-2.1.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:25792eac27877609e7bb06d42ff88278a6624fff2ba9bbb523c09616b117e80f", size = 225688, upload-time = "2026-08-03T21:20:38.959Z" }, + { url = "https://files.pythonhosted.org/packages/eb/d2/3b7176cb570a1d3e27faf67b72f591af508036e0d8b2be2ef9af9e8c84bb/cffi-2.1.1-cp314-cp314t-win32.whl", hash = "sha256:8ef53b2de9bcb9197d31854256575d59dbac0cba72ac627bb291ef5eceb74be4", size = 182868, upload-time = "2026-08-03T21:20:40.388Z" }, + { url = "https://files.pythonhosted.org/packages/56/78/31f00c1bcd97c9bbf55f1bfdf5bc809a5de8887473e90bb9960dca825e80/cffi-2.1.1-cp314-cp314t-win_amd64.whl", hash = "sha256:616f097f2fe415bc92a247f02e11f634e1f9e9a83d327e3c915c15089c87869e", size = 194104, upload-time = "2026-08-03T21:20:41.725Z" }, + { url = "https://files.pythonhosted.org/packages/7b/1b/58496f2ed0a35de575250c02a43ab3cc2c04d494a88fed31c1cabc0fd176/cffi-2.1.1-cp314-cp314t-win_arm64.whl", hash = "sha256:ad2c86c495b899d862ea0f4b42891b8713a3bd45dd4105c7fd51c2a72f39f3a5", size = 186402, upload-time = "2026-08-03T21:20:43.042Z" }, + { url = "https://files.pythonhosted.org/packages/c1/8f/9ebe220eab48a093d1a5a5e339ab0dc7316eef3bb04d63c42f0251b61f50/cffi-2.1.1-cp315-cp315-ios_13_0_arm64_iphoneos.whl", hash = "sha256:dddad92b554513a31f272570678ba307fb9f618f05e3d4a5eacafff9eae03e1d", size = 194043, upload-time = "2026-08-03T21:20:48.179Z" }, + { url = "https://files.pythonhosted.org/packages/ff/69/844bad3ece306c4782c2ecb93597035b6690d48704b803914c199da1e8b3/cffi-2.1.1-cp315-cp315-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:da0e573f9f97159390c89d9f1a9e41908b66d408cc5b58d08cf3847d844c531b", size = 196737, upload-time = "2026-08-03T21:20:49.457Z" }, + { url = "https://files.pythonhosted.org/packages/1b/8a/af668013284634733f02d683458a0728739c7d6ddb5e14cb0c20832266fe/cffi-2.1.1-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:fb92203a88b3d3053034db775110081c49d28be6551923805e039924093761e4", size = 184933, upload-time = "2026-08-03T21:20:50.639Z" }, + { url = "https://files.pythonhosted.org/packages/0c/75/2f5207ff6d1a613133b23a5203cc0c2a628313b5eb3974d7956ae3c57950/cffi-2.1.1-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:2ae64be792b8966f2c69538199728b290e34726562896df1e5dc8ffd8d8188e8", size = 185002, upload-time = "2026-08-03T21:20:52.173Z" }, + { url = "https://files.pythonhosted.org/packages/e2/31/9e1313b0a6e30e91b3b3d3fff51ae99c857c07738e3afcce1f7334e1b7ab/cffi-2.1.1-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:507a24c282e0f42f8ed737cf048572cbf580468da5555764a8331735e9c736b6", size = 222271, upload-time = "2026-08-03T21:20:53.462Z" }, + { url = "https://files.pythonhosted.org/packages/50/e3/f6234a833e6e08c7007003074723c406559eecf9b48dfc97471e5a8eb7a0/cffi-2.1.1-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:246fa40ce8645a614ff682e0b70f37134e460eaf93a775e0cbe3cca585a67a80", size = 209919, upload-time = "2026-08-03T21:20:54.783Z" }, + { url = "https://files.pythonhosted.org/packages/0d/fc/5f74e293fced6edb51af3a46c4ccf6c23c9943774ecb375ddbd522c76add/cffi-2.1.1-cp315-cp315-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:471cee653ae88de62096552e6d24ccb4a5adb8c8c9f10b5054d0122c15bf2779", size = 208529, upload-time = "2026-08-03T21:20:56.066Z" }, + { url = "https://files.pythonhosted.org/packages/44/16/29e6d01b388bef055ecd6ca8244b3f4d336bd09e92d5d892187b9601084e/cffi-2.1.1-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:aeae0e330c9f6acd681f647d46cefd30c29f93e3392882e792e82080c9691399", size = 221630, upload-time = "2026-08-03T21:20:57.336Z" }, + { url = "https://files.pythonhosted.org/packages/a4/18/fa7f1f6857d5eb88a4ca99ffcbfb7c387a287ccc154c64a73e86314745d7/cffi-2.1.1-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:42a494cee34437f05546455144f2b5d9ac09b1face62bcfce597d2e521066688", size = 225134, upload-time = "2026-08-03T21:20:58.675Z" }, + { url = "https://files.pythonhosted.org/packages/e0/9f/e8e3dfa04a1b4c241f8c91faacad872b4d4efd051d49764ad4e2fd4b9fea/cffi-2.1.1-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:cc572dace3f60ef98d7b12ff411d20f5362feb31a0439eab0085bbfd349982d7", size = 223197, upload-time = "2026-08-03T21:20:59.968Z" }, + { url = "https://files.pythonhosted.org/packages/f8/7e/8debeb04f1ab9fe2a6963964cd6f1aaf7192627b83926586a6a4e089c9fa/cffi-2.1.1-cp315-cp315-win32.whl", hash = "sha256:4f42141fc14250de6dde5ee7ea4432be017252d91f19c5ad043c084cea629cac", size = 177683, upload-time = "2026-08-03T21:21:14.901Z" }, + { url = "https://files.pythonhosted.org/packages/e0/31/5158704cc474ab65c1647932e88be78dc0873f47130e253be38bcaf13d01/cffi-2.1.1-cp315-cp315-win_amd64.whl", hash = "sha256:e6e8cff14d6fb0be70a09c0bdc58096f501952d04624ebf867e0e56da2df8960", size = 187897, upload-time = "2026-08-03T21:21:16.108Z" }, + { url = "https://files.pythonhosted.org/packages/cc/4b/b3a2da8570c704ffc0f9762cdc3ec0f02c8573798e0b5cf7f11c82bbb70f/cffi-2.1.1-cp315-cp315-win_arm64.whl", hash = "sha256:27350daa11d4f10c540e6e89dada4c54feb7256ad03e9a4dc075ebad7ba360d1", size = 182935, upload-time = "2026-08-03T21:21:17.271Z" }, + { url = "https://files.pythonhosted.org/packages/d0/ef/5443574510a1207e6f6bc38ba6e1f1de36cb48fef07b2728bb896a21f430/cffi-2.1.1-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:c26608d2222fb1e94487e4a387d85f13eb55d5ed725cb25a0c589ac4ee60e7bc", size = 188464, upload-time = "2026-08-03T21:21:01.163Z" }, + { url = "https://files.pythonhosted.org/packages/7e/ae/a56fa8c4686ad50e148fcbc8d3ae0d03915ff5c30d795058988c24118cef/cffi-2.1.1-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:4be96343e422f2dfcd12ab5c9f5aebe03f82f737c6bffeca6830b3875cb44aab", size = 188262, upload-time = "2026-08-03T21:21:02.382Z" }, + { url = "https://files.pythonhosted.org/packages/53/b2/6187f46f2912276a3ae284076109cc5c8680482f11f766ccf26db4a86427/cffi-2.1.1-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:937c0052c05a31ca1daf18de3158eed4dbfcb9cc107adbea227728d647be701e", size = 223779, upload-time = "2026-08-03T21:21:03.553Z" }, + { url = "https://files.pythonhosted.org/packages/8a/f6/c3ad28bd19f77047a03084424fbd4cbe997303267c14423737324be0385d/cffi-2.1.1-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:df423d40ee8654634421812bc3b196da3f9bd7d32929da813f8394c4348a5358", size = 211520, upload-time = "2026-08-03T21:21:04.863Z" }, + { url = "https://files.pythonhosted.org/packages/a0/cd/ccac9013a5bd9fd764de118674ab9c805b5ca10c19270d90ee273f8b2240/cffi-2.1.1-cp315-cp315t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:a730a083190634c65cca36ba5f489531576ebd79bcd5c8e172130f6453127231", size = 210673, upload-time = "2026-08-03T21:21:06.223Z" }, + { url = "https://files.pythonhosted.org/packages/52/86/2976131c639aead931c5bee5aba67e4b09fbeb8018b6f282f70803f923a7/cffi-2.1.1-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:363e05fa78e15116c3c32c210ee36884fd6b9afa6d440e47112c3bd511d64cb6", size = 223835, upload-time = "2026-08-03T21:21:07.539Z" }, + { url = "https://files.pythonhosted.org/packages/ac/0c/33a7aeab2f9c76918c52e084beb39c570db3588133412929e8ec06fab90b/cffi-2.1.1-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:770de9db11e84213beec501cfcaa013b019820ca881e03344dea5844f7876d94", size = 226705, upload-time = "2026-08-03T21:21:08.774Z" }, + { url = "https://files.pythonhosted.org/packages/e3/26/2cde30fdde421130bfc18f70395731a6e6b2053c6a1978a5258ff04e72fa/cffi-2.1.1-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:7da0c5eff80f0197f3b3d1232ec5a682a9325f4ae9016a78f5f5ca35f9ced1f5", size = 225539, upload-time = "2026-08-03T21:21:09.911Z" }, + { url = "https://files.pythonhosted.org/packages/6d/cd/a361394c94b2129d604bb846f624a8e88255a3ee33129c434a00d715e64f/cffi-2.1.1-cp315-cp315t-win32.whl", hash = "sha256:06c72bb76605a4b0cd0aad6930b69d4baf7dd5d806cfc409b824191099700e66", size = 182707, upload-time = "2026-08-03T21:21:11.226Z" }, + { url = "https://files.pythonhosted.org/packages/9b/b5/ba2b299993c26577d529b6ae29841f9e15b9fcf004d65f423f4fcf94ade9/cffi-2.1.1-cp315-cp315t-win_amd64.whl", hash = "sha256:d9c275eaacd24aa73f94ffd6de08fc3f932424d8b6c376f4bed7cde376fe7bc3", size = 193772, upload-time = "2026-08-03T21:21:12.39Z" }, + { url = "https://files.pythonhosted.org/packages/aa/29/35e016098c814cd93de9cd320c66b5bfba14dc6ecedd3cb518fa7c408c69/cffi-2.1.1-cp315-cp315t-win_arm64.whl", hash = "sha256:d18e5ac0f2f03f4f518d3e23db0f0cad7faa1da8620e9c09461d443bbf6e6692", size = 186360, upload-time = "2026-08-03T21:21:13.636Z" }, +] + +[[package]] +name = "charset-normalizer" +version = "3.5.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/33/1c/f41d4e74c28ab327ff3acd36053f7ea506c55872d7a90b0fa71aa3ab0c89/charset_normalizer-3.5.2.tar.gz", hash = "sha256:39de2a259fc954455c57274dc94c79d5842774e1247a016aff30bc0efed0f4ef", size = 172659, upload-time = "2026-09-30T04:39:23.398Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e7/c8/693809898870237d82785a03f3b2b58fe4c9f14669f84a7d4e623c92a59e/charset_normalizer-3.5.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:ed2a239c0ea213acc1908150a3037257083c7c083128f1a4cec2ec4b97dca491", size = 367780, upload-time = "2026-09-30T04:35:30.888Z" }, + { url = "https://files.pythonhosted.org/packages/c9/87/2fea8c13dc24b3ca9c6f803a5b2dfdeae73eb4f9e12c7885ed908ff0433c/charset_normalizer-3.5.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b91363207bd9dc966a691e959bb47f64b30f7ac4b072be9968b366982f7db77c", size = 246730, upload-time = "2026-09-30T04:35:32.286Z" }, + { url = "https://files.pythonhosted.org/packages/a8/9e/09efac30b937722f46d3110ba30b875b24b2e3a266ed746cc4e376a94d80/charset_normalizer-3.5.2-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:38a873987f3be698494da8b2e3085e29da02da7b633dce73e79c699a113d7bf0", size = 237707, upload-time = "2026-09-30T04:35:33.709Z" }, + { url = "https://files.pythonhosted.org/packages/9e/18/70d76670b13686237863a379928d60bd10e021f17d243ab3d7014c4a5f4e/charset_normalizer-3.5.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:355ad8011081dec5412240c087a9a0c9d4d5039f3ed11a3f13e18c2b29b56c51", size = 273050, upload-time = "2026-09-30T04:35:35.138Z" }, + { url = "https://files.pythonhosted.org/packages/54/e2/77a8b09d5adc013ed07b95b01b8b8fa5441c4e810e83ee7e4aae2fa4d91a/charset_normalizer-3.5.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ee21e28f0430bd6dc9086c6e525d5e818a44a5ad19720c8a0ef766792f3eb5e5", size = 270345, upload-time = "2026-09-30T04:35:36.502Z" }, + { url = "https://files.pythonhosted.org/packages/7f/c5/38806a25ab5e65fc178f39affeda20858efafede2fce1ffc2556cfc9fe73/charset_normalizer-3.5.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3d31298449090ab8d47b7b1b2a555ff73cac7ed438a08b7ac160980c7ebed649", size = 257601, upload-time = "2026-09-30T04:35:37.919Z" }, + { url = "https://files.pythonhosted.org/packages/ae/8d/213565184708fdb263ae55e2c04ee1ff748129dd65d48ed0e3502da9c85a/charset_normalizer-3.5.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5cde776b7cc66e4f6c99612cea4aa7269aa65863f7a15841b2c264f103822f4e", size = 252222, upload-time = "2026-09-30T04:35:39.544Z" }, + { url = "https://files.pythonhosted.org/packages/7e/24/76d2cefc25472531e4c5c7dfff68865eb1c39b78482f0fdc15b46f047830/charset_normalizer-3.5.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ae4f5fea5b8b8ccff88238cc8569303e5ee95efae67fa62922a311397a71f346", size = 248482, upload-time = "2026-09-30T04:35:41.088Z" }, + { url = "https://files.pythonhosted.org/packages/7d/dc/65a801b66ab4c197e22c433ab25e7ac24324ac6f45a2269aca42cce309bf/charset_normalizer-3.5.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:f7d486c83842422badd511868fd8a9a20e9407ace71564b6af47ce7e60a336c1", size = 241206, upload-time = "2026-09-30T04:35:42.59Z" }, + { url = "https://files.pythonhosted.org/packages/a7/95/ca9b5eabde673002c6f1e7ada1b223916fe18f6d661da7aabd4d643718f1/charset_normalizer-3.5.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:11a4d68a6ecda3292cb1e50239e111543ba5d709bb62a6b4ea1afcfa729d8875", size = 273190, upload-time = "2026-09-30T04:35:44.347Z" }, + { url = "https://files.pythonhosted.org/packages/2d/8b/803b4d2a3f6e1740f63f1e87b04d14b42f3d4fdfe6ed7d4db2d34102b14f/charset_normalizer-3.5.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:d6734d2ef8a50fbf8445c139477da401f50d62a0606bf00e20ec6d87773fefb1", size = 253527, upload-time = "2026-09-30T04:35:45.915Z" }, + { url = "https://files.pythonhosted.org/packages/a9/55/93c0e5dbd085ae0471346026abbe7e0db9ea2d6fea74e51f0b5a46f233a7/charset_normalizer-3.5.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:a815775b6c38d4e0ff7bcffbeba67feded90202bb6a226b8dd35f1c855217413", size = 271285, upload-time = "2026-09-30T04:35:47.49Z" }, + { url = "https://files.pythonhosted.org/packages/95/69/0dbd0e0b9b16cfa816cdfcb3e2e3854a1f680dc07fb1245ea125e7448060/charset_normalizer-3.5.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:23851fb4e1b85ed3f6c2a27b777cdfe2e19fb5b38429a8faf38c7542b7665869", size = 260010, upload-time = "2026-09-30T04:35:48.996Z" }, + { url = "https://files.pythonhosted.org/packages/58/9d/e7b88e7b1bf403590c3b573277b5e1e488c68c7a6fbacca310a2c324e90c/charset_normalizer-3.5.2-cp312-cp312-win32.whl", hash = "sha256:db19d07e2e0129e974a0e65d0064fc222a446cd5122c2fd4184d2af9fc734a9e", size = 184125, upload-time = "2026-09-30T04:35:50.777Z" }, + { url = "https://files.pythonhosted.org/packages/eb/e6/e6e083884cbcfd49c64865af05027fe7011be7b2d9179524f099a1b611f3/charset_normalizer-3.5.2-cp312-cp312-win_amd64.whl", hash = "sha256:780fbe7cab297b81dad9fb8dc5eb003c0468ffb0d9e5f65068c53a34661a96bc", size = 207486, upload-time = "2026-09-30T04:35:52.194Z" }, + { url = "https://files.pythonhosted.org/packages/c4/e3/017aea0911ada7405a825c7d937eb3a13009664e2f5b38e8c4bbf2abf894/charset_normalizer-3.5.2-cp312-cp312-win_arm64.whl", hash = "sha256:e2af3aad578aa6bd1384bcf4750fc285e5a9de53f40b7d41e5a0bf748edeb2b3", size = 196734, upload-time = "2026-09-30T04:35:53.636Z" }, + { url = "https://files.pythonhosted.org/packages/c5/34/68292d68512768591aaff07c59bb53ee31341c87759433a859c4641a50c2/charset_normalizer-3.5.2-cp313-cp313-android_24_arm64_v8a.whl", hash = "sha256:ed905975ab14056a2e5eb1c376cb2e1ebc5396baf84163939c518556fccde9f5", size = 225946, upload-time = "2026-09-30T04:35:55.313Z" }, + { url = "https://files.pythonhosted.org/packages/e3/80/bee0b01b90ccd5322ae1d0abb33fab1bd95b7c2eadaf02aeccf22e04ee83/charset_normalizer-3.5.2-cp313-cp313-android_24_x86_64.whl", hash = "sha256:a66c3bc5ab1f0ff2164fc9965ddd611ff0802173f4b9d24554c563f6ab7e1d6e", size = 238619, upload-time = "2026-09-30T04:35:56.863Z" }, + { url = "https://files.pythonhosted.org/packages/78/6e/60ce52a85a7fd631ae8482ae6d74521014ca2f255892679484dc04d7ef56/charset_normalizer-3.5.2-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:d2374b62878abb00cd8309b32af6c0b715cd02dec0ca74ef12e5069bdc64144a", size = 206701, upload-time = "2026-09-30T04:35:58.639Z" }, + { url = "https://files.pythonhosted.org/packages/36/8c/71aafad23f971afc84c2b295bc0c560739ce1dac558aad9fec22e39f3639/charset_normalizer-3.5.2-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:d376bbd28b3a8999db1a103b3b388aee6f1ddeb3e51bc2172993efdcd86e064d", size = 210506, upload-time = "2026-09-30T04:36:00.147Z" }, + { url = "https://files.pythonhosted.org/packages/91/da/3c5a7798c046df7d2d68ad653cf5b6c5a8bfee225055a843c6f2f42aac1a/charset_normalizer-3.5.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:6045373d5a89a5ec71afde535db987ca28e76dfa276c2d4c818265b375d4b055", size = 366626, upload-time = "2026-09-30T04:36:01.77Z" }, + { url = "https://files.pythonhosted.org/packages/e1/16/710ac3de2ee354e2bd1a9c94efe45a2d27b5c6ad39b2d6a905be2c094b6c/charset_normalizer-3.5.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:849df64e889b2e17230d58410a03dba311a65b163508fd33679b2b737d4b7858", size = 244624, upload-time = "2026-09-30T04:36:03.389Z" }, + { url = "https://files.pythonhosted.org/packages/d6/39/45c7439f5b63d24f7d5b2a1d760f34af7628782d7144b4cc8ded45c2d4bc/charset_normalizer-3.5.2-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:15c44f7edfd477b06f517a5cc317fc1707edb9de2c865f43d4b6513907473234", size = 235731, upload-time = "2026-09-30T04:36:04.987Z" }, + { url = "https://files.pythonhosted.org/packages/4d/34/38f3154785ce92e9f56eb226f4d35bdfae6b008480dd055f58837a89c810/charset_normalizer-3.5.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a89012d6d5476ee112d20d998570ed58df2260a852afb1758809cd6900411d21", size = 270352, upload-time = "2026-09-30T04:36:06.412Z" }, + { url = "https://files.pythonhosted.org/packages/04/f3/859f74e7babc977705026b30593b3be04049632a522fb7000f83c033d747/charset_normalizer-3.5.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:0c951d5e6dd9c2ff60609476752bee49da4206adde960ebc247766937f72e718", size = 267393, upload-time = "2026-09-30T04:36:07.865Z" }, + { url = "https://files.pythonhosted.org/packages/4b/85/41d27f234b82e47c167a5f6c0f62501dc0c640585ff4aba79e08a390336a/charset_normalizer-3.5.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7218e8f32b0956cfcd048fd42d9d5779809745ca1d86113ca56f66e7ae1549c4", size = 254660, upload-time = "2026-09-30T04:36:09.248Z" }, + { url = "https://files.pythonhosted.org/packages/58/ca/5d1a997587febe5b26d8daffe363b5c1a091cece19828eec6502fd09c5ef/charset_normalizer-3.5.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a19a731138fc27d5682277d3b9df22855cea1239bce7fcec5f78f42ef2d1f3c3", size = 249996, upload-time = "2026-09-30T04:36:10.73Z" }, + { url = "https://files.pythonhosted.org/packages/b3/1f/d1e78246f7ed60c8c8d606b4ac27f66ce49cc3e95f24893ccbeba9f77302/charset_normalizer-3.5.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:62603db9a7caa0802eaa28c1c46fecd7b3a263a774069c24c3c28c302448721c", size = 247028, upload-time = "2026-09-30T04:36:12.294Z" }, + { url = "https://files.pythonhosted.org/packages/8e/37/eba316edd4f0c4d3a5d945924c4eeeae59abac4056aa815d8a4268f863a2/charset_normalizer-3.5.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:b6856554c4f44d79fc2307d5768854310a8f0096e501c75637542c82292b0429", size = 240562, upload-time = "2026-09-30T04:36:13.887Z" }, + { url = "https://files.pythonhosted.org/packages/c8/8e/aaa037d40ca9ef045977f1a661048b1aa33f223adfce3452fe9be9f79d14/charset_normalizer-3.5.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:1bc0baf5ef96b6ede57d47f4b8fe4d9d84019c3bfcbeb20a41edc6a6ee341f1f", size = 271699, upload-time = "2026-09-30T04:36:15.41Z" }, + { url = "https://files.pythonhosted.org/packages/26/19/1c1c9f75974adf523b87f34b8a2adc5a435cd65916812bcbd0dfa45f9a29/charset_normalizer-3.5.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:56bc200a365efb37383b7852e4cc5898d3b2da5987289b543956cf8cad71018a", size = 252030, upload-time = "2026-09-30T04:36:16.839Z" }, + { url = "https://files.pythonhosted.org/packages/bc/90/0660ef18e18df0a4d2a1a0edff7dfbba42d4e50ef2425557a5bb7051f77b/charset_normalizer-3.5.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:2c9ad19a6cfcd5ea5c0d41161d22f9df1dcc277e9bef2751391334546a314c00", size = 268565, upload-time = "2026-09-30T04:36:18.468Z" }, + { url = "https://files.pythonhosted.org/packages/79/ba/57adc269824e8658f1a0f97a9e514c247445a9632b3419b97e0ba37f16dc/charset_normalizer-3.5.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e243bd13217235fc7290c621941c3f5cc8b66e4872495be821d7436ba2fb838d", size = 257627, upload-time = "2026-09-30T04:36:19.938Z" }, + { url = "https://files.pythonhosted.org/packages/9a/85/33abd4315c052d3d4f54c92b1ee49bfbc0dc7115a981e462a793b6d2ab87/charset_normalizer-3.5.2-cp313-cp313-pyemscripten_2025_0_wasm32.whl", hash = "sha256:a090bb2c68df85450502e3e20d665e3a5af9c65a84d6508ed477badd49166fd3", size = 143407, upload-time = "2026-09-30T04:36:21.376Z" }, + { url = "https://files.pythonhosted.org/packages/4f/de/6435e18d1aaa5d910b896d551411c96af1f42a0c56c29afc2016c61ccc2e/charset_normalizer-3.5.2-cp313-cp313-win32.whl", hash = "sha256:2b7b3bbfb4fe8ef40600792d762fbaa9057559f9d3fad209525b7a22b99e91fd", size = 183207, upload-time = "2026-09-30T04:36:22.776Z" }, + { url = "https://files.pythonhosted.org/packages/9c/76/b8ec57f4e9ee3253541abf95e4a462c0175fe8032dcd070f1f2421240942/charset_normalizer-3.5.2-cp313-cp313-win_amd64.whl", hash = "sha256:78456a747de8dc58360ffa581f30a002baf5aa28cb262536545e91f113ed7639", size = 206489, upload-time = "2026-09-30T04:36:24.306Z" }, + { url = "https://files.pythonhosted.org/packages/3e/60/c647c6ae47480221e875ea5d743ff94946f7416e3c69415ab772928e8d32/charset_normalizer-3.5.2-cp313-cp313-win_arm64.whl", hash = "sha256:11912e4bb14baae7c5d8791aa55ba0a3a03ec6729073307b0f57270abaa713d3", size = 195501, upload-time = "2026-09-30T04:36:25.846Z" }, + { url = "https://files.pythonhosted.org/packages/58/ca/7aa91362a2f77ac8e9e28a9b902a74f7d0e11a851ef0d27a74308da8cd90/charset_normalizer-3.5.2-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:1afb975bd5d68d5ce9f6b6d44fdf2f7e34b895a35e95708a7a91b20a3b51d187", size = 226010, upload-time = "2026-09-30T04:36:27.669Z" }, + { url = "https://files.pythonhosted.org/packages/a8/cf/ac8878d0322cf88a1aad4c7b147db32ca0bd806eb0060957b2e31486dbe6/charset_normalizer-3.5.2-cp314-cp314-android_24_x86_64.whl", hash = "sha256:bbbfc8e28816f19d7c0f1816664980c0a9875d01b27cdf8eedddb639d9e108ad", size = 239266, upload-time = "2026-09-30T04:36:29.434Z" }, + { url = "https://files.pythonhosted.org/packages/c9/6d/9a08d7e0b29b7208e2c6c01dc56c8e0520e7c7beadbbfb024b58fd69c8a5/charset_normalizer-3.5.2-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:7967d08cf06dee78443b874f98c98036f624f3a4e73e11f9f64f5be4d25393cf", size = 206783, upload-time = "2026-09-30T04:36:30.872Z" }, + { url = "https://files.pythonhosted.org/packages/82/44/b0aa350280e6ff5a5492d17cf10460dd39d5ee848f872f7ba2df10607f60/charset_normalizer-3.5.2-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:4c2b5031f63e331e3839b40aed2dd6f191e9c07edbde303e7876846ea1946995", size = 210449, upload-time = "2026-09-30T04:36:32.625Z" }, + { url = "https://files.pythonhosted.org/packages/7c/8a/40db9aa9f5907bb0e6f8b6d64064bf8852fb33d4b813ff9414911df7647c/charset_normalizer-3.5.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:fcff63213e8e6e47770541a4607175404f47cbb3ebea7b6058cc82d524a0e424", size = 367815, upload-time = "2026-09-30T04:36:34.197Z" }, + { url = "https://files.pythonhosted.org/packages/7f/72/9c5e7707b57c8ddfa9ddf7b0b1d009d7fbab9e9e887d5b721060f37e307d/charset_normalizer-3.5.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8d86d6fc60743dc916eb79e2eb1ec4818e21e427731543af40a3021851174a13", size = 247095, upload-time = "2026-09-30T04:36:35.803Z" }, + { url = "https://files.pythonhosted.org/packages/83/09/71e453691e927de4ddf792770cfaab3f49d494e222f66ea5e404bbd5e39c/charset_normalizer-3.5.2-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:7a881931aa470808df94a8c380eed2bbbc76cd9dc622310f99665658c821eb6d", size = 233942, upload-time = "2026-09-30T04:36:37.407Z" }, + { url = "https://files.pythonhosted.org/packages/9f/86/85c84e4da8b27dd409577d9437926ff581c5f9d3c66038dc68c1a526de51/charset_normalizer-3.5.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8024d00c3faf3fc0c16e07a69f4405e8eac7cc0ab15f65fe6cf43827c4cf72b4", size = 273762, upload-time = "2026-09-30T04:36:38.904Z" }, + { url = "https://files.pythonhosted.org/packages/92/08/564955a4b5f2ccb410ab480bbe8c6a18063ff27f2d35458731c4a5335df9/charset_normalizer-3.5.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4d48f2d08b9de5864e2c8744d4461b862fb149a18274abc8b698c45975573438", size = 269514, upload-time = "2026-09-30T04:36:40.469Z" }, + { url = "https://files.pythonhosted.org/packages/18/24/bad3ac4271589df29cf5ce2f5ae490518a5739358052bd0d61209e6fea54/charset_normalizer-3.5.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:34276fd796040bf0993ab33a369aa572e6979c7aab225a88893667ad8eac8f7a", size = 255056, upload-time = "2026-09-30T04:36:42.02Z" }, + { url = "https://files.pythonhosted.org/packages/d6/3e/350d89ad49916b86554d6f5f2d03ec1152148f87e5ff735106c6a03b1a36/charset_normalizer-3.5.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0521c5665880b33d603717defa76c094048900010897909952397feb3039da56", size = 251311, upload-time = "2026-09-30T04:36:43.577Z" }, + { url = "https://files.pythonhosted.org/packages/56/5b/4970a2d154df502e133402906dd04e3ae7cada7b3011283c88d0479a2585/charset_normalizer-3.5.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:eff0ac9dbe711a4aee69bf04a83896aa9b85f19641264053a9f6d48573abb7dd", size = 250190, upload-time = "2026-09-30T04:36:45.185Z" }, + { url = "https://files.pythonhosted.org/packages/88/8c/f1a91bddc8fb47c2889e29ea7ea49a194eb0d9868675d786806519c00d76/charset_normalizer-3.5.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:1503bccbeb36d5527790c3930327704c39af22de3112f1b1666a9f3ce15ee204", size = 237098, upload-time = "2026-09-30T04:36:46.689Z" }, + { url = "https://files.pythonhosted.org/packages/24/0e/bb5dace3cc7e79068425386a6589c19b5a2ab5fefc2a46abea6919683332/charset_normalizer-3.5.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:52aa6992700996af31f375de0c6bacd402b0097fe40b53c426b9f51a90ebabc7", size = 274228, upload-time = "2026-09-30T04:36:48.31Z" }, + { url = "https://files.pythonhosted.org/packages/9d/79/b849ad523017ea9f5a45581bbebed91439e0cf42fd2860a6f64e358eb5a6/charset_normalizer-3.5.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:e09a3942ecbdee5cce73ea9d42da82b81b72ac1bf031ce069b93b5adf4eac8cd", size = 252401, upload-time = "2026-09-30T04:36:50.091Z" }, + { url = "https://files.pythonhosted.org/packages/89/8c/75469d690cf47200bce8f6cad7655724fc23148e147abfc5ce78b5f65863/charset_normalizer-3.5.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:c7c9ab723cde841fefb34efbad91e87f00a674b1fe1cd0784fde742bf2c154dc", size = 271846, upload-time = "2026-09-30T04:36:51.719Z" }, + { url = "https://files.pythonhosted.org/packages/26/cd/6d52d3c7437cdcf2e310ce9f28f282e733d4ef60ed19105d1819c356255f/charset_normalizer-3.5.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ddc7dacc8ece3a182e7f15cb862d1fd616b46d076cb1ae9dd232b2c38b655874", size = 258151, upload-time = "2026-09-30T04:36:53.234Z" }, + { url = "https://files.pythonhosted.org/packages/f7/4c/070b38bdb5f49a70199fce923ec0726a49536a63ab262abbfcaaf351110b/charset_normalizer-3.5.2-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:ee43c17b173d46a3212baa6ead3ae258eeabdae48c263a01ccf0218c366dd655", size = 143570, upload-time = "2026-09-30T04:36:54.816Z" }, + { url = "https://files.pythonhosted.org/packages/81/84/9ebfc8ed6c8c4fcd8e726ff6bf220cc8deb3966e31dce9be8dd8aa017e64/charset_normalizer-3.5.2-cp314-cp314-win32.whl", hash = "sha256:4f87960d57feabfb618e4e0af6e7371645fa26a277860739d6e5d6e0012c92f0", size = 186412, upload-time = "2026-09-30T04:36:56.643Z" }, + { url = "https://files.pythonhosted.org/packages/d1/78/5ed86f743d4bc350db307e7636419a0a5ee1d91806d30c7f667bd5c80dae/charset_normalizer-3.5.2-cp314-cp314-win_amd64.whl", hash = "sha256:e4e81e09c1578b8df602e3db08b0b3ea0a6947ad612f52bf8dc5ea8d47691f0c", size = 210965, upload-time = "2026-09-30T04:36:58.205Z" }, + { url = "https://files.pythonhosted.org/packages/53/94/a3a7698e9b1a395e1eb99ccd9a324be9347973bff4e72db2a06496d7cd27/charset_normalizer-3.5.2-cp314-cp314-win_arm64.whl", hash = "sha256:80d02b6f04e92601a081dd97b23d3128033098bff5d35d392ddcc0476ea11253", size = 200406, upload-time = "2026-09-30T04:36:59.764Z" }, + { url = "https://files.pythonhosted.org/packages/c1/48/c5dd00d5ef7791f02666de250a5bb6071e29b7e133cf4b835800b6d3bc27/charset_normalizer-3.5.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:dca9ab98072a5a54ebacebdc45f53e645336b320c667410b061be1ca588ae709", size = 396871, upload-time = "2026-09-30T04:37:01.543Z" }, + { url = "https://files.pythonhosted.org/packages/12/c8/8379554b42e8368161d898476686947a0fdbd3e8865170d7909dcabfdee8/charset_normalizer-3.5.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f0aa869112ef88429ae17820d99c3dd9504c9e9c671d3c246f3d7442cb051084", size = 249711, upload-time = "2026-09-30T04:37:03.111Z" }, + { url = "https://files.pythonhosted.org/packages/4a/eb/2ddb1035d17320caa9f41682935123a9a250277b261c3efc86b2d2a21343/charset_normalizer-3.5.2-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c0afc6800ba57ccc350374c5bd6150419915d95ce93cdbab2d783d75eaf30ecb", size = 231146, upload-time = "2026-09-30T04:37:04.721Z" }, + { url = "https://files.pythonhosted.org/packages/4a/24/2ecb4bde104322cd7859d6594fcfa74649f8d90b3221c9feecbef149875b/charset_normalizer-3.5.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7dcd882da75ef9adf94903b1e3b9419e8aa8fb4c7396822b834b9ef7fb96954f", size = 271552, upload-time = "2026-09-30T04:37:06.295Z" }, + { url = "https://files.pythonhosted.org/packages/3f/98/9d5f6ebc3aee9fef5d30b4aff11fb2ab7a1222b4064f8ef2c7c87cde217a/charset_normalizer-3.5.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2e06a3a98f916dd41d27f3105e02e7a40181c98c94b9158733d03a6f80506c09", size = 269810, upload-time = "2026-09-30T04:37:07.905Z" }, + { url = "https://files.pythonhosted.org/packages/09/e1/a3b06a10461b1b7628853c934c644e03bc28e42767116afb52f19a56519b/charset_normalizer-3.5.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6bd128f206a7752ae1f2ab6c61bf8a24ba28913a10df8b14c2637b973ff97a80", size = 259845, upload-time = "2026-09-30T04:37:09.554Z" }, + { url = "https://files.pythonhosted.org/packages/fd/d3/6f561f74a296cf27d61775a1dc665ad13f3bff6a798810ca05907f37a7c4/charset_normalizer-3.5.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c8f3d67aeaf55f017982b73683f0e7342ba2f6635a78f69ce89ebb26aa411e5c", size = 252825, upload-time = "2026-09-30T04:37:11.274Z" }, + { url = "https://files.pythonhosted.org/packages/26/9f/69e13ca3b18f43e0eafcd34c04a45b732ae22a43b54a5fc9e119103356eb/charset_normalizer-3.5.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:fe9753dfee015c570d73df76f899f18444d41388bffcde097deba51c4fadbb9f", size = 252659, upload-time = "2026-09-30T04:37:12.941Z" }, + { url = "https://files.pythonhosted.org/packages/73/a9/ace29806a0dae18939919c76ba526472d83214afa101105fabff2cf30625/charset_normalizer-3.5.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:92888bb3187c5ba50500b00b3b310c9f2c651709d28036077680cb5255450a03", size = 238819, upload-time = "2026-09-30T04:37:14.659Z" }, + { url = "https://files.pythonhosted.org/packages/f8/c1/6116d52a2e3311ec80f21f5fb5e17b27405f10b9608af8f6e69516841a1b/charset_normalizer-3.5.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:d008d90a7f2471519aef0c90dfbe73b3e6e4d5e66ac48e19154c17e89e98b604", size = 272973, upload-time = "2026-09-30T04:37:16.346Z" }, + { url = "https://files.pythonhosted.org/packages/19/aa/9955c7e93bba10a9c7e8f7a5031b7ced66f3a1883a55c00712b8d5850ff3/charset_normalizer-3.5.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:31f3930700408d211f13378ccbe1c40845d8da54bd0681fac3a9b5aae81c7aa8", size = 253692, upload-time = "2026-09-30T04:37:18.212Z" }, + { url = "https://files.pythonhosted.org/packages/bb/33/2a6ae7fdc1b10cb581cef91addd8cdfc5f40d50abb5702309369d5834579/charset_normalizer-3.5.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:2a925889534b3748302dae5dead07cc13480de1dac3aea80a941b729b471ef93", size = 268757, upload-time = "2026-09-30T04:37:19.877Z" }, + { url = "https://files.pythonhosted.org/packages/a2/22/80992720a0282cd39bba1db35868e6b9c22f41281160143a836544bc1d8a/charset_normalizer-3.5.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:f5ec61164adcec446f8969a3358ec3f9b26bbda3b9213e5586d219afa8df2915", size = 262220, upload-time = "2026-09-30T04:37:21.583Z" }, + { url = "https://files.pythonhosted.org/packages/92/9f/181fd07e1bffea1d95cd80c84ac537354f50699c22cfc4d3c02b6fc16208/charset_normalizer-3.5.2-cp314-cp314t-win32.whl", hash = "sha256:598a11a2c7ebaa5334bf698bf29568c9c390abac6a154d8170fedecd1cea38c5", size = 197797, upload-time = "2026-09-30T04:37:23.235Z" }, + { url = "https://files.pythonhosted.org/packages/49/1c/25d8415ec1c4f2f41f1680435e4c87cfb378ff2f677d950946f2a45d0632/charset_normalizer-3.5.2-cp314-cp314t-win_amd64.whl", hash = "sha256:7fdde2c9fd9e3eca40631e024664cf2584272cc8f96308cbe5fdfc930f51d8bc", size = 222765, upload-time = "2026-09-30T04:37:24.891Z" }, + { url = "https://files.pythonhosted.org/packages/3e/b4/46b48f013dadfc0d0d33b375438e31bdf5a989dc68389c6bf627054d4df9/charset_normalizer-3.5.2-cp314-cp314t-win_arm64.whl", hash = "sha256:d1befeed746d247c81127bb14de9dc3d30edb6e5976d34f83f86ed262b1d9105", size = 210985, upload-time = "2026-09-30T04:37:26.634Z" }, + { url = "https://files.pythonhosted.org/packages/ca/e9/34e597dee616d0b8ee4b34d29399e85c2204ade174157a48505d42baa4ff/charset_normalizer-3.5.2-cp315-cp315-macosx_10_15_universal2.whl", hash = "sha256:87475fabc8d9996fd9c27debb395e642e8c838d78a00b6e932227a0e06b81e26", size = 367676, upload-time = "2026-09-30T04:37:28.329Z" }, + { url = "https://files.pythonhosted.org/packages/60/9f/a5d1c91c0263745e2cd344c5a4415d787c575501ab1d449f1148ac6b495d/charset_normalizer-3.5.2-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9409a8bf35cf78353942504b24a57de3d75b708997a1e4bd8db71ac8633ce364", size = 248414, upload-time = "2026-09-30T04:37:30.167Z" }, + { url = "https://files.pythonhosted.org/packages/26/79/e697f77464748a3ee3cf490c83d592459400d4898380d66c38366b03080c/charset_normalizer-3.5.2-cp315-cp315-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:498dc3188ca05a68231ac3fdbfc7f57eb67e1343c30e0fea17f8218c1599b253", size = 244775, upload-time = "2026-09-30T04:37:31.964Z" }, + { url = "https://files.pythonhosted.org/packages/ca/87/3d42a42e18ea066e2513936fd678a00696e77878b5ae04528976abdbcb83/charset_normalizer-3.5.2-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e242bb1c5e76e97dfa9e7f209a71e93a01d7f19ffdd5cfbb2e2d55b4f08f8ab0", size = 273877, upload-time = "2026-09-30T04:37:33.661Z" }, + { url = "https://files.pythonhosted.org/packages/c3/76/8a28136f3938ba9836f84280ce0c4d61ed1cf15a036b2034900c62634162/charset_normalizer-3.5.2-cp315-cp315-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:def79fa35ef0cef8d2accec024f4fdc7ead3012ff02f5215c783f39f03ef8cfc", size = 269455, upload-time = "2026-09-30T04:37:35.573Z" }, + { url = "https://files.pythonhosted.org/packages/a0/a1/4fbf5d0f0f1b2a080474c1cf9a2f12c4c6531bb0e8ba591055e846d2b4e9/charset_normalizer-3.5.2-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3df041de8887954562c9b261cba85ca0e9ded74048daf125f45edcfaa4832229", size = 257336, upload-time = "2026-09-30T04:37:37.397Z" }, + { url = "https://files.pythonhosted.org/packages/ba/a2/8b50aa320adb880ad579518e6f718f24944804b42a88b83d267d5d444125/charset_normalizer-3.5.2-cp315-cp315-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:04851f73ae72b8413dddadb16a49dfee95263553741fd42d546f7d66907e6be5", size = 257584, upload-time = "2026-09-30T04:37:39.522Z" }, + { url = "https://files.pythonhosted.org/packages/a5/57/50e3fed84e175f40349bd0da7a4fce94c87f0378f52d74f511d89e0bdc20/charset_normalizer-3.5.2-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:183b88127acdb4fabe59d951ab424faf1af7b63cdbb5f776186c1ea2ffcaed98", size = 251854, upload-time = "2026-09-30T04:37:41.23Z" }, + { url = "https://files.pythonhosted.org/packages/d6/54/f7fbb3493c9f49091213b9c2d6dd65800696f1ce1a3f196a4205f50417b1/charset_normalizer-3.5.2-cp315-cp315-musllinux_1_2_armv7l.whl", hash = "sha256:16fa0eccf81304b79c5cd87f9271c3b85dd9dd99245e4422ae9c0dd45e0f99d3", size = 248767, upload-time = "2026-09-30T04:37:42.883Z" }, + { url = "https://files.pythonhosted.org/packages/d9/37/b3a6385acc5a1e45b39ae9c90bfb9cf838a09b9dd37ef2740ab4c6b4a2eb/charset_normalizer-3.5.2-cp315-cp315-musllinux_1_2_ppc64le.whl", hash = "sha256:7441d755b7ab94f8d4eb3e43ec05482d760842fd263d003a99102d742cd835e2", size = 274479, upload-time = "2026-09-30T04:37:44.658Z" }, + { url = "https://files.pythonhosted.org/packages/89/44/809913e2cfd279e635a9294fdbbfb1b1dc62a8189d473d561f649fce98d8/charset_normalizer-3.5.2-cp315-cp315-musllinux_1_2_riscv64.whl", hash = "sha256:ca403d7e4798f525fdfc78e258820419cbbd0f0ecbab9de7840e3c017cf6b8cf", size = 258310, upload-time = "2026-09-30T04:37:46.529Z" }, + { url = "https://files.pythonhosted.org/packages/af/a2/f28400ab13359d91bd39179df8e149376b9bf36588e739a3a4f9de2b84b2/charset_normalizer-3.5.2-cp315-cp315-musllinux_1_2_s390x.whl", hash = "sha256:df29a0a7107f7011e77f4eebdddec4c7331e24d787a0b21a46d63bdf7445da95", size = 271100, upload-time = "2026-09-30T04:37:48.399Z" }, + { url = "https://files.pythonhosted.org/packages/e9/89/9bab37955edf0adb3b66f8a3a6617d9f2f487e0d56f295a6a286cb640aa6/charset_normalizer-3.5.2-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:f3c96f633825733f735c5a9cf21d21a257d8e1edf0b1cee0a064b9c424ca0f7d", size = 259936, upload-time = "2026-09-30T04:37:50.023Z" }, + { url = "https://files.pythonhosted.org/packages/23/b5/4459e08d45a679f903d50fea08bc52cfa728cca4d7bd02c757b5e5abda2e/charset_normalizer-3.5.2-cp315-cp315-win32.whl", hash = "sha256:281cb91036248400f4cc957495cccd44c275c2e0c5854f7e45ac5cf7dc193847", size = 186301, upload-time = "2026-09-30T04:37:51.722Z" }, + { url = "https://files.pythonhosted.org/packages/98/e8/55d5fd3935b4bce6da4fe0df61898e8c82653e317e677bd58aceb9c60f13/charset_normalizer-3.5.2-cp315-cp315-win_amd64.whl", hash = "sha256:89b53f3cda69831909888e0494f4fa0bcd3537e3e138dabeb620bd6ad946bae8", size = 210858, upload-time = "2026-09-30T04:37:53.427Z" }, + { url = "https://files.pythonhosted.org/packages/a9/5b/974423c2fd8e524c7a7f64318c1e02240ef954912fa2b4d70344107b9c68/charset_normalizer-3.5.2-cp315-cp315-win_arm64.whl", hash = "sha256:6be488a102b8cf28d0391d8c4ba7748938ae28b78ad901f8585520fca33ead1a", size = 200309, upload-time = "2026-09-30T04:37:55.015Z" }, + { url = "https://files.pythonhosted.org/packages/ee/f9/00ee0195db1013d8f7c416fd770fbeb560bb46eb2e36b054d05cb56f6cfa/charset_normalizer-3.5.2-cp315-cp315t-macosx_10_15_universal2.whl", hash = "sha256:915563965d418f986e7e145accc592eae9e1a1be3566ff98a05d7a9ec42a76e1", size = 395343, upload-time = "2026-09-30T04:37:56.743Z" }, + { url = "https://files.pythonhosted.org/packages/04/3a/c00b50e94c964cf934c7899cd47c97952fc11dad71cc5884b3c61795b09b/charset_normalizer-3.5.2-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:65cd72beeeca9d3aaea1201e5923859f308f952f9c71de93f06063c79f0f7a3b", size = 250114, upload-time = "2026-09-30T04:37:58.607Z" }, + { url = "https://files.pythonhosted.org/packages/50/27/d102dc880bbcffd0479ab64dfc1fb96777a854355a55e2bda72a71efadcb/charset_normalizer-3.5.2-cp315-cp315t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b7fd005a73d9e657273b7a10dc71a9e03c8fb9ee6999798d6918ce095b81ac7f", size = 239997, upload-time = "2026-09-30T04:38:00.511Z" }, + { url = "https://files.pythonhosted.org/packages/a5/4a/bf7ef45794dd293fab5f98a9309817977fbb845b9998f171b8cc5d8437a3/charset_normalizer-3.5.2-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e54da4baf05720032d527874d40b65fa4d7e5c6c6a43d0c3adbeffcaf275a2b3", size = 272313, upload-time = "2026-09-30T04:38:02.509Z" }, + { url = "https://files.pythonhosted.org/packages/e8/ee/008a2837737991474c5754bb3191010007663860979701990982a502cbaf/charset_normalizer-3.5.2-cp315-cp315t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:124fbf1a8ff966d87ae05bb8bd45a71f966055ed8bba320d0c7cf450bc5f4d0e", size = 269986, upload-time = "2026-09-30T04:38:04.435Z" }, + { url = "https://files.pythonhosted.org/packages/93/ad/bd74a283940dc910c5b14f8e4f80a248082bc9c0fcbe1f54530cb6d9cc5e/charset_normalizer-3.5.2-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:28b4f0d66fb834ff90f28209ac7bce77868c45d8c93e26f906709d9b7c2e1af9", size = 260202, upload-time = "2026-09-30T04:38:06.549Z" }, + { url = "https://files.pythonhosted.org/packages/8a/7b/ed341c66f69f688723501fac752be3d63c7159ca0d0d4174fc611e5710bb/charset_normalizer-3.5.2-cp315-cp315t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:58ca3755ee7ff7f59b57789ec9833c9de9ea275405cdd240eda1f193112e398a", size = 259083, upload-time = "2026-09-30T04:38:08.311Z" }, + { url = "https://files.pythonhosted.org/packages/cc/9d/e41588b777965e5031a43128a1e96173ebb35ac75fc53ec3b517e7c21cd4/charset_normalizer-3.5.2-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:443eae2bf318abeaf6f15d785138f71fd6de770e99a92158b8b814265e079115", size = 253081, upload-time = "2026-09-30T04:38:10.402Z" }, + { url = "https://files.pythonhosted.org/packages/81/35/b761eb6d8c1eb218b9b42b9b4d5ac902afdc399fb6dac6f9a9aac7bda589/charset_normalizer-3.5.2-cp315-cp315t-musllinux_1_2_armv7l.whl", hash = "sha256:58f361dcbab699cf8f42db3f47c8e7fd1036f138c23a5d08de9fde5f425a730c", size = 248712, upload-time = "2026-09-30T04:38:12.317Z" }, + { url = "https://files.pythonhosted.org/packages/4d/2c/147169a041b747759f37405c0a97157e8e92de967968373101ff14915cba/charset_normalizer-3.5.2-cp315-cp315t-musllinux_1_2_ppc64le.whl", hash = "sha256:1b4cbc7c3491ccb4aa17fcd8165649d01cf39f76de1696da8631b5f71b85401d", size = 274176, upload-time = "2026-09-30T04:38:14.138Z" }, + { url = "https://files.pythonhosted.org/packages/f0/2d/0ff8db0d373ba8538db686db11cd7e8912031490b9e4f383b41912e8d594/charset_normalizer-3.5.2-cp315-cp315t-musllinux_1_2_riscv64.whl", hash = "sha256:ba0b1d2620edf869789c3879223f52bf2afc5d31b3cb47cc57b3a12c05e2aa9d", size = 259413, upload-time = "2026-09-30T04:38:15.841Z" }, + { url = "https://files.pythonhosted.org/packages/8a/8e/b4a085fb47c9d3a7e43576a4784fdd8fe23f907514a972de8086edaf7a48/charset_normalizer-3.5.2-cp315-cp315t-musllinux_1_2_s390x.whl", hash = "sha256:5e2b6b57e9733d39f0c9fd3185efa6b8e29652c4cd8fe94180272cf6ed9a78c4", size = 269434, upload-time = "2026-09-30T04:38:17.626Z" }, + { url = "https://files.pythonhosted.org/packages/83/1c/d8d8d7322a7c3eecdf3237a4a419cf41d2eaad8e006ce7dfdd9d4c8fa2eb/charset_normalizer-3.5.2-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:51cf45226a9b588d0d2b4880c62d686934b63ab0bd79ca23ab0e9762eb27441b", size = 262716, upload-time = "2026-09-30T04:38:19.214Z" }, + { url = "https://files.pythonhosted.org/packages/a0/16/0e4c6ba9b44e97a2da150e52d331e8f9c968b21b358fbffa6c856cebcd89/charset_normalizer-3.5.2-cp315-cp315t-win32.whl", hash = "sha256:5fb29fb8cd1a46c27a1bf9613ad5ec2599310d46b4025d9556404a6b6a292800", size = 197159, upload-time = "2026-09-30T04:38:21.037Z" }, + { url = "https://files.pythonhosted.org/packages/be/33/e90bc2b1374f7f36ef106f56620de5a783907e19ca857efe2277e31cac3e/charset_normalizer-3.5.2-cp315-cp315t-win_amd64.whl", hash = "sha256:a192e2c40070d92c3ccf777e3a5c4ff515573cd2bb7ed0c537fdadbbec5bbf21", size = 221322, upload-time = "2026-09-30T04:38:22.886Z" }, + { url = "https://files.pythonhosted.org/packages/66/89/dfa6dcb08c200b7830ab56439e8c1890f2971d51aafbb3937894a2e7fcfc/charset_normalizer-3.5.2-cp315-cp315t-win_arm64.whl", hash = "sha256:749e97e1b32313717a565abbe321bc2190bc8b35f1a67e4cdbc7c56c8d8ffe58", size = 209192, upload-time = "2026-09-30T04:38:24.648Z" }, + { url = "https://files.pythonhosted.org/packages/8c/ab/176fbfd5b64939c55d652366aa5b9ef1d767af207a3aa6ebeb0d226c484d/charset_normalizer-3.5.2-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:4275811936e2f06feff5e598fb42a1b7ae852da8e39605211892b56b81a34efd", size = 331815, upload-time = "2026-09-30T04:38:26.216Z" }, + { url = "https://files.pythonhosted.org/packages/7e/84/371eac6b30bdbcbf2d632a1a01809103459216fcaae61b8b8d922c1bfb8a/charset_normalizer-3.5.2-cp37-abi3-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:1c50fe28bbc2ced33386f298650d91218076c05420e6cbd790b913adc41659e7", size = 253276, upload-time = "2026-09-30T04:38:28.032Z" }, + { url = "https://files.pythonhosted.org/packages/43/6f/c4fbae58febff71709c51bc7e18fdfa55341dc382704740f9f0cbf03817b/charset_normalizer-3.5.2-cp37-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d19fbd981a488e22cd04883659ca6b08f50b5974f9fd7c95655ef6a043e5893f", size = 241239, upload-time = "2026-09-30T04:38:29.732Z" }, + { url = "https://files.pythonhosted.org/packages/61/71/458c3f42164a07d0c5210798e9e704b39e540a6793b05aba67f3a35243a9/charset_normalizer-3.5.2-cp37-abi3-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:0fed1d06615f022ee3b13caf5e8b180cfea32bb2c5aded8a9d44277afc040f93", size = 231121, upload-time = "2026-09-30T04:38:31.462Z" }, + { url = "https://files.pythonhosted.org/packages/09/54/ab9e89367076f6331bb6c65c4bf14a5361fa5191cb6561bf534f18504e1b/charset_normalizer-3.5.2-cp37-abi3-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:838dcc90063569a0448120554591a1d6c4a4ffe11babf048908793154ab86ade", size = 260350, upload-time = "2026-09-30T04:38:33.239Z" }, + { url = "https://files.pythonhosted.org/packages/7c/c1/061431ecc688d9d76602502cb57cc01e691e682c18f1beb45f9673b5bbd2/charset_normalizer-3.5.2-cp37-abi3-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2ce45c6627b22c47e390bc91a41c3d13032192e699fa0bea96e9671b373d69b0", size = 255430, upload-time = "2026-09-30T04:38:34.865Z" }, + { url = "https://files.pythonhosted.org/packages/8d/1f/20c8949f0676f7ab811abdeb7f4d7f1cbc6e61ff20bef08b44edeb092bc8/charset_normalizer-3.5.2-cp37-abi3-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0774bf9bf620249fee3e0b8b9fd3065de213be30f3aa94ce2494b3b638949e26", size = 250612, upload-time = "2026-09-30T04:38:36.649Z" }, + { url = "https://files.pythonhosted.org/packages/2b/9e/46f2fa4c431fc98c4ae76a8cb5bdca54e0341e3cfc3fcfd8e82740250818/charset_normalizer-3.5.2-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:1db38f4c5496827c1a501846d64d14c3b80c7e6714e406cd7dc36a9899fa1011", size = 242083, upload-time = "2026-09-30T04:38:38.26Z" }, + { url = "https://files.pythonhosted.org/packages/bd/39/559be29a0c0f086e0bba6922babd38916cc5e0b58ced4de13ee01ea05508/charset_normalizer-3.5.2-cp37-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:304d8e4d493af723536393eee0c689eb7813f4a474c8b479dee63f1fdd98f621", size = 232738, upload-time = "2026-09-30T04:38:39.81Z" }, + { url = "https://files.pythonhosted.org/packages/ff/6c/387b0e4f756a282831c1d9fc6aeb6c51ca4507ca202767c8de15ce9b12e2/charset_normalizer-3.5.2-cp37-abi3-musllinux_1_2_ppc64le.whl", hash = "sha256:9b7f416ff0978e2f2249330527f0ad6fa02f4932e6199692d3b52da2048c19e4", size = 260703, upload-time = "2026-09-30T04:38:41.346Z" }, + { url = "https://files.pythonhosted.org/packages/96/92/1fdf015f09ef449f50d3ac4b67c90887c9c318b727daa95cc4f866e6521d/charset_normalizer-3.5.2-cp37-abi3-musllinux_1_2_riscv64.whl", hash = "sha256:01077390b03f7988f11d700a2194e69b119741a86b1a638b1db88891e3eced8e", size = 247622, upload-time = "2026-09-30T04:38:42.937Z" }, + { url = "https://files.pythonhosted.org/packages/dc/3c/8e7b8a5671ad5d433669fb2a76f1a0164df2d9b1718b0206bc2a16d840cc/charset_normalizer-3.5.2-cp37-abi3-musllinux_1_2_s390x.whl", hash = "sha256:7e841fb9010836c992c9f12fcbd43a831de93a5f726fc1ccd8ca1d0268c5014c", size = 257500, upload-time = "2026-09-30T04:38:44.604Z" }, + { url = "https://files.pythonhosted.org/packages/b4/f0/45b579df5cabc1d5d53ea1cc35e8437d3ca768c0acccc7041517cb6fbb32/charset_normalizer-3.5.2-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:9cae88599c7219005d879f98e5ed53341e9a122af585e1091200358a3003d2a0", size = 255100, upload-time = "2026-09-30T04:38:46.289Z" }, + { url = "https://files.pythonhosted.org/packages/31/68/fdec18a343f5fb3f310588dd478b09ac4799e0b187dbade3a8cd776f03ef/charset_normalizer-3.5.2-cp37-abi3-win32.whl", hash = "sha256:01b0c0d2262a9e28e8484a278c7e1b5d650e3ac8cf2683d2967e25899f208bdf", size = 174499, upload-time = "2026-09-30T04:38:47.999Z" }, + { url = "https://files.pythonhosted.org/packages/9d/8a/b618149cc5207943a0242068d7a27897f56a62947b5a039085f2a22029f8/charset_normalizer-3.5.2-cp37-abi3-win_amd64.whl", hash = "sha256:9f56f72050826f63dcee7a7f55b0a77168cb3bfc553fd405e7f8f9ece75a4036", size = 200092, upload-time = "2026-09-30T04:38:49.707Z" }, + { url = "https://files.pythonhosted.org/packages/03/cf/4c66866fa9e2b1c78e3c911516d1de497a677b7ac60f1eceda74ce777ca3/charset_normalizer-3.5.2-cp37-abi3-win_arm64.whl", hash = "sha256:40ab6bffa02ae10a0581e6c198be7d2d8ca5c2a0c64e4ed3465d766df457573e", size = 294363, upload-time = "2026-09-30T04:38:51.312Z" }, + { url = "https://files.pythonhosted.org/packages/fc/ad/d07d7862a62ffa6d79d68074d14823243dd235a77c45262acbf6adeb28bf/charset_normalizer-3.5.2-py3-none-any.whl", hash = "sha256:b6b751274acb69d77b3323d6b7dbaa3c7fdfc1eb829b7eb61d262f32e1af9685", size = 68872, upload-time = "2026-09-30T04:39:21.828Z" }, +] + +[[package]] +name = "click" +version = "8.5.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c7/0e/7fa0ef50764b67090eca4114772a2abf8b6148198475e54c660b97caeee6/click-8.5.0.tar.gz", hash = "sha256:ba0d2089de75ea0310e2dde03160e6ca10009947fb95a182f9b54021bb272e34", size = 382235, upload-time = "2026-08-26T13:33:14.56Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/58/50/6c0d534c5f134586a8e1ba4e330569e32f057e33372ae556463212fb4cd3/click-8.5.0-py3-none-any.whl", hash = "sha256:255bc9599cf7748b4b1a446ccc735421bd08a2ae529a8b88597d3de5664ee360", size = 125251, upload-time = "2026-08-26T13:33:12.928Z" }, +] + +[[package]] +name = "fastapi" +version = "0.141.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-doc" }, + { name = "pydantic" }, + { name = "starlette" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8a/02/91e3416a8fdd715abb903a952a6bec7cdd8d14eed55d415fc8595524c319/fastapi-0.141.1.tar.gz", hash = "sha256:e8822fc40db1e1858054d7a949a888695bc9bdce70139178e33bd2871a453ca1", size = 425799, upload-time = "2026-07-29T17:18:05.568Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/03/10388a42375ee7e4ac9b94eb2c5c569c8b5795e377e701c9ac3ad63de890/fastapi-0.141.1-py3-none-any.whl", hash = "sha256:bfb91aa2d334c61cb35ba9a116fc123b3d3df31640b801cf57a7a78ec3f603b3", size = 131954, upload-time = "2026-07-29T17:18:04.364Z" }, +] + +[[package]] +name = "googleapis-common-protos" +version = "1.76.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/62/b6/bc095b23a3e0d978e75c8c18cabe4c11f791837410670e4486f0d539bfba/googleapis_common_protos-1.76.0.tar.gz", hash = "sha256:71db0e44a1d4c2dd024ebad44ca67129cc1e2892e4861df466764749d5ef8642", size = 156250, upload-time = "2026-10-22T23:43:36.263Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/05/6d/78dc67a9e779e594b49bf62604034d3a6aa428a55afc729cafda4b7333e1/googleapis_common_protos-1.76.0-py3-none-any.whl", hash = "sha256:59e13344a9f3651aa201a03e2bee815a3bf43a4baf1f9ff8a9b62a840ce25fac", size = 307707, upload-time = "2026-10-22T23:43:10.445Z" }, +] + +[[package]] +name = "h11" +version = "0.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/ee/02a2c011bdab74c6fb3c75474d40b3052059d95df7e73351460c8588d963/h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1", size = 101250, upload-time = "2025-04-24T03:35:25.427Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, +] + +[[package]] +name = "idna" +version = "3.20" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f5/08/8eea9d4b8302028f3abb2c0813953f7aec26d33b7a8960ed760e65ff29fa/idna-3.20.tar.gz", hash = "sha256:a7db850025b95ded1eae8a46181a1a6c56c92c96f0e2b005d9ff8dc0210cab44", size = 216463, upload-time = "2026-09-17T14:11:04.752Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/58/a2/bb081bab032533a855d44de1d56f8e8426114ff1ba5d1f07a438a0a654f8/idna-3.20-py3-none-any.whl", hash = "sha256:ab7ae7122974553370f0bdb919e1a960b2cd1bc1ef0276416d896db81c14582c", size = 69583, upload-time = "2026-09-17T14:11:03.168Z" }, +] + +[[package]] +name = "importlib-metadata" +version = "8.7.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "zipp" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f3/49/3b30cad09e7771a4982d9975a8cbf64f00d4a1ececb53297f1d9a7be1b10/importlib_metadata-8.7.1.tar.gz", hash = "sha256:49fef1ae6440c182052f407c8d34a68f72efc36db9ca90dc0113398f2fdde8bb", size = 57107, upload-time = "2025-12-21T10:00:19.278Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fa/5e/f8e9a1d23b9c20a551a8a02ea3637b4642e22c2626e3a13a9a29cdea99eb/importlib_metadata-8.7.1-py3-none-any.whl", hash = "sha256:5a1f80bf1daa489495071efbb095d75a634cf28a8bc299581244063b53176151", size = 27865, upload-time = "2025-12-21T10:00:18.329Z" }, +] + +[[package]] +name = "jmespath" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d3/59/322338183ecda247fb5d1763a6cbe46eff7222eaeebafd9fa65d4bf5cb11/jmespath-1.1.0.tar.gz", hash = "sha256:472c87d80f36026ae83c6ddd0f1d05d4e510134ed462851fd5f754c8c3cbb88d", size = 27377, upload-time = "2026-01-22T16:35:26.279Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/14/2f/967ba146e6d58cf6a652da73885f52fc68001525b4197effc174321d70b4/jmespath-1.1.0-py3-none-any.whl", hash = "sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64", size = 20419, upload-time = "2026-01-22T16:35:24.919Z" }, +] + +[[package]] +name = "mtusi2026lab6" +version = "1.6.0" +source = { virtual = "." } +dependencies = [ + { name = "argon2-cffi" }, + { name = "asyncpg" }, + { name = "boto3" }, + { name = "fastapi" }, + { name = "opentelemetry-exporter-otlp-proto-http" }, + { name = "opentelemetry-sdk" }, + { name = "pika" }, + { name = "pillow" }, + { name = "uvicorn" }, +] + +[package.metadata] +requires-dist = [ + { name = "argon2-cffi", specifier = "==25.1.0" }, + { name = "asyncpg", specifier = "==0.31.0" }, + { name = "boto3", specifier = "==1.43.115" }, + { name = "fastapi", specifier = "==0.141.1" }, + { name = "opentelemetry-exporter-otlp-proto-http", specifier = "==1.36.0" }, + { name = "opentelemetry-sdk", specifier = "==1.36.0" }, + { name = "pika", specifier = "==1.3.2" }, + { name = "pillow", specifier = "==13.0.0" }, + { name = "uvicorn", specifier = "==0.40.0" }, +] + +[[package]] +name = "opentelemetry-api" +version = "1.36.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "importlib-metadata" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/27/d2/c782c88b8afbf961d6972428821c302bd1e9e7bc361352172f0ca31296e2/opentelemetry_api-1.36.0.tar.gz", hash = "sha256:9a72572b9c416d004d492cbc6e61962c0501eaf945ece9b5a0f56597d8348aa0", size = 64780, upload-time = "2025-07-29T15:12:06.02Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bb/ee/6b08dde0a022c463b88f55ae81149584b125a42183407dc1045c486cc870/opentelemetry_api-1.36.0-py3-none-any.whl", hash = "sha256:02f20bcacf666e1333b6b1f04e647dc1d5111f86b8e510238fcc56d7762cda8c", size = 65564, upload-time = "2025-07-29T15:11:47.998Z" }, +] + +[[package]] +name = "opentelemetry-exporter-otlp-proto-common" +version = "1.36.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-proto" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/34/da/7747e57eb341c59886052d733072bc878424bf20f1d8cf203d508bbece5b/opentelemetry_exporter_otlp_proto_common-1.36.0.tar.gz", hash = "sha256:6c496ccbcbe26b04653cecadd92f73659b814c6e3579af157d8716e5f9f25cbf", size = 20302, upload-time = "2025-07-29T15:12:07.71Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d0/ed/22290dca7db78eb32e0101738366b5bbda00d0407f00feffb9bf8c3fdf87/opentelemetry_exporter_otlp_proto_common-1.36.0-py3-none-any.whl", hash = "sha256:0fc002a6ed63eac235ada9aa7056e5492e9a71728214a61745f6ad04b923f840", size = 18349, upload-time = "2025-07-29T15:11:51.327Z" }, +] + +[[package]] +name = "opentelemetry-exporter-otlp-proto-http" +version = "1.36.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "googleapis-common-protos" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-exporter-otlp-proto-common" }, + { name = "opentelemetry-proto" }, + { name = "opentelemetry-sdk" }, + { name = "requests" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/25/85/6632e7e5700ba1ce5b8a065315f92c1e6d787ccc4fb2bdab15139eaefc82/opentelemetry_exporter_otlp_proto_http-1.36.0.tar.gz", hash = "sha256:dd3637f72f774b9fc9608ab1ac479f8b44d09b6fb5b2f3df68a24ad1da7d356e", size = 16213, upload-time = "2025-07-29T15:12:08.932Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7f/41/a680d38b34f8f5ddbd78ed9f0042e1cc712d58ec7531924d71cb1e6c629d/opentelemetry_exporter_otlp_proto_http-1.36.0-py3-none-any.whl", hash = "sha256:3d769f68e2267e7abe4527f70deb6f598f40be3ea34c6adc35789bea94a32902", size = 18752, upload-time = "2025-07-29T15:11:53.164Z" }, +] + +[[package]] +name = "opentelemetry-proto" +version = "1.36.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fd/02/f6556142301d136e3b7e95ab8ea6a5d9dc28d879a99f3dd673b5f97dca06/opentelemetry_proto-1.36.0.tar.gz", hash = "sha256:0f10b3c72f74c91e0764a5ec88fd8f1c368ea5d9c64639fb455e2854ef87dd2f", size = 46152, upload-time = "2025-07-29T15:12:15.717Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/57/3361e06136225be8180e879199caea520f38026f8071366241ac458beb8d/opentelemetry_proto-1.36.0-py3-none-any.whl", hash = "sha256:151b3bf73a09f94afc658497cf77d45a565606f62ce0c17acb08cd9937ca206e", size = 72537, upload-time = "2025-07-29T15:12:02.243Z" }, +] + +[[package]] +name = "opentelemetry-sdk" +version = "1.36.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "opentelemetry-semantic-conventions" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/4c/85/8567a966b85a2d3f971c4d42f781c305b2b91c043724fa08fd37d158e9dc/opentelemetry_sdk-1.36.0.tar.gz", hash = "sha256:19c8c81599f51b71670661ff7495c905d8fdf6976e41622d5245b791b06fa581", size = 162557, upload-time = "2025-07-29T15:12:16.76Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0b/59/7bed362ad1137ba5886dac8439e84cd2df6d087be7c09574ece47ae9b22c/opentelemetry_sdk-1.36.0-py3-none-any.whl", hash = "sha256:19fe048b42e98c5c1ffe85b569b7073576ad4ce0bcb6e9b4c6a39e890a6c45fb", size = 119995, upload-time = "2025-07-29T15:12:03.181Z" }, +] + +[[package]] +name = "opentelemetry-semantic-conventions" +version = "0.57b0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7e/31/67dfa252ee88476a29200b0255bda8dfc2cf07b56ad66dc9a6221f7dc787/opentelemetry_semantic_conventions-0.57b0.tar.gz", hash = "sha256:609a4a79c7891b4620d64c7aac6898f872d790d75f22019913a660756f27ff32", size = 124225, upload-time = "2025-07-29T15:12:17.873Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/05/75/7d591371c6c39c73de5ce5da5a2cc7b72d1d1cd3f8f4638f553c01c37b11/opentelemetry_semantic_conventions-0.57b0-py3-none-any.whl", hash = "sha256:757f7e76293294f124c827e514c2a3144f191ef175b069ce8d1211e1e38e9e78", size = 201627, upload-time = "2025-07-29T15:12:04.174Z" }, +] + +[[package]] +name = "pika" +version = "1.3.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/db/db/d4102f356af18f316c67f2cead8ece307f731dd63140e2c71f170ddacf9b/pika-1.3.2.tar.gz", hash = "sha256:b2a327ddddf8570b4965b3576ac77091b850262d34ce8c1d8cb4e4146aa4145f", size = 145029, upload-time = "2023-05-05T14:25:43.368Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f9/f3/f412836ec714d36f0f4ab581b84c491e3f42c6b5b97a6c6ed1817f3c16d0/pika-1.3.2-py3-none-any.whl", hash = "sha256:0779a7c1fafd805672796085560d290213a465e4f6f76a6fb19e378d8041a14f", size = 155415, upload-time = "2023-05-05T14:25:41.484Z" }, +] + +[[package]] +name = "pillow" +version = "13.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ce/a4/9cc30690a394a8305e8be4c9036aa16d0dfddadd09079860b582f562496d/pillow-13.0.0.tar.gz", hash = "sha256:cdcd1ba258178f7356d74170c9a81ba4eab6a5d55748bc402a5a2a6fe6be3510", size = 47058631, upload-time = "2026-10-16T22:18:09.718Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7a/b8/58d79332229482eb0d27c82973850808c19d37e5bdc14443d3417bd373f5/pillow-13.0.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:67c8d44ad89fa7eb88de5d6ef6199b1e03dd24974f85a67639fc8badb801fe79", size = 5568643, upload-time = "2026-10-16T22:15:44.474Z" }, + { url = "https://files.pythonhosted.org/packages/39/4b/f2ed53d83aa44ac19599ac0a54897f867fafb95f406980d1e24083f8cbe3/pillow-13.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc481f742ccda20e9ed760b8f978dff50f2a4e9bc253ac167fb0a05b0783ef52", size = 4852329, upload-time = "2026-10-16T22:15:46.284Z" }, + { url = "https://files.pythonhosted.org/packages/3b/bb/9e30e8ed18c20800ee327a12107d5249ad8dc444867b3adfcc21654e3c71/pillow-13.0.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8bf295fe459343bc6093c8e5e0df60bfa189e0fcb775a22e2c2488fdeccc6c0d", size = 6519309, upload-time = "2026-10-16T22:15:48.14Z" }, + { url = "https://files.pythonhosted.org/packages/e6/99/fdf65cc539e266ab93546c261bdf98411fe165c6e4e13f9dbf597097192b/pillow-13.0.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:42d63be3af4f9b951b199b0a9d23365d37ab14687e6d77b74940432c8515c4ca", size = 7144542, upload-time = "2026-10-16T22:15:50.093Z" }, + { url = "https://files.pythonhosted.org/packages/26/9b/f38b7c4d8f599537e2d1a6d546cec9beef7536ff8f5c49a3f9c78179d8ac/pillow-13.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4bd024d567eca931b668978d672dba914d4b8e2628e2c3073a2ad178de8a2620", size = 6618042, upload-time = "2026-10-16T22:15:52.054Z" }, + { url = "https://files.pythonhosted.org/packages/a8/7c/45ee28531f89ee4582967d8a17f7a26d71ca64cd902b93c269590cd60a00/pillow-13.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:23edf4ec4b6f4f240e8df6e4985e9ee2ced86474632433e55fb1e2548744bf40", size = 7228137, upload-time = "2026-10-16T22:15:54.009Z" }, + { url = "https://files.pythonhosted.org/packages/01/bc/9c2a969cb1b70201d04fa17fd5aedf02a732870b8da86bceaaa30b0ca385/pillow-13.0.0-cp312-cp312-win32.whl", hash = "sha256:02464da86f561f410e13b061ec37c3068a9cb557a095494157742d07b3c24a27", size = 6515423, upload-time = "2026-10-16T22:15:56.25Z" }, + { url = "https://files.pythonhosted.org/packages/ff/cd/73a79ecdc822076002eb24b5e1e4a530b0bb5310fcb6d239abbec877110b/pillow-13.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:ff3fe9eb621fbb9db4c4be33ab946f6bac5f98ab50ca8d3260503653656c961b", size = 7272166, upload-time = "2026-10-16T22:15:58.597Z" }, + { url = "https://files.pythonhosted.org/packages/4a/a3/be5a1236efd085b3984e4305da8f179ae9dab80c991d2ad78f2bd5c4c843/pillow-13.0.0-cp312-cp312-win_arm64.whl", hash = "sha256:c3bc8ee0271387e0b1e3ad0ff893645990a656e79bf9c42f4f734a3642b11d51", size = 2698943, upload-time = "2026-10-16T22:16:00.637Z" }, + { url = "https://files.pythonhosted.org/packages/82/e2/9132d38df320e8099e2642d9fa538f18a7dfebf68a875a31725dc2a115aa/pillow-13.0.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:e98b4f91f99b24b1da72e10621ba181f7281922384a7403094debb4199de6c00", size = 4190620, upload-time = "2026-10-16T22:16:02.527Z" }, + { url = "https://files.pythonhosted.org/packages/cb/be/436dac0996876f65ef1916ffb711c5ef4af2ebc57414b6f16642e07f6b47/pillow-13.0.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:34fb3db1149c879343861831728c243c432d4a5afc8f1878ca7612ef8de2a033", size = 4290286, upload-time = "2026-10-16T22:16:04.861Z" }, + { url = "https://files.pythonhosted.org/packages/60/16/7c1007162e4169ea0d5bec47e2e0b2174f56c50cf5b8c02a1dc23e41037e/pillow-13.0.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:2d50500a4a3bc6469adcdb319cf05d688b08ca3bf0f22a05339f3b25b433a599", size = 3723169, upload-time = "2026-10-16T22:16:06.956Z" }, + { url = "https://files.pythonhosted.org/packages/08/bc/578ab2eaa04705a8978bd9be0fe15c3d43a8fc95097c4f7e39540e79e4fa/pillow-13.0.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:ef5fe6b7db3f501c56000ba8fe7c7e1a35ca1d45824fbeb95a67daa85cff4dbe", size = 5568469, upload-time = "2026-10-16T22:16:08.664Z" }, + { url = "https://files.pythonhosted.org/packages/98/c8/ccede3dcc4f274b80fb49199eed1ef34ba05566c8b71976b9d1f111da2e3/pillow-13.0.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f11a3c1358bb3c68012a7b1f41b223c3db9f7bfde90b481df028cfe1c3363313", size = 4852187, upload-time = "2026-10-16T22:16:10.693Z" }, + { url = "https://files.pythonhosted.org/packages/fb/3c/cd0770e96d20bfd990db024bcbd43897ff3e88da1f4782cf90ea231b1d4f/pillow-13.0.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:19b66ca0a4d25cc0200b381ce824c7ea090db61427315e73cfb05dbf1b35c369", size = 6515908, upload-time = "2026-10-16T22:16:12.896Z" }, + { url = "https://files.pythonhosted.org/packages/ef/fe/f5cdaa2d1c4086e757430f368c271a58ee29cfe7378e7ae7b9d964015c25/pillow-13.0.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5f56499c1a74d2b862beb0c33b8016f99d095858455df9d21583bbbc1c15b582", size = 7142544, upload-time = "2026-10-16T22:16:14.844Z" }, + { url = "https://files.pythonhosted.org/packages/fb/b5/a54f6d44adbcd23737c8c12953f42be107e4cca64e247c0ca3a5b563061c/pillow-13.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a2a14cd73d5e6c927f644b0110ee81f59c4ef506259ec9183904747f2caacc81", size = 6613097, upload-time = "2026-10-16T22:16:16.893Z" }, + { url = "https://files.pythonhosted.org/packages/e6/b4/9e6c1207c21b2e91db0d15c16d30f3701ce5add1bd4da0e787761e66cb6e/pillow-13.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:08b826a9bf1265c76de04a42061d61ea6b109418b7826f28ff1ecdfaa26c5502", size = 7224846, upload-time = "2026-10-16T22:16:19.195Z" }, + { url = "https://files.pythonhosted.org/packages/78/9f/d3c3cb0518f3dd1eee0ef501535f53d15597a53c5c4118d178086143397a/pillow-13.0.0-cp313-cp313-win32.whl", hash = "sha256:12b0fa15879fe5a9d9eebe66e4e2d74128a773b1a4448010c9c714f45e04b9d8", size = 6516013, upload-time = "2026-10-16T22:16:21.389Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ed/1b16fb66a37e1b9e41370b93b2c614c7da761a5b2a41dc333f7af4639962/pillow-13.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:9dda0788a9932776a75d8bcc020076edc308ce2508b51f6739a45e3158b3fc42", size = 7284777, upload-time = "2026-10-16T22:16:23.393Z" }, + { url = "https://files.pythonhosted.org/packages/87/7d/8d02eba9f7ecc87f8e90f731e032cc4773802f2a843763afd0c66a535ff4/pillow-13.0.0-cp313-cp313-win_arm64.whl", hash = "sha256:40a266371ece581e5de13b6635ef78d7487824af80561b6ca27694a88c19bc0f", size = 2698826, upload-time = "2026-10-16T22:16:25.382Z" }, + { url = "https://files.pythonhosted.org/packages/8e/45/fbad9c7f5e2f1745c373a434d46072bc84eddfefaa28b852425ac957fda6/pillow-13.0.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:44629d4498c9cb6b2051806155f22141df0d8a929c56134b166655232dfb2f8c", size = 4190721, upload-time = "2026-10-16T22:16:27.524Z" }, + { url = "https://files.pythonhosted.org/packages/a2/bf/2074902a259a90692c5dbf132885d21671644cbbd3f690047516243a55db/pillow-13.0.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:bf97ba1b5542050e9b0c74e632323b6d8136cc99d64e193abebee22fe6013c76", size = 4290236, upload-time = "2026-10-16T22:16:29.562Z" }, + { url = "https://files.pythonhosted.org/packages/ff/a5/3d1bb93b8ea38d15d21356c1b8e9f50971ac592aaa248f7fb71739094c0e/pillow-13.0.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:aa694c49787040943570d61c9b8fc01271bd2a2cfdd55f0d896f8aecdce1a981", size = 3723093, upload-time = "2026-10-16T22:16:31.249Z" }, + { url = "https://files.pythonhosted.org/packages/84/a1/be060135d85187b25fa8b2573786c0e45c62ae27527fff5257675eb97f4a/pillow-13.0.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:49c12af515073e9bf511e4ef736743bf70d531a643668722279314b937742ba4", size = 5572253, upload-time = "2026-10-16T22:16:33.103Z" }, + { url = "https://files.pythonhosted.org/packages/b5/f4/0bf6f1ac9ba7ee2ffe9914aeb410a1035a0c1a3c42f122388d7fc950bc4b/pillow-13.0.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:d9b0a2bee3f3e03e9f29057aedb807dbf1e1d96ee1b9d0f422d9db6c8583271d", size = 4852269, upload-time = "2026-10-16T22:16:35.102Z" }, + { url = "https://files.pythonhosted.org/packages/83/16/b603274617b3e6728da3a0133d4fba1ea8d7fbe0c4ea5fef9cbffa398b8b/pillow-13.0.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5933fcea5a5ba64c686f55c886a115cbfd92f2900460784a1e79909fdbafd7b6", size = 6516058, upload-time = "2026-10-16T22:16:37.228Z" }, + { url = "https://files.pythonhosted.org/packages/1d/f2/1371707ce438ba9a43f9d24089828494ec6f9396e28cd7903aa704dd6306/pillow-13.0.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e91c2b1cbeae768c98752e67e717e283d68dae45717b41f2d91eaee648e37b7d", size = 7142574, upload-time = "2026-10-16T22:16:39.172Z" }, + { url = "https://files.pythonhosted.org/packages/b9/a9/62cf68587f910bc87c5951bcb6e9430c7643a65076adf5f18b4bf350e2bd/pillow-13.0.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7f6299e494e46253d40c34653cac369412f44ca53b5f85d848d3f95b1ed0546c", size = 6614295, upload-time = "2026-10-16T22:16:41.117Z" }, + { url = "https://files.pythonhosted.org/packages/6c/c9/9e9b48178649730299be0bafbf5f810a5b6977538f5a92d47e4ba5f5d608/pillow-13.0.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f9bd7d260d986f3553518c5d723544346fa48031f8aef3265d786aeb59845bc0", size = 7224841, upload-time = "2026-10-16T22:16:43.118Z" }, + { url = "https://files.pythonhosted.org/packages/8f/48/c44e91384b1b6c072a714a63d00608acc16a6a968735e478e4be4384e5ac/pillow-13.0.0-cp314-cp314-win32.whl", hash = "sha256:2e31c4e68cffd7b1f116127626813b82fcba19edf6b940a7b0c266d6484d0055", size = 6508694, upload-time = "2026-10-16T22:16:45.883Z" }, + { url = "https://files.pythonhosted.org/packages/50/e2/7ad17747517e4c813dca78257d6bb593466f8231baf1384bc98d92745edb/pillow-13.0.0-cp314-cp314-win_amd64.whl", hash = "sha256:f7cbb7a64e7ec664775195e4f81923bc8a93d9451b963f844e59118f63147a7a", size = 7281830, upload-time = "2026-10-16T22:16:47.725Z" }, + { url = "https://files.pythonhosted.org/packages/41/9f/e5b51f2ce8ef4563215334035403ce61a244f93a2e65c24a940131618f87/pillow-13.0.0-cp314-cp314-win_arm64.whl", hash = "sha256:a6aaa7871aebca081eb5c45f83e845bbd9712ae99fa9286dbfdcee835e63f4b4", size = 2695468, upload-time = "2026-10-16T22:16:49.515Z" }, + { url = "https://files.pythonhosted.org/packages/97/73/a53502f731f6c84805a4ce87a8881e86d07a65aee64728b900d112dc703e/pillow-13.0.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:7b999816b5850a35ada08443e6e03945f1f6488144a83afe2d37b240a552aafa", size = 5578196, upload-time = "2026-10-16T22:16:51.291Z" }, + { url = "https://files.pythonhosted.org/packages/21/50/7e949959a56b5559c78ef172b70fc6025706acb289f3f648a15329e4fa6f/pillow-13.0.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:67871a13cf47275201553b536c025da777f997fa562a19eb798af4ad4b97c18f", size = 4857276, upload-time = "2026-10-16T22:16:53.399Z" }, + { url = "https://files.pythonhosted.org/packages/c9/52/43ded0ac08e77e39d17bd6d894162572201b5e8fdc4a022b14c55a1960a4/pillow-13.0.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e8ca8f69b7c77fa96dad99f8638c1fc4f1c0bed4acaa1baf7a3c31bdb80a0ab1", size = 6607008, upload-time = "2026-10-16T22:16:55.525Z" }, + { url = "https://files.pythonhosted.org/packages/0d/2e/4a57ed216d5bf877387cba4d44c02aa87abe51ad2e06404aa6b597c4da39/pillow-13.0.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6bce6a29c3898191f4c9998e4d4e8f69790caeabde46e54da4bc235ce5bde683", size = 7224281, upload-time = "2026-10-16T22:16:57.738Z" }, + { url = "https://files.pythonhosted.org/packages/5c/3d/2929b248e135752d63aaeabf53f6de797e905b7b6263f9c1e2adade20aba/pillow-13.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:816752a5ef76a80484c8c441c85593b7ea9304f6231fc5776ad1fbff7e773058", size = 6695600, upload-time = "2026-10-16T22:16:59.842Z" }, + { url = "https://files.pythonhosted.org/packages/bc/33/fb0da992cf084ed1ee6ee05e96b6191e35c351eb707ca2df874feb24ce47/pillow-13.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5bdb38ba220b61b4ec31dd1046dea6b016d3c6850a23f43ed8b58332b2fc668e", size = 7300589, upload-time = "2026-10-16T22:17:01.686Z" }, + { url = "https://files.pythonhosted.org/packages/bb/af/c30dd9ef6a3dce84ecb1b2393cac629047a32e66279056c6cb46c3ffd65c/pillow-13.0.0-cp314-cp314t-win32.whl", hash = "sha256:06903aac074d00c988e77a1fc435286dc498506d4ca8d6f69df9babddf4ecc15", size = 6517204, upload-time = "2026-10-16T22:17:03.722Z" }, + { url = "https://files.pythonhosted.org/packages/85/d5/ef3a8ae7609c8ab26570abf18194882534b480d245a9148b895a2262c004/pillow-13.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:9a635068d9793c7f0d929f39f18b58f1bd86c236e8e7b64167b86af9232ebca0", size = 7286695, upload-time = "2026-10-16T22:17:05.543Z" }, + { url = "https://files.pythonhosted.org/packages/84/3c/0861db0c69a03689ebe3e7599d35668a414deb5aaafd8255215e4e62e3d3/pillow-13.0.0-cp314-cp314t-win_arm64.whl", hash = "sha256:0d1a9cf7bc6917538b20f8d06b9ab2147e542f68849e96fc58eb66bdf6acc692", size = 2698098, upload-time = "2026-10-16T22:17:07.674Z" }, + { url = "https://files.pythonhosted.org/packages/6e/eb/e86a41962a3a31ad495a3d6a912d9002b01bba4a6fa71c28703abdca7f69/pillow-13.0.0-cp315-cp315-ios_13_0_arm64_iphoneos.whl", hash = "sha256:7d5e097eabd5d4915c27ac9f6e9325bb1e01a7c94d7348b5c4451d3fc30e8521", size = 4191213, upload-time = "2026-10-16T22:17:09.449Z" }, + { url = "https://files.pythonhosted.org/packages/dd/04/7e7445e706afec84ef06dc25b2c77f9cecff242c23730c62928ad1089d4a/pillow-13.0.0-cp315-cp315-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:51ebb529fc5e8abdd55302f6c3f8943037348ec2384329a3e55677f69ae2fdfc", size = 4290650, upload-time = "2026-10-16T22:17:11.196Z" }, + { url = "https://files.pythonhosted.org/packages/46/85/d534089068ac40c0f61f650389a296c812eab745a8a733f81feec39ae955/pillow-13.0.0-cp315-cp315-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:c87a04e7f152ee4db5dbe15f0bfa3fe628a1aeedbe66b81c43b87f52c813f9ac", size = 3723078, upload-time = "2026-10-16T22:17:12.772Z" }, + { url = "https://files.pythonhosted.org/packages/ce/2b/00b5c3b401df538b5dbe311d3899730fae5ba1e21cb8b80a128887ac6747/pillow-13.0.0-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:20722cd5375533ad754053363a80b8fb30c79d37930a6f7228030e237ce3bd5d", size = 5572292, upload-time = "2026-10-16T22:17:14.539Z" }, + { url = "https://files.pythonhosted.org/packages/72/f5/a6b0610f946e9b54ae888ba7ad29d6e2f8d9b11f69e212d70416b7186c09/pillow-13.0.0-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:04a63720f79e22fd54d897c233a68314c50ca0603360f54ce2bd5db8f0f43782", size = 4852222, upload-time = "2026-10-16T22:17:16.521Z" }, + { url = "https://files.pythonhosted.org/packages/1e/d0/c0ff25bb0427ec70f32aac995453321fc3c4494a46c7292ff02d9d260b04/pillow-13.0.0-cp315-cp315-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4f8db54177035a2b5f717f94475b74907e12a96dc74f6d487f3966b0a4694071", size = 6527349, upload-time = "2026-10-16T22:17:18.534Z" }, + { url = "https://files.pythonhosted.org/packages/85/11/e46fa0bcc0fc117f1eacc206b3d51f04d583fab328b4b740672d0438d83f/pillow-13.0.0-cp315-cp315-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8f36aa77f74b14a2103622b196402c71330e4e7cf4122ee45c854e9a01e0eecf", size = 7152873, upload-time = "2026-10-16T22:17:20.415Z" }, + { url = "https://files.pythonhosted.org/packages/7c/53/2bdeca705f3a58679fa9201233193c86a30238340ca29b3a152c3fcc719b/pillow-13.0.0-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:149a1c323ba366f90b48e6f1a6db6582eb50512df374588de56fc909abc0f0f3", size = 6626871, upload-time = "2026-10-16T22:17:22.361Z" }, + { url = "https://files.pythonhosted.org/packages/14/de/3829030f209722ade3753bb456b805a0e7f885cf885b483407365c82e9ca/pillow-13.0.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:e892354a424b1e82a78ba16785822bdd40de06f19446a3d4a28047df1a4f8932", size = 7236439, upload-time = "2026-10-16T22:17:24.352Z" }, + { url = "https://files.pythonhosted.org/packages/44/b1/67fc37388382bc036a74f04da5bb6fcbf90ec3ac1f45893e06176c373fd3/pillow-13.0.0-cp315-cp315-win32.whl", hash = "sha256:8e459b613cf31f8980f584387b4fe47bf08e076e288d61aa8eba1f7fd4ddbb81", size = 6508688, upload-time = "2026-10-16T22:17:26.314Z" }, + { url = "https://files.pythonhosted.org/packages/44/05/fdb6a79aee10861165526b41b7b440e36ee1ff58613b2fbefe3acbf06d4f/pillow-13.0.0-cp315-cp315-win_amd64.whl", hash = "sha256:d4737f50d23ddd11d6685faacbb9252887dde0bc7997127757247983d74399d6", size = 7281107, upload-time = "2026-10-16T22:17:28.198Z" }, + { url = "https://files.pythonhosted.org/packages/e6/9c/6c6f7dc2a566467baf121394e83de25ac287cffe34cc030726beb4be0224/pillow-13.0.0-cp315-cp315-win_arm64.whl", hash = "sha256:2310e2adfc13c6d457d7d0f2ffdd5c56a74a317650cf765887df8d515a647ac0", size = 2695513, upload-time = "2026-10-16T22:17:30.013Z" }, + { url = "https://files.pythonhosted.org/packages/23/16/e9120defdb8bff693b582d035312acc6f530cdc2bc5238a87f410efccd82/pillow-13.0.0-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:21c32061a5512dc13f87cc7303e120e4d77c2f70df89a35af0e3ce246e5ac57f", size = 5578129, upload-time = "2026-10-16T22:17:31.642Z" }, + { url = "https://files.pythonhosted.org/packages/47/22/76faf2bb2448f48a84cf894c1e2bb1090d7d2b1aa521ece131cb2f05d631/pillow-13.0.0-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:1c774b9c0ba0d0b356f71530677b39028878a2cae5a93e0a13779cfd17120f88", size = 4857428, upload-time = "2026-10-16T22:17:33.43Z" }, + { url = "https://files.pythonhosted.org/packages/b3/64/0861ea4e03c50b14af9ed9dd8b9f388ab42f445c623a9fcbab19aef7aa55/pillow-13.0.0-cp315-cp315t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:20822140c9a2ba9c5e42df342b0a892ab2e102fc0e3db967a133bb9ce3f898bb", size = 6612582, upload-time = "2026-10-16T22:17:35.415Z" }, + { url = "https://files.pythonhosted.org/packages/5c/20/cd1e9b3f60a9f3d5ebe46da36955c26004092a501f7f2c42f9fcc26611bc/pillow-13.0.0-cp315-cp315t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ffea6b2986a797c84b6d1d27d4f43e0b83c73fc16281a930891cd237cf0d08fe", size = 7230176, upload-time = "2026-10-16T22:17:37.362Z" }, + { url = "https://files.pythonhosted.org/packages/b6/ef/ffb0a2fc85690714fb7ac662956949645d57f15bf4426f520da361e7b6e3/pillow-13.0.0-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:05cd2cd39055d0b9650060e4513ef4a9a0aea870083950bd046a0d6490f93a22", size = 6703640, upload-time = "2026-10-16T22:17:39.313Z" }, + { url = "https://files.pythonhosted.org/packages/e0/c2/6402ee3ccb3363aeed8fdb3f9fe56703ac5377a539f0b81b53999d7e121f/pillow-13.0.0-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:c6fd031333dedcad7ce3e03d244093f43504f209e86925ebdb2b868fa4037630", size = 7309231, upload-time = "2026-10-16T22:17:41.021Z" }, + { url = "https://files.pythonhosted.org/packages/fc/96/d6b82e8b8c06ad143b66b12e0b31051e4bf489b4fabca1f390a14fe1ebde/pillow-13.0.0-cp315-cp315t-win32.whl", hash = "sha256:c50c80e3591e5cf9a8dfa6007d4f10732f02e32263850999286e56ccd650cfb2", size = 6517200, upload-time = "2026-10-16T22:17:42.886Z" }, + { url = "https://files.pythonhosted.org/packages/60/d4/0062e6005e9b270b165015aafd343364ac378ec5a685431ea42baad33357/pillow-13.0.0-cp315-cp315t-win_amd64.whl", hash = "sha256:adb3c2124c68e480f4632d06cb7caa3b48a51678475f483b4edb1c6dee29ea43", size = 7287488, upload-time = "2026-10-16T22:17:45.56Z" }, + { url = "https://files.pythonhosted.org/packages/e4/81/b8eb349746be7bce47dd95337c6634e84910af91b59aaa9159c034c614fa/pillow-13.0.0-cp315-cp315t-win_arm64.whl", hash = "sha256:4a25756244bdcce3b50580baadf320fa48297f0eeb8e114ba64139a40d4d0fc8", size = 2698073, upload-time = "2026-10-16T22:17:47.556Z" }, + { url = "https://files.pythonhosted.org/packages/60/da/f6d644e631c5b370c5d0b899f3ccadd2780aab070b29a110527cf4e1fd2c/pillow-13.0.0-pp312-pypy312_pp80-macosx_10_15_x86_64.whl", hash = "sha256:8619685cad8efc4c38c66d3b36aad854523297c43d748b3b6f0cc03c90ba3463", size = 5523188, upload-time = "2026-10-16T22:17:59.334Z" }, + { url = "https://files.pythonhosted.org/packages/e4/b0/2594f202d743a35d4fe80682bfd96f277e5ca7aecb27cf64ef5bfc1d000b/pillow-13.0.0-pp312-pypy312_pp80-macosx_11_0_arm64.whl", hash = "sha256:7e91e97a929232cdb828889ee6ad0e48d7f40f4ec38818f4e4a8968b2b1af22e", size = 4806808, upload-time = "2026-10-16T22:18:01.188Z" }, + { url = "https://files.pythonhosted.org/packages/8e/ed/d504165ef79af6fafe25c76726cf30f544cc236002f7ed054f1d51adc4ff/pillow-13.0.0-pp312-pypy312_pp80-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:93ff9c9c3663ce23aaa5d0b6b6abac55359e2a0c06c7e30f3771829af816be63", size = 5464358, upload-time = "2026-10-16T22:18:02.779Z" }, + { url = "https://files.pythonhosted.org/packages/7b/fe/a10aba5503ca13e9e009b8bd0a1a4ce5584bdce246245fd6f3a83701369c/pillow-13.0.0-pp312-pypy312_pp80-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e65dd6e75de67b2362a4fa3e16aa627595f65054dc4ec73d64680c84d592fa61", size = 6043514, upload-time = "2026-10-16T22:18:04.538Z" }, + { url = "https://files.pythonhosted.org/packages/c0/37/66b12dc476041fe523bca29c9d58fbf1ea92a4eaf73acfb9e993948f1191/pillow-13.0.0-pp312-pypy312_pp80-win_amd64.whl", hash = "sha256:567f20f1c75897d7663830da6ea8af719e03d712b1d9ee7f813319f9ece42e5a", size = 7366030, upload-time = "2026-10-16T22:18:06.638Z" }, +] + +[[package]] +name = "protobuf" +version = "6.33.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/66/70/e908e9c5e52ef7c3a6c7902c9dfbb34c7e29c25d2f81ade3856445fd5c94/protobuf-6.33.6.tar.gz", hash = "sha256:a6768d25248312c297558af96a9f9c929e8c4cee0659cb07e780731095f38135", size = 444531, upload-time = "2026-03-18T19:05:00.988Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fc/9f/2f509339e89cfa6f6a4c4ff50438db9ca488dec341f7e454adad60150b00/protobuf-6.33.6-cp310-abi3-win32.whl", hash = "sha256:7d29d9b65f8afef196f8334e80d6bc1d5d4adedb449971fefd3723824e6e77d3", size = 425739, upload-time = "2026-03-18T19:04:48.373Z" }, + { url = "https://files.pythonhosted.org/packages/76/5d/683efcd4798e0030c1bab27374fd13a89f7c2515fb1f3123efdfaa5eab57/protobuf-6.33.6-cp310-abi3-win_amd64.whl", hash = "sha256:0cd27b587afca21b7cfa59a74dcbd48a50f0a6400cfb59391340ad729d91d326", size = 437089, upload-time = "2026-03-18T19:04:50.381Z" }, + { url = "https://files.pythonhosted.org/packages/5c/01/a3c3ed5cd186f39e7880f8303cc51385a198a81469d53d0fdecf1f64d929/protobuf-6.33.6-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:9720e6961b251bde64edfdab7d500725a2af5280f3f4c87e57c0208376aa8c3a", size = 427737, upload-time = "2026-03-18T19:04:51.866Z" }, + { url = "https://files.pythonhosted.org/packages/ee/90/b3c01fdec7d2f627b3a6884243ba328c1217ed2d978def5c12dc50d328a3/protobuf-6.33.6-cp39-abi3-manylinux2014_aarch64.whl", hash = "sha256:e2afbae9b8e1825e3529f88d514754e094278bb95eadc0e199751cdd9a2e82a2", size = 324610, upload-time = "2026-03-18T19:04:53.096Z" }, + { url = "https://files.pythonhosted.org/packages/9b/ca/25afc144934014700c52e05103c2421997482d561f3101ff352e1292fb81/protobuf-6.33.6-cp39-abi3-manylinux2014_s390x.whl", hash = "sha256:c96c37eec15086b79762ed265d59ab204dabc53056e3443e702d2681f4b39ce3", size = 339381, upload-time = "2026-03-18T19:04:54.616Z" }, + { url = "https://files.pythonhosted.org/packages/16/92/d1e32e3e0d894fe00b15ce28ad4944ab692713f2e7f0a99787405e43533a/protobuf-6.33.6-cp39-abi3-manylinux2014_x86_64.whl", hash = "sha256:e9db7e292e0ab79dd108d7f1a94fe31601ce1ee3f7b79e0692043423020b0593", size = 323436, upload-time = "2026-03-18T19:04:55.768Z" }, + { url = "https://files.pythonhosted.org/packages/c4/72/02445137af02769918a93807b2b7890047c32bfb9f90371cbc12688819eb/protobuf-6.33.6-py3-none-any.whl", hash = "sha256:77179e006c476e69bf8e8ce866640091ec42e1beb80b213c3900006ecfba6901", size = 170656, upload-time = "2026-03-18T19:04:59.826Z" }, +] + +[[package]] +name = "pycparser" +version = "3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ac/d3/eb1d3bc30dda12f7e69640ae2ac8cb10240b71fb73024ad528b7d2ae73da/pycparser-3.1.tar.gz", hash = "sha256:b3fc6dec06a8b2fefa0ed4ff92285306a5e3be9987bc5603c9edbdc4e492418f", size = 105946, upload-time = "2026-10-08T12:26:34.349Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/99/ce/b3ae9ee0324d991c860187be2a6ee436d27a3d02397eaddbb101ec901f3d/pycparser-3.1-py3-none-any.whl", hash = "sha256:f09d358c840bd147b79e55f2bc494f18ea869dc897f5852a8f5766b74f787882", size = 48709, upload-time = "2026-10-08T12:26:32.97Z" }, +] + +[[package]] +name = "pydantic" +version = "2.13.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-types" }, + { name = "pydantic-core" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/53/ef/fc4f868f4e2cee79f863883abffceff107875f569b848507319842d2a681/pydantic-2.13.5.tar.gz", hash = "sha256:51a9c5f7b2f8e636f04c6cada605d9b6a3bf1348fdf945a3d8869b19bba0ee08", size = 845750, upload-time = "2026-08-28T14:04:00.916Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/eb/47/c95ffc2009878c7aac0c5e08528022dcb885933252a88b5f170058014464/pydantic-2.13.5-py3-none-any.whl", hash = "sha256:346a034f080da3755d8e9cb5e00e8b07de1d39e4f6e2c87d8ab7cafa0b269a73", size = 472589, upload-time = "2026-08-28T14:03:59.136Z" }, +] + +[[package]] +name = "pydantic-core" +version = "2.46.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/af/f9/8a06bea35ef8daf588f707784c973a7046e0034c8d8cfb08828eeffb8b75/pydantic_core-2.46.5.tar.gz", hash = "sha256:10416c15b8839ecc4ef4d0885da76da6fd0f67333a0eb8aff6d93c4b8f2910fc", size = 472262, upload-time = "2026-08-28T10:01:31.677Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/82/3f/76358795aa7a8c6d4f36e2cb828ad1c90ee118e1393a9281664f5aade9d4/pydantic_core-2.46.5-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:b9fe6fb92520e3fd61f2e49000b6911b188824f089b75973ea06d6267f0b476d", size = 2076516, upload-time = "2026-08-28T09:58:21.576Z" }, + { url = "https://files.pythonhosted.org/packages/db/50/26b091836076ce4cb2fac264186936acc069e0595772cfd02a563bc4761a/pydantic_core-2.46.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a39ac25a9a2fa4072efdb429833c4a4c8009a51ff9eea3eeae131713cd27991e", size = 1922874, upload-time = "2026-08-28T09:58:23.766Z" }, + { url = "https://files.pythonhosted.org/packages/09/f0/2a8ce3849e299d44e2d2c196b6082643a3235565a735cb51db7a6261f614/pydantic_core-2.46.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4fdc8b93a41521988916eeaa271173fcca7fa0803d62f87675aac8dcec1c8e29", size = 1951772, upload-time = "2026-08-28T09:58:25.435Z" }, + { url = "https://files.pythonhosted.org/packages/87/46/ac0dc8bdd9e6048183a14eb127764e7ad9240021c17513074a4711b0e31e/pydantic_core-2.46.5-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b98134087d9de723658d17a42c7d0da8d6e2ef08015dee7dc93889047315f5e4", size = 2031832, upload-time = "2026-08-28T09:58:27.102Z" }, + { url = "https://files.pythonhosted.org/packages/c4/c2/339de5bef7be36301a2231eaa52e62163742c2281f11b5f4892bc79785cd/pydantic_core-2.46.5-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e652ab17569c94bff5475520f907b7148b8c24036a8ebbe5cf7cf7493d28579a", size = 2208645, upload-time = "2026-08-28T09:58:28.948Z" }, + { url = "https://files.pythonhosted.org/packages/7b/a0/9ff22b797724262da14427abaed4dd1d864a139693fc5e7809114376a716/pydantic_core-2.46.5-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d925f3d9afd05a8c0fb3a1031463a8d59ebe5e2afad297e29c78be19e13b4e62", size = 2265935, upload-time = "2026-08-28T09:58:30.625Z" }, + { url = "https://files.pythonhosted.org/packages/c0/a4/eb9409ec0736e50aa70a412f16c204ed149516846912f7e6724d4c73ee53/pydantic_core-2.46.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0fc5be0abd4a407e200d844b404e33639a554e7bd0d448e7b9ae181be4789ac2", size = 2066284, upload-time = "2026-08-28T09:58:32.289Z" }, + { url = "https://files.pythonhosted.org/packages/c0/02/7f6156ffc926857f1c37c07d9a388682865a81830ab6a1b637082c25e399/pydantic_core-2.46.5-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:816ff0a6550ffc06c098ccd2e0698600f9aa7da192a79eaa6f9af504a35db869", size = 2105889, upload-time = "2026-08-28T09:58:33.986Z" }, + { url = "https://files.pythonhosted.org/packages/92/b1/e781d357ebe09fc929f995700f1b3503e8897f1cece183ecb1300d4d67e9/pydantic_core-2.46.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c7ea57fc63aa7da93a1bd2d644e6577befae10c52c4e36377635eea1056a74f5", size = 2158006, upload-time = "2026-08-28T09:58:35.647Z" }, + { url = "https://files.pythonhosted.org/packages/70/0a/644597d84ab400e50609c192120b85c9681c22d3a20461b9060a79be0a7a/pydantic_core-2.46.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:efd62a42486f1bda5d24cb4f63d15a3c7768375fe83d36f9417b4ad7a2fb20b3", size = 2158408, upload-time = "2026-08-28T09:58:37.38Z" }, + { url = "https://files.pythonhosted.org/packages/1e/ee/ca3b7b3a4b3769ffe9ce9432a7c9be755de9593a46d3b0d54d0409323e44/pydantic_core-2.46.5-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:2bc9419666990c06d7397831f2126a1ecc3594aaa3ff7de5bf2d066802f4e07b", size = 2309609, upload-time = "2026-08-28T09:58:39.22Z" }, + { url = "https://files.pythonhosted.org/packages/ce/52/39fa1f451486019524ca685020390e7ca351832fd874530ba30c8628e6dc/pydantic_core-2.46.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:18a09e1e1011b462f2e32774f25859ef1223d5c2b0546a633cf56654710721e0", size = 2342618, upload-time = "2026-08-28T09:58:40.89Z" }, + { url = "https://files.pythonhosted.org/packages/81/5e/468fc630568c61dcef3cd47ad32ffbeed9af643f49208d1ea86ab4f890c4/pydantic_core-2.46.5-cp312-cp312-win32.whl", hash = "sha256:5cb482e9e84c851f4e623fe4acc1ced89168cf1fe18f7089db4548c8f5bbb65b", size = 1939475, upload-time = "2026-08-28T09:58:42.591Z" }, + { url = "https://files.pythonhosted.org/packages/cf/c9/4c19f41b84cf6b622a72fbeed7665b25d47a187d68d47d0d430c07f23268/pydantic_core-2.46.5-cp312-cp312-win_amd64.whl", hash = "sha256:5e81740c09e310f5aa5cbd3e434a01c154d4bef93241c7877b39f211d2b78ba8", size = 2043140, upload-time = "2026-08-28T09:58:44.272Z" }, + { url = "https://files.pythonhosted.org/packages/af/dd/0c1a050299147c746e5256db16d645ab5efd4f78c59937d581a0524e74a2/pydantic_core-2.46.5-cp312-cp312-win_arm64.whl", hash = "sha256:f7b0ec93a2893de856652154d73b7ba622f26fa97726487dcac373de5f4c6084", size = 1997729, upload-time = "2026-08-28T09:58:46.13Z" }, + { url = "https://files.pythonhosted.org/packages/f5/37/5abe39a8372a61d3dc3c1338fc504281c01b32fdb3169cd7187153b56d3e/pydantic_core-2.46.5-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:b7ca9034437b6022f941f4857459562ee00a560b97e7cce8a0ec5a74fc6766e0", size = 2075885, upload-time = "2026-08-28T09:58:47.856Z" }, + { url = "https://files.pythonhosted.org/packages/21/43/6323b1f8b217780454c61304bcd2b38ae4762f50754414124603ccc90bb2/pydantic_core-2.46.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f332f0e72a5a0400141f830744e141bf9f97917878dbe968669e8a7fefea78ff", size = 1922768, upload-time = "2026-08-28T09:58:49.58Z" }, + { url = "https://files.pythonhosted.org/packages/0f/a3/c05ca796e1197618a774b01e596aeedfefc2f7d8c01ae3054e910b120e8a/pydantic_core-2.46.5-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:193375f3548919d3f0b60936ca113ada3e38f264f91b9b8e0508efaad57be931", size = 1951241, upload-time = "2026-08-28T09:58:51.511Z" }, + { url = "https://files.pythonhosted.org/packages/68/32/33bc39ac705c52cffc908e8389f9754fdb208aea5c69cceddf4eb3ce99af/pydantic_core-2.46.5-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:79bdfa52f843137045b2d081cc05c120ba6665d29b7559c2c47690906f39279f", size = 2031975, upload-time = "2026-08-28T09:58:53.166Z" }, + { url = "https://files.pythonhosted.org/packages/b0/70/2333e885c0f6a67bc105c5916965dac9b57f2718ee20d81d1a06a4ebdc13/pydantic_core-2.46.5-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:24922243639cbdac66c75fcb6fd6495a9cb52b213d62f9a0d16f0310b1ff8038", size = 2208542, upload-time = "2026-08-28T09:58:55.017Z" }, + { url = "https://files.pythonhosted.org/packages/f7/ea/296debfb4264207bbda5936133892e027c0a58875ad53ebd512fba8ec3a2/pydantic_core-2.46.5-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c76fe65e607be28c7fd4d56fc3c42b1583aa058ce3408b7ad0fd540171d31f9f", size = 2264692, upload-time = "2026-08-28T09:58:56.767Z" }, + { url = "https://files.pythonhosted.org/packages/d3/f2/9e4de77a6271e07a76d2d58b11c091a979c191ed2939bf80067568b369d2/pydantic_core-2.46.5-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6f7b393a8b3da82f5c1fc0751e6d01ac6c55b93c18226a60bdfba4a724efafd1", size = 2066633, upload-time = "2026-08-28T09:58:58.531Z" }, + { url = "https://files.pythonhosted.org/packages/8d/db/f9e9d0c97445987b2084823d5c240de88087338f04fc2cfaa2df186b8049/pydantic_core-2.46.5-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:7ac031912d54f3d83ef3b3eb98dfabc1608802e2202263d25957eeed40b94761", size = 2105235, upload-time = "2026-08-28T09:59:00.421Z" }, + { url = "https://files.pythonhosted.org/packages/07/c5/79169b047b3b2c3e99e04bc76372af9637e0bf6db638274fa927df96369e/pydantic_core-2.46.5-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:837b396ca3d7b74091ca623f6cbd8351bd42d670a79c2683e79fb089f06a2de5", size = 2157367, upload-time = "2026-08-28T09:59:02.442Z" }, + { url = "https://files.pythonhosted.org/packages/26/b5/ba6057afb7c291bd449f51b867f95aef2072941c4ce4e5c31d6ffd132d3b/pydantic_core-2.46.5-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:5ee239d575f80b08eca11f6e20f90c4c695de7825c67eefe6091fbf20dda648e", size = 2158420, upload-time = "2026-08-28T09:59:04.2Z" }, + { url = "https://files.pythonhosted.org/packages/6e/28/2057abecaafdc22912afa819603a51f0a62d40643b7c4871c51721fea9be/pydantic_core-2.46.5-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:e80675d75ae2cd14372cb65cad5400d9347a3d3f6c13000183f22dfd027283ed", size = 2309588, upload-time = "2026-08-28T09:59:06.048Z" }, + { url = "https://files.pythonhosted.org/packages/71/9d/881156dc404e27479c4246128d73538464cab4a239bec61995e227644c30/pydantic_core-2.46.5-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:9c4b71f10dd532fb7a5cbc8f58707779e64f03a258c2bf8bfbaecfcd9970b519", size = 2341866, upload-time = "2026-08-28T09:59:08.539Z" }, + { url = "https://files.pythonhosted.org/packages/5a/38/d66f443a259f84d13babdceae568e572b0ed26da17ca5d0a649ebb110a67/pydantic_core-2.46.5-cp313-cp313-win32.whl", hash = "sha256:97bf8de4d541598c94a59344eeb988a94c08ff76b5723c41f6567ec18c7892ea", size = 1938580, upload-time = "2026-08-28T09:59:10.402Z" }, + { url = "https://files.pythonhosted.org/packages/2c/1e/1d5371213f4cc9a7ed70c0bfcc7911de22311ee99a662a56077d7292d2ac/pydantic_core-2.46.5-cp313-cp313-win_amd64.whl", hash = "sha256:15f4a94963c95accac15b7b657bb177d3ad82bb90b0d0526d9a9b85079925db5", size = 2041980, upload-time = "2026-08-28T09:59:12.396Z" }, + { url = "https://files.pythonhosted.org/packages/5a/48/4222d90b1c67568bace4dec6dca6271449c66de3595d72b6d098f5fde597/pydantic_core-2.46.5-cp313-cp313-win_arm64.whl", hash = "sha256:d22a945598fb91236b4dd793a6e42e4f3dd7740bb5aace5ebd7d4c08d13bb575", size = 1997213, upload-time = "2026-08-28T09:59:14.245Z" }, + { url = "https://files.pythonhosted.org/packages/8e/8a/14596f2a8367da50cf7cbac48169ee5d9c8e11d486a3b527082384630c72/pydantic_core-2.46.5-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:c1c43ad4339643d70ebb8124e1305a7dab423001eff58bb41a0f731adbc98355", size = 2074081, upload-time = "2026-08-28T09:59:16.141Z" }, + { url = "https://files.pythonhosted.org/packages/ae/d5/d8a4eb6d6c7f66b91dd37c576d76e9e60fba900caf5372c17bcf949febc2/pydantic_core-2.46.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1a353f84de772f423b5ffb11d7ae352fbbef0f446f3c0b0af0f8236d7233606e", size = 1920497, upload-time = "2026-08-28T09:59:18.065Z" }, + { url = "https://files.pythonhosted.org/packages/8e/26/092079428f86e927e030b2c0ced87df69dbb1c875cdeaa67bf42ea2be746/pydantic_core-2.46.5-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5086029a57366b8cf81b130a43908738095c270c21a8d7f0e8bdfdb89718e2f3", size = 1952130, upload-time = "2026-08-28T09:59:20.476Z" }, + { url = "https://files.pythonhosted.org/packages/08/c3/8ec0e290a9ebaebd64047bf5fda94be835c6b1551b02437e4b76778fbcd7/pydantic_core-2.46.5-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:46c25dda9d092a06c08db76ffe0a197107904d0dfac653f7d5306bbcd6d6119c", size = 2026371, upload-time = "2026-08-28T09:59:22.227Z" }, + { url = "https://files.pythonhosted.org/packages/01/72/4fd20ad520fb8da0157f95b27a7eb05a72790ef08138e7701ac972c342ea/pydantic_core-2.46.5-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:37ea7b83c935e5b0d68c9449b82651accf78a10828b2c02b2f2d9e9496446c21", size = 2202822, upload-time = "2026-08-28T09:59:24.277Z" }, + { url = "https://files.pythonhosted.org/packages/31/b0/d16e0771206b29314f0d52198b720be21e8a99ab2bf11e3bc0d7c9cebdff/pydantic_core-2.46.5-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e64e88d5585bea9ce95861079de72006c7fa6d3df4e3a3b65ba31eb979c15c9f", size = 2262756, upload-time = "2026-08-28T09:59:26.608Z" }, + { url = "https://files.pythonhosted.org/packages/2c/9b/59634b7ac631c63b2a37760eb6943af3e29573d6b59a4abc5e7f019d4cee/pydantic_core-2.46.5-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:54d510bac3ee52247af28ed4bb18a1e799f040ac60fd2bf5ccd4c92f1fbe786f", size = 2068352, upload-time = "2026-08-28T09:59:29.044Z" }, + { url = "https://files.pythonhosted.org/packages/08/7c/570abb1ad2155348dc754ea91be22e5aaa18eb6d69a6068f7c6f2679a6ed/pydantic_core-2.46.5-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:a2a5e1d0ff29adddc9f6d6821a66302e4493f8ca898b715b6b1182c2c201ea0a", size = 2104777, upload-time = "2026-08-28T09:59:30.95Z" }, + { url = "https://files.pythonhosted.org/packages/8e/25/5bf74adc65a1ac5b7be3f6cb0bcb5433615c1598a801c19d830d84c98ded/pydantic_core-2.46.5-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:03b9666e41e35d8909852ba191a0607520f81b74eaf12ccf8737005dbb313821", size = 2156312, upload-time = "2026-08-28T09:59:32.604Z" }, + { url = "https://files.pythonhosted.org/packages/90/6a/2ef38830675e050121040618135564ed56b860b45433b02d9b4ebece46f3/pydantic_core-2.46.5-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:a91c17edf6eea2402cb5457b4c89e99bc5ed1004aa34c4adf1d4258c1a5c22c2", size = 2150067, upload-time = "2026-08-28T09:59:34.453Z" }, + { url = "https://files.pythonhosted.org/packages/90/ef/a7dbb03a14a64c2a4621f989c615ed9a892535a6cad938fc27079f919d80/pydantic_core-2.46.5-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:b49924c73a235e969511bf2aabdff3beebf9820931f646c80274d5d780010c47", size = 2304516, upload-time = "2026-08-28T09:59:36.194Z" }, + { url = "https://files.pythonhosted.org/packages/68/f8/6bb4c4b80e8a6fde1904c64a51c62a1d04fcdfa3ea521a66b2ddefa1d885/pydantic_core-2.46.5-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:2cbd9a5eff05e51c447c34dfa4632145b26b09120cf04bd0c871e44c1a5e1c9a", size = 2335223, upload-time = "2026-08-28T09:59:37.931Z" }, + { url = "https://files.pythonhosted.org/packages/2a/80/f46b8c681195190b2c1f1c7c0a81abce60663e987613e09ef64d433dd96b/pydantic_core-2.46.5-cp314-cp314-win32.whl", hash = "sha256:2d5d76654becf5efd62c9e51c3756c67b49498b0c9a40884934c40807adbd074", size = 1934827, upload-time = "2026-08-28T09:59:39.836Z" }, + { url = "https://files.pythonhosted.org/packages/f7/3c/60674207246bc0a4009d2391b7c7251c7159f279c8d2ab8aae8ef46f3dee/pydantic_core-2.46.5-cp314-cp314-win_amd64.whl", hash = "sha256:fa10ef4112775900e7a0661068635eb67b2ab824fbde764de6e0e21982a93db0", size = 2042648, upload-time = "2026-08-28T09:59:41.792Z" }, + { url = "https://files.pythonhosted.org/packages/69/0c/117c562c7c1babdf44576b72a5e496906506c93690387ecfbca7c729ae2e/pydantic_core-2.46.5-cp314-cp314-win_arm64.whl", hash = "sha256:045ab3b6d308439e32b81cc173bba5b9018bc6ed896afd0c65b3b009b1699af5", size = 1989652, upload-time = "2026-08-28T09:59:43.702Z" }, + { url = "https://files.pythonhosted.org/packages/e8/66/9336ae58f9eb68c41d121894e52c4c89eccb07eb8f602a04ee9c3f37736a/pydantic_core-2.46.5-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:8816f3d218beb4b787de5c9759c259b8fa61f9dec42dc7811f320a33771778b7", size = 2065829, upload-time = "2026-08-28T09:59:45.364Z" }, + { url = "https://files.pythonhosted.org/packages/c5/02/bc19b47a96c2d3109760711acf22369e56bd7e405ca52f7ade164d2ead57/pydantic_core-2.46.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:bce57638e08ac148e5778cce7feb968307a727d66f8e2274a543d0cf0c9ad6a3", size = 1905716, upload-time = "2026-08-28T09:59:47.18Z" }, + { url = "https://files.pythonhosted.org/packages/52/a4/70b47c0509923dd98ccfed04fb3e32ea3849c82a0ff2205bb41009b43c00/pydantic_core-2.46.5-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:976e1128455aa595ea04c79ccfedff1aaeab96ee013fcc916bed120c4f0ad94f", size = 1934216, upload-time = "2026-08-28T09:59:49.241Z" }, + { url = "https://files.pythonhosted.org/packages/52/ab/aa03b65f7bb198585edf806b906c3223ecf1795543e39e23aec4cce27ad2/pydantic_core-2.46.5-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e7b891faeedeafba41b2983e5001a81b6a915b69544c7e7570d1989ce1c36ac7", size = 2010635, upload-time = "2026-08-28T09:59:51.692Z" }, + { url = "https://files.pythonhosted.org/packages/3c/8b/0da06343f30b84ec549aafd309c6456223d5dc8bd36af504c573faad561d/pydantic_core-2.46.5-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5f194189415698233dd1114a093a9b56e61e2c57e11b469be3b0506f46f0771c", size = 2209369, upload-time = "2026-08-28T09:59:53.582Z" }, + { url = "https://files.pythonhosted.org/packages/d6/5b/844c4defaa34a3df66eb9257087d121d70c201298b96abdf9f492fc2f1bf/pydantic_core-2.46.5-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:82a36973cf8a2ef5406f4fe2edbf8ed0c99629535d959e0b100c76a32535a111", size = 2253238, upload-time = "2026-08-28T09:59:55.484Z" }, + { url = "https://files.pythonhosted.org/packages/f4/64/a4e536cb16d7f61a7fd3120b46c577fc7fa7325992f69c4f52bc786d77d8/pydantic_core-2.46.5-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cdbb78909f52b981d3b2d56b97328d71eb0b974c36bd77c920123a7ebb192829", size = 2065740, upload-time = "2026-08-28T09:59:58.038Z" }, + { url = "https://files.pythonhosted.org/packages/5f/75/aaa38c6bc2d085f6605b34eabdc6a8a4e0b2e61fc9c8e6e52b28e97b3125/pydantic_core-2.46.5-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:52e24eacdb536cade636aa90fb851835222becff8484b7001fdc78cb0290f2aa", size = 2087425, upload-time = "2026-08-28T09:59:59.898Z" }, + { url = "https://files.pythonhosted.org/packages/55/ae/fcab4cfc39aba3689e1d20c8b5250ad280957022c09af2ed9cd585602a5e/pydantic_core-2.46.5-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:37ae34309d7bd8c0d61ab839668058f2a7962ea1fc51d105d2db228fe0618034", size = 2139306, upload-time = "2026-08-28T10:00:03.057Z" }, + { url = "https://files.pythonhosted.org/packages/2d/f4/f1d03a4bc9d9acbc62f4d742b8a319af52f71885079868b2ff8e48a651ee/pydantic_core-2.46.5-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:0cdbada856a1c69a7624a64d3d9aefe79300bd6ef827b43a4f265010b9b55184", size = 2144589, upload-time = "2026-08-28T10:00:05.645Z" }, + { url = "https://files.pythonhosted.org/packages/83/f3/7a53bb1356de514a4cd295f25b6ac39237895620c0462d2592b76c16e114/pydantic_core-2.46.5-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:545f26c504b27c3758439a5e6d9349931f0a04f855668d5fe323c89e82300a38", size = 2288882, upload-time = "2026-08-28T10:00:07.931Z" }, + { url = "https://files.pythonhosted.org/packages/cd/94/5a81583660c175c59d49ffb09f4b3a44debeaf86a19fca664ae1cdd9ee32/pydantic_core-2.46.5-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:ff218293c9c806138dca139765e3b067621be52bcd93cdc14c7711be7ddc90a9", size = 2335210, upload-time = "2026-08-28T10:00:10.177Z" }, + { url = "https://files.pythonhosted.org/packages/5a/9f/5d685c2693b972d1a59c998586e8823712b66603aeff47ee60a4bdaafd37/pydantic_core-2.46.5-cp314-cp314t-win32.whl", hash = "sha256:97cf3eb53a8cccacf9d46686a0926186c9bfb5574f2ed66d3639d5fe117cd3a9", size = 1921180, upload-time = "2026-08-28T10:00:12.35Z" }, + { url = "https://files.pythonhosted.org/packages/70/12/5c94ee16d65a37a15f9e869f5e6256df111154491173801a4c5e800ab548/pydantic_core-2.46.5-cp314-cp314t-win_amd64.whl", hash = "sha256:d2f9fc07a8042a8f95925b35c4f04f469707c981fc33245b6ca187cf5d2dd290", size = 2020515, upload-time = "2026-08-28T10:00:14.774Z" }, + { url = "https://files.pythonhosted.org/packages/63/19/67830dda664e6bdf9285ee2e40f355d0d7d6b92aa0c42e8d217bb8d33d36/pydantic_core-2.46.5-cp314-cp314t-win_arm64.whl", hash = "sha256:acf8a67ba51f4ca9ddbd0e6b3000a65ac51ab734661778b3e7ba64d99a710f2f", size = 1989276, upload-time = "2026-08-28T10:00:16.984Z" }, + { url = "https://files.pythonhosted.org/packages/df/dd/053c2e4303f791f3b8f8a14ab0b22008e8eb21d868c0c90b4f9be705b76a/pydantic_core-2.46.5-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:013d6f3483d81e02e7c328831808f336c8596ee33b4bd4026b9ffb1e960b8942", size = 2062540, upload-time = "2026-08-28T10:01:00.318Z" }, + { url = "https://files.pythonhosted.org/packages/d7/dd/a18df751a5e37dd51bfad7f68e766999125bebe68c9e1d10a493ad01bd63/pydantic_core-2.46.5-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:e9c134bb666dd54b778b9fc0d2b50cbb7f979b9e3716f26a88c9ab3b6fc1dd0f", size = 1902040, upload-time = "2026-08-28T10:01:02.529Z" }, + { url = "https://files.pythonhosted.org/packages/b7/13/01d40f9d07ce8a779fd6e0bd8ad4fba91309500dd67b869e2e219d261a6d/pydantic_core-2.46.5-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:347ec774390c87326a2e4929d58d3f7e8763a104d5d35f4cd595a4c952366433", size = 1967479, upload-time = "2026-08-28T10:01:05.004Z" }, + { url = "https://files.pythonhosted.org/packages/fa/04/c81d4841331c2178b6fb09ae225425e110ed72d990c9fe556c4ec03d1013/pydantic_core-2.46.5-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8e24d8f05fa2d28513d94e877e9c75ad66175376209b3977f916e240e623193c", size = 2111034, upload-time = "2026-08-28T10:01:07.345Z" }, +] + +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, +] + +[[package]] +name = "requests" +version = "2.34.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "charset-normalizer" }, + { name = "idna" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ac/c3/e2a2b89f2d3e2179abd6d00ebd70bff6273f37fb3e0cc209f48b39d00cbf/requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed", size = 142856, upload-time = "2026-05-14T19:25:27.735Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", size = 73075, upload-time = "2026-05-14T19:25:26.443Z" }, +] + +[[package]] +name = "s3transfer" +version = "0.19.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/49/c5/0a5ce655109cb96b8dfad36908a4e108d15f5dc2eff8fbc3146c4312e704/s3transfer-0.19.3.tar.gz", hash = "sha256:8326acc3d056d7f7b8a37110c7d2aea41f67e83f6ae79cd3b5fbcca34c726406", size = 166087, upload-time = "2026-10-19T19:19:49.751Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e2/a4/88f9e10498759121060b8050a1554753dfc04df39bbbcc7aad9ecc8d9440/s3transfer-0.19.3-py3-none-any.whl", hash = "sha256:24eb7535804306355e31c952a90022bf5b2f9d760f9a11062c7493a217c08553", size = 90252, upload-time = "2026-10-19T19:19:48.309Z" }, +] + +[[package]] +name = "six" +version = "1.17.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, +] + +[[package]] +name = "starlette" +version = "1.7.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7b/2b/3850dc6bf7ef71b088962eba31dafc6cffd2f96e577ebb0bb316df96da3e/starlette-1.7.0.tar.gz", hash = "sha256:c79f74ea63cff761804fbbfb182f1e0b440c2d07b164d24700c5a1bab5d6ff5d", size = 2736246, upload-time = "2026-09-23T07:30:26.35Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4e/d6/1ec1b290f9e0fb067899b61e1d37a30c923068bad260b216dbe37a7d2967/starlette-1.7.0-py3-none-any.whl", hash = "sha256:67f8e99895493dd2911a03f11314af6ceebeae4e704bb9f43dfc6a9db151c93e", size = 78980, upload-time = "2026-09-23T07:30:24.567Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f6/cc/6253133b5bb138fc3306cebfbda2c520f545d36b5be2c7255cc528bb45d6/typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5", size = 113555, upload-time = "2026-07-02T08:40:05.92Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", size = 45571, upload-time = "2026-07-02T08:40:04.659Z" }, +] + +[[package]] +name = "typing-inspection" +version = "0.4.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a3/26/b09b8010994eccc3c09092e6b34058f36a460eea2d4c3e8b910c695975a0/typing_inspection-0.4.4.tar.gz", hash = "sha256:547274fa6b0a561ccf549cc9524b999a578e737d015d8709d021f9d0d13bea47", size = 76928, upload-time = "2026-08-12T12:37:25.997Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/67/81/4add07e5172b7ac40d8ed5ff580409a7801a4fe26d529bdd915401dabfbe/typing_inspection-0.4.4-py3-none-any.whl", hash = "sha256:65b8397ba37ccbce054456aaccddfc91e6e3083c92824df348d96ca832f3f147", size = 14750, upload-time = "2026-08-12T12:37:24.648Z" }, +] + +[[package]] +name = "urllib3" +version = "2.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e3/05/b17359e1cefb4f909b5e40b1b90a496d987258916dbbf88e842c729f510e/urllib3-2.8.0.tar.gz", hash = "sha256:63bf2ead4c879426ebf22ef2a781eeb4aa3b4ae798a0435506f8687fd5bb9b63", size = 458972, upload-time = "2026-09-15T19:29:36.253Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/92/9d/c4e665119135114480843e7ab388fa94d8480650450e6f8e26b70d323a4c/urllib3-2.8.0-py3-none-any.whl", hash = "sha256:0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3", size = 135717, upload-time = "2026-09-15T19:29:34.577Z" }, +] + +[[package]] +name = "uvicorn" +version = "0.40.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "h11" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c3/d1/8f3c683c9561a4e6689dd3b1d345c815f10f86acd044ee1fb9a4dcd0b8c5/uvicorn-0.40.0.tar.gz", hash = "sha256:839676675e87e73694518b5574fd0f24c9d97b46bea16df7b8c05ea1a51071ea", size = 81761, upload-time = "2025-12-21T14:16:22.45Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3d/d8/2083a1daa7439a66f3a48589a57d576aa117726762618f6bb09fe3798796/uvicorn-0.40.0-py3-none-any.whl", hash = "sha256:c6c8f55bc8bf13eb6fa9ff87ad62308bbbc33d0b67f84293151efe87e0d5f2ee", size = 68502, upload-time = "2025-12-21T14:16:21.041Z" }, +] + +[[package]] +name = "zipp" +version = "4.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/18/05/ba1458dee0ac485b2a01071c1b88f5199fe878a485df482e10e1ef2205cc/zipp-4.1.2.tar.gz", hash = "sha256:cf474b90b07ea932637e17c6b9479de2b2882900b0d311bdaf4f5c000a15a97f", size = 27864, upload-time = "2026-10-21T18:57:04.743Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b8/44/7bd2007260da11cccca29ba2d4c345f1287aa663db38ee7cefeabf6386a5/zipp-4.1.2-py3-none-any.whl", hash = "sha256:b30cb6cd253122169c47de245ca24a86bac2a5d3802ba15bf7ba6414eb9e144d", size = 10581, upload-time = "2026-10-21T18:57:03.79Z" }, +] -- 2.54.0 From 7b6c10cde9f1e354809a0b39f3e70f2749711d4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9C=D0=B0=D1=82=D0=B2=D0=B5=D0=B9?= Date: Sat, 7 Nov 2026 15:00:53 +0300 Subject: [PATCH 2/3] add level 3 --- .env.example | 3 + REPORT.md | 84 ++-- compose.yaml | 379 +++++++++++++++--- contracts/trust-matrix.md | 24 +- evidence/lab7-clean-start.json | 12 + evidence/lab7-failover.json | 11 + evidence/lab7-persistence.json | 43 ++ infra/api-tls.cfg | 17 + infra/broker-tcp.cfg | 10 + infra/grafana-entrypoint.sh | 7 + .../provisioning/datasources/datasources.yaml | 13 +- infra/haproxy.cfg | 13 +- infra/loki-tls.cfg | 11 + infra/loki/loki.yaml | 3 +- infra/openbao/openbao.hcl | 15 + infra/otel-tls.cfg | 26 ++ infra/otel/collector.yaml | 12 +- infra/postgres-pg_hba.conf | 4 + infra/postgres-tls-entrypoint.sh | 6 + infra/prometheus-tls.cfg | 11 + infra/prometheus/prometheus.yaml | 4 + infra/rabbitmq-secret-entrypoint.sh | 25 ++ infra/rabbitmq.conf | 7 + infra/s3-tcp.cfg | 24 ++ infra/tempo-tls.cfg | 17 + infra/tempo/tempo.yaml | 8 +- scripts/benchmark-argon2.py | 25 ++ scripts/bootstrap-internal-pki.sh | 49 +++ scripts/bootstrap-pki.sh | 38 ++ scripts/init-secrets.py | 24 ++ scripts/openbao-operator.py | 57 +++ scripts/openbao-provision.py | 54 +++ scripts/restart.sh | 15 +- scripts/test-internal-mtls.sh | 29 ++ security/THREAT_MODEL.md | 18 +- src/bao_entrypoint.py | 37 ++ src/broker.py | 13 +- src/login_limit.py | 23 ++ src/main.py | 22 +- src/provision_db_roles.py | 41 ++ src/routes.py | 3 + src/storage/postgres.py | 10 +- src/storage/s3.py | 5 +- student-tests/dependency_tls.py | 40 ++ student-tests/lab7_limits.py | 67 ++++ student-tests/lab7_security.py | 49 +++ student-tests/mtls_probe.py | 31 ++ student-tests/observability_tls.py | 32 ++ student-tests/openbao_policy.py | 37 ++ student-tests/persistence_evidence.py | 33 ++ student-tests/postgres_tls.py | 47 +++ student-tests/s3_prefix_permissions.py | 40 ++ student-tests/service_permissions.py | 55 +++ student-tests/telemetry_tls.py | 30 ++ 54 files changed, 1593 insertions(+), 120 deletions(-) create mode 100644 evidence/lab7-clean-start.json create mode 100644 evidence/lab7-failover.json create mode 100644 evidence/lab7-persistence.json create mode 100644 infra/api-tls.cfg create mode 100644 infra/broker-tcp.cfg create mode 100644 infra/grafana-entrypoint.sh create mode 100644 infra/loki-tls.cfg create mode 100644 infra/openbao/openbao.hcl create mode 100644 infra/otel-tls.cfg create mode 100644 infra/postgres-pg_hba.conf create mode 100644 infra/postgres-tls-entrypoint.sh create mode 100644 infra/prometheus-tls.cfg create mode 100644 infra/rabbitmq.conf create mode 100644 infra/s3-tcp.cfg create mode 100644 infra/tempo-tls.cfg create mode 100644 scripts/benchmark-argon2.py create mode 100644 scripts/bootstrap-internal-pki.sh create mode 100644 scripts/bootstrap-pki.sh create mode 100644 scripts/openbao-operator.py create mode 100644 scripts/openbao-provision.py create mode 100644 scripts/test-internal-mtls.sh create mode 100644 src/bao_entrypoint.py create mode 100644 src/login_limit.py create mode 100644 src/provision_db_roles.py create mode 100644 student-tests/dependency_tls.py create mode 100644 student-tests/lab7_limits.py create mode 100644 student-tests/lab7_security.py create mode 100644 student-tests/mtls_probe.py create mode 100644 student-tests/observability_tls.py create mode 100644 student-tests/openbao_policy.py create mode 100644 student-tests/persistence_evidence.py create mode 100644 student-tests/postgres_tls.py create mode 100644 student-tests/s3_prefix_permissions.py create mode 100644 student-tests/service_permissions.py create mode 100644 student-tests/telemetry_tls.py diff --git a/.env.example b/.env.example index 2e9f054..70954db 100644 --- a/.env.example +++ b/.env.example @@ -30,6 +30,9 @@ OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab # TLS and OpenBao: actual paths/ports depend on your implemented trust matrix. PUBLIC_BASE_URL=https://localhost:8443 +PUBLIC_HTTPS_PORT=8443 +BAO_HOST_PORT=18200 +GRAFANA_HOST_PORT=3000 TLS_CA_FILE=/run/secrets/ca.crt TLS_CERT_FILE=/run/secrets/service.crt TLS_KEY_FILE=/run/secrets/service.key diff --git a/REPORT.md b/REPORT.md index e85e1d1..e289fe1 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,41 +1,77 @@ # Отчёт — лабораторная 7 -- Автор, группа: Kozyrev, BVT2301 -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор: Kozyrev, группа BVT2301. +- Целевая оценка: 3. +- База: собственная lab6, commit `7fe48e3`; исходники, конфигурация и тесты перенесены без пользовательских данных и секретов. +- Использована помощь Codex. Документация: [OpenBao certificate auth](https://openbao.org/docs/auth/cert/), [OpenBao configuration](https://openbao.org/docs/configuration/), [RabbitMQ access control](https://www.rabbitmq.com/docs/access-control/); остальные источники — в README задания. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Нужны Docker с Compose, Python 3.9+ для операторских и публичных скриптов и OpenSSL. Порты по умолчанию: HTTPS API `127.0.0.1:8443`, OpenBao `127.0.0.1:18200`, Grafana `127.0.0.1:3000`. Если порт занят, измените `PUBLIC_HTTPS_PORT`, `BAO_HOST_PORT` или `GRAFANA_HOST_PORT` в `.env`; для операторского скрипта при изменённом порте OpenBao задайте `BAO_OPERATOR_ADDR=https://localhost:<порт>`. -## Архитектура и решения +Чистый запуск проверен на Docker Desktop для macOS. Права на файловые Compose secrets при запуске на Linux отдельно не проверены; non-root процессы должны иметь доступ только к назначенным им файлам. Переменные в `.env.example` являются подсказками задания: сетевые адреса контейнеров и пути секретов фактически заданы в `compose.yaml`. -За основу перенесена реализация lab6 из локального репозитория, commit `7fe48e3`: исходники, миграции, Dockerfile, Compose, инфраструктурные конфигурации, собственные тесты и вспомогательные скрипты. Контракт и публичные проверки lab7 сохранены. Секреты и данные стенда не переносились. +```sh +cp .env.example .env +python3 scripts/init-secrets.py +sh scripts/bootstrap-pki.sh +sh scripts/bootstrap-internal-pki.sh +docker compose --env-file .env up -d openbao +# Дождаться ответа статуса, затем один раз инициализировать: +python3 scripts/openbao-operator.py status +python3 scripts/openbao-operator.py init +python3 scripts/openbao-operator.py unseal +python3 scripts/openbao-provision.py +docker compose --env-file .env up -d --build +make test CA_FILE="$PWD/secrets/ca.crt" +make test-security CA_FILE="$PWD/secrets/ca.crt" +``` -Перенесённая конфигурация пока относится к lab6: она использует HTTP на ряде соединений и не реализует требования lab7 к TLS/mTLS и OpenBao. Её необходимо переработать до проверки lab7; примеры и тесты lab6 могут содержать адреса HTTP. Работоспособность перенесённого стенда после копирования ещё не подтверждена. +При обычном перезапуске OpenBao остаётся sealed: оператор выполняет `python3 scripts/openbao-operator.py unseal`. Повторно выполнять `init` нельзя: его результат хранится в `secrets/openbao-operator.json` с локальными правами 0600. Файл содержит root token и unseal shares, не передаётся приложениям и требует защищённого резервного копирования оператором. Хранение всех shares на одной учебной машине — ограничение стенда, не схема разделения полномочий. Если OpenBao недоступен или sealed, новый процесс API/worker не получает секреты и не запускается; уже запущенный процесс использует полученные при старте рабочие credentials. Secret values, CA key и ключи сервисов не коммитятся и не копируются в образ. -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab6, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +Для безопасной остановки без удаления данных: `docker compose --env-file .env down`. Команда намеренного уничтожения **всех томов данного Compose-проекта**, включая OpenBao и данные сообщений: `docker compose --env-file .env down -v`; запускать только когда учебные данные больше не нужны. При новом OpenBao после удаления его тома старые operator credentials нужно отдельно архивировать и провести новую инициализацию. -## Доказательства критериев +## Архитектура -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +Клиент → HTTPS HAProxy → mTLS-прокси каждой API-реплики → loopback API. PostgreSQL принимает только TLS-соединения; роли API, worker, outbox и reconcile раздельны. S3 принимает TLS и имеет разные bucket-scoped credentials API, worker и reconcile; прокси ограничивает запись по префиксам, внутренние интерфейсы SeaweedFS доступны только в его сетевом пространстве. RabbitMQ принимает AMQPS через TCP-прокси, служебные порты ограничены loopback, а API, worker и outbox имеют отдельные роли и ограничения на очереди. PostgreSQL хранит пользователей, сессии, сообщения и метаданные; S3 — файлы; RabbitMQ — устойчивую очередь, outbox обеспечивает повторную публикацию. -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +OpenBao использует persistent file storage вне dev-mode. API и worker аутентифицируются отдельными сертификатами, получают часовой token с policy только на свой KV path и считывают нужные секреты при старте. Миграция, S3 bootstrap и DB provisioning используют отдельные операторские credentials. OTel, Grafana и хранилища сигналов общаются через TLS-прокси с проверкой сертификатов сервисных ролей. Локальные plaintext hops прокси ограничены loopback внутри соответствующего сетевого пространства. Подробности и ограничения — в [матрице доверия](contracts/trust-matrix.md). -## Эксперименты и отказы +Сервисные leaf certificates действуют 90 дней, локальный CA — 365 дней. Автоматический renewal и ротация не реализованы. Локальный CA не предназначен для внешнего production-развёртывания. -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +## Проверки -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +Публичные проверки используют HTTPS и CA certificate. Для стенда с нестандартным портом передайте `BASE_URL=https://localhost:<порт>`; для тестов с `docker compose` задайте `COMPOSE_FILE`, если используются дополнительные Compose-файлы. -## Ограничения и следующие шаги +| Сценарий | Команда или проверка | Результат локального запуска | +| --- | --- | --- | +| Структура | `make check` | PASS | +| Чистый запуск | Команды из раздела запуска в отдельном временном Compose-проекте | PASS: новая инициализация OpenBao, пустые тома, HTTPS smoke, mTLS и S3 ACL; `evidence/lab7-clean-start.json` | +| Публичный сценарий | `make test CA_FILE="$PWD/secrets/ca.crt"` | PASS через HTTPS, в том числе загрузка и thumbnail | +| Внешний TLS | `make test-security CA_FILE="$PWD/secrets/ca.crt"` | PASS: недоверенный CA отклонён | +| Внутренний mTLS | `sh scripts/test-internal-mtls.sh` | PASS на обеих репликах: без cert, чужой CA и неправильная роль отклонены; прямой порт API недоступен | +| PostgreSQL TLS | `docker compose exec -T api-a /app/.venv/bin/python - < student-tests/postgres_tls.py` | PASS: TLS, отказ plaintext и недоверенного CA | +| TLS зависимостей | `docker compose exec -T api-a /app/.venv/bin/python - < student-tests/dependency_tls.py` | PASS: проверка CA S3, broker, OpenBao; закрыты внутренние порты и S3 POST | +| Политики OpenBao | `docker compose exec -T api-a /app/.venv/bin/python - < student-tests/openbao_policy.py` и аналогично `worker` | PASS: свой path разрешён, чужой path и запись policy запрещены | +| Права DB/broker | `docker compose exec -T api-a /app/.venv/bin/python - < student-tests/service_permissions.py` и аналогично `worker` | PASS: отказ лишних операций | +| S3 prefix ACL | `docker compose exec -T api-a /app/.venv/bin/python - < student-tests/s3_prefix_permissions.py` и аналогично `worker` | PASS: запись в staging разрешена; запись API в originals и worker в bootstrap отклонена | +| Collector | `docker compose exec -T api-a /app/.venv/bin/python - < student-tests/telemetry_tls.py` | PASS: разрешённый publisher проходит; без cert и с чужой ролью отказ | +| Observability | `python3 student-tests/observability_tls.py` | PASS: Grafana запрашивает данные Prometheus, Loki и Tempo через TLS; `GRAFANA_URL` меняет адрес | +| ACL и expiry | `python3 student-tests/lab7_security.py --base-url https://localhost:8443 --ca-file "$PWD/secrets/ca.crt"` | PASS: чужое вложение, hostname и истечение синтетической сессии | +| Лимиты | `python3 student-tests/lab7_limits.py` в окружении зависимостей приложения | PASS: лимит запросов и 429/Retry-After | +| Сохранность | `python3 student-tests/persistence_evidence.py` с `BASE_URL` и `CA_FILE` | PASS: данные, вложение и прежняя сессия сохранились; время запуска хранилищ и API до/после записано в `evidence/lab7-persistence.json` | +| Отказ API | `make test-failover ACTION_SCRIPT=scripts/stop-one.sh CA_FILE="$PWD/secrets/ca.crt"` | PASS отдельно для обеих реплик и повторно для API-A после финальной S3 ACL; `evidence/lab7-failover.json`. После теста остановленную реплику нужно восстановить | -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +Локальные запуски использовали порты API `18443` и Grafana `13000` через временный Compose override из-за занятых портов. Для failover первой реплики за 15 секунд было 92 подтверждённых записи, 2 неуспешные попытки, восстановление записи через 5,910 с; для второй — 111, 1 и 2,887 с соответственно. Это результаты отдельных запусков публичного closed-loop сценария, а не показатели максимальной производительности. Raw-вывод отдельных запусков не сохранён. + +Argon2id настроен на `m=19456 KiB`, `t=2`, `p=1`. Один запуск `docker compose exec -T api-a /app/.venv/bin/python - < scripts/benchmark-argon2.py` выполнил по 10 последовательных хеширований и проверок: медианы 11,84 и 11,27 мс. Контейнер API ограничен 0,5 CPU и 512 MiB; Docker сообщил 10 CPU и 10 416 644 096 байт памяти. Параметры соответствуют минимуму задания, а замер показывает задержку одной операции на этом стенде; выбор не исследован при параллельной нагрузке и не является оценкой устойчивости к DoS. + +`docker inspect` показал для API и worker пользователя `65534:65534`, `privileged=false` и `cap_drop=[ALL]`; лимиты API — 0,5 CPU/512 MiB, worker — 2 CPU/4 GiB. Конфигурация остальных сервисов также содержит CPU и memory limits. Для защиты необходима отдельная демонстрация отсутствия лишних опубликованных портов и утечек в telemetry. + +## Ограничения + +Rate limit локален каждой API-реплике: 60 попыток за 60 секунд на процесс и может затрагивать добросовестных клиентов. S3-прокси поддерживает только SigV4 и запрещает экранированные и неоднозначные пути и POST; проверки ACL ограничены заявленными сценариями. Токены и секреты OpenBao читаются при запуске, автоматическая ротация и renewal не реализованы. Поведение при потере одного хоста и отказ отдельных хранилищ не исследованы. Один узел PostgreSQL, RabbitMQ, S3 и OpenBao не образует отказоустойчивый кластер. + +Для SeaweedFS опубликовано [сообщение о обходе bucket policy через POST Object](https://github.com/seaweedfs/seaweedfs/security/advisories/GHSA-qmx6-fpcw-q3xg), в котором указаны затронутые версии до 4.45 включительно и отсутствие исправленной версии на дату публикации. Стенд использует 4.17. В приложении POST Object не нужен; прокси запрещает HTTP POST, что проверяет `student-tests/dependency_tls.py`. Это ограничение конкретного пути, а не утверждение об устранении всех уязвимостей SeaweedFS. + +Не проверены все возможные пути утечки секретов в телеметрию и все сценарии отказа зависимостей. Оценка не подтверждается одним smoke-тестом. diff --git a/compose.yaml b/compose.yaml index 4d58c8e..bfa75d1 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1,21 +1,29 @@ x-api: &api build: context: . + cap_drop: [ALL] + security_opt: [no-new-privileges:true] + command: ["/app/.venv/bin/python", "-m", "src.bao_entrypoint", "/app/.venv/bin/python", "-m", "src.main"] environment: - APP_HOST: 0.0.0.0 + APP_HOST: 127.0.0.1 APP_PORT: 8080 APP_ENV: lab OTEL_ENABLED: "1" - OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318 - DATABASE_URL_FILE: /run/secrets/database_url + OTEL_EXPORTER_OTLP_ENDPOINT: https://otel-collector:4318 + OTEL_EXPORTER_OTLP_CERTIFICATE: /run/secrets/ca_cert + OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE: /run/secrets/otel_api_cert + OTEL_EXPORTER_OTLP_CLIENT_KEY: /run/secrets/otel_api_cert + DATABASE_CA_FILE: /run/secrets/ca_cert + BROKER_CA_FILE: /run/secrets/ca_cert + BAO_ADDR: https://openbao:8200 + BAO_CACERT: /run/secrets/ca_cert + BAO_CLIENT_CERT: /run/secrets/bao_api_cert + BAO_SERVICE_ROLE: api SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400} - CURSOR_SECRET_FILE: /run/secrets/cursor_secret - S3_ENDPOINT: http://s3:8333 + S3_ENDPOINT: https://s3:8333 + AWS_CA_BUNDLE: /run/secrets/ca_cert S3_BUCKET: messenger-attachments - S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id - S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key - BROKER_URL_FILE: /run/secrets/broker_url MAX_UPLOAD_BYTES: "10485760" MAX_IMAGE_PIXELS: "40000000" UPLOAD_CONCURRENCY: "1" @@ -24,20 +32,22 @@ x-api: &api OUTBOX_FAULT_AFTER_COMMIT: ${OUTBOX_FAULT_AFTER_COMMIT:-0} MAX_PENDING_JOBS: ${MAX_PENDING_JOBS:-100} secrets: - - database_url - - cursor_secret - - s3_access_key_id - - s3_secret_access_key - - broker_url + - ca_cert + - bao_api_cert + - otel_api_cert depends_on: + openbao: + condition: service_started s3-init: condition: service_completed_successfully migrate: condition: service_completed_successfully + db-provision: + condition: service_completed_successfully broker: condition: service_healthy - expose: - - "8080" + broker-tcp: + condition: service_started stop_grace_period: 10s healthcheck: test: ["CMD", "/app/.venv/bin/python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2).read()"] @@ -52,19 +62,48 @@ x-api: &api memory: 512M services: + openbao: + image: openbao/openbao:2.3.2 + user: "0:0" + entrypoint: ["bao"] + command: ["server", "-config=/bao/config/openbao.hcl"] + environment: + BAO_ADDR: https://openbao:8200 + BAO_CACERT: /run/secrets/ca.crt + ports: + - "127.0.0.1:${BAO_HOST_PORT:-18200}:8200" + volumes: + - ./infra/openbao/openbao.hcl:/bao/config/openbao.hcl:ro + - ./secrets/openbao.pem:/run/secrets/openbao.pem:ro + - ./secrets/ca.crt:/run/secrets/ca.crt:ro + - openbao_data:/bao/file + cap_drop: [ALL] + security_opt: [no-new-privileges:true] + mem_limit: 256m + cpus: "0.50" + broker: image: rabbitmq:4.1.2-alpine + hostname: broker + extra_hosts: ["broker:127.0.0.1"] environment: RABBITMQ_DEFAULT_USER: messenger + RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS: "-kernel inet_dist_use_interface {127,0,0,1}" + ERL_EPMD_ADDRESS: 127.0.0.1 entrypoint: ["/bin/sh", "/usr/local/bin/rabbitmq-secret-entrypoint.sh"] command: ["rabbitmq-server"] secrets: - rabbitmq_password + - broker_tls + - broker_api_password_hash + - broker_worker_password_hash + - broker_outbox_password_hash volumes: - rabbitmq_data:/var/lib/rabbitmq - ./infra/rabbitmq-secret-entrypoint.sh:/usr/local/bin/rabbitmq-secret-entrypoint.sh:ro + - ./infra/rabbitmq.conf:/etc/rabbitmq/rabbitmq.conf:ro healthcheck: - test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"] + test: ["CMD-SHELL", "test -f /tmp/rabbitmq-tls/provisioned && rabbitmq-diagnostics -q ping"] interval: 5s timeout: 3s retries: 20 @@ -75,16 +114,29 @@ services: cpus: "0.50" memory: 256M + broker-tcp: + image: haproxy:3.0.11-alpine + network_mode: service:broker + depends_on: + broker: + condition: service_healthy + volumes: + - ./infra/broker-tcp.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + restart: unless-stopped + mem_limit: 128m + cpus: "0.25" + s3: image: chrislusf/seaweedfs:4.17@sha256:186de7ef977a20343ee9a5544073f081976a29e2d29ecf8379891e7bf177fbe9 - command: ["server", "-dir=/data", "-ip=s3", "-ip.bind=0.0.0.0", "-s3", "-s3.config=/run/secrets/s3_config", "-master.volumeSizeLimitMB=128", "-volume.max=64", "-s3.port.iceberg=0", "-filer.exposeDirectoryData=false"] + command: ["server", "-dir=/data", "-ip=127.0.0.1", "-ip.bind=127.0.0.1", "-s3", "-s3.port=18333", "-s3.config=/run/secrets/s3_config", "-s3.cert.file=/run/secrets/s3_tls", "-s3.key.file=/run/secrets/s3_tls", "-master.volumeSizeLimitMB=128", "-volume.max=64", "-s3.port.iceberg=0", "-filer.exposeDirectoryData=false"] secrets: - s3_config + - s3_tls volumes: - s3_data:/data - ./infra/filer.toml:/etc/seaweedfs/filer.toml:ro healthcheck: - test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://localhost:9333/cluster/status"] + test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:9333/cluster/status"] interval: 5s timeout: 3s retries: 20 @@ -95,34 +147,60 @@ services: cpus: "1.0" memory: 512M + s3-tcp: + image: haproxy:3.0.11-alpine + network_mode: service:s3 + depends_on: + s3: + condition: service_healthy + volumes: + - ./infra/s3-tcp.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + - ./secrets/s3.pem:/run/secrets/server.pem:ro + - ./secrets/ca.crt:/run/secrets/ca.crt:ro + - ./secrets/s3_admin_acl:/run/secrets/admin_acl:ro + - ./secrets/s3_api_acl:/run/secrets/api_acl:ro + - ./secrets/s3_worker_acl:/run/secrets/worker_acl:ro + - ./secrets/s3_reconcile_acl:/run/secrets/reconcile_acl:ro + restart: unless-stopped + mem_limit: 128m + cpus: "0.25" + s3-init: build: context: . command: ["/app/.venv/bin/python", "-m", "src.bootstrap_s3"] environment: - S3_ENDPOINT: http://s3:8333 + S3_ENDPOINT: https://s3:8333 + AWS_CA_BUNDLE: /run/secrets/ca_cert S3_BUCKET: messenger-attachments S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_admin_key S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_admin_secret secrets: + - ca_cert - s3_admin_key - s3_admin_secret depends_on: - s3: - condition: service_healthy + s3-tcp: + condition: service_started restart: "no" + mem_limit: 256m + cpus: "0.50" postgres: image: postgres:17.6-alpine - command: ["postgres", "-c", "track_commit_timestamp=on"] + entrypoint: ["/bin/sh", "/usr/local/bin/postgres-tls-entrypoint.sh"] + command: ["postgres", "-c", "track_commit_timestamp=on", "-c", "ssl=on", "-c", "ssl_cert_file=/tmp/postgres-tls/server.pem", "-c", "ssl_key_file=/tmp/postgres-tls/server.pem", "-c", "hba_file=/etc/postgresql/pg_hba.conf"] environment: POSTGRES_DB: messenger POSTGRES_USER: messenger POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password secrets: - postgres_password + - postgres_tls volumes: - postgres_data:/var/lib/postgresql/data + - ./infra/postgres-tls-entrypoint.sh:/usr/local/bin/postgres-tls-entrypoint.sh:ro + - ./infra/postgres-pg_hba.conf:/etc/postgresql/pg_hba.conf:ro healthcheck: test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"] interval: 5s @@ -141,8 +219,10 @@ services: command: ["/app/.venv/bin/python", "-m", "src.migrate"] environment: DATABASE_URL_FILE: /run/secrets/database_url + DATABASE_CA_FILE: /run/secrets/ca_cert CURSOR_SECRET_FILE: /run/secrets/cursor_secret secrets: + - ca_cert - database_url - cursor_secret depends_on: @@ -155,6 +235,24 @@ services: cpus: "0.25" memory: 128M + db-provision: + build: + context: . + command: ["/app/.venv/bin/python", "-m", "src.provision_db_roles"] + secrets: + - database_url + - database_api_password + - database_worker_password + - database_outbox_password + - database_reconcile_password + - ca_cert + depends_on: + migrate: + condition: service_completed_successfully + restart: "no" + mem_limit: 128m + cpus: "0.25" + reconcile: healthcheck: disable: true @@ -164,23 +262,28 @@ services: command: ["--apply", "--loop"] restart: unless-stopped environment: - DATABASE_URL_FILE: /run/secrets/database_url + DATABASE_URL_FILE: /run/secrets/database_reconcile_url + DATABASE_CA_FILE: /run/secrets/ca_cert CURSOR_SECRET_FILE: /run/secrets/cursor_secret - S3_ENDPOINT: http://s3:8333 + S3_ENDPOINT: https://s3:8333 + AWS_CA_BUNDLE: /run/secrets/ca_cert S3_BUCKET: messenger-attachments - S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_admin_key - S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_admin_secret + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_reconcile_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_reconcile_secret_access_key STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400} secrets: - - database_url + - ca_cert + - database_reconcile_url - cursor_secret - - s3_admin_key - - s3_admin_secret + - s3_reconcile_access_key_id + - s3_reconcile_secret_access_key depends_on: postgres: condition: service_healthy s3: condition: service_healthy + s3-tcp: + condition: service_started deploy: resources: limits: @@ -193,8 +296,38 @@ services: api-b: <<: *api + api-a-tls: + image: haproxy:3.0.11-alpine + network_mode: service:api-a + depends_on: + api-a: + condition: service_healthy + volumes: + - ./infra/api-tls.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + - ./secrets/api-a.pem:/run/secrets/api.pem:ro + - ./secrets/ca.crt:/run/secrets/ca.crt:ro + restart: unless-stopped + mem_limit: 128m + cpus: "0.25" + + api-b-tls: + image: haproxy:3.0.11-alpine + network_mode: service:api-b + depends_on: + api-b: + condition: service_healthy + volumes: + - ./infra/api-tls.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + - ./secrets/api-b.pem:/run/secrets/api.pem:ro + - ./secrets/ca.crt:/run/secrets/ca.crt:ro + restart: unless-stopped + mem_limit: 128m + cpus: "0.25" + worker: restart: unless-stopped + cap_drop: [ALL] + security_opt: [no-new-privileges:true] healthcheck: test: ["CMD", "/app/.venv/bin/python", "-c", "import time,pathlib; assert time.time()-float(pathlib.Path('/tmp/worker-heartbeat').read_text()) < 15"] interval: 5s @@ -207,18 +340,23 @@ services: memory: 4G build: context: . - command: ["/app/.venv/bin/python", "-m", "src.worker"] + command: ["/app/.venv/bin/python", "-m", "src.bao_entrypoint", "/app/.venv/bin/python", "-m", "src.worker"] environment: APP_ENV: lab OTEL_ENABLED: "1" - OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318 - DATABASE_URL_FILE: /run/secrets/database_url - CURSOR_SECRET_FILE: /run/secrets/cursor_secret - S3_ENDPOINT: http://s3:8333 + OTEL_EXPORTER_OTLP_ENDPOINT: https://otel-collector:4318 + OTEL_EXPORTER_OTLP_CERTIFICATE: /run/secrets/ca_cert + OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE: /run/secrets/otel_worker_cert + OTEL_EXPORTER_OTLP_CLIENT_KEY: /run/secrets/otel_worker_cert + DATABASE_CA_FILE: /run/secrets/ca_cert + BROKER_CA_FILE: /run/secrets/ca_cert + BAO_ADDR: https://openbao:8200 + BAO_CACERT: /run/secrets/ca_cert + BAO_CLIENT_CERT: /run/secrets/bao_worker_cert + BAO_SERVICE_ROLE: worker + S3_ENDPOINT: https://s3:8333 + AWS_CA_BUNDLE: /run/secrets/ca_cert S3_BUCKET: messenger-attachments - S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id - S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key - BROKER_URL_FILE: /run/secrets/broker_url WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2} WORKER_FAULT_AFTER_S3_PUT: ${WORKER_FAULT_AFTER_S3_PUT:-0} RENDER_MEMORY_MB: ${RENDER_MEMORY_MB:-768} @@ -227,21 +365,29 @@ services: JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60} STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400} secrets: - - database_url - - cursor_secret - - s3_access_key_id - - s3_secret_access_key - - broker_url + - ca_cert + - bao_worker_cert + - otel_worker_cert depends_on: + openbao: + condition: service_started + db-provision: + condition: service_completed_successfully s3-init: condition: service_completed_successfully migrate: condition: service_completed_successfully broker: condition: service_healthy + broker-tcp: + condition: service_started stop_grace_period: 30s outbox: + mem_limit: 256m + cpus: "0.50" + cap_drop: [ALL] + security_opt: [no-new-privileges:true] healthcheck: disable: true build: @@ -250,36 +396,52 @@ services: environment: APP_ENV: lab OTEL_ENABLED: "1" - OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318 - DATABASE_URL_FILE: /run/secrets/database_url + OTEL_EXPORTER_OTLP_ENDPOINT: https://otel-collector:4318 + OTEL_EXPORTER_OTLP_CERTIFICATE: /run/secrets/ca_cert + OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE: /run/secrets/otel_outbox_cert + OTEL_EXPORTER_OTLP_CLIENT_KEY: /run/secrets/otel_outbox_cert + DATABASE_URL_FILE: /run/secrets/database_outbox_url + DATABASE_CA_FILE: /run/secrets/ca_cert CURSOR_SECRET_FILE: /run/secrets/cursor_secret - BROKER_URL_FILE: /run/secrets/broker_url + BROKER_URL_FILE: /run/secrets/broker_outbox_url + BROKER_CA_FILE: /run/secrets/ca_cert OUTBOX_DISPATCH_INTERVAL_SECONDS: "1" JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} secrets: - - database_url + - ca_cert + - database_outbox_url - cursor_secret - - broker_url + - broker_outbox_url + - otel_outbox_cert depends_on: migrate: condition: service_completed_successfully broker: condition: service_healthy + broker-tcp: + condition: service_started restart: unless-stopped load_balancer: image: haproxy:3.0.11-alpine ports: - - "127.0.0.1:8080:8080" + - "127.0.0.1:${PUBLIC_HTTPS_PORT:-8443}:8443" volumes: - ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + - ./secrets/ingress.pem:/run/secrets/ingress.pem:ro + - ./secrets/lb.pem:/run/secrets/lb.pem:ro + - ./secrets/ca.crt:/run/secrets/ca.crt:ro depends_on: api-a: condition: service_healthy api-b: condition: service_healthy + api-a-tls: + condition: service_started + api-b-tls: + condition: service_started healthcheck: - test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:8080/health/ready"] + test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:8404/health/ready"] interval: 5s timeout: 3s retries: 12 @@ -295,45 +457,157 @@ services: command: ["--config=/etc/otelcol/config.yaml"] volumes: - ./infra/otel/collector.yaml:/etc/otelcol/config.yaml:ro + secrets: [ca_cert, telemetry_collector_cert] mem_limit: 256m + cpus: "0.50" restart: unless-stopped + otel-tls: + image: haproxy:3.0.11-alpine + network_mode: service:otel-collector + depends_on: [otel-collector] + volumes: + - ./infra/otel-tls.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + - ./secrets/otel-collector.pem:/run/secrets/server.pem:ro + - ./secrets/ca.crt:/run/secrets/ca.crt:ro + restart: unless-stopped + mem_limit: 128m + cpus: "0.25" prometheus: image: prom/prometheus:v3.2.1 - command: ["--config.file=/etc/prometheus/prometheus.yaml", "--storage.tsdb.path=/prometheus", "--storage.tsdb.retention.time=7d"] + mem_limit: 512m + cpus: "0.50" + command: ["--config.file=/etc/prometheus/prometheus.yaml", "--web.listen-address=127.0.0.1:19090", "--storage.tsdb.path=/prometheus", "--storage.tsdb.retention.time=7d"] + secrets: [ca_cert, telemetry_prometheus_cert] volumes: - ./infra/prometheus/prometheus.yaml:/etc/prometheus/prometheus.yaml:ro - ./infra/prometheus/alerts.yaml:/etc/prometheus/alerts.yaml:ro - prometheus_data:/prometheus restart: unless-stopped + prometheus-tls: + image: haproxy:3.0.11-alpine + network_mode: service:prometheus + depends_on: [prometheus] + volumes: + - ./infra/prometheus-tls.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + - ./secrets/prometheus.pem:/run/secrets/server.pem:ro + - ./secrets/ca.crt:/run/secrets/ca.crt:ro + restart: unless-stopped + mem_limit: 128m + cpus: "0.25" loki: image: grafana/loki:3.4.2 + mem_limit: 512m + cpus: "0.50" command: ["-config.file=/etc/loki/loki.yaml"] volumes: - ./infra/loki/loki.yaml:/etc/loki/loki.yaml:ro - loki_data:/loki restart: unless-stopped + loki-tls: + image: haproxy:3.0.11-alpine + network_mode: service:loki + depends_on: [loki] + volumes: + - ./infra/loki-tls.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + - ./secrets/loki.pem:/run/secrets/server.pem:ro + - ./secrets/ca.crt:/run/secrets/ca.crt:ro + restart: unless-stopped + mem_limit: 128m + cpus: "0.25" tempo: image: grafana/tempo:2.7.1 + mem_limit: 512m + cpus: "0.50" command: ["-config.file=/etc/tempo/tempo.yaml"] volumes: - ./infra/tempo/tempo.yaml:/etc/tempo/tempo.yaml:ro - tempo_data:/var/tempo restart: unless-stopped + tempo-tls: + image: haproxy:3.0.11-alpine + network_mode: service:tempo + depends_on: [tempo] + volumes: + - ./infra/tempo-tls.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + - ./secrets/tempo.pem:/run/secrets/server.pem:ro + - ./secrets/ca.crt:/run/secrets/ca.crt:ro + restart: unless-stopped + mem_limit: 128m + cpus: "0.25" grafana: image: grafana/grafana:11.5.2 - ports: ["127.0.0.1:3000:3000"] + mem_limit: 512m + cpus: "0.50" + ports: ["127.0.0.1:${GRAFANA_HOST_PORT:-3000}:3000"] environment: GF_SECURITY_ADMIN_USER: admin GF_SECURITY_ADMIN_PASSWORD__FILE: /run/secrets/grafana_password GF_USERS_ALLOW_SIGN_UP: "false" - secrets: [grafana_password] + GF_SERVER_PROTOCOL: https + GF_SERVER_CERT_FILE: /run/secrets/grafana_tls + GF_SERVER_CERT_KEY: /run/secrets/grafana_tls + entrypoint: ["/bin/sh", "/etc/grafana/lab-entrypoint.sh"] + secrets: [grafana_password, grafana_tls, ca_cert, telemetry_grafana_cert] volumes: - grafana_data:/var/lib/grafana - ./infra/grafana/provisioning:/etc/grafana/provisioning:ro - ./infra/grafana/dashboards:/var/lib/grafana/dashboards:ro + - ./infra/grafana-entrypoint.sh:/etc/grafana/lab-entrypoint.sh:ro restart: unless-stopped secrets: + database_outbox_password: + file: ./secrets/database_outbox_password + database_reconcile_password: + file: ./secrets/database_reconcile_password + database_outbox_url: + file: ./secrets/database_outbox_url + database_reconcile_url: + file: ./secrets/database_reconcile_url + s3_reconcile_access_key_id: + file: ./secrets/s3_reconcile_access_key_id + s3_reconcile_secret_access_key: + file: ./secrets/s3_reconcile_secret_access_key + telemetry_grafana_cert: + file: ./secrets/telemetry-grafana.pem + telemetry_collector_cert: + file: ./secrets/telemetry-collector.pem + telemetry_prometheus_cert: + file: ./secrets/telemetry-prometheus.pem + grafana_tls: + file: ./secrets/grafana.pem + otel_api_cert: + file: ./secrets/otel-api.pem + otel_worker_cert: + file: ./secrets/otel-worker.pem + otel_outbox_cert: + file: ./secrets/otel-outbox.pem + database_api_password: + file: ./secrets/database_api_password + database_worker_password: + file: ./secrets/database_worker_password + broker_api_password_hash: + file: ./secrets/broker_api_password_hash + broker_worker_password_hash: + file: ./secrets/broker_worker_password_hash + broker_outbox_password_hash: + file: ./secrets/broker_outbox_password_hash + broker_outbox_url: + file: ./secrets/broker_outbox_url + broker_tls: + file: ./secrets/broker.pem + broker_tls_url: + file: ./secrets/broker_tls_url + s3_tls: + file: ./secrets/s3.pem + bao_api_cert: + file: ./secrets/bao-api.pem + bao_worker_cert: + file: ./secrets/bao-worker.pem + ca_cert: + file: ./secrets/ca.crt + postgres_tls: + file: ./secrets/postgres.pem grafana_password: file: ./secrets/grafana_password s3_config: @@ -358,6 +632,7 @@ secrets: file: ./secrets/broker_url volumes: + openbao_data: prometheus_data: loki_data: tempo_data: diff --git a/contracts/trust-matrix.md b/contracts/trust-matrix.md index 0c2a977..11821d4 100644 --- a/contracts/trust-matrix.md +++ b/contracts/trust-matrix.md @@ -4,18 +4,18 @@ | Связь | Минимум на 3 | Дополнительно на 4 | Доказательство / конфиг | | --- | --- | --- | --- | -| Клиент → ingress/LB | TLS, CA + hostname, Bearer | Проверка expiry/revocation сессии и общего login rate limit | TODO | -| LB → API (все реплики) | mTLS + allowlist LB identity | Ротация leaf без обхода проверки | TODO | -| API/worker → PostgreSQL | TLS verify-full + раздельные ограниченные DB roles | Ротация credentials; mTLS — допустимое усиление | TODO | -| API/worker → S3 | TLS + разные scoped credentials/prefix rights | Проверка denied prefix и ротация | TODO | -| API/worker → broker | TLS + разные producer/consumer identities и ACL | mTLS + ограничение допустимых SAN/ролей | TODO | -| API/worker → Collector | TLS; сеть/аутентификация запрещает произвольный ingest | mTLS с раздельными identities | TODO | -| Collector → telemetry backends | TLS + scoped backend credentials или mTLS | Ротация и проверка least privilege | TODO | -| Grafana → data sources | TLS с проверкой сервера, ограниченная identity | Отрицательные проверки доступа | TODO | -| API/worker/agent → OpenBao | TLS + сервисный auth + scoped policy | Renewal/expiry/revocation | TODO | -| Оператор → OpenBao/Grafana/S3 admin | TLS + authentication, доступ локальный/через защищённый канал | Разделение operator/application roles | TODO | -| API → Redis/Valkey, если используется | TLS + ACL/пароль с минимальными правами | Ротация; mTLS по возможности | TODO | -| Replication/кластерные каналы, если добавлены | TLS/mTLS либо подтверждённый шифрованный transport | Проверка заявленной модели отказа | TODO | +| Клиент → ingress/LB | TLS, CA + hostname, Bearer | Проверка expiry/revocation сессии и общего login rate limit | `https://localhost:8443` (порт настраивается), DNS/IP SAN `localhost`/`127.0.0.1`, CA `secrets/ca.crt`; `make test-security`, `student-tests/lab7_security.py`; session TTL по умолчанию 3600 с. | +| LB → API (все реплики) | mTLS + allowlist LB identity | Ротация leaf без обхода проверки | `infra/api-tls.cfg`, `infra/haproxy.cfg`; TCP 8444, TLS-прокси каждой реплики проверяет CA и роль `CN=lb, OU=load-balancer`, LB проверяет DNS identity `api-a`/`api-b`; `sh scripts/test-internal-mtls.sh`. Сертификаты на 90 дней, renewal пока ручной. API порт 8080 доступен только через loopback в сетевом пространстве реплики. | +| API/worker → PostgreSQL | TLS verify-full + раздельные ограниченные DB roles | Ротация credentials; mTLS — допустимое усиление | TCP 5432, server DNS `postgres`, CA `/run/secrets/ca_cert`; роли `messenger_api` и `messenger_worker`, права в `src/provision_db_roles.py`. Plaintext отклоняется `infra/postgres-pg_hba.conf`. Проверки `student-tests/postgres_tls.py`, `student-tests/service_permissions.py`. Пароли статические до ротации оператором. | +| API/worker → S3 | TLS + разные scoped credentials/prefix rights | Проверка denied prefix и ротация | `https://s3:8333`, CA `/run/secrets/ca_cert`, server DNS `s3`; отдельные ключи API/worker ограничены bucket `messenger-attachments`. Прокси по SigV4 identity разрешает API писать только в `staging/`, worker — в `staging/`, `originals/`, `variants/`; backend проверяет подпись. Прокси проверяет TLS на обоих соединениях, filer/master loopback-only. Тест `student-tests/s3_prefix_permissions.py`; срок статических ключей не ограничен. | +| API/worker → broker | TLS + разные producer/consumer identities и ACL | mTLS + ограничение допустимых SAN/ролей | `amqps://broker:5671`, CA `/run/secrets/ca_cert`, DNS `broker`; TCP-прокси передаёт TLS к loopback 15671. Erlang distribution, EPMD и metrics listener привязаны к loopback. Отдельные роли `messenger_api`, `messenger_worker`, `messenger_outbox` и ограничения очередей в `infra/rabbitmq-secret-entrypoint.sh`. Проверки `student-tests/service_permissions.py` и `student-tests/dependency_tls.py`; пароли статические до ротации. | +| API/worker → Collector | TLS; сеть/аутентификация запрещает произвольный ingest | mTLS с раздельными identities | `https://otel-collector:4318`, отдельные сертификаты API/worker/outbox с OU `telemetry-publisher`, CA и server DNS проверяются клиентом; `infra/otel-tls.cfg`, `student-tests/telemetry_tls.py`. Сертификаты на 90 дней. | +| Collector → telemetry backends | TLS + scoped backend credentials или mTLS | Ротация и проверка least privilege | Loki TCP 3100 и Tempo TCP 4317: проверка server DNS/CA, клиентская роль `telemetry-writer`; Prometheus получает метрики через TLS на Collector 8888/8889 с ролью `telemetry-scraper`. TLS завершается локальными sidecar; `infra/*-tls.cfg`, `student-tests/observability_tls.py`. Сертификаты на 90 дней. | +| Grafana → data sources | TLS с проверкой сервера, ограниченная identity | Отрицательные проверки доступа | `https://prometheus:9090`, `https://loki:3100`, `https://tempo:3200`; CA, DNS и клиентская роль `telemetry-reader`; datasource provisioning и `student-tests/observability_tls.py`. Сертификат на 90 дней. | +| API/worker/agent → OpenBao | TLS + сервисный auth + scoped policy | Renewal/expiry/revocation | `https://openbao:8200`, CA и DNS `openbao`, отдельные клиентские cert auth identities и политики `lab/data/api`, `lab/data/worker`; срок service token 1 час, сертификаты 90 дней; `student-tests/openbao_policy.py`. Secret values считываются при старте. | +| Оператор → OpenBao/Grafana/S3 admin | TLS + authentication, доступ локальный/через защищённый канал | Разделение operator/application roles | OpenBao `https://localhost:18200`, Grafana `https://localhost:3000`, порты привязаны к loopback. Операторские ключи/пароль хранятся в `secrets/`; S3 bootstrap работает только во внутренней сети с отдельным admin credential. | +| API → Redis/Valkey, если используется | TLS + ACL/пароль с минимальными правами | Ротация; mTLS по возможности | N/A: сессии хранятся в PostgreSQL. | +| Replication/кластерные каналы, если добавлены | TLS/mTLS либо подтверждённый шифрованный transport | Проверка заявленной модели отказа | Кластеры или репликация хранилищ не настроены. Встроенные локальные процессы SeaweedFS и Loki/Tempo не являются проверкой отказа хоста. | Нет выбранного компонента — отметьте N/A с причиной, а не оставляйте пустую строку. Внутренний публичный health listener, если нужен, минимален и изолирован; через него нельзя обращаться к бизнес-API без mTLS. Отдельно задокументируйте scrape/health/bootstrap каналы: они тоже не должны создавать обход. diff --git a/evidence/lab7-clean-start.json b/evidence/lab7-clean-start.json new file mode 100644 index 0000000..3d9ab5c --- /dev/null +++ b/evidence/lab7-clean-start.json @@ -0,0 +1,12 @@ +{ + "date": "2026-11-07", + "environment": "Docker Desktop on local machine; separate temporary Compose project", + "procedure": "Generated new local secrets and CA, initialized and unsealed a new OpenBao, created fresh PostgreSQL, RabbitMQ and S3 volumes, and started the Compose stack.", + "checks": { + "public_https_smoke": "PASS", + "external_tls_untrusted_ca": "PASS", + "internal_mtls_both_api_replicas": "PASS", + "api_and_worker_s3_prefix_acl": "PASS" + }, + "note": "Two initial RabbitMQ bootstrap races were corrected before the final successful clean run. The final run included S3 prefix ACL. Temporary test data were isolated from the main project and removed." +} diff --git a/evidence/lab7-failover.json b/evidence/lab7-failover.json new file mode 100644 index 0000000..3e6ad63 --- /dev/null +++ b/evidence/lab7-failover.json @@ -0,0 +1,11 @@ +{ + "date": "2026-11-07", + "scenario": "tests/resilience.py failover; separate runs stopping each API replica", + "transport": "HTTPS with the local CA", + "runs": [ + {"stopped_service": "api-a", "observed_instances_before": 2, "successful_writes": 92, "failed_attempts": 2, "first_recovery_seconds": 5.91, "status": "PASS"}, + {"stopped_service": "api-b", "observed_instances_before": 2, "successful_writes": 111, "failed_attempts": 1, "first_recovery_seconds": 2.887, "status": "PASS"}, + {"stopped_service": "api-a", "observed_instances_before": 2, "successful_writes": 91, "failed_attempts": 2, "first_recovery_seconds": 4.784, "status": "PASS", "configuration": "after S3 prefix ACL"} + ], + "note": "Closed-loop public test; first two runs preceded the final S3 prefix ACL. Results are not a throughput benchmark." +} diff --git a/evidence/lab7-persistence.json b/evidence/lab7-persistence.json new file mode 100644 index 0000000..93e3e84 --- /dev/null +++ b/evidence/lab7-persistence.json @@ -0,0 +1,43 @@ +{ + "status": "PASS", + "command": "make test-persistence ACTION_SCRIPT=scripts/restart.sh", + "services": { + "postgres": { + "before": "2026-11-07T11:21:11.514449215Z", + "after": "2026-11-07T11:31:51.644584012Z" + }, + "s3": { + "before": "2026-11-07T11:21:14.448491467Z", + "after": "2026-11-07T11:31:54.493811096Z" + }, + "broker": { + "before": "2026-11-07T11:21:12.477210382Z", + "after": "2026-11-07T11:31:52.565561429Z" + }, + "openbao": { + "before": "2026-11-07T11:21:11.728365965Z", + "after": "2026-11-07T11:31:51.983157637Z" + }, + "prometheus": { + "before": "2026-11-07T11:21:11.498290965Z", + "after": "2026-11-07T11:31:51.625424303Z" + }, + "loki": { + "before": "2026-11-07T11:21:14.486210217Z", + "after": "2026-11-07T11:31:53.332075471Z" + }, + "tempo": { + "before": "2026-11-07T11:21:13.305379716Z", + "after": "2026-11-07T11:31:52.261244679Z" + }, + "api-a": { + "before": "2026-11-07T11:21:36.434313963Z", + "after": "2026-11-07T11:32:15.545949676Z" + }, + "api-b": { + "before": "2026-11-07T11:21:36.45464788Z", + "after": "2026-11-07T11:32:15.598332884Z" + } + }, + "test_output": "python3 tests/resilience.py --mode persistence --action-script \"scripts/restart.sh\" --base-url \"https://localhost:18443\" --ca-file \"/Users/garfieldmk/Desktop/study/highLoad/mtusi2026lab7/secrets/ca.crt\"\nExecuting explicit persistence hook; hook output suppressed to avoid leaking secrets.\nPASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence." +} diff --git a/infra/api-tls.cfg b/infra/api-tls.cfg new file mode 100644 index 0000000..a85ab04 --- /dev/null +++ b/infra/api-tls.cfg @@ -0,0 +1,17 @@ +global + log stdout format raw local0 + +defaults + mode http + log global + timeout connect 2s + timeout client 30s + timeout server 30s + +frontend internal_mtls + bind :8444 ssl crt /run/secrets/api.pem ca-file /run/secrets/ca.crt verify required ssl-min-ver TLSv1.2 + http-request deny unless { ssl_c_s_dn(OU) -m str load-balancer } { ssl_c_s_dn(CN) -m str lb } + default_backend local_api + +backend local_api + server app 127.0.0.1:8080 diff --git a/infra/broker-tcp.cfg b/infra/broker-tcp.cfg new file mode 100644 index 0000000..c88da45 --- /dev/null +++ b/infra/broker-tcp.cfg @@ -0,0 +1,10 @@ +defaults + mode tcp + timeout connect 2s + timeout client 60s + timeout server 60s +frontend broker_tls + bind :5671 + default_backend broker_local +backend broker_local + server broker 127.0.0.1:15671 check diff --git a/infra/grafana-entrypoint.sh b/infra/grafana-entrypoint.sh new file mode 100644 index 0000000..fd31d79 --- /dev/null +++ b/infra/grafana-entrypoint.sh @@ -0,0 +1,7 @@ +#!/bin/sh +set -eu +LAB_CA="$(cat /run/secrets/ca_cert)" +export LAB_CA +LAB_CLIENT_PEM="$(cat /run/secrets/telemetry_grafana_cert)" +export LAB_CLIENT_PEM +exec /run.sh diff --git a/infra/grafana/provisioning/datasources/datasources.yaml b/infra/grafana/provisioning/datasources/datasources.yaml index 2264ab3..fce7fbe 100644 --- a/infra/grafana/provisioning/datasources/datasources.yaml +++ b/infra/grafana/provisioning/datasources/datasources.yaml @@ -4,22 +4,29 @@ datasources: uid: prometheus type: prometheus access: proxy - url: http://prometheus:9090 + url: https://prometheus:9090 + jsonData: {tlsAuthWithCACert: true, tlsAuth: true} + secureJsonData: {tlsCACert: '$LAB_CA', tlsClientCert: '$LAB_CLIENT_PEM', tlsClientKey: '$LAB_CLIENT_PEM'} isDefault: true - name: Loki uid: loki type: loki access: proxy - url: http://loki:3100 + url: https://loki:3100 jsonData: + tlsAuthWithCACert: true + tlsAuth: true derivedFields: - name: TraceID matcherType: label matcherRegex: '^trace_?id$' url: '$${__value.raw}' datasourceUid: tempo + secureJsonData: {tlsCACert: '$LAB_CA', tlsClientCert: '$LAB_CLIENT_PEM', tlsClientKey: '$LAB_CLIENT_PEM'} - name: Tempo uid: tempo type: tempo access: proxy - url: http://tempo:3200 + url: https://tempo:3200 + jsonData: {tlsAuthWithCACert: true, tlsAuth: true} + secureJsonData: {tlsCACert: '$LAB_CA', tlsClientCert: '$LAB_CLIENT_PEM', tlsClientKey: '$LAB_CLIENT_PEM'} diff --git a/infra/haproxy.cfg b/infra/haproxy.cfg index 00f0b6c..e83a213 100644 --- a/infra/haproxy.cfg +++ b/infra/haproxy.cfg @@ -20,8 +20,13 @@ resolvers docker hold valid 2s hold obsolete 2s -frontend http - bind :8080 +frontend https + bind :8443 ssl crt /run/secrets/ingress.pem ssl-min-ver TLSv1.2 + default_backend api + +frontend container_health + bind 127.0.0.1:8404 + http-request deny unless { path -i /health/ready } { method GET } default_backend api backend api @@ -30,6 +35,6 @@ backend api http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host localhost http-check expect status 200 default-server check inter 2s fall 1 rise 2 resolvers docker resolve-prefer ipv4 init-addr libc,none - server api-a api-a:8080 - server api-b api-b:8080 + server api-a api-a:8444 ssl verify required ca-file /run/secrets/ca.crt crt /run/secrets/lb.pem verifyhost api-a + server api-b api-b:8444 ssl verify required ca-file /run/secrets/ca.crt crt /run/secrets/lb.pem verifyhost api-b diff --git a/infra/loki-tls.cfg b/infra/loki-tls.cfg new file mode 100644 index 0000000..00fc7cf --- /dev/null +++ b/infra/loki-tls.cfg @@ -0,0 +1,11 @@ +defaults + mode tcp + timeout connect 2s + timeout client 30s + timeout server 30s +frontend https + bind :3100 ssl crt /run/secrets/server.pem ca-file /run/secrets/ca.crt verify required ssl-min-ver TLSv1.2 + tcp-request content reject unless { ssl_c_s_dn(OU) -m str telemetry-reader telemetry-writer } + default_backend local +backend local + server loki 127.0.0.1:13100 diff --git a/infra/loki/loki.yaml b/infra/loki/loki.yaml index ec83888..b66d573 100644 --- a/infra/loki/loki.yaml +++ b/infra/loki/loki.yaml @@ -1,6 +1,7 @@ auth_enabled: false -server: {http_listen_port: 3100} +server: {http_listen_address: "127.0.0.1", http_listen_port: 13100, grpc_listen_address: "127.0.0.1"} common: + instance_addr: 127.0.0.1 path_prefix: /loki replication_factor: 1 ring: {kvstore: {store: inmemory}} diff --git a/infra/openbao/openbao.hcl b/infra/openbao/openbao.hcl new file mode 100644 index 0000000..9dc6fc8 --- /dev/null +++ b/infra/openbao/openbao.hcl @@ -0,0 +1,15 @@ +ui = false +disable_mlock = true +api_addr = "https://openbao:8200" + +storage "file" { + path = "/bao/file" +} + +listener "tcp" { + address = "0.0.0.0:8200" + tls_cert_file = "/run/secrets/openbao.pem" + tls_key_file = "/run/secrets/openbao.pem" + tls_client_ca_file = "/run/secrets/ca.crt" + tls_min_version = "tls12" +} diff --git a/infra/otel-tls.cfg b/infra/otel-tls.cfg new file mode 100644 index 0000000..64e2f58 --- /dev/null +++ b/infra/otel-tls.cfg @@ -0,0 +1,26 @@ +defaults + mode tcp + timeout connect 2s + timeout client 30s + timeout server 30s + +frontend ingest + bind :4318 ssl crt /run/secrets/server.pem ca-file /run/secrets/ca.crt verify required ssl-min-ver TLSv1.2 + tcp-request content reject unless { ssl_c_s_dn(OU) -m str telemetry-publisher } + default_backend ingest_local +backend ingest_local + server collector 127.0.0.1:14318 + +frontend metrics + bind :8889 ssl crt /run/secrets/server.pem ca-file /run/secrets/ca.crt verify required ssl-min-ver TLSv1.2 + tcp-request content reject unless { ssl_c_s_dn(OU) -m str telemetry-scraper } + default_backend metrics_local +backend metrics_local + server collector 127.0.0.1:18889 + +frontend internal_metrics + bind :8888 ssl crt /run/secrets/server.pem ca-file /run/secrets/ca.crt verify required ssl-min-ver TLSv1.2 + tcp-request content reject unless { ssl_c_s_dn(OU) -m str telemetry-scraper } + default_backend internal_metrics_local +backend internal_metrics_local + server collector 127.0.0.1:18888 diff --git a/infra/otel/collector.yaml b/infra/otel/collector.yaml index d29d6fb..c7fa48e 100644 --- a/infra/otel/collector.yaml +++ b/infra/otel/collector.yaml @@ -1,22 +1,22 @@ receivers: otlp: protocols: - grpc: {endpoint: "0.0.0.0:4317"} - http: {endpoint: "0.0.0.0:4318"} + http: {endpoint: "127.0.0.1:14318"} processors: memory_limiter: {check_interval: 1s, limit_mib: 192, spike_limit_mib: 48} batch: {timeout: 2s, send_batch_size: 512, send_batch_max_size: 1024} exporters: prometheus: - endpoint: "0.0.0.0:8889" + endpoint: "127.0.0.1:18889" resource_to_telemetry_conversion: {enabled: true} otlp/tempo: endpoint: tempo:4317 - tls: {insecure: true} + tls: {ca_file: /run/secrets/ca_cert, cert_file: /run/secrets/telemetry_collector_cert, key_file: /run/secrets/telemetry_collector_cert, server_name_override: tempo} sending_queue: {enabled: true, queue_size: 128, num_consumers: 2} retry_on_failure: {enabled: true, initial_interval: 1s, max_interval: 10s, max_elapsed_time: 60s} otlphttp/loki: - endpoint: http://loki:3100/otlp + endpoint: https://loki:3100/otlp + tls: {ca_file: /run/secrets/ca_cert, cert_file: /run/secrets/telemetry_collector_cert, key_file: /run/secrets/telemetry_collector_cert} sending_queue: {enabled: true, queue_size: 128, num_consumers: 2} retry_on_failure: {enabled: true, initial_interval: 1s, max_interval: 10s, max_elapsed_time: 60s} service: @@ -25,7 +25,7 @@ service: readers: - pull: exporter: - prometheus: {host: 0.0.0.0, port: 8888} + prometheus: {host: 127.0.0.1, port: 18888} pipelines: metrics: {receivers: [otlp], processors: [memory_limiter, batch], exporters: [prometheus]} traces: {receivers: [otlp], processors: [memory_limiter, batch], exporters: [otlp/tempo]} diff --git a/infra/postgres-pg_hba.conf b/infra/postgres-pg_hba.conf new file mode 100644 index 0000000..a48b459 --- /dev/null +++ b/infra/postgres-pg_hba.conf @@ -0,0 +1,4 @@ +# Local maintenance stays inside the database container. Network clients must use TLS. +local all all trust +hostssl all all all scram-sha-256 +hostnossl all all all reject diff --git a/infra/postgres-tls-entrypoint.sh b/infra/postgres-tls-entrypoint.sh new file mode 100644 index 0000000..668202c --- /dev/null +++ b/infra/postgres-tls-entrypoint.sh @@ -0,0 +1,6 @@ +#!/bin/sh +set -eu +# PostgreSQL requires its key to be owned by the postgres user and mode 0600. +install -d -o postgres -g postgres -m 0700 /tmp/postgres-tls +install -o postgres -g postgres -m 0600 /run/secrets/postgres_tls /tmp/postgres-tls/server.pem +exec /usr/local/bin/docker-entrypoint.sh "$@" diff --git a/infra/prometheus-tls.cfg b/infra/prometheus-tls.cfg new file mode 100644 index 0000000..e8ff45f --- /dev/null +++ b/infra/prometheus-tls.cfg @@ -0,0 +1,11 @@ +defaults + mode tcp + timeout connect 2s + timeout client 30s + timeout server 30s +frontend https + bind :9090 ssl crt /run/secrets/server.pem ca-file /run/secrets/ca.crt verify required ssl-min-ver TLSv1.2 + tcp-request content reject unless { ssl_c_s_dn(OU) -m str telemetry-reader } + default_backend local +backend local + server prometheus 127.0.0.1:19090 diff --git a/infra/prometheus/prometheus.yaml b/infra/prometheus/prometheus.yaml index 4db25df..759865b 100644 --- a/infra/prometheus/prometheus.yaml +++ b/infra/prometheus/prometheus.yaml @@ -2,6 +2,10 @@ global: {scrape_interval: 5s, evaluation_interval: 5s} rule_files: ['/etc/prometheus/alerts.yaml'] scrape_configs: - job_name: collector + scheme: https + tls_config: {ca_file: /run/secrets/ca_cert, cert_file: /run/secrets/telemetry_prometheus_cert, key_file: /run/secrets/telemetry_prometheus_cert, server_name: otel-collector} static_configs: [{targets: ['otel-collector:8889']}] - job_name: collector-internal + scheme: https + tls_config: {ca_file: /run/secrets/ca_cert, cert_file: /run/secrets/telemetry_prometheus_cert, key_file: /run/secrets/telemetry_prometheus_cert, server_name: otel-collector} static_configs: [{targets: ['otel-collector:8888']}] diff --git a/infra/rabbitmq-secret-entrypoint.sh b/infra/rabbitmq-secret-entrypoint.sh index 8ed866f..e86460c 100644 --- a/infra/rabbitmq-secret-entrypoint.sh +++ b/infra/rabbitmq-secret-entrypoint.sh @@ -9,4 +9,29 @@ if [ -z "$RABBITMQ_DEFAULT_PASS" ]; then exit 1 fi export RABBITMQ_DEFAULT_PASS +install -d -o rabbitmq -g rabbitmq -m 0700 /tmp/rabbitmq-tls +rm -f /tmp/rabbitmq-tls/provisioned +install -o rabbitmq -g rabbitmq -m 0600 /run/secrets/broker_tls /tmp/rabbitmq-tls/server.pem +( + set -e + # Let the server create its own Erlang cookie before starting CLI tools. + until [ -s /var/lib/rabbitmq/.erlang.cookie ]; do sleep 1; done + until rabbitmqctl -q await_startup >/dev/null 2>&1; do sleep 2; done + for role in api worker outbox; do + user="messenger_$role" + if ! rabbitmqctl -q list_users | cut -f1 | grep -Fxq "$user"; then + if ! rabbitmqctl -q add_user "$user" "$(cat "/run/secrets/broker_${role}_password_hash")" --pre-hashed-password >/dev/null 2>&1; then + echo 'RabbitMQ user provisioning failed' >&2 + exit 1 + fi + fi + case "$role" in + api) write='^$'; read='^$' ;; + worker) write='^amq\.default$'; read='^attachment\.process\.v1$' ;; + outbox) write='^amq\.default$'; read='^$' ;; + esac + rabbitmqctl -q set_permissions -p / "$user" '^attachment\.process\.v1(\.dlq)?$' "$write" "$read" + done + touch /tmp/rabbitmq-tls/provisioned +) & exec /usr/local/bin/docker-entrypoint.sh "$@" diff --git a/infra/rabbitmq.conf b/infra/rabbitmq.conf new file mode 100644 index 0000000..fa11cc6 --- /dev/null +++ b/infra/rabbitmq.conf @@ -0,0 +1,7 @@ +listeners.tcp = none +listeners.ssl.1 = 127.0.0.1:15671 +prometheus.tcp.ip = 127.0.0.1 +ssl_options.certfile = /tmp/rabbitmq-tls/server.pem +ssl_options.keyfile = /tmp/rabbitmq-tls/server.pem +ssl_options.verify = verify_none +ssl_options.fail_if_no_peer_cert = false diff --git a/infra/s3-tcp.cfg b/infra/s3-tcp.cfg new file mode 100644 index 0000000..4c5d8e8 --- /dev/null +++ b/infra/s3-tcp.cfg @@ -0,0 +1,24 @@ +defaults + mode http + timeout connect 2s + timeout client 30s + timeout server 30s +frontend s3_tls + bind :8333 ssl crt /run/secrets/server.pem ssl-min-ver TLSv1.2 + # Multipart form POST is not used by this application; deny it at the boundary. + http-request deny if { method POST } + # Only the four locally provisioned SigV4 identities may use this endpoint. + acl admin_auth req.hdr(Authorization) -m reg -f /run/secrets/admin_acl + acl api_auth req.hdr(Authorization) -m reg -f /run/secrets/api_acl + acl worker_auth req.hdr(Authorization) -m reg -f /run/secrets/worker_acl + acl reconcile_auth req.hdr(Authorization) -m reg -f /run/secrets/reconcile_acl + http-request deny unless admin_auth or api_auth or worker_auth or reconcile_auth + # Application-generated keys contain no escapes or dot segments. Reject + # ambiguous paths so the prefix checks and the backend see the same path. + http-request deny if { path -m sub % } or { path -m sub /../ } or { path -m sub /./ } or { path -m sub // } or { path_end /.. } or { path_end /. } + http-request deny if api_auth { method PUT DELETE } !{ path_beg /messenger-attachments/staging/ } + http-request deny if worker_auth { method PUT DELETE } !{ path_beg /messenger-attachments/staging/ /messenger-attachments/originals/ /messenger-attachments/variants/ } + http-request deny if reconcile_auth { method PUT DELETE } !{ path_beg /messenger-attachments/staging/ /messenger-attachments/originals/ /messenger-attachments/variants/ } + default_backend s3_local +backend s3_local + server s3 127.0.0.1:18333 ssl verify required ca-file /run/secrets/ca.crt verifyhost s3 check diff --git a/infra/tempo-tls.cfg b/infra/tempo-tls.cfg new file mode 100644 index 0000000..27406b9 --- /dev/null +++ b/infra/tempo-tls.cfg @@ -0,0 +1,17 @@ +defaults + mode tcp + timeout connect 2s + timeout client 30s + timeout server 30s +frontend http_tls + bind :3200 ssl crt /run/secrets/server.pem ca-file /run/secrets/ca.crt verify required ssl-min-ver TLSv1.2 + tcp-request content reject unless { ssl_c_s_dn(OU) -m str telemetry-reader } + default_backend http_local +backend http_local + server tempo 127.0.0.1:13200 +frontend grpc_tls + bind :4317 ssl crt /run/secrets/server.pem ca-file /run/secrets/ca.crt verify required ssl-min-ver TLSv1.2 alpn h2 + tcp-request content reject unless { ssl_c_s_dn(OU) -m str telemetry-writer } + default_backend grpc_local +backend grpc_local + server tempo 127.0.0.1:14317 diff --git a/infra/tempo/tempo.yaml b/infra/tempo/tempo.yaml index 2e501ea..c8395cb 100644 --- a/infra/tempo/tempo.yaml +++ b/infra/tempo/tempo.yaml @@ -1,10 +1,12 @@ -server: {http_listen_port: 3200} +server: {http_listen_address: "127.0.0.1", http_listen_port: 13200, grpc_listen_address: "127.0.0.1"} distributor: receivers: otlp: protocols: - grpc: {endpoint: "0.0.0.0:4317"} -ingester: {trace_idle_period: 10s} + grpc: {endpoint: "127.0.0.1:14317"} +ingester: + trace_idle_period: 10s + lifecycler: {address: "127.0.0.1"} compactor: {compaction: {block_retention: 168h}} storage: trace: diff --git a/scripts/benchmark-argon2.py b/scripts/benchmark-argon2.py new file mode 100644 index 0000000..212c66d --- /dev/null +++ b/scripts/benchmark-argon2.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 +"""Measure local Argon2id hashing and verification without recording secrets.""" +from statistics import median +from time import perf_counter + +from argon2 import PasswordHasher + + +def main(): + hasher = PasswordHasher(time_cost=2, memory_cost=19456, parallelism=1) + password = "synthetic-benchmark-password" + hashes, verifications = [], [] + for _ in range(10): + started = perf_counter() + encoded = hasher.hash(password) + hashes.append((perf_counter() - started) * 1000) + started = perf_counter() + assert hasher.verify(encoded, password) + verifications.append((perf_counter() - started) * 1000) + print("Argon2id m=19456 KiB, t=2, p=1; samples=10") + print(f"hash median={median(hashes):.2f} ms; verify median={median(verifications):.2f} ms") + + +if __name__ == "__main__": + main() diff --git a/scripts/bootstrap-internal-pki.sh b/scripts/bootstrap-internal-pki.sh new file mode 100644 index 0000000..d3b227c --- /dev/null +++ b/scripts/bootstrap-internal-pki.sh @@ -0,0 +1,49 @@ +#!/bin/sh +# Issue separate internal API server identities and a load-balancer client role. +set -eu +umask 077 +cd "$(dirname "$0")/.." +test -s secrets/ca.key && test -s secrets/ca.crt || { echo 'Run bootstrap-pki.sh first' >&2; exit 1; } +work=$(mktemp -d "${TMPDIR:-/tmp}/lab7-mtls.XXXXXXXX") +trap 'rm -rf "$work"' EXIT HUP INT TERM +issue() { + name=$1; usage=$2; subject=$3; san=$4 + if [ -s "secrets/$name.pem" ]; then return; fi + if [ -e "secrets/$name.pem" ]; then echo "Incomplete $name.pem" >&2; exit 1; fi + cat > "$work/ext" </dev/null + openssl x509 -req -in "$work/$name.csr" -CA secrets/ca.crt -CAkey secrets/ca.key \ + -set_serial "0x$(openssl rand -hex 16)" -days 90 -sha256 -extfile "$work/ext" \ + -out "$work/$name.crt" 2>/dev/null + openssl verify -CAfile secrets/ca.crt "$work/$name.crt" >/dev/null + cat "$work/$name.crt" "$work/$name.key" > "secrets/$name.pem" +} +issue api-a serverAuth '/CN=api-a' api-a +issue api-b serverAuth '/CN=api-b' api-b +issue lb clientAuth '/CN=lb/OU=load-balancer' lb +issue openbao serverAuth '/CN=openbao' 'openbao,DNS:localhost,IP:127.0.0.1' +issue bao-api clientAuth '/CN=bao-api/OU=api' bao-api +issue bao-worker clientAuth '/CN=bao-worker/OU=worker' bao-worker +issue postgres serverAuth '/CN=postgres' postgres +issue s3 serverAuth '/CN=s3' s3 +issue broker serverAuth '/CN=broker' broker +issue otel-collector serverAuth '/CN=otel-collector' otel-collector +issue otel-api clientAuth '/CN=otel-api/OU=telemetry-publisher' otel-api +issue otel-worker clientAuth '/CN=otel-worker/OU=telemetry-publisher' otel-worker +issue otel-outbox clientAuth '/CN=otel-outbox/OU=telemetry-publisher' otel-outbox +issue tempo serverAuth '/CN=tempo' tempo +issue loki serverAuth '/CN=loki' loki +issue prometheus serverAuth '/CN=prometheus' prometheus +issue grafana serverAuth '/CN=localhost' 'localhost,IP:127.0.0.1' +issue telemetry-grafana clientAuth '/CN=grafana/OU=telemetry-reader' grafana +issue telemetry-collector clientAuth '/CN=collector/OU=telemetry-writer' collector +issue telemetry-prometheus clientAuth '/CN=prometheus/OU=telemetry-scraper' prometheus +echo 'Local internal identities ready.' diff --git a/scripts/bootstrap-pki.sh b/scripts/bootstrap-pki.sh new file mode 100644 index 0000000..0e75dfd --- /dev/null +++ b/scripts/bootstrap-pki.sh @@ -0,0 +1,38 @@ +#!/bin/sh +# Local-only CA and ingress certificate. Never commit the generated secrets. +set -eu +umask 077 +cd "$(dirname "$0")/.." +mkdir -p secrets +if [ -e secrets/ca.key ] || [ -e secrets/ca.crt ] || [ -e secrets/ingress.pem ]; then + if [ -s secrets/ca.key ] && [ -s secrets/ca.crt ] && [ -s secrets/ingress.pem ]; then + echo 'Local PKI already exists; preserving it.' + exit 0 + fi + echo 'Incomplete PKI in secrets/: inspect it before retrying.' >&2 + exit 1 +fi +work=$(mktemp -d "${TMPDIR:-/tmp}/lab7-pki.XXXXXXXX") +trap 'rm -rf "$work"' EXIT HUP INT TERM +cat > "$work/extensions.cnf" <<'EOF' +basicConstraints=critical,CA:FALSE +keyUsage=critical,digitalSignature,keyEncipherment +extendedKeyUsage=serverAuth +subjectAltName=DNS:localhost,IP:127.0.0.1 +subjectKeyIdentifier=hash +authorityKeyIdentifier=keyid,issuer +EOF +openssl req -x509 -newkey rsa:3072 -nodes -sha256 -days 365 \ + -subj '/CN=Lab7 local CA' -addext 'basicConstraints=critical,CA:TRUE' \ + -addext 'keyUsage=critical,keyCertSign,cRLSign' -addext 'subjectKeyIdentifier=hash' \ + -keyout "$work/ca.key" -out "$work/ca.crt" 2>/dev/null +openssl req -new -newkey rsa:2048 -nodes -sha256 -subj '/CN=localhost' \ + -keyout "$work/ingress.key" -out "$work/ingress.csr" 2>/dev/null +openssl x509 -req -in "$work/ingress.csr" -CA "$work/ca.crt" -CAkey "$work/ca.key" \ + -CAcreateserial -days 90 -sha256 -extfile "$work/extensions.cnf" -out "$work/ingress.crt" 2>/dev/null +openssl verify -CAfile "$work/ca.crt" "$work/ingress.crt" >/dev/null +cat "$work/ingress.crt" "$work/ingress.key" > "$work/ingress.pem" +cp "$work/ca.key" secrets/ca.key +cp "$work/ca.crt" secrets/ca.crt +cp "$work/ingress.pem" secrets/ingress.pem +echo 'Local ingress PKI generated in secrets/.' diff --git a/scripts/init-secrets.py b/scripts/init-secrets.py index 616933d..5cba3b8 100644 --- a/scripts/init-secrets.py +++ b/scripts/init-secrets.py @@ -1,6 +1,8 @@ #!/usr/bin/env python3 """Create local credentials without printing or overwriting existing values.""" import json +import base64 +import hashlib from pathlib import Path import secrets @@ -21,19 +23,41 @@ def write(name, value): write("grafana_password", secrets.token_urlsafe(24)) password = write("postgres_password", secrets.token_hex(32)) write("database_url", f"postgresql://messenger:{password}@postgres:5432/messenger") +for role in ("api", "worker", "outbox", "reconcile"): + role_password = write(f"database_{role}_password", secrets.token_hex(32)) + write(f"database_{role}_url", f"postgresql://messenger_{role}:{role_password}@postgres:5432/messenger") write("cursor_secret", secrets.token_hex(32)) admin_key = write("s3_admin_key", secrets.token_hex(16)) admin_secret = write("s3_admin_secret", secrets.token_hex(32)) app_key = write("s3_access_key_id", secrets.token_hex(16)) app_secret = write("s3_secret_access_key", secrets.token_hex(32)) +worker_key = write("s3_worker_access_key_id", secrets.token_hex(16)) +worker_secret = write("s3_worker_secret_access_key", secrets.token_hex(32)) +reconcile_key = write("s3_reconcile_access_key_id", secrets.token_hex(16)) +reconcile_secret = write("s3_reconcile_secret_access_key", secrets.token_hex(32)) +for role, key in (("admin", admin_key), ("api", app_key), ("worker", worker_key), ("reconcile", reconcile_key)): + # Used by the S3 proxy to select a write-prefix policy; the gateway still + # verifies the complete SigV4 signature before accepting a request. + write(f"s3_{role}_acl", "^AWS4-HMAC-SHA256[[:space:]]+Credential=" + key + "/") rabbitmq_password = write("rabbitmq_password", secrets.token_urlsafe(32)) write("broker_url", f"amqp://messenger:{rabbitmq_password}@broker:5672/%2F") +write("broker_tls_url", f"amqps://messenger:{rabbitmq_password}@broker:5671/%2F") +for role in ("api", "worker", "outbox"): + role_password = write(f"broker_{role}_password", secrets.token_hex(32)) + write(f"broker_{role}_url", f"amqps://messenger_{role}:{role_password}@broker:5671/%2F") + salt = secrets.token_bytes(4) + digest = base64.b64encode(salt + hashlib.sha256(salt + role_password.encode()).digest()).decode() + write(f"broker_{role}_password_hash", digest) config_path = root / "s3.json" config_path.write_text(json.dumps({"identities": [ {"name": "bootstrap", "credentials": [{"accessKey": admin_key, "secretKey": admin_secret}], "actions": ["Admin", "Read", "Write", "List", "Tagging"]}, {"name": "api", "credentials": [{"accessKey": app_key, "secretKey": app_secret}], "actions": ["Read:messenger-attachments", "Write:messenger-attachments", "List:messenger-attachments"]}, + {"name": "worker", "credentials": [{"accessKey": worker_key, "secretKey": worker_secret}], + "actions": ["Read:messenger-attachments", "Write:messenger-attachments", "List:messenger-attachments"]}, + {"name": "reconcile", "credentials": [{"accessKey": reconcile_key, "secretKey": reconcile_secret}], + "actions": ["Read:messenger-attachments", "Write:messenger-attachments", "List:messenger-attachments"]}, ]}, indent=2) + "\n") config_path.chmod(0o600) print("Local secrets ready; existing values preserved.") diff --git a/scripts/openbao-operator.py b/scripts/openbao-operator.py new file mode 100644 index 0000000..24024e4 --- /dev/null +++ b/scripts/openbao-operator.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +"""Initialize/unseal the local OpenBao as operator without printing credentials.""" +import argparse +import json +import os +from pathlib import Path +import ssl +import urllib.error +import urllib.request + +ROOT = Path(__file__).resolve().parents[1] +DATA = ROOT / 'secrets' / 'openbao-operator.json' +BASE = os.environ.get('BAO_OPERATOR_ADDR', 'https://localhost:18200').rstrip('/') +TLS = ssl.create_default_context(cafile=str(ROOT / 'secrets' / 'ca.crt')) + + +def request(path, body=None): + data = None if body is None else json.dumps(body).encode() + req = urllib.request.Request(BASE + '/v1/' + path, data=data, + headers={'Content-Type': 'application/json'}, + method='GET' if body is None else 'PUT') + with urllib.request.urlopen(req, context=TLS, timeout=10) as response: + return json.load(response) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('action', choices=['init', 'unseal', 'status']) + args = parser.parse_args() + if args.action == 'init': + if DATA.exists(): + raise SystemExit('Operator credentials file exists; refusing to overwrite it.') + if request('sys/init')['initialized']: + raise SystemExit('OpenBao is already initialized. Recover operator credentials; do not reinitialize.') + fd = os.open(DATA, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(fd, 'w') as stream: + result = request('sys/init', {'secret_shares': 3, 'secret_threshold': 2}) + json.dump(result, stream) + stream.flush() + os.fsync(stream.fileno()) + print('Initialized. Operator credentials stored locally in secrets/openbao-operator.json; protect and back them up.') + elif args.action == 'unseal': + data = json.loads(DATA.read_text()) + for key in data['keys_base64'][:2]: + state = request('sys/unseal', {'key': key}) + if not state['sealed']: + break + if state['sealed']: + raise SystemExit('OpenBao remains sealed.') + print('OpenBao unsealed.') + else: + state = request('sys/seal-status') + print('initialized=' + str(state['initialized']) + ' sealed=' + str(state['sealed'])) + + +if __name__ == '__main__': + main() diff --git a/scripts/openbao-provision.py b/scripts/openbao-provision.py new file mode 100644 index 0000000..4045ade --- /dev/null +++ b/scripts/openbao-provision.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +"""Operator provisions certificate auth and scoped KV v2 paths; no credentials printed.""" +import json +import os +from pathlib import Path +import ssl +import subprocess +import urllib.request + +ROOT = Path(__file__).resolve().parents[1] +SECRETS = ROOT / 'secrets' +BASE = os.environ.get('BAO_OPERATOR_ADDR', 'https://localhost:18200').rstrip('/') +TLS = ssl.create_default_context(cafile=str(SECRETS / 'ca.crt')) +TOKEN = json.loads((SECRETS / 'openbao-operator.json').read_text())['root_token'] + + +def call(path, body=None): + data = None if body is None else json.dumps(body).encode() + request = urllib.request.Request(BASE + '/v1/' + path, data=data, + headers={'Content-Type': 'application/json', 'X-Vault-Token': TOKEN}, + method='GET' if body is None else 'POST') + with urllib.request.urlopen(request, context=TLS, timeout=10) as response: + payload = response.read() + return json.loads(payload) if payload else {} + + +def main(): + if 'cert/' not in call('sys/auth')['data']: + call('sys/auth/cert', {'type': 'cert'}) + if 'lab/' not in call('sys/mounts')['data']: + call('sys/mounts/lab', {'type': 'kv', 'options': {'version': '2'}}) + files = {'DATABASE_URL': 'database_api_url', 'CURSOR_SECRET': 'cursor_secret', + 'S3_ACCESS_KEY_ID': 's3_access_key_id', 'S3_SECRET_ACCESS_KEY': 's3_secret_access_key', + 'BROKER_URL': 'broker_api_url'} + for role in ('api', 'worker'): + role_files = dict(files) + if role == 'worker': + role_files.update(DATABASE_URL='database_worker_url', + S3_ACCESS_KEY_ID='s3_worker_access_key_id', + S3_SECRET_ACCESS_KEY='s3_worker_secret_access_key', + BROKER_URL='broker_worker_url') + values = {name: (SECRETS / file).read_text().strip() for name, file in role_files.items()} + policy = f'path "lab/data/{role}" {{ capabilities = ["read"] }}' + call('sys/policies/acl/' + role, {'policy': policy}) + certificate = subprocess.check_output(['openssl', 'x509', '-in', str(SECRETS / ('bao-' + role + '.pem'))], text=True) + call('auth/cert/certs/' + role, {'certificate': certificate, 'token_policies': [role], + 'token_ttl': '1h', 'token_max_ttl': '1h', + 'token_no_default_policy': True}) + call('lab/data/' + role, {'data': values}) + print('Scoped API and worker policies, certificate identities and secret paths configured.') + + +if __name__ == '__main__': + main() diff --git a/scripts/restart.sh b/scripts/restart.sh index 958b700..0d18972 100755 --- a/scripts/restart.sh +++ b/scripts/restart.sh @@ -3,8 +3,19 @@ set -eu cd "$(dirname "$0")/.." # Deliberately preserves named volumes and test data. -docker compose restart postgres s3 broker +docker compose restart postgres s3 broker openbao prometheus loki tempo grafana otel-collector +attempt=0 +until python3 scripts/openbao-operator.py unseal; do + attempt=$((attempt + 1)) + [ "$attempt" -lt 30 ] || exit 1 + sleep 1 +done docker compose up -d --wait postgres s3 broker +docker compose up -d --no-deps --force-recreate broker-tcp +docker compose up -d --no-deps --force-recreate s3-tcp docker compose run --rm --no-deps s3-init -docker compose restart worker outbox reconcile api-a api-b load_balancer +docker compose up -d --no-deps --force-recreate otel-tls prometheus-tls loki-tls tempo-tls +docker compose restart worker outbox reconcile api-a api-b +docker compose up -d --no-deps --force-recreate api-a-tls api-b-tls +docker compose restart load_balancer docker compose up -d --wait worker outbox reconcile api-a api-b load_balancer diff --git a/scripts/test-internal-mtls.sh b/scripts/test-internal-mtls.sh new file mode 100644 index 0000000..00b4438 --- /dev/null +++ b/scripts/test-internal-mtls.sh @@ -0,0 +1,29 @@ +#!/bin/sh +set -eu +umask 077 +cd "$(dirname "$0")/.." +test -s secrets/ca.key && test -s secrets/lb.pem || { echo 'Bootstrap PKI first' >&2; exit 1; } +work=$(mktemp -d "${TMPDIR:-/tmp}/lab7-mtls-test.XXXXXXXX") +trap 'rm -rf "$work"' EXIT HUP INT TERM +cat > "$work/ext" <<'EOF' +basicConstraints=critical,CA:FALSE +keyUsage=critical,digitalSignature,keyEncipherment +extendedKeyUsage=clientAuth +subjectKeyIdentifier=hash +authorityKeyIdentifier=keyid,issuer +EOF +openssl req -new -newkey rsa:2048 -nodes -subj '/CN=lb/OU=worker' \ + -keyout "$work/wrong.key" -out "$work/wrong.csr" 2>/dev/null +openssl x509 -req -in "$work/wrong.csr" -CA secrets/ca.crt -CAkey secrets/ca.key \ + -set_serial "0x$(openssl rand -hex 16)" -days 1 -extfile "$work/ext" -out "$work/wrong.crt" 2>/dev/null +cat "$work/wrong.crt" "$work/wrong.key" > "$work/wrong-role.pem" +openssl req -x509 -newkey rsa:2048 -nodes -days 1 -subj '/CN=foreign' \ + -keyout "$work/foreign.key" -out "$work/foreign.crt" 2>/dev/null +cat "$work/foreign.crt" "$work/foreign.key" > "$work/foreign.pem" +docker run --rm --network "${COMPOSE_PROJECT_NAME:-$(basename "$PWD")}_default" --user 0 \ + -v "$PWD/secrets/ca.crt:/certs/ca.crt:ro" \ + -v "$PWD/secrets/lb.pem:/certs/lb.pem:ro" \ + -v "$work/wrong-role.pem:/certs/wrong-role.pem:ro" \ + -v "$work/foreign.pem:/certs/foreign.pem:ro" \ + -v "$PWD/student-tests/mtls_probe.py:/probe.py:ro" \ + "${COMPOSE_PROJECT_NAME:-$(basename "$PWD")}-api-a" /app/.venv/bin/python /probe.py diff --git a/security/THREAT_MODEL.md b/security/THREAT_MODEL.md index 31da287..65063d1 100644 --- a/security/THREAT_MODEL.md +++ b/security/THREAT_MODEL.md @@ -1,19 +1,27 @@ -# Модель угроз — заполнить +# Модель угроз — лабораторная 7 + +Статус проверок и оставшиеся ограничения приведены в [отчёте](../REPORT.md); полная проверка всех каналов ещё не завершена. ## Активы и акторы -TODO: пользовательские credentials/сессии, сообщения/вложения, service identities, CA/secrets, telemetry. Акторы: участник чата, посторонний пользователь, внешний клиент, скомпрометированный сервис, оператор. Опишите доступ атакующего и границы учебного стенда. +Активы: пароли и сессии, сообщения и вложения, ключи CA и сервисные credentials, метаданные и telemetry. Акторы: участник чата, посторонний зарегистрированный пользователь, внешний клиент, скомпрометированный сервис и оператор. Внешний клиент может отправлять произвольные запросы к ingress; пользователь может менять заголовки и идентификаторы в запросах. Компрометация хоста и оператора не рассматривается как выдерживаемый отказ. ## Границы доверия -TODO: схема и ссылка на заполненную contracts/trust-matrix.md; публичные/внутренние интерфейсы, процессы, секреты и их выдача. +Границы: клиент → ingress, ingress → API, приложения → хранилища и брокер, приложения → Collector, компоненты → OpenBao, оператор → административные интерфейсы. Конкретные endpoint, identity и проверки перечислены в [матрице доверия](../contracts/trust-matrix.md). | Угроза и конкретный путь | Нарушаемый инвариант | Контроль | Отрицательный тест | Остаточное ограничение | | --- | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | TODO | +| Посторонний пользователь подставляет ID чужого вложения | Доступ только участникам чата | Проверка Bearer и членства для metadata/content и сообщений | Авторизованным посторонним запросить оба endpoint и приложить чужой ID | Не защищает от компрометации участника | +| Клиент подменяет заголовок X-User-Id | Идентичность определяется сессией | Игнорировать клиентский X-User-Id | Передать заголовок с ID другого пользователя и проверить отказ | Украденный Bearer остаётся действителен до expiry/logout | +| Клиент соединяется с поддельным ingress | Клиент не передаёт секреты чужому серверу | TLS и проверка CA/hostname | Соединение с чужим CA и неверным hostname отклоняется | Компрометированный доверенный CA вне заявленной защиты | +| Сервис с чужой identity подключается напрямую к API | Только LB вызывает внутренний API | mTLS и проверка разрешённой identity | `sh scripts/test-internal-mtls.sh`: без сертификата, чужой CA, доверенный CA с неверной identity — отказ | Отзыв и ротация identity под нагрузкой не проверены | +| Скомпрометированный worker читает чужой секрет | Каждому сервису доступен только нужный набор секретов | Отдельная identity и scoped policy OpenBao | Запросить разрешённый и чужой путь, попробовать изменить policy | Из разрешённых worker данных возможна утечка | +| Автоматические попытки входа перегружают KDF | Вход ограничен по частоте | Лимит попыток с 429 и Retry-After | Превысить лимит и проверить ответ, затем дождаться окна | Текущий лимит локален процессу и общий для всех клиентов | +| Загрузка чрезмерного файла исчерпывает память или диск | Размер и число пикселей ограничены | Потоковый лимит и проверка изображения | Отправить файл свыше лимита и изображение со слишком большим числом пикселей | Остальные лимиты ресурсов стенда нужно проверить | На 4 минимум 6 конкретных угроз. Не заменяйте сценарии словами «SQL injection», «безопасный Docker», «mTLS включён» без пути и проверки. ## Lifecycle и инциденты -TODO: bootstrap, expiry/renewal, revoke/rotation, компрометация identity, sealed OpenBao и восстановление. Указать владельца CA/unseal и способ защиты bootstrap credentials. Сопоставить scan findings с реальной применимостью и исправлением/остаточным риском. +Владелец CA и unseal — оператор стенда; bootstrap и поведение при sealed OpenBao описаны в [отчёте](../REPORT.md). Сертификаты действуют 90 дней, CA — 365 дней; автоматический renewal и отзыв не реализованы. Сканирование и разбор результатов не проводились. diff --git a/src/bao_entrypoint.py b/src/bao_entrypoint.py new file mode 100644 index 0000000..bd5568b --- /dev/null +++ b/src/bao_entrypoint.py @@ -0,0 +1,37 @@ +"""Obtain scoped service configuration using a certificate identity in OpenBao.""" +import json +import os +import ssl +import sys +import urllib.request + + +def main(): + role = os.environ['BAO_SERVICE_ROLE'] + if role not in {'api', 'worker'}: + raise ValueError('unsupported service identity') + base = os.environ.get('BAO_ADDR', 'https://openbao:8200').rstrip('/') + context = ssl.create_default_context(cafile=os.environ['BAO_CACERT']) + context.load_cert_chain(os.environ['BAO_CLIENT_CERT']) + login = urllib.request.Request(base + '/v1/auth/cert/login', + data=json.dumps({'name': role}).encode(), + headers={'Content-Type': 'application/json'}, method='POST') + with urllib.request.urlopen(login, context=context, timeout=5) as response: + token = json.load(response)['auth']['client_token'] + request = urllib.request.Request(base + '/v1/lab/data/' + role, + headers={'X-Vault-Token': token}) + with urllib.request.urlopen(request, context=context, timeout=5) as response: + values = json.load(response)['data']['data'] + allowed = {'DATABASE_URL', 'CURSOR_SECRET', 'S3_ACCESS_KEY_ID', 'S3_SECRET_ACCESS_KEY', 'BROKER_URL'} + if set(values) != allowed or not all(isinstance(v, str) and v for v in values.values()): + raise ValueError('invalid service configuration from OpenBao') + for key, value in values.items(): + os.environ.pop(key + '_FILE', None) + os.environ[key] = value + if len(sys.argv) < 2: + raise SystemExit('missing command') + os.execv(sys.argv[1], sys.argv[1:]) + + +if __name__ == '__main__': + main() diff --git a/src/broker.py b/src/broker.py index ed306b5..a54d86c 100644 --- a/src/broker.py +++ b/src/broker.py @@ -1,6 +1,7 @@ """Small durable RabbitMQ adapter; broker messages contain no attachment bytes.""" import json import os +import ssl from pathlib import Path import pika @@ -16,13 +17,19 @@ DLQ = "attachment.process.v1.dlq" def broker_url() -> str: path = os.environ.get("BROKER_URL_FILE") value = Path(path).read_text(encoding="utf-8").strip() if path else os.environ.get("BROKER_URL", "") - if not value.startswith(("amqp://", "amqps://")): - raise ValueError("BROKER_URL или BROKER_URL_FILE должен содержать AMQP URL") + if not value.startswith("amqps://"): + raise ValueError("BROKER_URL или BROKER_URL_FILE должен содержать AMQPS URL") return value def _connection() -> pika.BlockingConnection: - connection = pika.BlockingConnection(pika.URLParameters(broker_url())) + url = broker_url() + parameters = pika.URLParameters(url) + ca_file = os.environ.get("BROKER_CA_FILE") + if not ca_file: + raise ValueError("BROKER_CA_FILE is required for verified broker TLS") + parameters.ssl_options = pika.SSLOptions(ssl.create_default_context(cafile=ca_file), parameters.host) + connection = pika.BlockingConnection(parameters) channel = connection.channel() channel.queue_declare(queue=QUEUE, durable=True) channel.queue_declare(queue=DLQ, durable=True) diff --git a/src/login_limit.py b/src/login_limit.py new file mode 100644 index 0000000..c952c34 --- /dev/null +++ b/src/login_limit.py @@ -0,0 +1,23 @@ +"""Bounded per-process login attempt limit, using a monotonic clock.""" +import math +import time +from collections import deque + + +class LoginLimiter: + def __init__(self, attempts: int = 60, window_seconds: int = 60, clock=time.monotonic): + if attempts < 1 or window_seconds < 1: + raise ValueError("login limit must be positive") + self.attempts = attempts + self.window_seconds = window_seconds + self.clock = clock + self.timestamps = deque() + + def retry_after(self) -> int: + now = self.clock() + while self.timestamps and self.timestamps[0] <= now - self.window_seconds: + self.timestamps.popleft() + if len(self.timestamps) >= self.attempts: + return max(1, math.ceil(self.timestamps[0] + self.window_seconds - now)) + self.timestamps.append(now) + return 0 diff --git a/src/main.py b/src/main.py index 004b3e6..b3844a0 100644 --- a/src/main.py +++ b/src/main.py @@ -1,6 +1,7 @@ import asyncio import logging import os +import re import sys import time from contextlib import asynccontextmanager @@ -21,6 +22,7 @@ from .attachments import router as attachment_router from .storage.s3 import S3Settings, S3Store from .storage.postgres import PostgresStore from . import telemetry as tel +from .login_limit import LoginLimiter tel.setup("messenger-api") @@ -48,6 +50,7 @@ async def lifespan(application: FastAPI): await store.connect() application.state.store = store application.state.settings = settings + application.state.login_limiter = LoginLimiter() application.state.s3 = S3Store(S3Settings.load()) concurrency = int(os.environ.get("UPLOAD_CONCURRENCY", "1")) if not 1 <= concurrency <= 8: @@ -75,7 +78,7 @@ async def lifespan(application: FastAPI): app = FastAPI( title="MTUSI Messenger", - version="1.6.0", + version="1.7.0", redoc_url=None, lifespan=lifespan, ) @@ -90,7 +93,22 @@ async def add_request_id(request: Request, call_next) -> Response: status_code = 500 with tel.tracer.start_as_current_span("HTTP request", context=tel.extract(dict(request.headers)), kind=tel.SpanKind.SERVER, record_exception=False, set_status_on_exception=False) as span: try: - response = await call_next(request) + upload = request.method == "POST" and re.fullmatch(r"/api/v1/chats/[^/]+/attachments", request.url.path) + if request.method in {"POST", "PUT", "PATCH"} and not upload: + # Buffer only bounded non-upload bodies; upload has its own streaming limit. + chunks = [] + size = 0 + async for chunk in request.stream(): + size += len(chunk) + if size > 65536: + response = error_response(request, 413, "payload_too_large", "Тело запроса слишком велико") + break + chunks.append(chunk) + else: + request._body = b"".join(chunks) + response = await call_next(request) + else: + response = await call_next(request) status_code = response.status_code response.headers["X-Request-Id"] = request.state.request_id response.headers["X-Instance-Id"] = request.app.state.settings.instance_id diff --git a/src/provision_db_roles.py b/src/provision_db_roles.py new file mode 100644 index 0000000..9fdb4cb --- /dev/null +++ b/src/provision_db_roles.py @@ -0,0 +1,41 @@ +"""Create limited application roles after migrations using operator DB credentials.""" +import asyncio +from pathlib import Path +import re +import ssl +import asyncpg + + +async def main(): + secret = lambda name: Path('/run/secrets/' + name).read_text().strip() + connection = await asyncpg.connect(secret('database_url'), + ssl=ssl.create_default_context(cafile='/run/secrets/ca_cert')) + try: + for role in ('api', 'worker', 'outbox', 'reconcile'): + user = 'messenger_' + role + password = secret('database_' + role + '_password') + if not re.fullmatch(r'[0-9a-f]{64}', password): + raise ValueError('invalid generated password') + exists = await connection.fetchval('SELECT 1 FROM pg_roles WHERE rolname=$1', user) + if not exists: + await connection.execute(f'CREATE ROLE {user} LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION PASSWORD \'{password}\'') + await connection.execute(f'GRANT USAGE ON SCHEMA public TO {user}') + await connection.execute('REVOKE CREATE ON SCHEMA public FROM PUBLIC') + await connection.execute('GRANT SELECT, INSERT ON users, chats, chat_members, messages TO messenger_api') + await connection.execute('GRANT SELECT, INSERT, DELETE ON sessions TO messenger_api') + await connection.execute('GRANT SELECT, INSERT, UPDATE, DELETE ON idempotency_records TO messenger_api') + await connection.execute('GRANT SELECT, INSERT, UPDATE ON attachments, attachment_jobs TO messenger_api') + await connection.execute('GRANT SELECT ON attachment_variants TO messenger_api') + await connection.execute('GRANT INSERT ON attachment_outbox, attachment_slo TO messenger_api') + await connection.execute('GRANT SELECT, UPDATE ON attachments, attachment_jobs, attachment_slo TO messenger_worker') + await connection.execute('GRANT SELECT, INSERT, UPDATE ON attachment_variants TO messenger_worker') + await connection.execute('GRANT SELECT, UPDATE ON attachments, attachment_jobs, attachment_slo TO messenger_outbox') + await connection.execute('GRANT SELECT, INSERT, UPDATE ON attachment_outbox TO messenger_outbox') + await connection.execute('GRANT SELECT ON attachments, attachment_variants TO messenger_reconcile') + print('Database API and worker roles configured.') + finally: + await connection.close() + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/src/routes.py b/src/routes.py index fcd2e74..27722ab 100644 --- a/src/routes.py +++ b/src/routes.py @@ -100,6 +100,9 @@ async def create_session( authorization: str | None = Header(default=None), store: PostgresStore = Depends(get_store), ) -> dict: + retry_after = request.app.state.login_limiter.retry_after() + if retry_after: + raise ApiError(429, "rate_limited", "Слишком много попыток входа", {"Retry-After": str(retry_after)}) if await request.body(): raise ApiError(400, "invalid_request", "Тело запроса не требуется") username, password = parse_basic(authorization) diff --git a/src/storage/postgres.py b/src/storage/postgres.py index 9e6edfe..876e0ee 100644 --- a/src/storage/postgres.py +++ b/src/storage/postgres.py @@ -2,6 +2,8 @@ from datetime import datetime, timedelta, timezone from contextlib import asynccontextmanager from hashlib import sha256 import json +import os +import ssl from pathlib import Path from secrets import token_urlsafe from uuid import UUID, uuid4 @@ -16,7 +18,7 @@ from .. import telemetry as tel ROOT = Path(__file__).resolve().parents[2] MIGRATIONS = ROOT / "migrations" -PASSWORD_HASHER = PasswordHasher() +PASSWORD_HASHER = PasswordHasher(time_cost=2, memory_cost=19456, parallelism=1) DB_ERRORS = (asyncpg.PostgresError, asyncpg.InterfaceError, OSError, TimeoutError) @@ -27,8 +29,12 @@ class PostgresStore: async def connect(self) -> None: try: + ca_file = os.environ.get("DATABASE_CA_FILE") + if not ca_file: + raise ValueError("DATABASE_CA_FILE is required for verified PostgreSQL TLS") + tls = ssl.create_default_context(cafile=ca_file) self._pool = await asyncpg.create_pool( - self._database_url, min_size=1, max_size=10, command_timeout=5 + self._database_url, min_size=1, max_size=10, command_timeout=5, ssl=tls ) except DB_ERRORS as error: raise RuntimeError("PostgreSQL is unavailable") from error diff --git a/src/storage/s3.py b/src/storage/s3.py index 7188fb2..e46b898 100644 --- a/src/storage/s3.py +++ b/src/storage/s3.py @@ -36,7 +36,10 @@ class S3Settings: max_pixels = int(os.environ.get("MAX_IMAGE_PIXELS", "40000000")) if not 1 <= max_bytes <= 10485760 or not 1 <= max_pixels <= 40000000: raise ValueError("Image limits exceed contract") - return cls(os.environ.get("S3_ENDPOINT", "http://s3:8333"), + endpoint = os.environ.get("S3_ENDPOINT", "https://s3:8333") + if not endpoint.startswith("https://") or not os.environ.get("AWS_CA_BUNDLE"): + raise ValueError("S3 requires HTTPS and AWS_CA_BUNDLE") + return cls(endpoint, os.environ.get("S3_REGION", "us-east-1"), os.environ.get("S3_BUCKET", "messenger-attachments"), secret("S3_ACCESS_KEY_ID"), secret("S3_SECRET_ACCESS_KEY"), diff --git a/student-tests/dependency_tls.py b/student-tests/dependency_tls.py new file mode 100644 index 0000000..f1d9e93 --- /dev/null +++ b/student-tests/dependency_tls.py @@ -0,0 +1,40 @@ +"""Probe trusted and untrusted TLS plus inaccessible plaintext dependency ports.""" +import os +import socket +import ssl +import urllib.error +import urllib.request + +ca = os.environ['BAO_CACERT'] +for host, port in [('s3', 8333), ('broker', 5671), ('openbao', 8200)]: + trusted = ssl.create_default_context(cafile=ca) + with socket.create_connection((host, port), timeout=5) as raw: + with trusted.wrap_socket(raw, server_hostname=host) as connection: + assert connection.version() in {'TLSv1.2', 'TLSv1.3'} + untrusted = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + try: + with socket.create_connection((host, port), timeout=5) as raw: + with untrusted.wrap_socket(raw, server_hostname=host): + pass + except ssl.SSLCertVerificationError: + pass + else: + raise AssertionError(f'{host}: untrusted CA accepted') + print(f'PASS {host}: verified TLS; untrusted CA rejected') + +for host, port in [('s3', 8888), ('s3', 9333), ('s3', 18333), ('broker', 5672), ('broker', 4369), ('broker', 25672), ('broker', 15692), ('loki', 9095), ('tempo', 9095)]: + try: + socket.create_connection((host, port), timeout=3).close() + except ConnectionRefusedError: + pass + else: + raise AssertionError(f'{host}:{port}: unexpected accessible plaintext port') +print('PASS: S3 internal and broker plaintext ports are closed') +request = urllib.request.Request('https://s3:8333/', data=b'', method='POST') +try: + urllib.request.urlopen(request, context=ssl.create_default_context(cafile=ca), timeout=5) +except urllib.error.HTTPError as error: + assert error.code == 403 +else: + raise AssertionError('S3 proxy allowed the blocked POST method') +print('PASS: S3 proxy blocks POST') diff --git a/student-tests/lab7_limits.py b/student-tests/lab7_limits.py new file mode 100644 index 0000000..cde0be9 --- /dev/null +++ b/student-tests/lab7_limits.py @@ -0,0 +1,67 @@ +"""Offline tests of login throttling and bounded request bodies.""" +import asyncio +import sys +import unittest +from pathlib import Path +from types import SimpleNamespace + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from src.login_limit import LoginLimiter +from src.main import app + + +async def request(chunks, limiter=None): + app.state.settings = SimpleNamespace(instance_id="test") + app.state.login_limiter = limiter or LoginLimiter(attempts=1, window_seconds=60) + app.state.store = object() + messages = [{"type": "http.request", "body": chunk, "more_body": i < len(chunks) - 1} + for i, chunk in enumerate(chunks)] + response = [] + async def receive(): + if messages: + return messages.pop(0) + await asyncio.Event().wait() + async def send(message): + response.append(message) + scope = {"type": "http", "asgi": {"version": "3.0"}, "http_version": "1.1", + "method": "POST", "scheme": "http", "path": "/api/v1/auth/sessions", + "raw_path": b"/api/v1/auth/sessions", "query_string": b"", "headers": [], + "client": ("127.0.0.1", 1234), "server": ("localhost", 8080), "root_path": ""} + await app(scope, receive, send) + return response + + +class LimitsTests(unittest.TestCase): + def test_window_and_retry(self): + now = [100.0] + limiter = LoginLimiter(2, 60, lambda: now[0]) + self.assertEqual([limiter.retry_after() for _ in range(3)], [0, 0, 60]) + now[0] = 159.5 + self.assertEqual(limiter.retry_after(), 1) + now[0] = 160 + self.assertEqual(limiter.retry_after(), 0) + + def test_chunked_oversized_body(self): + result = asyncio.run(request([b"x" * 40000, b"x" * 30000])) + start = result[0] + self.assertEqual(start["status"], 413) + headers = dict(start["headers"]) + self.assertIn(b"x-request-id", headers) + self.assertIn(b"x-instance-id", headers) + self.assertIn(b'"payload_too_large"', b"".join(x.get("body", b"") for x in result)) + + def test_small_body_reaches_route(self): + result = asyncio.run(request([b""])) + self.assertEqual(result[0]["status"], 401) + + def test_rate_limit_returns_retry_after(self): + limiter = LoginLimiter(attempts=1, window_seconds=60) + first = asyncio.run(request([b""], limiter)) + second = asyncio.run(request([b""], limiter)) + self.assertEqual(first[0]["status"], 401) + self.assertEqual(second[0]["status"], 429) + self.assertGreater(int(dict(second[0]["headers"])[b"retry-after"]), 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/student-tests/lab7_security.py b/student-tests/lab7_security.py new file mode 100644 index 0000000..3058228 --- /dev/null +++ b/student-tests/lab7_security.py @@ -0,0 +1,49 @@ +"""TLS hostname, cross-chat attachment ACL, and actual session expiry.""" +import hashlib +from pathlib import Path +import socket +import ssl +import subprocess +import sys +import time +from urllib.parse import urlsplit +from uuid import uuid4 + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client, arguments, run + + +def main(): + args = arguments(__doc__) + client = Client(args.base_url, args.ca_file) + target = urlsplit(client.base) + context = ssl.create_default_context(cafile=args.ca_file) + try: + with socket.create_connection((target.hostname, target.port or 443), timeout=5) as raw: + with context.wrap_socket(raw, server_hostname='wrong-host.invalid'): + pass + except ssl.SSLCertVerificationError: + pass + else: + raise AssertionError('TLS accepted wrong server hostname') + alice, bob, eve, chat = client.bootstrap() + attachment, _ = client.attachment(alice, chat) + other_chat = client.call('POST', '/api/v1/chats', 201, actor=eve, + body={'title': 'Other chat', 'member_ids': [eve['id']]}, schema='Chat')['json'] + client.call('POST', '/api/v1/chats/' + other_chat['id'] + '/messages', 404, actor=eve, + body={'text': '', 'attachment_ids': [attachment['id']]}, + headers={'Idempotency-Key': uuid4().hex}) + client.call('GET', '/api/v1/users/me', actor=alice, schema='User') + digest = hashlib.sha256(alice['token'].encode('ascii')).hexdigest() + sql = "UPDATE sessions SET expires_at=now()+interval '2 seconds' WHERE token_hash='" + digest + "';\n" + result = subprocess.run(['docker', 'compose', 'exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', '-q'], + cwd=ROOT, input=sql, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True) + del result + time.sleep(3) + client.call('GET', '/api/v1/users/me', 401, actor=alice) + print('PASS: wrong hostname rejected; cross-chat attachment denied; expired session rejected') + + +if __name__ == '__main__': + run(main) diff --git a/student-tests/mtls_probe.py b/student-tests/mtls_probe.py new file mode 100644 index 0000000..f8e2131 --- /dev/null +++ b/student-tests/mtls_probe.py @@ -0,0 +1,31 @@ +"""Run inside the Compose network using temporary test identities.""" +import socket +import ssl + + +def call(host, cert=None): + context = ssl.create_default_context(cafile="/certs/ca.crt") + if cert: + context.load_cert_chain(cert) + with socket.create_connection((host, 8444), timeout=5) as raw: + with context.wrap_socket(raw, server_hostname=host) as conn: + conn.sendall(b"GET /health/ready HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n") + return conn.recv(4096).split(b"\r\n", 1)[0] + + +for host in ("api-a", "api-b"): + assert b" 200 " in call(host, "/certs/lb.pem"), f"{host}: valid LB certificate rejected" + for certificate in (None, "/certs/foreign.pem"): + try: + result = call(host, certificate) + except (ssl.SSLError, ConnectionResetError, BrokenPipeError): + continue + raise AssertionError(f"{host}: bad certificate unexpectedly received {result!r}") + assert b" 403 " in call(host, "/certs/wrong-role.pem"), f"{host}: wrong role not denied" + try: + socket.create_connection((host, 8080), timeout=3).close() + except ConnectionRefusedError: + pass + else: + raise AssertionError(f"{host}: application port reachable without mTLS") + print(f"PASS {host}: valid identity accepted; no cert, foreign CA and wrong role rejected; direct API closed") diff --git a/student-tests/observability_tls.py b/student-tests/observability_tls.py new file mode 100644 index 0000000..c67ea54 --- /dev/null +++ b/student-tests/observability_tls.py @@ -0,0 +1,32 @@ +"""Query Grafana's three TLS-protected datasources without printing telemetry.""" +import base64 +import json +import os +from pathlib import Path +import ssl +import time +import urllib.parse +import urllib.request + +root = Path(__file__).resolve().parents[1] +context = ssl.create_default_context(cafile=str(root / 'secrets/ca.crt')) +password = (root / 'secrets/grafana_password').read_text().strip() +auth = base64.b64encode(('admin:' + password).encode()).decode() +base = os.environ.get('GRAFANA_URL', 'https://localhost:3000').rstrip('/') + '/api/datasources/proxy/uid/' +now = time.time_ns() +paths = { + 'prometheus': '/api/v1/query?' + urllib.parse.urlencode({'query': 'up'}), + 'loki': '/loki/api/v1/query_range?' + urllib.parse.urlencode({ + 'query': '{service_name="messenger-api"}', 'start': now - 600_000_000_000, + 'end': now, 'limit': 5}), + 'tempo': '/api/search?' + urllib.parse.urlencode({'limit': 5, 'start': now // 1_000_000_000 - 120, 'end': now // 1_000_000_000}), +} +for name, path in paths.items(): + req = urllib.request.Request(base + name + path, headers={'Authorization': 'Basic ' + auth}) + with urllib.request.urlopen(req, context=context, timeout=15) as response: + payload = json.load(response) + values = payload.get('data', {}).get('result', payload.get('traces', [])) + assert values, f'{name}: no data returned' + if name == 'prometheus': + assert all(item['value'][1] == '1' for item in values), 'Prometheus scrape target down' + print(f'PASS: Grafana connected to {name}; data present') diff --git a/student-tests/openbao_policy.py b/student-tests/openbao_policy.py new file mode 100644 index 0000000..bd35e23 --- /dev/null +++ b/student-tests/openbao_policy.py @@ -0,0 +1,37 @@ +"""Run inside API and worker containers; print only allow/deny outcomes.""" +import json +import os +from pathlib import Path +import ssl +import urllib.error +import urllib.request + +role = os.environ['BAO_SERVICE_ROLE'] +other = {'api': 'worker', 'worker': 'api'}[role] +base = os.environ['BAO_ADDR'].rstrip('/') +context = ssl.create_default_context(cafile=os.environ['BAO_CACERT']) +context.load_cert_chain(os.environ['BAO_CLIENT_CERT']) + + +def request(path, token=None, body=None): + headers = {'Content-Type': 'application/json'} + if token: + headers['X-Vault-Token'] = token + data = json.dumps(body).encode() if body is not None else None + req = urllib.request.Request(base + '/v1/' + path, data=data, headers=headers, + method='POST' if body is not None else 'GET') + with urllib.request.urlopen(req, context=context, timeout=5) as response: + return json.load(response) + + +token = request('auth/cert/login', body={'name': role})['auth']['client_token'] +assert request('lab/data/' + role, token)['data']['data'] +for path, body in [('lab/data/' + other, None), ('sys/policies/acl/forbidden', {'policy': 'path "*" { capabilities = ["read"] }'})]: + try: + request(path, token, body) + except urllib.error.HTTPError as error: + assert error.code == 403, f'unexpected status {error.code}' + else: + raise AssertionError('cross-role read or policy write allowed') +assert not Path('/run/secrets/openbao-operator.json').exists() +print(f'PASS {role}: own path allowed; other path and policy write denied') diff --git a/student-tests/persistence_evidence.py b/student-tests/persistence_evidence.py new file mode 100644 index 0000000..f7277db --- /dev/null +++ b/student-tests/persistence_evidence.py @@ -0,0 +1,33 @@ +"""Run the persistence hook and record actual service restart timestamps.""" +import json +import os +from pathlib import Path +import subprocess + +ROOT = Path(__file__).resolve().parents[1] +SERVICES = ('postgres', 's3', 'broker', 'openbao', 'prometheus', 'loki', 'tempo', 'api-a', 'api-b') + + +def started(service): + container = subprocess.check_output(['docker', 'compose', 'ps', '-q', service], cwd=ROOT, text=True).strip() + if not container: + raise RuntimeError(f'{service} is not running') + return subprocess.check_output(['docker', 'inspect', '--format', '{{.State.StartedAt}}', container], text=True).strip() + + +before = {service: started(service) for service in SERVICES} +result = subprocess.run(['make', 'test-persistence', 'ACTION_SCRIPT=scripts/restart.sh'], cwd=ROOT, + text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) +print(result.stdout, end='') +if result.returncode: + raise SystemExit(result.returncode) +after = {service: started(service) for service in SERVICES} +for service in SERVICES: + if after[service] <= before[service]: + raise RuntimeError(f'{service} did not restart') +evidence = {'status': 'PASS', 'command': 'make test-persistence ACTION_SCRIPT=scripts/restart.sh', + 'services': {service: {'before': before[service], 'after': after[service]} for service in SERVICES}, + 'test_output': result.stdout.strip()} +path = ROOT / 'evidence' / 'lab7-persistence.json' +path.write_text(json.dumps(evidence, ensure_ascii=False, indent=2) + '\n') +print('PASS: restart timestamps recorded in evidence/lab7-persistence.json') diff --git a/student-tests/postgres_tls.py b/student-tests/postgres_tls.py new file mode 100644 index 0000000..7aed55d --- /dev/null +++ b/student-tests/postgres_tls.py @@ -0,0 +1,47 @@ +"""Execute inside an API container to test the PostgreSQL transport.""" +import asyncio +import json +import os +from pathlib import Path +import ssl +import urllib.request +import asyncpg + + +async def main(): + if Path('/run/secrets/database_url').exists(): + dsn = Path('/run/secrets/database_url').read_text().strip() + else: + bao_tls = ssl.create_default_context(cafile=os.environ['BAO_CACERT']) + bao_tls.load_cert_chain(os.environ['BAO_CLIENT_CERT']) + base = os.environ['BAO_ADDR'].rstrip('/') + role = os.environ['BAO_SERVICE_ROLE'] + request = urllib.request.Request(base + '/v1/auth/cert/login', + data=json.dumps({'name': role}).encode(), headers={'Content-Type': 'application/json'}) + with urllib.request.urlopen(request, context=bao_tls) as response: + token = json.load(response)['auth']['client_token'] + request = urllib.request.Request(base + '/v1/lab/data/' + role, headers={'X-Vault-Token': token}) + with urllib.request.urlopen(request, context=bao_tls) as response: + dsn = json.load(response)['data']['data']['DATABASE_URL'] + context = ssl.create_default_context(cafile='/run/secrets/ca_cert') + connection = await asyncpg.connect(dsn, ssl=context) + try: + assert await connection.fetchval('select ssl from pg_stat_ssl where pid=pg_backend_pid()') + finally: + await connection.close() + try: + await asyncpg.connect(dsn, ssl=False) + except asyncpg.InvalidAuthorizationSpecificationError: + pass + else: + raise AssertionError('plaintext PostgreSQL connection was accepted') + try: + await asyncpg.connect(dsn, ssl=ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)) + except ssl.SSLCertVerificationError: + pass + else: + raise AssertionError('untrusted CA was accepted') + print('PASS: PostgreSQL TLS verified, plaintext and untrusted CA rejected') + + +asyncio.run(main()) diff --git a/student-tests/s3_prefix_permissions.py b/student-tests/s3_prefix_permissions.py new file mode 100644 index 0000000..d82b270 --- /dev/null +++ b/student-tests/s3_prefix_permissions.py @@ -0,0 +1,40 @@ +"""Check allowed and denied S3 prefixes using the current service identity.""" +import json +import os +import ssl +import urllib.request +from uuid import uuid4 + +import boto3 +from botocore.exceptions import ClientError + +role = os.environ['BAO_SERVICE_ROLE'] +ctx = ssl.create_default_context(cafile=os.environ['BAO_CACERT']) +ctx.load_cert_chain(os.environ['BAO_CLIENT_CERT']) +base = os.environ['BAO_ADDR'].rstrip('/') +request = urllib.request.Request(base + '/v1/auth/cert/login', data=json.dumps({'name': role}).encode(), + headers={'Content-Type': 'application/json'}) +with urllib.request.urlopen(request, context=ctx) as response: + token = json.load(response)['auth']['client_token'] +request = urllib.request.Request(base + '/v1/lab/data/' + role, headers={'X-Vault-Token': token}) +with urllib.request.urlopen(request, context=ctx) as response: + values = json.load(response)['data']['data'] +client = boto3.client('s3', endpoint_url=os.environ['S3_ENDPOINT'], region_name='us-east-1', + aws_access_key_id=values['S3_ACCESS_KEY_ID'], + aws_secret_access_key=values['S3_SECRET_ACCESS_KEY']) +bucket = os.environ['S3_BUCKET'] +probe = uuid4().hex +allowed = 'staging/security-probe-' + probe +denied = ('originals/' if role == 'api' else '_bootstrap/') + 'security-probe-' + probe +client.put_object(Bucket=bucket, Key=allowed, Body=b'test') +client.delete_object(Bucket=bucket, Key=allowed) +try: + client.put_object(Bucket=bucket, Key=denied, Body=b'test') +except ClientError as error: + assert error.response['ResponseMetadata']['HTTPStatusCode'] == 403 +else: + client.delete_object(Bucket=bucket, Key=denied) + raise AssertionError('disallowed prefix was writable') +finally: + client.close() +print(f'PASS {role}: allowed staging write; forbidden prefix denied') diff --git a/student-tests/service_permissions.py b/student-tests/service_permissions.py new file mode 100644 index 0000000..bb81d93 --- /dev/null +++ b/student-tests/service_permissions.py @@ -0,0 +1,55 @@ +"""Check scoped DB and broker permissions from the running service identity.""" +import asyncio +import json +import os +import ssl +import urllib.request + +import asyncpg +import pika + +role = os.environ['BAO_SERVICE_ROLE'] +context = ssl.create_default_context(cafile=os.environ['BAO_CACERT']) +context.load_cert_chain(os.environ['BAO_CLIENT_CERT']) +base = os.environ['BAO_ADDR'].rstrip('/') +req = urllib.request.Request(base + '/v1/auth/cert/login', data=json.dumps({'name': role}).encode(), + headers={'Content-Type': 'application/json'}) +with urllib.request.urlopen(req, context=context) as response: + token = json.load(response)['auth']['client_token'] +req = urllib.request.Request(base + '/v1/lab/data/' + role, headers={'X-Vault-Token': token}) +with urllib.request.urlopen(req, context=context) as response: + values = json.load(response)['data']['data'] + + +async def database(): + db = await asyncpg.connect(values['DATABASE_URL'], ssl=ssl.create_default_context(cafile=os.environ['DATABASE_CA_FILE'])) + try: + assert await db.fetchval('SELECT current_user') == 'messenger_' + role + try: + await db.execute('SELECT password_hash FROM users LIMIT 1' if role == 'worker' else 'CREATE TABLE unauthorized_probe (id integer)') + except asyncpg.InsufficientPrivilegeError: + pass + else: + raise AssertionError('unexpected database privilege') + finally: + await db.close() + + +asyncio.run(database()) +params = pika.URLParameters(values['BROKER_URL']) +params.ssl_options = pika.SSLOptions(ssl.create_default_context(cafile=os.environ['BROKER_CA_FILE']), params.host) +connection = pika.BlockingConnection(params) +try: + channel = connection.channel() + try: + if role == 'api': + channel.basic_get('attachment.process.v1', auto_ack=False) + else: + channel.queue_declare(queue='unauthorized.probe', durable=True) + except pika.exceptions.ChannelClosedByBroker as error: + assert error.reply_code == 403 + else: + raise AssertionError('unexpected broker privilege') +finally: + connection.close() +print(f'PASS {role}: DB identity and denied privilege; broker denied unauthorized operation') diff --git a/student-tests/telemetry_tls.py b/student-tests/telemetry_tls.py new file mode 100644 index 0000000..82250f0 --- /dev/null +++ b/student-tests/telemetry_tls.py @@ -0,0 +1,30 @@ +"""Verify the Collector accepts its publisher role and rejects other callers.""" +import os +import socket +import ssl + + +def request(certificate): + ctx = ssl.create_default_context(cafile=os.environ['OTEL_EXPORTER_OTLP_CERTIFICATE']) + if certificate: + ctx.load_cert_chain(certificate) + with socket.create_connection(('otel-collector', 4318), timeout=5) as raw: + with ctx.wrap_socket(raw, server_hostname='otel-collector') as conn: + conn.sendall(b'GET / HTTP/1.1\r\nHost: otel-collector\r\nConnection: close\r\n\r\n') + return conn.recv(1024) + + +assert b'404' in request(os.environ['OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE']) +for cert in (None, os.environ['BAO_CLIENT_CERT']): + try: + result = request(cert) + except (ssl.SSLError, ConnectionResetError, BrokenPipeError): + continue + assert not result, 'Collector accepted an unauthorized caller' +try: + socket.create_connection(('otel-collector', 14318), timeout=2).close() +except ConnectionRefusedError: + pass +else: + raise AssertionError('Collector plaintext listener reachable from outside') +print('PASS: Collector publisher accepted; no cert and wrong role rejected; local listener closed') -- 2.54.0 From 9cb708335d1c4dbbff6f4916af40a12c7fdb6e78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9C=D0=B0=D1=82=D0=B2=D0=B5=D0=B9?= Date: Sat, 7 Nov 2026 22:00:19 +0300 Subject: [PATCH 3/3] add level 3 --- REPORT.md | 15 +- compose.yaml | 29 +++ evidence/lab7-failover-audit.json | 19 ++ evidence/lab7-persistence.json | 36 ++-- evidence/lab7-strict-audit.json | 232 +++++++++++++++++++++++ infra/haproxy.cfg | 3 +- infra/s3-tcp.cfg | 2 + scripts/init-secrets.py | 1 + scripts/prepare-secret-access.py | 39 ++++ security/THREAT_MODEL.md | 3 +- student-tests/ingress_redaction.py | 35 ++++ student-tests/runtime_hardening.py | 29 +++ student-tests/s3_prefix_permissions.py | 26 +++ student-tests/secret_file_permissions.py | 35 ++++ 14 files changed, 482 insertions(+), 22 deletions(-) create mode 100644 evidence/lab7-failover-audit.json create mode 100644 evidence/lab7-strict-audit.json create mode 100644 scripts/prepare-secret-access.py create mode 100644 student-tests/ingress_redaction.py create mode 100644 student-tests/runtime_hardening.py create mode 100644 student-tests/secret_file_permissions.py diff --git a/REPORT.md b/REPORT.md index e289fe1..7ebad11 100644 --- a/REPORT.md +++ b/REPORT.md @@ -9,13 +9,14 @@ Нужны Docker с Compose, Python 3.9+ для операторских и публичных скриптов и OpenSSL. Порты по умолчанию: HTTPS API `127.0.0.1:8443`, OpenBao `127.0.0.1:18200`, Grafana `127.0.0.1:3000`. Если порт занят, измените `PUBLIC_HTTPS_PORT`, `BAO_HOST_PORT` или `GRAFANA_HOST_PORT` в `.env`; для операторского скрипта при изменённом порте OpenBao задайте `BAO_OPERATOR_ADDR=https://localhost:<порт>`. -Чистый запуск проверен на Docker Desktop для macOS. Права на файловые Compose secrets при запуске на Linux отдельно не проверены; non-root процессы должны иметь доступ только к назначенным им файлам. Переменные в `.env.example` являются подсказками задания: сетевые адреса контейнеров и пути секретов фактически заданы в `compose.yaml`. +Чистый запуск проверен на Docker Desktop для macOS. После генерации secrets выполните `scripts/prepare-secret-access.py`: каталог остаётся 0700, отдельно монтируемые файлы получают 0640 и группу оператора; её GID записывается в `.env` как `LAB_SECRET_GID` и добавляется контейнерам через `group_add`. CA key и operator JSON остаются 0600 и не монтируются приложениям. После повторной генерации secrets повторите этот шаг. Это учитывает [ограничение Compose для file secrets](https://docs.docker.com/reference/compose-file/services/#secrets): uid/gid/mode не переназначаются для bind-mounted файлов. Полный запуск на отдельном Linux-хосте пока не проверен. Переменные в `.env.example` являются подсказками задания: сетевые адреса контейнеров и пути секретов фактически заданы в `compose.yaml`. ```sh cp .env.example .env python3 scripts/init-secrets.py sh scripts/bootstrap-pki.sh sh scripts/bootstrap-internal-pki.sh +python3 scripts/prepare-secret-access.py docker compose --env-file .env up -d openbao # Дождаться ответа статуса, затем один раз инициализировать: python3 scripts/openbao-operator.py status @@ -66,7 +67,7 @@ OpenBao использует persistent file storage вне dev-mode. API и wor Argon2id настроен на `m=19456 KiB`, `t=2`, `p=1`. Один запуск `docker compose exec -T api-a /app/.venv/bin/python - < scripts/benchmark-argon2.py` выполнил по 10 последовательных хеширований и проверок: медианы 11,84 и 11,27 мс. Контейнер API ограничен 0,5 CPU и 512 MiB; Docker сообщил 10 CPU и 10 416 644 096 байт памяти. Параметры соответствуют минимуму задания, а замер показывает задержку одной операции на этом стенде; выбор не исследован при параллельной нагрузке и не является оценкой устойчивости к DoS. -`docker inspect` показал для API и worker пользователя `65534:65534`, `privileged=false` и `cap_drop=[ALL]`; лимиты API — 0,5 CPU/512 MiB, worker — 2 CPU/4 GiB. Конфигурация остальных сервисов также содержит CPU и memory limits. Для защиты необходима отдельная демонстрация отсутствия лишних опубликованных портов и утечек в telemetry. +`docker inspect` показал для API и worker пользователя `65534:65534`, `privileged=false` и `cap_drop=[ALL]`; лимиты API — 0,5 CPU/512 MiB, worker — 2 CPU/4 GiB. Конфигурация остальных сервисов также содержит CPU и memory limits. Повторная проверка `python3 student-tests/runtime_hardening.py` подтверждает ограничения фактически работающих контейнеров. `student-tests/ingress_redaction.py` проверяет отсутствие синтетических секретов в access-логах; `student-tests/telemetry_unit.py` — фильтрацию полей OTLP. Это не исчерпывающий аудит всех каналов утечки. ## Ограничения @@ -75,3 +76,13 @@ Rate limit локален каждой API-реплике: 60 попыток з Для SeaweedFS опубликовано [сообщение о обходе bucket policy через POST Object](https://github.com/seaweedfs/seaweedfs/security/advisories/GHSA-qmx6-fpcw-q3xg), в котором указаны затронутые версии до 4.45 включительно и отсутствие исправленной версии на дату публикации. Стенд использует 4.17. В приложении POST Object не нужен; прокси запрещает HTTP POST, что проверяет `student-tests/dependency_tls.py`. Это ограничение конкретного пути, а не утверждение об устранении всех уязвимостей SeaweedFS. Не проверены все возможные пути утечки секретов в телеметрию и все сценарии отказа зависимостей. Оценка не подтверждается одним smoke-тестом. + +## Повторный строгий аудит + +Обнаружена и воспроизведена запись синтетического секрета из query string в stdout HAProxy. `infra/haproxy.cfg` теперь пишет только статус, размер, backend/server и код завершения; URL и заголовки исключены. Регрессия: `BASE_URL=https://localhost:18443 python3 student-tests/ingress_redaction.py`. Проверка касается этих access-логов, а не всех потенциальных каналов утечки. + +Добавлены подготовка прав файлов secrets для non-root контейнеров и зависимость outbox/reconcile от успешного `db-provision`, чтобы запуск не опережал выдачу прав БД. S3-прокси явно отклоняет повторные Authorization; в проверенном исходном сценарии обход ACL не воспроизвёлся (403), поэтому это усиление, а не подтверждённая уязвимость. + +Повторные проверки сохранены в [evidence/lab7-strict-audit.json](evidence/lab7-strict-audit.json): HTTPS smoke, внешний TLS, mTLS обеих реплик, ACL/expiry/hostname, scoped policies и права API/worker, TLS зависимостей, лимиты, OTLP redaction и доступ Grafana к трём источникам. Linux DAC проверен в одноразовом контейнере на синтетических файлах (`student-tests/secret_file_permissions.py`), без утверждения о полном развёртывании на отдельном Linux-хосте. + +После исправлений повторно прошёл persistence с фактическими временами рестарта (`evidence/lab7-persistence.json`). Повторный последовательный failover и восстановление API-A/API-B фиксируются отдельно в `evidence/lab7-failover-audit.json`. Публичные тесты и контракт не изменялись. diff --git a/compose.yaml b/compose.yaml index bfa75d1..0e9ff04 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1,4 +1,5 @@ x-api: &api + group_add: ["${LAB_SECRET_GID:-65534}"] build: context: . cap_drop: [ALL] @@ -63,6 +64,7 @@ x-api: &api services: openbao: + group_add: ["${LAB_SECRET_GID:-65534}"] image: openbao/openbao:2.3.2 user: "0:0" entrypoint: ["bao"] @@ -83,6 +85,7 @@ services: cpus: "0.50" broker: + group_add: ["${LAB_SECRET_GID:-65534}"] image: rabbitmq:4.1.2-alpine hostname: broker extra_hosts: ["broker:127.0.0.1"] @@ -115,6 +118,7 @@ services: memory: 256M broker-tcp: + group_add: ["${LAB_SECRET_GID:-65534}"] image: haproxy:3.0.11-alpine network_mode: service:broker depends_on: @@ -127,6 +131,7 @@ services: cpus: "0.25" s3: + group_add: ["${LAB_SECRET_GID:-65534}"] image: chrislusf/seaweedfs:4.17@sha256:186de7ef977a20343ee9a5544073f081976a29e2d29ecf8379891e7bf177fbe9 command: ["server", "-dir=/data", "-ip=127.0.0.1", "-ip.bind=127.0.0.1", "-s3", "-s3.port=18333", "-s3.config=/run/secrets/s3_config", "-s3.cert.file=/run/secrets/s3_tls", "-s3.key.file=/run/secrets/s3_tls", "-master.volumeSizeLimitMB=128", "-volume.max=64", "-s3.port.iceberg=0", "-filer.exposeDirectoryData=false"] secrets: @@ -148,6 +153,7 @@ services: memory: 512M s3-tcp: + group_add: ["${LAB_SECRET_GID:-65534}"] image: haproxy:3.0.11-alpine network_mode: service:s3 depends_on: @@ -166,6 +172,7 @@ services: cpus: "0.25" s3-init: + group_add: ["${LAB_SECRET_GID:-65534}"] build: context: . command: ["/app/.venv/bin/python", "-m", "src.bootstrap_s3"] @@ -187,6 +194,7 @@ services: cpus: "0.50" postgres: + group_add: ["${LAB_SECRET_GID:-65534}"] image: postgres:17.6-alpine entrypoint: ["/bin/sh", "/usr/local/bin/postgres-tls-entrypoint.sh"] command: ["postgres", "-c", "track_commit_timestamp=on", "-c", "ssl=on", "-c", "ssl_cert_file=/tmp/postgres-tls/server.pem", "-c", "ssl_key_file=/tmp/postgres-tls/server.pem", "-c", "hba_file=/etc/postgresql/pg_hba.conf"] @@ -214,6 +222,7 @@ services: memory: 512M migrate: + group_add: ["${LAB_SECRET_GID:-65534}"] build: context: . command: ["/app/.venv/bin/python", "-m", "src.migrate"] @@ -236,6 +245,7 @@ services: memory: 128M db-provision: + group_add: ["${LAB_SECRET_GID:-65534}"] build: context: . command: ["/app/.venv/bin/python", "-m", "src.provision_db_roles"] @@ -254,6 +264,7 @@ services: cpus: "0.25" reconcile: + group_add: ["${LAB_SECRET_GID:-65534}"] healthcheck: disable: true build: @@ -278,6 +289,8 @@ services: - s3_reconcile_access_key_id - s3_reconcile_secret_access_key depends_on: + db-provision: + condition: service_completed_successfully postgres: condition: service_healthy s3: @@ -297,6 +310,7 @@ services: <<: *api api-a-tls: + group_add: ["${LAB_SECRET_GID:-65534}"] image: haproxy:3.0.11-alpine network_mode: service:api-a depends_on: @@ -311,6 +325,7 @@ services: cpus: "0.25" api-b-tls: + group_add: ["${LAB_SECRET_GID:-65534}"] image: haproxy:3.0.11-alpine network_mode: service:api-b depends_on: @@ -325,6 +340,7 @@ services: cpus: "0.25" worker: + group_add: ["${LAB_SECRET_GID:-65534}"] restart: unless-stopped cap_drop: [ALL] security_opt: [no-new-privileges:true] @@ -384,6 +400,7 @@ services: stop_grace_period: 30s outbox: + group_add: ["${LAB_SECRET_GID:-65534}"] mem_limit: 256m cpus: "0.50" cap_drop: [ALL] @@ -414,6 +431,8 @@ services: - broker_outbox_url - otel_outbox_cert depends_on: + db-provision: + condition: service_completed_successfully migrate: condition: service_completed_successfully broker: @@ -423,6 +442,7 @@ services: restart: unless-stopped load_balancer: + group_add: ["${LAB_SECRET_GID:-65534}"] image: haproxy:3.0.11-alpine ports: - "127.0.0.1:${PUBLIC_HTTPS_PORT:-8443}:8443" @@ -453,6 +473,7 @@ services: memory: 128M otel-collector: + group_add: ["${LAB_SECRET_GID:-65534}"] image: otel/opentelemetry-collector-contrib:0.120.0 command: ["--config=/etc/otelcol/config.yaml"] volumes: @@ -462,6 +483,7 @@ services: cpus: "0.50" restart: unless-stopped otel-tls: + group_add: ["${LAB_SECRET_GID:-65534}"] image: haproxy:3.0.11-alpine network_mode: service:otel-collector depends_on: [otel-collector] @@ -473,6 +495,7 @@ services: mem_limit: 128m cpus: "0.25" prometheus: + group_add: ["${LAB_SECRET_GID:-65534}"] image: prom/prometheus:v3.2.1 mem_limit: 512m cpus: "0.50" @@ -484,6 +507,7 @@ services: - prometheus_data:/prometheus restart: unless-stopped prometheus-tls: + group_add: ["${LAB_SECRET_GID:-65534}"] image: haproxy:3.0.11-alpine network_mode: service:prometheus depends_on: [prometheus] @@ -495,6 +519,7 @@ services: mem_limit: 128m cpus: "0.25" loki: + group_add: ["${LAB_SECRET_GID:-65534}"] image: grafana/loki:3.4.2 mem_limit: 512m cpus: "0.50" @@ -504,6 +529,7 @@ services: - loki_data:/loki restart: unless-stopped loki-tls: + group_add: ["${LAB_SECRET_GID:-65534}"] image: haproxy:3.0.11-alpine network_mode: service:loki depends_on: [loki] @@ -515,6 +541,7 @@ services: mem_limit: 128m cpus: "0.25" tempo: + group_add: ["${LAB_SECRET_GID:-65534}"] image: grafana/tempo:2.7.1 mem_limit: 512m cpus: "0.50" @@ -524,6 +551,7 @@ services: - tempo_data:/var/tempo restart: unless-stopped tempo-tls: + group_add: ["${LAB_SECRET_GID:-65534}"] image: haproxy:3.0.11-alpine network_mode: service:tempo depends_on: [tempo] @@ -535,6 +563,7 @@ services: mem_limit: 128m cpus: "0.25" grafana: + group_add: ["${LAB_SECRET_GID:-65534}"] image: grafana/grafana:11.5.2 mem_limit: 512m cpus: "0.50" diff --git a/evidence/lab7-failover-audit.json b/evidence/lab7-failover-audit.json new file mode 100644 index 0000000..a8cb56e --- /dev/null +++ b/evidence/lab7-failover-audit.json @@ -0,0 +1,19 @@ +{ + "results": [ + { + "stopped": [ + "api-a" + ], + "status": "PASS", + "output": "python3 tests/resilience.py --mode failover --action-script \"scripts/stop-one.sh\" --base-url \"https://localhost:18443\" --ca-file \"/Users/garfieldmk/Desktop/study/highLoad/mtusi2026lab7/secrets/ca.crt\"\nExecuting explicit failover hook; hook output suppressed to avoid leaking secrets.\n{\"observed_instances_before\": 2, \"successful_writes\": 90, \"failed_attempts\": 2, \"first_recovery_seconds\": 2.58}\nPASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.\n" + }, + { + "stopped": [ + "api-b" + ], + "status": "PASS", + "output": "python3 tests/resilience.py --mode failover --action-script \"scripts/stop-one.sh\" --base-url \"https://localhost:18443\" --ca-file \"/Users/garfieldmk/Desktop/study/highLoad/mtusi2026lab7/secrets/ca.crt\"\nExecuting explicit failover hook; hook output suppressed to avoid leaking secrets.\n{\"observed_instances_before\": 2, \"successful_writes\": 108, \"failed_attempts\": 1, \"first_recovery_seconds\": 2.573}\nPASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.\n" + } + ], + "both_replicas_tested": true +} diff --git a/evidence/lab7-persistence.json b/evidence/lab7-persistence.json index 93e3e84..fac49bb 100644 --- a/evidence/lab7-persistence.json +++ b/evidence/lab7-persistence.json @@ -3,40 +3,40 @@ "command": "make test-persistence ACTION_SCRIPT=scripts/restart.sh", "services": { "postgres": { - "before": "2026-11-07T11:21:11.514449215Z", - "after": "2026-11-07T11:31:51.644584012Z" + "before": "2026-11-07T18:37:40.342662055Z", + "after": "2026-11-07T18:40:29.165485675Z" }, "s3": { - "before": "2026-11-07T11:21:14.448491467Z", - "after": "2026-11-07T11:31:54.493811096Z" + "before": "2026-11-07T18:37:40.258400055Z", + "after": "2026-11-07T18:40:32.078899093Z" }, "broker": { - "before": "2026-11-07T11:21:12.477210382Z", - "after": "2026-11-07T11:31:52.565561429Z" + "before": "2026-11-07T18:37:40.181643097Z", + "after": "2026-11-07T18:40:30.133283175Z" }, "openbao": { - "before": "2026-11-07T11:21:11.728365965Z", - "after": "2026-11-07T11:31:51.983157637Z" + "before": "2026-11-07T18:37:25.793975132Z", + "after": "2026-11-07T18:40:29.375343008Z" }, "prometheus": { - "before": "2026-11-07T11:21:11.498290965Z", - "after": "2026-11-07T11:31:51.625424303Z" + "before": "2026-11-07T18:37:40.556532722Z", + "after": "2026-11-07T18:40:29.121133508Z" }, "loki": { - "before": "2026-11-07T11:21:14.486210217Z", - "after": "2026-11-07T11:31:53.332075471Z" + "before": "2026-11-07T18:37:40.63945868Z", + "after": "2026-11-07T18:40:30.996157426Z" }, "tempo": { - "before": "2026-11-07T11:21:13.305379716Z", - "after": "2026-11-07T11:31:52.261244679Z" + "before": "2026-11-07T18:37:40.724841472Z", + "after": "2026-11-07T18:40:30.060878259Z" }, "api-a": { - "before": "2026-11-07T11:21:36.434313963Z", - "after": "2026-11-07T11:32:15.545949676Z" + "before": "2026-11-07T18:39:11.623110833Z", + "after": "2026-11-07T18:40:53.924133089Z" }, "api-b": { - "before": "2026-11-07T11:21:36.45464788Z", - "after": "2026-11-07T11:32:15.598332884Z" + "before": "2026-11-07T18:39:11.708902667Z", + "after": "2026-11-07T18:40:53.958697422Z" } }, "test_output": "python3 tests/resilience.py --mode persistence --action-script \"scripts/restart.sh\" --base-url \"https://localhost:18443\" --ca-file \"/Users/garfieldmk/Desktop/study/highLoad/mtusi2026lab7/secrets/ca.crt\"\nExecuting explicit persistence hook; hook output suppressed to avoid leaking secrets.\nPASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence." diff --git a/evidence/lab7-strict-audit.json b/evidence/lab7-strict-audit.json new file mode 100644 index 0000000..5bcd277 --- /dev/null +++ b/evidence/lab7-strict-audit.json @@ -0,0 +1,232 @@ +{ + "time": "2026-11-07T18:39:57.470382+00:00", + "results": [ + { + "test": "structure", + "command": [ + "make", + "check" + ], + "status": "PASS", + "output": "python3 scripts/check_contract.py\nPASS: offline skeleton structure, local refs and Python syntax; application NOT tested.\n" + }, + { + "test": "smoke", + "command": [ + "make", + "test" + ], + "status": "PASS", + "output": "python3 tests/smoke.py --base-url \"https://localhost:18443\" --ca-file \"/Users/garfieldmk/Desktop/study/highLoad/mtusi2026lab7/secrets/ca.crt\"\nPASS: lab7 public API scenario; infrastructure and grading evidence remain separate.\n" + }, + { + "test": "external TLS", + "command": [ + "make", + "test-security" + ], + "status": "PASS", + "output": "python3 tests/security.py --base-url \"https://localhost:18443\" --ca-file \"/Users/garfieldmk/Desktop/study/highLoad/mtusi2026lab7/secrets/ca.crt\"\nPASS: trusted HTTPS works; untrusted CA rejected. This does NOT verify internal mTLS, OpenBao or hardening.\n" + }, + { + "test": "mTLS", + "command": [ + "sh", + "scripts/test-internal-mtls.sh" + ], + "status": "PASS", + "output": "PASS api-a: valid identity accepted; no cert, foreign CA and wrong role rejected; direct API closed\nPASS api-b: valid identity accepted; no cert, foreign CA and wrong role rejected; direct API closed\n" + }, + { + "test": "access log redaction", + "command": [ + "python3", + "student-tests/ingress_redaction.py" + ], + "status": "PASS", + "output": "PASS: ingress/API access records present; query, path and header markers absent\n" + }, + { + "test": "ACL expiry hostname", + "command": [ + "python3", + "student-tests/lab7_security.py", + "--base-url", + "https://localhost:18443", + "--ca-file", + "/Users/garfieldmk/Desktop/study/highLoad/mtusi2026lab7/secrets/ca.crt" + ], + "status": "PASS", + "output": "PASS: wrong hostname rejected; cross-chat attachment denied; expired session rejected\n" + }, + { + "test": "api-a openbao_policy", + "command": [ + "docker", + "compose", + "exec", + "-T", + "api-a", + "/app/.venv/bin/python", + "-" + ], + "status": "PASS", + "output": "PASS api: own path allowed; other path and policy write denied\n" + }, + { + "test": "api-a service_permissions", + "command": [ + "docker", + "compose", + "exec", + "-T", + "api-a", + "/app/.venv/bin/python", + "-" + ], + "status": "PASS", + "output": "PASS api: DB identity and denied privilege; broker denied unauthorized operation\n" + }, + { + "test": "api-a s3_prefix_permissions", + "command": [ + "docker", + "compose", + "exec", + "-T", + "api-a", + "/app/.venv/bin/python", + "-" + ], + "status": "PASS", + "output": "PASS api: allowed staging write; forbidden prefix denied\nPASS api: duplicate Authorization denied\n" + }, + { + "test": "worker openbao_policy", + "command": [ + "docker", + "compose", + "exec", + "-T", + "worker", + "/app/.venv/bin/python", + "-" + ], + "status": "PASS", + "output": "PASS worker: own path allowed; other path and policy write denied\n" + }, + { + "test": "worker service_permissions", + "command": [ + "docker", + "compose", + "exec", + "-T", + "worker", + "/app/.venv/bin/python", + "-" + ], + "status": "PASS", + "output": "PASS worker: DB identity and denied privilege; broker denied unauthorized operation\n" + }, + { + "test": "worker s3_prefix_permissions", + "command": [ + "docker", + "compose", + "exec", + "-T", + "worker", + "/app/.venv/bin/python", + "-" + ], + "status": "PASS", + "output": "PASS worker: allowed staging write; forbidden prefix denied\nPASS worker: duplicate Authorization denied\n" + }, + { + "test": "postgres_tls", + "command": [ + "docker", + "compose", + "exec", + "-T", + "api-a", + "/app/.venv/bin/python", + "-" + ], + "status": "PASS", + "output": "PASS: PostgreSQL TLS verified, plaintext and untrusted CA rejected\n" + }, + { + "test": "dependency_tls", + "command": [ + "docker", + "compose", + "exec", + "-T", + "api-a", + "/app/.venv/bin/python", + "-" + ], + "status": "PASS", + "output": "PASS s3: verified TLS; untrusted CA rejected\nPASS broker: verified TLS; untrusted CA rejected\nPASS openbao: verified TLS; untrusted CA rejected\nPASS: S3 internal and broker plaintext ports are closed\nPASS: S3 proxy blocks POST\n" + }, + { + "test": "telemetry_tls", + "command": [ + "docker", + "compose", + "exec", + "-T", + "api-a", + "/app/.venv/bin/python", + "-" + ], + "status": "PASS", + "output": "PASS: Collector publisher accepted; no cert and wrong role rejected; local listener closed\n" + }, + { + "test": "lab7_limits", + "command": [ + "docker", + "compose", + "exec", + "-T", + "api-a", + "/app/.venv/bin/python", + "-" + ], + "status": "PASS", + "output": "{\"timestamp\": \"2026-11-07T18:39:56.443007+00:00\", \"severity\": \"INFO\", \"service.name\": \"messenger-api\", \"event\": \"http.request\", \"trace_id\": \"ed16f2869c1055cfa8eba8a2c8ddcc26\", \"span_id\": \"3322ff50bd52d312\", \"request_id\": \"791140e4-e019-4902-8943-715409df0a3a\", \"method\": \"POST\", \"route\": \"unmatched\", \"status\": 413}\n.{\"timestamp\": \"2026-11-07T18:39:56.447785+00:00\", \"severity\": \"INFO\", \"service.name\": \"messenger-api\", \"event\": \"http.request\", \"trace_id\": \"8ee51e4ac84311da20f7e9bf76cd8dbc\", \"span_id\": \"5d24a10e81cf49e1\", \"request_id\": \"b009f765-9103-4510-b4a8-452388d746c1\", \"method\": \"POST\", \"route\": \"/api/v1/auth/sessions\", \"status\": 401}\n{\"timestamp\": \"2026-11-07T18:39:56.448485+00:00\", \"severity\": \"INFO\", \"service.name\": \"messenger-api\", \"event\": \"http.request\", \"trace_id\": \"7cf78e312d771b81e3296f8d0f396c72\", \"span_id\": \"76966e6c01b4dedf\", \"request_id\": \"1f2dc31e-17be-433b-b6eb-12a3d548b7cf\", \"method\": \"POST\", \"route\": \"/api/v1/auth/sessions\", \"status\": 429}\n.{\"timestamp\": \"2026-11-07T18:39:56.449173+00:00\", \"severity\": \"INFO\", \"service.name\": \"messenger-api\", \"event\": \"http.request\", \"trace_id\": \"efbb722e22211c38bd5772aab61a12a4\", \"span_id\": \"4d9aa2c1437a6693\", \"request_id\": \"c4a1c350-7dc3-456e-8415-5d0df3644b33\", \"method\": \"POST\", \"route\": \"/api/v1/auth/sessions\", \"status\": 401}\n..\n----------------------------------------------------------------------\nRan 4 tests in 0.026s\n\nOK\n" + }, + { + "test": "telemetry_unit", + "command": [ + "docker", + "compose", + "exec", + "-T", + "api-a", + "/app/.venv/bin/python", + "-" + ], + "status": "PASS", + "output": "......\n----------------------------------------------------------------------\nRan 6 tests in 0.091s\n\nOK\n" + }, + { + "test": "observability", + "command": [ + "python3", + "student-tests/observability_tls.py" + ], + "status": "PASS", + "output": "PASS: Grafana connected to prometheus; data present\nPASS: Grafana connected to loki; data present\nPASS: Grafana connected to tempo; data present\n" + } + ], + "additional_checks": { + "linux_synthetic_DAC": "PASS: student-tests/secret_file_permissions.py, disposable Linux container; not a full native Linux deployment", + "runtime_hardening": "PASS: student-tests/runtime_hardening.py", + "validate": "PASS: make validate", + "credential_scan": "PASS: current tracked files compared to local generated credentials and private-key markers; Git history and image layers not scanned" + } +} diff --git a/infra/haproxy.cfg b/infra/haproxy.cfg index e83a213..8ec4f80 100644 --- a/infra/haproxy.cfg +++ b/infra/haproxy.cfg @@ -4,7 +4,8 @@ global defaults mode http log global - option httplog + # Never log raw URLs, query strings, headers or request bodies. + log-format "ingress status=%ST bytes=%B backend=%b server=%s termination=%tsc" option forwardfor timeout connect 1s timeout client 30s diff --git a/infra/s3-tcp.cfg b/infra/s3-tcp.cfg index 4c5d8e8..9d14ba5 100644 --- a/infra/s3-tcp.cfg +++ b/infra/s3-tcp.cfg @@ -7,6 +7,8 @@ frontend s3_tls bind :8333 ssl crt /run/secrets/server.pem ssl-min-ver TLSv1.2 # Multipart form POST is not used by this application; deny it at the boundary. http-request deny if { method POST } + # Reject repeated Authorization fields; commas inside SigV4 are valid. + http-request deny unless { req.fhdr_cnt(Authorization) eq 1 } # Only the four locally provisioned SigV4 identities may use this endpoint. acl admin_auth req.hdr(Authorization) -m reg -f /run/secrets/admin_acl acl api_auth req.hdr(Authorization) -m reg -f /run/secrets/api_acl diff --git a/scripts/init-secrets.py b/scripts/init-secrets.py index 5cba3b8..216b491 100644 --- a/scripts/init-secrets.py +++ b/scripts/init-secrets.py @@ -8,6 +8,7 @@ import secrets root = Path(__file__).resolve().parents[1] / "secrets" root.mkdir(mode=0o700, exist_ok=True) +root.chmod(0o700) def write(name, value): diff --git a/scripts/prepare-secret-access.py b/scripts/prepare-secret-access.py new file mode 100644 index 0000000..763a8ad --- /dev/null +++ b/scripts/prepare-secret-access.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 +"""Give containers group-read access only to individually mounted secret files. + +The host directory stays owner-only; CA and operator credentials stay 0600. +Compose bind-mounted file secrets preserve host ownership on Linux. +""" +import os +from pathlib import Path +import re + +ROOT = Path(__file__).resolve().parents[1] +secret_dir = ROOT / 'secrets' +if secret_dir.is_symlink(): + raise SystemExit('secrets directory must not be a symlink') +secret_dir.chmod(0o700) +compose = (ROOT / 'compose.yaml').read_text() +names = set(re.findall(r'\./secrets/([A-Za-z0-9_.-]+)', compose)) +gid = os.getgid() +for name in sorted(names): + path = secret_dir / name + if path.is_symlink() or not path.is_file(): + raise SystemExit('Missing or unsafe mounted secret: ' + name) + os.chown(path, -1, gid) + path.chmod(0o640) +for name in ('ca.key', 'openbao-operator.json'): + path = secret_dir / name + if path.is_symlink(): + raise SystemExit('Operator credential must not be a symlink') + if path.exists(): + path.chmod(0o600) +env = ROOT / '.env' +if env.is_symlink(): + raise SystemExit('.env must not be a symlink') +text = env.read_text() if env.exists() else (ROOT / '.env.example').read_text() +text = re.sub(r'^LAB_SECRET_GID=.*\n?', '', text, flags=re.M) +with env.open('w') as stream: + env.chmod(0o600) + stream.write(text.rstrip() + '\nLAB_SECRET_GID=' + str(gid) + '\n') +print('Mounted secrets: group-readable; host directory: 0700; operator files: 0600.') diff --git a/security/THREAT_MODEL.md b/security/THREAT_MODEL.md index 65063d1..72d4d3b 100644 --- a/security/THREAT_MODEL.md +++ b/security/THREAT_MODEL.md @@ -18,7 +18,8 @@ | Сервис с чужой identity подключается напрямую к API | Только LB вызывает внутренний API | mTLS и проверка разрешённой identity | `sh scripts/test-internal-mtls.sh`: без сертификата, чужой CA, доверенный CA с неверной identity — отказ | Отзыв и ротация identity под нагрузкой не проверены | | Скомпрометированный worker читает чужой секрет | Каждому сервису доступен только нужный набор секретов | Отдельная identity и scoped policy OpenBao | Запросить разрешённый и чужой путь, попробовать изменить policy | Из разрешённых worker данных возможна утечка | | Автоматические попытки входа перегружают KDF | Вход ограничен по частоте | Лимит попыток с 429 и Retry-After | Превысить лимит и проверить ответ, затем дождаться окна | Текущий лимит локален процессу и общий для всех клиентов | -| Загрузка чрезмерного файла исчерпывает память или диск | Размер и число пикселей ограничены | Потоковый лимит и проверка изображения | Отправить файл свыше лимита и изображение со слишком большим числом пикселей | Остальные лимиты ресурсов стенда нужно проверить | +| Загрузка чрезмерного файла исчерпывает память или диск | Размер и число пикселей ограничены | Потоковый лимит и проверка изображения | Отправить файл свыше лимита и изображение со слишком большим числом пикселей | Лимиты контейнеров проверяются `student-tests/runtime_hardening.py`; устойчивость под произвольной нагрузкой не доказана | +| Клиент передаёт секрет в URL, который попадает в access-log | Секреты не сохраняются в наблюдаемости | Ingress пишет только статус, размер и служебные поля; API пишет шаблон маршрута | `student-tests/ingress_redaction.py`: маркеры в path/query/headers отсутствуют в реальных логах | Тест не доказывает отсутствие всех каналов утечки | На 4 минимум 6 конкретных угроз. Не заменяйте сценарии словами «SQL injection», «безопасный Docker», «mTLS включён» без пути и проверки. diff --git a/student-tests/ingress_redaction.py b/student-tests/ingress_redaction.py new file mode 100644 index 0000000..92e318b --- /dev/null +++ b/student-tests/ingress_redaction.py @@ -0,0 +1,35 @@ +"""Check that request-controlled secrets are absent from actual access logs.""" +from datetime import datetime, timezone +import os +from pathlib import Path +import ssl +import subprocess +import time +import urllib.error +import urllib.request +from uuid import uuid4 + +root = Path(__file__).resolve().parents[1] +ctx = ssl.create_default_context(cafile=str(root / 'secrets/ca.crt')) +base = os.environ.get('BASE_URL', 'https://localhost:8443').rstrip('/') +marker = 'audit-private-' + uuid4().hex +since = datetime.now(timezone.utc).isoformat() +for path in ('/health/ready?token=' + marker, '/missing/' + marker): + request = urllib.request.Request(base + path, headers={ + 'Authorization': 'Bearer ' + marker, 'X-Request-Id': marker}) + try: + with urllib.request.urlopen(request, context=ctx, timeout=10) as response: + response.read() + except urllib.error.HTTPError as error: + assert error.code == 404 + error.read() +for attempt in range(20): + logs = subprocess.check_output(['docker', 'compose', 'logs', '--since', since, + 'load_balancer', 'api-a', 'api-b'], cwd=root, text=True) + assert marker not in logs, 'request secret leaked into container logs' + if 'ingress status=200' in logs and 'ingress status=404' in logs and 'http.request' in logs: + break + time.sleep(0.25) +else: + raise AssertionError('missing positive evidence of access logging') +print('PASS: ingress/API access records present; query, path and header markers absent') diff --git a/student-tests/runtime_hardening.py b/student-tests/runtime_hardening.py new file mode 100644 index 0000000..d9e9693 --- /dev/null +++ b/student-tests/runtime_hardening.py @@ -0,0 +1,29 @@ +"""Inspect actual container privileges, mounts, published ports and limits.""" +import json +import subprocess + +ids = subprocess.check_output(['docker','compose','ps','-q'], text=True).split() +assert ids, 'stack not running' +containers = json.loads(subprocess.check_output(['docker','inspect',*ids], text=True)) +seen = set() +for c in containers: + role=c['Config']['Labels']['com.docker.compose.service'] + seen.add(role) + host=c['HostConfig'] + assert not host['Privileged'], role + assert host['NetworkMode'] != 'host', role + assert host['Memory'] > 0 and host['NanoCpus'] > 0, role + for mount in c['Mounts']: + assert 'docker.sock' not in mount['Source'] and 'docker.sock' not in mount['Destination'], role + assert not mount['Source'].endswith(('/ca.key','/openbao-operator.json')), role + for ports in (c['NetworkSettings'].get('Ports') or {}).values(): + for port in ports or []: + assert role in {'load_balancer','openbao','grafana'}, role + assert port['HostIp']=='127.0.0.1', role + if role in {'api-a','api-b','worker'}: + assert c['Config']['User'].split(':')[0] not in {'','0','root'}, role + assert 'ALL' in host['CapDrop'], role + assert not host['CapAdd'], role + assert 'no-new-privileges:true' in host['SecurityOpt'], role +assert {'api-a','api-b','worker','load_balancer'} <= seen +print('PASS runtime: scoped ports, no privileged/host/socket/operator keys; resource limits; non-root app/worker') diff --git a/student-tests/s3_prefix_permissions.py b/student-tests/s3_prefix_permissions.py index d82b270..f74fff8 100644 --- a/student-tests/s3_prefix_permissions.py +++ b/student-tests/s3_prefix_permissions.py @@ -1,5 +1,10 @@ """Check allowed and denied S3 prefixes using the current service identity.""" import json +import http.client +import urllib.parse +from botocore.auth import S3SigV4Auth +from botocore.awsrequest import AWSRequest +from botocore.credentials import Credentials import os import ssl import urllib.request @@ -38,3 +43,24 @@ else: finally: client.close() print(f'PASS {role}: allowed staging write; forbidden prefix denied') + +# Repeated Authorization fields must fail even for an otherwise allowed read. +endpoint = urllib.parse.urlsplit(os.environ['S3_ENDPOINT']) +path = '/' + bucket + '/' + allowed +req = AWSRequest(method='GET', url=os.environ['S3_ENDPOINT'] + path) +S3SigV4Auth(Credentials(values['S3_ACCESS_KEY_ID'], values['S3_SECRET_ACCESS_KEY']), + 's3', 'us-east-1').add_auth(req) +conn = http.client.HTTPSConnection(endpoint.hostname, endpoint.port, + context=ssl.create_default_context(cafile=os.environ['AWS_CA_BUNDLE']), timeout=10) +try: + conn.putrequest('GET', path) + for key, value in req.headers.items(): + conn.putheader(key, value) + conn.putheader('Authorization', req.headers['Authorization']) + conn.endheaders() + response = conn.getresponse() + assert response.status == 403, 'duplicate Authorization not denied' + response.read() +finally: + conn.close() +print(f'PASS {role}: duplicate Authorization denied') diff --git a/student-tests/secret_file_permissions.py b/student-tests/secret_file_permissions.py new file mode 100644 index 0000000..122f486 --- /dev/null +++ b/student-tests/secret_file_permissions.py @@ -0,0 +1,35 @@ +"""Linux DAC regression; run as root in a disposable container with synthetic data.""" +import os +from pathlib import Path +import subprocess +import sys +import tempfile + +with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + root.chmod(0o755) + mounted = root / 'mounted' + mounted.write_text('synthetic-secret') + os.chown(mounted, 1000, 12345) + mounted.chmod(0o640) + host = root / 'operator' + host.mkdir(mode=0o700) + os.chown(host, 1000, 12345) + protected = host / 'secret' + protected.write_text('synthetic-secret') + os.chown(protected, 1000, 12345) + protected.chmod(0o640) + def can_read(path, groups): + def identity(): + os.setgroups(groups) + os.setgid(65534) + os.setuid(65534) + return subprocess.run([sys.executable, '-c', + 'from pathlib import Path; import sys; Path(sys.argv[1]).read_bytes()', str(path)], + preexec_fn=identity, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL).returncode == 0 + assert not can_read(mounted, []) + assert can_read(mounted, [12345]) + assert not can_read(protected, [12345]) + mounted.chmod(0o600) + assert not can_read(mounted, [12345]) +print('PASS Linux DAC: original failure reproduced; group access works; host directory protected') -- 2.54.0