From 66026d3ec993058211b2bc0e53aa9428ceecc700 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9C=D0=B0=D1=82=D0=B2=D0=B5=D0=B9?= Date: Thu, 5 Nov 2026 21:17:50 +0300 Subject: [PATCH 1/4] add lab5 --- .env.example | 5 +- Dockerfile | 23 + REPORT.md | 22 +- compose.yaml | 311 ++++++++++++ infra/filer.toml | 3 + infra/haproxy.cfg | 35 ++ infra/rabbitmq-secret-entrypoint.sh | 12 + migrations/001_initial.sql | 37 ++ migrations/002_history_cache_version.sql | 1 + migrations/003_idempotency.sql | 12 + migrations/004_attachments.sql | 18 + migrations/005_async_attachments.sql | 34 ++ migrations/006_level4_reliability.sql | 17 + migrations/007_job_measurements.sql | 1 + pyproject.toml | 15 + scripts/audit.py | 70 +++ scripts/init-secrets.py | 38 ++ scripts/record-environment.py | 44 ++ scripts/restart.sh | 10 + scripts/run-track1-profiles.py | 31 ++ scripts/stop-one.sh | 18 + scripts/track1.py | 217 +++++++++ scripts/verify-persistence.py | 27 + scripts/verify-resilience.py | 48 ++ src/__init__.py | 1 + src/attachments.py | 141 ++++++ src/bootstrap_s3.py | 51 ++ src/broker.py | 83 ++++ src/config.py | 125 +++++ src/cursor.py | 81 +++ src/errors.py | 12 + src/main.py | 172 +++++++ src/migrate.py | 18 + src/models.py | 151 ++++++ src/outbox.py | 33 ++ src/pipeline_metrics.py | 37 ++ src/reconcile.py | 104 ++++ src/render.py | 27 + src/replay.py | 24 + src/routes.py | 185 +++++++ src/storage/__init__.py | 2 + src/storage/postgres.py | 531 ++++++++++++++++++++ src/storage/s3.py | 120 +++++ src/worker.py | 234 +++++++++ student-tests/attachment_resilience.py | 120 +++++ student-tests/attachments.py | 113 +++++ student-tests/idempotency_concurrency.py | 32 ++ student-tests/job_contract.py | 31 ++ student-tests/pipeline_critical.py | 107 ++++ student-tests/pipeline_integration.py | 205 ++++++++ student-tests/pipeline_invariants.py | 141 ++++++ student-tests/readiness_failover.py | 89 ++++ student-tests/readiness_unavailable.py | 16 + student-tests/reconcile.py | 152 ++++++ student-tests/reconcile_audit.py | 159 ++++++ student-tests/reconcile_benchmark.py | 74 +++ student-tests/remaining_regressions.py | 127 +++++ student-tests/review_regressions.py | 32 ++ student-tests/s3_permissions.py | 54 ++ student-tests/session_expiry.py | 20 + student-tests/upload_memory.py | 106 ++++ uv.lock | 596 +++++++++++++++++++++++ 62 files changed, 5351 insertions(+), 4 deletions(-) create mode 100644 Dockerfile create mode 100644 compose.yaml create mode 100644 infra/filer.toml create mode 100644 infra/haproxy.cfg create mode 100644 infra/rabbitmq-secret-entrypoint.sh create mode 100644 migrations/001_initial.sql create mode 100644 migrations/002_history_cache_version.sql create mode 100644 migrations/003_idempotency.sql create mode 100644 migrations/004_attachments.sql create mode 100644 migrations/005_async_attachments.sql create mode 100644 migrations/006_level4_reliability.sql create mode 100644 migrations/007_job_measurements.sql create mode 100644 pyproject.toml create mode 100644 scripts/audit.py create mode 100644 scripts/init-secrets.py create mode 100644 scripts/record-environment.py create mode 100755 scripts/restart.sh create mode 100644 scripts/run-track1-profiles.py create mode 100755 scripts/stop-one.sh create mode 100755 scripts/track1.py create mode 100644 scripts/verify-persistence.py create mode 100644 scripts/verify-resilience.py create mode 100644 src/__init__.py create mode 100644 src/attachments.py create mode 100644 src/bootstrap_s3.py create mode 100644 src/broker.py create mode 100644 src/config.py create mode 100644 src/cursor.py create mode 100644 src/errors.py create mode 100644 src/main.py create mode 100644 src/migrate.py create mode 100644 src/models.py create mode 100644 src/outbox.py create mode 100644 src/pipeline_metrics.py create mode 100644 src/reconcile.py create mode 100644 src/render.py create mode 100644 src/replay.py create mode 100644 src/routes.py create mode 100644 src/storage/__init__.py create mode 100644 src/storage/postgres.py create mode 100644 src/storage/s3.py create mode 100644 src/worker.py create mode 100644 student-tests/attachment_resilience.py create mode 100644 student-tests/attachments.py create mode 100755 student-tests/idempotency_concurrency.py create mode 100644 student-tests/job_contract.py create mode 100644 student-tests/pipeline_critical.py create mode 100644 student-tests/pipeline_integration.py create mode 100644 student-tests/pipeline_invariants.py create mode 100644 student-tests/readiness_failover.py create mode 100644 student-tests/readiness_unavailable.py create mode 100644 student-tests/reconcile.py create mode 100644 student-tests/reconcile_audit.py create mode 100644 student-tests/reconcile_benchmark.py create mode 100644 student-tests/remaining_regressions.py create mode 100644 student-tests/review_regressions.py create mode 100644 student-tests/s3_permissions.py create mode 100644 student-tests/session_expiry.py create mode 100644 student-tests/upload_memory.py create mode 100644 uv.lock diff --git a/.env.example b/.env.example index 2b32655..c4e69b8 100644 --- a/.env.example +++ b/.env.example @@ -11,7 +11,7 @@ SESSION_STORE=postgres CACHE_URL_FILE=/run/secrets/cache_url API_REPLICAS=2 IDEMPOTENCY_TTL_SECONDS=86400 -S3_ENDPOINT=http://s3:9000 +S3_ENDPOINT=http://s3:8333 S3_REGION=us-east-1 S3_BUCKET=messenger-attachments S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id @@ -20,9 +20,12 @@ MAX_UPLOAD_BYTES=10485760 MAX_IMAGE_PIXELS=40000000 BROKER_URL_FILE=/run/secrets/broker_url WORKER_CONCURRENCY=2 +WORKER_PREFETCH=2 JOB_MAX_ATTEMPTS=5 JOB_TIMEOUT_SECONDS=60 STAGING_RETENTION_SECONDS=86400 +MAX_PENDING_JOBS=100 +RENDER_MEMORY_MB=768 OTEL_SERVICE_NAME=messenger-api # Worker must set a distinct service.name. OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318 diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..ab272d3 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,23 @@ +FROM python:3.12.12-slim + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 + +WORKDIR /app + +RUN pip install --no-cache-dir uv==0.11.28 + +COPY pyproject.toml uv.lock ./ +RUN uv sync --frozen --no-dev --no-install-project + +COPY src ./src +COPY migrations ./migrations + +USER 65534:65534 + +EXPOSE 8080 + +HEALTHCHECK --interval=10s --timeout=3s --start-period=5s --retries=3 \ + CMD ["/app/.venv/bin/python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2).read()"] + +CMD ["/app/.venv/bin/python", "-m", "src.main"] diff --git a/REPORT.md b/REPORT.md index b97ae42..0f8f976 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,17 +1,33 @@ # Отчёт — лабораторная 6 -- Автор, группа: TODO +- Автор, группа: Kozyrev, BVT2301 - Целевая оценка; выбранный трек 5, если нужен: TODO - Commit/PR: TODO после публикации - Использованные источники, библиотеки и помощь ИИ: TODO ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Перенесённый стенд lab5 требует Docker Engine с Compose v2 и Python 3 для генерации локальных секретов. Порт `127.0.0.1:8080` должен быть свободен. Версии образов указаны в `Dockerfile` и `compose.yaml`. + +```sh +python3 scripts/init-secrets.py +docker compose config --quiet +docker compose up -d --build +docker compose ps -a +curl --fail http://localhost:8080/health/ready +make check +make test +``` + +Миграции и создание S3 bucket выполняют сервисы `migrate` и `s3-init`. Секреты создаются в игнорируемом каталоге `secrets/`; `.env` для значений Compose по умолчанию не обязателен. `docker compose down` останавливает стенд с сохранением volumes. Команда `docker compose down -v` намеренно удаляет учебные данные PostgreSQL, S3 и RabbitMQ. + +После переноса `docker compose config --quiet` и `make check` выполнены успешно. Запуск стенда и `make test` ещё не проверены: Docker Engine был недоступен. Телеметрия lab6 пока не реализована. ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +Перенесены исходники, миграции, инфраструктура, собственные тесты и вспомогательные скрипты из lab5, commit `6bc0712`. Контракты и публичные тесты lab6 сохранены. Клиент обращается через HAProxy к двум API-репликам; PostgreSQL хранит данные и outbox, SeaweedFS — файлы, RabbitMQ передаёт задания worker. Dispatcher публикует задания из outbox; worker обрабатывает изображения. S3 и PostgreSQL не образуют общую транзакцию; сервис reconcile отвечает за сверку и очистку. Текущий Compose не содержит Collector, хранилища телеметрии и Grafana. + +`.env.example` описывает логические настройки; `compose.yaml` использует только часть значений из окружения, остальные заданы в нём явно. Конфигурация OpenTelemetry в примере пока не подключена к приложению. ## Доказательства критериев diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..8b0b05d --- /dev/null +++ b/compose.yaml @@ -0,0 +1,311 @@ +x-api: &api + build: + context: . + environment: + APP_HOST: 0.0.0.0 + APP_PORT: 8080 + APP_ENV: lab5 + DATABASE_URL_FILE: /run/secrets/database_url + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400} + CURSOR_SECRET_FILE: /run/secrets/cursor_secret + S3_ENDPOINT: http://s3:8333 + S3_BUCKET: messenger-attachments + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + BROKER_URL_FILE: /run/secrets/broker_url + MAX_UPLOAD_BYTES: "10485760" + MAX_IMAGE_PIXELS: "40000000" + UPLOAD_CONCURRENCY: "1" + UPLOAD_TIMEOUT_SECONDS: "30" + UPLOAD_FAULT_AFTER_S3_PUT: ${UPLOAD_FAULT_AFTER_S3_PUT:-0} + OUTBOX_FAULT_AFTER_COMMIT: ${OUTBOX_FAULT_AFTER_COMMIT:-0} + MAX_PENDING_JOBS: ${MAX_PENDING_JOBS:-100} + secrets: + - database_url + - cursor_secret + - s3_access_key_id + - s3_secret_access_key + - broker_url + depends_on: + s3-init: + condition: service_completed_successfully + migrate: + condition: service_completed_successfully + broker: + condition: service_healthy + expose: + - "8080" + stop_grace_period: 10s + healthcheck: + test: ["CMD", "/app/.venv/bin/python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2).read()"] + interval: 5s + timeout: 3s + retries: 12 + start_period: 10s + deploy: + resources: + limits: + cpus: "0.50" + memory: 512M + +services: + broker: + image: rabbitmq:4.1.2-alpine + environment: + RABBITMQ_DEFAULT_USER: messenger + entrypoint: ["/bin/sh", "/usr/local/bin/rabbitmq-secret-entrypoint.sh"] + command: ["rabbitmq-server"] + secrets: + - rabbitmq_password + volumes: + - rabbitmq_data:/var/lib/rabbitmq + - ./infra/rabbitmq-secret-entrypoint.sh:/usr/local/bin/rabbitmq-secret-entrypoint.sh:ro + healthcheck: + test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"] + interval: 5s + timeout: 3s + retries: 20 + start_period: 10s + deploy: + resources: + limits: + cpus: "0.50" + memory: 256M + + s3: + image: chrislusf/seaweedfs:4.17@sha256:186de7ef977a20343ee9a5544073f081976a29e2d29ecf8379891e7bf177fbe9 + command: ["server", "-dir=/data", "-ip=s3", "-ip.bind=0.0.0.0", "-s3", "-s3.config=/run/secrets/s3_config", "-master.volumeSizeLimitMB=128", "-volume.max=64", "-s3.port.iceberg=0", "-filer.exposeDirectoryData=false"] + secrets: + - s3_config + volumes: + - s3_data:/data + - ./infra/filer.toml:/etc/seaweedfs/filer.toml:ro + healthcheck: + test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://localhost:9333/cluster/status"] + interval: 5s + timeout: 3s + retries: 20 + start_period: 15s + deploy: + resources: + limits: + cpus: "1.0" + memory: 512M + + s3-init: + build: + context: . + command: ["/app/.venv/bin/python", "-m", "src.bootstrap_s3"] + environment: + S3_ENDPOINT: http://s3:8333 + S3_BUCKET: messenger-attachments + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_admin_key + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_admin_secret + secrets: + - s3_admin_key + - s3_admin_secret + depends_on: + s3: + condition: service_healthy + restart: "no" + + postgres: + image: postgres:17.6-alpine + environment: + POSTGRES_DB: messenger + POSTGRES_USER: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + secrets: + - postgres_password + volumes: + - postgres_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"] + interval: 5s + timeout: 3s + retries: 12 + start_period: 5s + deploy: + resources: + limits: + cpus: "0.75" + memory: 512M + + migrate: + build: + context: . + command: ["/app/.venv/bin/python", "-m", "src.migrate"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + CURSOR_SECRET_FILE: /run/secrets/cursor_secret + secrets: + - database_url + - cursor_secret + depends_on: + postgres: + condition: service_healthy + restart: "no" + deploy: + resources: + limits: + cpus: "0.25" + memory: 128M + + reconcile: + healthcheck: + disable: true + build: + context: . + entrypoint: ["/app/.venv/bin/python", "-m", "src.reconcile"] + command: ["--apply", "--loop"] + restart: unless-stopped + environment: + DATABASE_URL_FILE: /run/secrets/database_url + CURSOR_SECRET_FILE: /run/secrets/cursor_secret + S3_ENDPOINT: http://s3:8333 + S3_BUCKET: messenger-attachments + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_admin_key + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_admin_secret + STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400} + secrets: + - database_url + - cursor_secret + - s3_admin_key + - s3_admin_secret + depends_on: + postgres: + condition: service_healthy + s3: + condition: service_healthy + deploy: + resources: + limits: + cpus: "0.25" + memory: 128M + + api-a: + <<: *api + + api-b: + <<: *api + + worker: + restart: unless-stopped + healthcheck: + test: ["CMD", "/app/.venv/bin/python", "-c", "import time,pathlib; assert time.time()-float(pathlib.Path('/tmp/worker-heartbeat').read_text()) < 15"] + interval: 5s + timeout: 3s + retries: 3 + deploy: + resources: + limits: + cpus: "2.0" + memory: 4G + build: + context: . + command: ["/app/.venv/bin/python", "-m", "src.worker"] + environment: + APP_ENV: lab5 + DATABASE_URL_FILE: /run/secrets/database_url + CURSOR_SECRET_FILE: /run/secrets/cursor_secret + S3_ENDPOINT: http://s3:8333 + S3_BUCKET: messenger-attachments + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + BROKER_URL_FILE: /run/secrets/broker_url + WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2} + WORKER_FAULT_AFTER_S3_PUT: ${WORKER_FAULT_AFTER_S3_PUT:-0} + RENDER_MEMORY_MB: ${RENDER_MEMORY_MB:-768} + WORKER_PREFETCH: ${WORKER_PREFETCH:-2} + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60} + STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400} + secrets: + - database_url + - cursor_secret + - s3_access_key_id + - s3_secret_access_key + - broker_url + depends_on: + s3-init: + condition: service_completed_successfully + migrate: + condition: service_completed_successfully + broker: + condition: service_healthy + stop_grace_period: 30s + + outbox: + healthcheck: + disable: true + build: + context: . + command: ["/app/.venv/bin/python", "-m", "src.outbox"] + environment: + APP_ENV: lab5 + DATABASE_URL_FILE: /run/secrets/database_url + CURSOR_SECRET_FILE: /run/secrets/cursor_secret + BROKER_URL_FILE: /run/secrets/broker_url + OUTBOX_DISPATCH_INTERVAL_SECONDS: "1" + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + secrets: + - database_url + - cursor_secret + - broker_url + depends_on: + migrate: + condition: service_completed_successfully + broker: + condition: service_healthy + restart: unless-stopped + + load_balancer: + image: haproxy:3.0.11-alpine + ports: + - "127.0.0.1:8080:8080" + volumes: + - ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + depends_on: + api-a: + condition: service_healthy + api-b: + condition: service_healthy + healthcheck: + test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:8080/health/ready"] + interval: 5s + timeout: 3s + retries: 12 + start_period: 5s + deploy: + resources: + limits: + cpus: "0.25" + memory: 128M + +secrets: + s3_config: + file: ./secrets/s3.json + s3_admin_key: + file: ./secrets/s3_admin_key + s3_admin_secret: + file: ./secrets/s3_admin_secret + s3_access_key_id: + file: ./secrets/s3_access_key_id + s3_secret_access_key: + file: ./secrets/s3_secret_access_key + postgres_password: + file: ./secrets/postgres_password + database_url: + file: ./secrets/database_url + cursor_secret: + file: ./secrets/cursor_secret + rabbitmq_password: + file: ./secrets/rabbitmq_password + broker_url: + file: ./secrets/broker_url + +volumes: + s3_data: + postgres_data: + rabbitmq_data: diff --git a/infra/filer.toml b/infra/filer.toml new file mode 100644 index 0000000..85b2149 --- /dev/null +++ b/infra/filer.toml @@ -0,0 +1,3 @@ +[leveldb2] +enabled = true +dir = "/data/filer" diff --git a/infra/haproxy.cfg b/infra/haproxy.cfg new file mode 100644 index 0000000..00f0b6c --- /dev/null +++ b/infra/haproxy.cfg @@ -0,0 +1,35 @@ +global + log stdout format raw local0 + +defaults + mode http + log global + option httplog + option forwardfor + timeout connect 1s + timeout client 30s + timeout server 10s + timeout check 2s + retries 2 + +resolvers docker + nameserver dns 127.0.0.11:53 + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + hold valid 2s + hold obsolete 2s + +frontend http + bind :8080 + default_backend api + +backend api + balance roundrobin + option httpchk + http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host localhost + http-check expect status 200 + default-server check inter 2s fall 1 rise 2 resolvers docker resolve-prefer ipv4 init-addr libc,none + server api-a api-a:8080 + server api-b api-b:8080 + diff --git a/infra/rabbitmq-secret-entrypoint.sh b/infra/rabbitmq-secret-entrypoint.sh new file mode 100644 index 0000000..8ed866f --- /dev/null +++ b/infra/rabbitmq-secret-entrypoint.sh @@ -0,0 +1,12 @@ +#!/bin/sh +set -eu + +# RabbitMQ 4 no longer supports *_FILE. Keep the password in a Docker +# secret and export it only inside the running container, never in Compose. +RABBITMQ_DEFAULT_PASS="$(cat /run/secrets/rabbitmq_password)" +if [ -z "$RABBITMQ_DEFAULT_PASS" ]; then + echo "RabbitMQ password secret must not be empty" >&2 + exit 1 +fi +export RABBITMQ_DEFAULT_PASS +exec /usr/local/bin/docker-entrypoint.sh "$@" diff --git a/migrations/001_initial.sql b/migrations/001_initial.sql new file mode 100644 index 0000000..c03d5fd --- /dev/null +++ b/migrations/001_initial.sql @@ -0,0 +1,37 @@ +CREATE TABLE users ( + id uuid PRIMARY KEY, + username varchar(32) NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'), + display_name varchar(100) NOT NULL CHECK (char_length(btrim(display_name)) > 0), + password_hash text NOT NULL, + created_at timestamptz NOT NULL DEFAULT now() +); + +CREATE TABLE chats ( + id uuid PRIMARY KEY, + title varchar(100) NOT NULL CHECK (char_length(btrim(title)) > 0), + created_at timestamptz NOT NULL DEFAULT now() +); + +CREATE TABLE chat_members ( + chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + user_id uuid NOT NULL REFERENCES users(id) ON DELETE RESTRICT, + PRIMARY KEY (chat_id, user_id) +); +CREATE INDEX chat_members_user_chat_idx ON chat_members (user_id, chat_id); + +CREATE TABLE messages ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + sender_id uuid NOT NULL REFERENCES users(id) ON DELETE RESTRICT, + text varchar(4000) NOT NULL CHECK (char_length(btrim(text)) > 0), + created_at timestamptz NOT NULL DEFAULT now() +); +CREATE INDEX messages_chat_created_id_idx ON messages (chat_id, created_at, id); + +CREATE TABLE sessions ( + token_hash char(64) PRIMARY KEY, + user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE, + expires_at timestamptz NOT NULL, + created_at timestamptz NOT NULL DEFAULT now() +); +CREATE INDEX sessions_expiry_idx ON sessions (expires_at); diff --git a/migrations/002_history_cache_version.sql b/migrations/002_history_cache_version.sql new file mode 100644 index 0000000..7b7972c --- /dev/null +++ b/migrations/002_history_cache_version.sql @@ -0,0 +1 @@ +ALTER TABLE chats ADD COLUMN history_version bigint NOT NULL DEFAULT 0; diff --git a/migrations/003_idempotency.sql b/migrations/003_idempotency.sql new file mode 100644 index 0000000..7fcf665 --- /dev/null +++ b/migrations/003_idempotency.sql @@ -0,0 +1,12 @@ +CREATE TABLE idempotency_records ( + user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE, + method varchar(8) NOT NULL, + path text NOT NULL, + idempotency_key varchar(128) NOT NULL, + request_hash char(64) NOT NULL, + message_id uuid REFERENCES messages(id) ON DELETE RESTRICT, + created_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (user_id, method, path, idempotency_key) +); + +CREATE INDEX idempotency_records_created_at_idx ON idempotency_records (created_at); diff --git a/migrations/004_attachments.sql b/migrations/004_attachments.sql new file mode 100644 index 0000000..2fe0f9e --- /dev/null +++ b/migrations/004_attachments.sql @@ -0,0 +1,18 @@ +CREATE TABLE attachments ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats(id), + owner_id uuid NOT NULL REFERENCES users(id), + filename varchar(255) NOT NULL, + content_type text NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg')), + size_bytes integer NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760), + sha256 char(64) NOT NULL, + object_key text NOT NULL UNIQUE, + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + UNIQUE (id, chat_id) +); +ALTER TABLE messages DROP CONSTRAINT messages_text_check; +ALTER TABLE messages ADD COLUMN attachment_ids uuid[] NOT NULL DEFAULT '{}'; +ALTER TABLE messages ADD CONSTRAINT messages_content_check + CHECK (char_length(btrim(text)) > 0 OR cardinality(attachment_ids) > 0); +CREATE INDEX attachments_chat_idx ON attachments(chat_id); diff --git a/migrations/005_async_attachments.sql b/migrations/005_async_attachments.sql new file mode 100644 index 0000000..fa3bcca --- /dev/null +++ b/migrations/005_async_attachments.sql @@ -0,0 +1,34 @@ +ALTER TABLE attachments + RENAME COLUMN object_key TO staging_key; +ALTER TABLE attachments + ADD COLUMN original_key text, + ADD COLUMN status text NOT NULL DEFAULT 'queued' + CHECK (status IN ('queued', 'processing', 'ready', 'failed')), + ADD COLUMN error_code text, + ADD COLUMN pipeline_version integer NOT NULL DEFAULT 1, + ADD COLUMN processing_started_at timestamptz; + +CREATE TABLE attachment_variants ( + attachment_id uuid NOT NULL REFERENCES attachments(id) ON DELETE CASCADE, + pipeline_version integer NOT NULL, + name text NOT NULL CHECK (name IN ('thumbnail', 'crop', 'optimized')), + object_key text NOT NULL UNIQUE, + content_type text NOT NULL, + size_bytes integer NOT NULL CHECK (size_bytes > 0), + width integer NOT NULL CHECK (width > 0), + height integer NOT NULL CHECK (height > 0), + sha256 char(64) NOT NULL, + PRIMARY KEY (attachment_id, pipeline_version, name) +); + +CREATE TABLE attachment_jobs ( + job_id uuid PRIMARY KEY, + attachment_id uuid NOT NULL UNIQUE REFERENCES attachments(id) ON DELETE CASCADE, + pipeline_version integer NOT NULL DEFAULT 1, + attempt integer NOT NULL DEFAULT 1 CHECK (attempt >= 1), + status text NOT NULL DEFAULT 'queued' CHECK (status IN ('queued', 'processing', 'ready', 'failed')), + last_error text, + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now() +); +CREATE INDEX attachment_jobs_status_idx ON attachment_jobs(status, updated_at); diff --git a/migrations/006_level4_reliability.sql b/migrations/006_level4_reliability.sql new file mode 100644 index 0000000..3be954c --- /dev/null +++ b/migrations/006_level4_reliability.sql @@ -0,0 +1,17 @@ +ALTER TABLE attachment_jobs + ADD COLUMN next_attempt_at timestamptz NOT NULL DEFAULT now(), + ADD COLUMN lease_until timestamptz, + ADD COLUMN replayed_at timestamptz; +CREATE INDEX attachment_jobs_due_idx ON attachment_jobs(status, next_attempt_at); +CREATE INDEX attachment_jobs_lease_idx ON attachment_jobs(status, lease_until); + +CREATE TABLE attachment_outbox ( + id uuid PRIMARY KEY, + job_id uuid NOT NULL REFERENCES attachment_jobs(job_id) ON DELETE CASCADE, + payload jsonb NOT NULL, + created_at timestamptz NOT NULL DEFAULT now(), + published_at timestamptz, + attempts integer NOT NULL DEFAULT 0, + UNIQUE (job_id, published_at) +); +CREATE INDEX attachment_outbox_pending_idx ON attachment_outbox(created_at) WHERE published_at IS NULL; diff --git a/migrations/007_job_measurements.sql b/migrations/007_job_measurements.sql new file mode 100644 index 0000000..12fc193 --- /dev/null +++ b/migrations/007_job_measurements.sql @@ -0,0 +1 @@ +ALTER TABLE attachment_jobs ADD COLUMN first_started_at timestamptz; diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..20c78ef --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,15 @@ +[project] +name = "mtusi2026lab4" +version = "0.1.0" +description = "MTUSI high-load messenger, laboratory 4" +readme = "README.md" +requires-python = ">=3.12" +dependencies = [ + "argon2-cffi==25.1.0", + "asyncpg==0.31.0", + "boto3==1.43.115", + "fastapi==0.141.1", + "pillow==13.0.0", + "pika==1.3.2", + "uvicorn==0.40.0", +] diff --git a/scripts/audit.py b/scripts/audit.py new file mode 100644 index 0000000..5c15de2 --- /dev/null +++ b/scripts/audit.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 +"""Sequential lab3-5 verification; capture actual output and exit status, fail fast.""" +from datetime import datetime, timezone +import argparse +import json +from pathlib import Path +import subprocess +import sys +import time + +ROOT = Path(__file__).resolve().parents[1] +DESTINATION = ROOT / "evidence/audit" +CHECKS = [ + ("contracts", ["make", "check", "validate", "PYTHON=.venv/bin/python"]), + ("compose", ["docker", "compose", "config", "--quiet"]), + ("smoke", ["make", "test"]), + ("attachments", ["make", "test-attachments"]), + ("idempotency", [sys.executable, "student-tests/idempotency_concurrency.py"]), + ("regressions", [sys.executable, "student-tests/review_regressions.py"]), + ("private-bucket", ["make", "test-private"]), + ("s3-permissions", ["make", "test-s3-permissions"]), + ("bootstrap", ["docker", "compose", "run", "--rm", "--no-deps", "s3-init"]), + ("upload-failover", ["make", "test-upload-failover"]), + ("public-failover", ["make", "test-failover", "ACTION_SCRIPT=scripts/stop-one.sh"]), + ("restore-replicas", ["docker", "compose", "up", "-d", "--wait", "api-a", "api-b", "load_balancer"]), + ("s3-outage", ["make", "test-s3-outage"]), + ("persistence", [sys.executable, "scripts/verify-persistence.py"]), + ("reconcile", ["make", "test-reconcile"]), + ("reconcile-audit", ["make", "test-reconcile-audit"]), + ("upload-memory", ["make", "test-upload-memory"]), + ("reconcile-benchmark", ["make", "test-reconcile-benchmark"]), + ("final-smoke", ["make", "test"]), + ("environment", [sys.executable, "scripts/record-environment.py"]), +] + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--resume-from', choices=[name for name, _ in CHECKS]) + args = parser.parse_args() + DESTINATION.mkdir(parents=True, exist_ok=True) + results = [] + checks = CHECKS + if args.resume_from: + offset = [name for name, _ in CHECKS].index(args.resume_from) + previous = json.loads((DESTINATION / 'results.json').read_text())['checks'] + results = previous[:offset] + if [item['name'] for item in results] != [name for name, _ in CHECKS[:offset]] or any(item['exit_code'] for item in results): + parser.error('resume requires successful evidence for every earlier step') + checks = CHECKS[offset:] + for name, command in checks: + print(f"RUN {name}", flush=True) + started = time.monotonic() + with (DESTINATION / f"{name}.txt").open("w") as stream: + result = subprocess.run(command, cwd=ROOT, stdout=stream, stderr=subprocess.STDOUT) + results.append({"name": name, "command": command, "exit_code": result.returncode, + "seconds": round(time.monotonic() - started, 3)}) + (DESTINATION / "results.json").write_text(json.dumps({ + "recorded_at_utc": datetime.now(timezone.utc).isoformat(), + "checks": results, "complete": len(results) == len(CHECKS), + }, indent=2) + "\n") + print(f"{'PASS' if result.returncode == 0 else 'FAIL'} {name}: exit={result.returncode}", flush=True) + if result.returncode: + print((DESTINATION / f"{name}.txt").read_text()[-8000:], flush=True) + return result.returncode + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/init-secrets.py b/scripts/init-secrets.py new file mode 100644 index 0000000..a416049 --- /dev/null +++ b/scripts/init-secrets.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +"""Create local credentials without printing or overwriting existing values.""" +import json +from pathlib import Path +import secrets + +root = Path(__file__).resolve().parents[1] / "secrets" +root.mkdir(mode=0o700, exist_ok=True) + + +def write(name, value): + path = root / name + if path.exists(): + return path.read_text().strip() + with path.open("x") as stream: + path.chmod(0o600) + stream.write(value + "\n") + return value + + +password = write("postgres_password", secrets.token_hex(32)) +write("database_url", f"postgresql://messenger:{password}@postgres:5432/messenger") +write("cursor_secret", secrets.token_hex(32)) +admin_key = write("s3_admin_key", secrets.token_hex(16)) +admin_secret = write("s3_admin_secret", secrets.token_hex(32)) +app_key = write("s3_access_key_id", secrets.token_hex(16)) +app_secret = write("s3_secret_access_key", secrets.token_hex(32)) +rabbitmq_password = write("rabbitmq_password", secrets.token_urlsafe(32)) +write("broker_url", f"amqp://messenger:{rabbitmq_password}@broker:5672/%2F") +config_path = root / "s3.json" +config_path.write_text(json.dumps({"identities": [ + {"name": "bootstrap", "credentials": [{"accessKey": admin_key, "secretKey": admin_secret}], + "actions": ["Admin", "Read", "Write", "List", "Tagging"]}, + {"name": "api", "credentials": [{"accessKey": app_key, "secretKey": app_secret}], + "actions": ["Read:messenger-attachments", "Write:messenger-attachments", "List:messenger-attachments"]}, +]}, indent=2) + "\n") +config_path.chmod(0o600) +print("Local secrets ready; existing values preserved.") diff --git a/scripts/record-environment.py b/scripts/record-environment.py new file mode 100644 index 0000000..961dc2f --- /dev/null +++ b/scripts/record-environment.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +"""Sanitized environment and source fingerprint for local test evidence.""" +from datetime import datetime, timezone +from hashlib import sha256 +import json +from pathlib import Path +import platform +import subprocess + +root = Path(__file__).resolve().parents[1] + + +def command(*args): + return subprocess.check_output(args, cwd=root, text=True).strip() + + +paths = [root / name for name in ('Dockerfile', 'compose.yaml', 'pyproject.toml', 'uv.lock', 'Makefile')] +for directory in ('src', 'migrations', 'infra', 'scripts', 'student-tests'): + paths.extend(path for path in (root / directory).rglob('*') + if path.is_file() and '__pycache__' not in path.parts) +report = { + 'recorded_at_utc': datetime.now(timezone.utc).isoformat(), + 'base_commit': command('git', 'rev-parse', 'HEAD'), + 'branch': command('git', 'branch', '--show-current'), + 'note': 'Results apply to this working tree, not only the base commit. Per-profile load settings are recorded in real-c*-p*-s*.json.', + 'host': platform.platform(), + 'docker': json.loads(command('docker', 'info', '--format', + '{"server":"{{.ServerVersion}}","cpus":{{.NCPU}},"memory_bytes":{{.MemTotal}}}')), + 'compose_version': command('docker', 'compose', 'version', '--short'), + 'sources_sha256': {str(path.relative_to(root)): sha256(path.read_bytes()).hexdigest() for path in sorted(paths)}, + 'services': {}, +} +for service in ('postgres', 's3', 'broker', 'worker', 'outbox', 'reconcile', 'api-a', 'api-b', 'load_balancer'): + container = command('docker', 'compose', 'ps', '-q', service) + info = json.loads(command('docker', 'inspect', container))[0] + report['services'][service] = { + 'image': info['Config']['Image'], 'image_id': info['Image'], + 'state': info['State']['Status'], 'health': info['State'].get('Health', {}).get('Status'), + 'memory_limit_bytes': info['HostConfig']['Memory'], + 'nano_cpus_limit': info['HostConfig']['NanoCpus'], + 'volumes': [mount['Name'] for mount in info['Mounts'] if mount['Type'] == 'volume'], + } +(root / 'evidence/environment.json').write_text(json.dumps(report, ensure_ascii=False, indent=2) + '\n') +print('PASS: sanitized environment and source hashes saved') diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 0000000..958b700 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,10 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." + +# Deliberately preserves named volumes and test data. +docker compose restart postgres s3 broker +docker compose up -d --wait postgres s3 broker +docker compose run --rm --no-deps s3-init +docker compose restart worker outbox reconcile api-a api-b load_balancer +docker compose up -d --wait worker outbox reconcile api-a api-b load_balancer diff --git a/scripts/run-track1-profiles.py b/scripts/run-track1-profiles.py new file mode 100644 index 0000000..ba3f140 --- /dev/null +++ b/scripts/run-track1-profiles.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python3 +"""Run real, sequential Docker profiles; restore normal settings on exit.""" +import os +from pathlib import Path +import subprocess +import sys + +ROOT = Path(__file__).resolve().parents[1] + + +def main(): + try: + for workers in (1, 2, 4): + env = dict(os.environ, WORKER_CONCURRENCY=str(workers), + WORKER_PREFETCH=str(workers), MAX_PENDING_JOBS='32') + subprocess.run(['docker', 'compose', 'up', '-d', '--no-deps', '--wait', + 'worker', 'api-a', 'api-b'], cwd=ROOT, env=env, check=True) + for size in (512, 2400): + subprocess.run([sys.executable, 'scripts/track1.py', + '--run-name', f'real-c{workers}-p{workers}-s{size}', + '--workers', str(workers), '--prefetch', str(workers), + '--max-pending', '32', '--images', '160', + '--concurrency', '16', '--size', str(size), + '--require-backpressure'], cwd=ROOT, check=True) + finally: + subprocess.run(['docker', 'compose', 'up', '-d', '--no-deps', '--wait', + 'worker', 'api-a', 'api-b'], cwd=ROOT, check=True) + + +if __name__ == '__main__': + main() diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 0000000..befd6cf --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,18 @@ +#!/bin/sh +set -eu + +: "${TARGET_INSTANCE_ID:?test provides target API instance alias}" +cd "$(dirname "$0")/.." + +for service in api-a api-b; do + container="$(docker compose ps -q "$service")" + [ -n "$container" ] || continue + hostname="$(docker inspect --format '{{.Config.Hostname}}' "$container")" + if [ "$hostname" = "$TARGET_INSTANCE_ID" ]; then + docker compose stop "$service" + exit 0 + fi +done + +printf '%s\n' 'TARGET_INSTANCE_ID does not match a running API replica' >&2 +exit 2 diff --git a/scripts/track1.py b/scripts/track1.py new file mode 100755 index 0000000..d72240d --- /dev/null +++ b/scripts/track1.py @@ -0,0 +1,217 @@ +#!/usr/bin/env python3 +"""Closed-loop upload burst with concurrent readiness observations and drain proof.""" +from concurrent.futures import ThreadPoolExecutor, as_completed +from io import BytesIO +import json +import math +from pathlib import Path +import subprocess +import sys +import threading +import time + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, Failure, arguments, check, run + + +def percentile(values, point): + values = sorted(values) + return round(values[max(0, math.ceil(len(values) * point) - 1)] * 1000, 3) if values else None + + +def command_json(*args): + result = subprocess.run(args, cwd=ROOT, text=True, capture_output=True, timeout=20) + if result.returncode: + raise Failure("metrics command failed: " + args[0]) + try: + return json.loads(result.stdout) + except ValueError: + raise Failure("metrics command returned invalid JSON") from None + + +def metrics(chat_id): + return command_json("docker", "compose", "exec", "-T", "worker", + "/app/.venv/bin/python", "-m", "src.pipeline_metrics", chat_id) + + +def broker_depth(): + rows = command_json("docker", "compose", "exec", "-T", "broker", + "rabbitmqctl", "list_queues", "name", "messages_ready", + "messages_unacknowledged", "--formatter", "json") + return next(row for row in rows if row["name"] == "attachment.process.v1") + + +def container_info(service): + result = subprocess.run(['docker', 'compose', 'ps', '-q', service], cwd=ROOT, + capture_output=True, text=True, timeout=10, check=True) + ids = result.stdout.split() + check(len(ids) == 1, 'expected exactly one container: ' + service) + return command_json('docker', 'inspect', ids[0])[0] + + +def main(): + def configure(p): + p.add_argument("--run-name", required=True) + p.add_argument("--workers", type=int, required=True) + p.add_argument("--prefetch", type=int, required=True) + p.add_argument("--max-pending", type=int, required=True) + p.add_argument("--images", type=int, default=40) + p.add_argument("--concurrency", type=int, default=8) + p.add_argument("--size", type=int, default=1600) + p.add_argument("--max-ready-seconds", type=float, default=180) + p.add_argument("--require-backpressure", action="store_true") + p.add_argument("--output") + args = arguments(__doc__, configure) + check(1 <= args.images <= 1000 and 1 <= args.concurrency <= 128, "invalid workload") + check(1 <= args.size <= 6000 and 0 < args.max_ready_seconds <= 3600, "invalid size/deadline") + check(args.workers > 0 and args.prefetch > 0 and args.max_pending > 0, "invalid profile") + from PIL import Image + # Reproducible, nonuniform synthetic image; no real personal data. + row = bytes(v for x in range(args.size) for v in ((x * 13) % 256, (x * 37) % 256, (x * 71) % 256)) + image = Image.frombytes("RGB", (args.size, args.size), row * args.size) + stream = BytesIO() + image.save(stream, "PNG") + data = stream.getvalue() + client = Client(args.base_url, args.ca_file, timeout=30) + alice, bob, _, chat = client.bootstrap() + baseline = metrics(chat["id"]) + check(baseline["concurrency"] == args.workers and baseline["prefetch"] == args.prefetch, + "worker settings do not match requested profile") + check(baseline["pending"] == 0, "use an idle isolated stack") + for service in ("api-a", "api-b"): + env = dict(item.split('=', 1) for item in container_info(service)['Config']['Env']) + check(int(env['MAX_PENDING_JOBS']) == args.max_pending, + 'running API MAX_PENDING_JOBS does not match profile') + resources = container_info('worker')['HostConfig'] + path = "/api/v1/chats/" + chat["id"] + "/attachments?filename=track1.png" + pending, records, samples, errors = {}, [], [], [] + lock, finished = threading.Lock(), threading.Event() + started = time.monotonic() + def observe(): + local = Client(args.base_url, args.ca_file, timeout=5) + while not finished.wait(.1): + with lock: + batch = list(pending.items()) + for key, record in batch: + if finished.is_set(): + break + try: + att = local.call("GET", "/api/v1/attachments/" + key, actor=bob, schema="Attachment")["json"] + if att["status"] in ("ready", "failed"): + with lock: + record["terminal"] = att["status"] + record["terminal_at"] = time.monotonic() + pending.pop(key, None) + except Failure: + continue + def sample(): + while not finished.is_set(): + try: + sample = metrics(chat["id"]) + sample["queue"] = broker_depth() + sample["seconds"] = time.monotonic() - started + with lock: + samples.append(sample) + except (Failure, OSError, subprocess.TimeoutExpired) as exc: + with lock: + errors.append(type(exc).__name__) + if finished.wait(1): + break + observer = threading.Thread(target=observe, daemon=True) + sampler = threading.Thread(target=sample, daemon=True) + observer.start() + sampler.start() + def upload(index): + local = Client(args.base_url, args.ca_file, timeout=30) + before = time.monotonic() + try: + response = local.call("POST", path, (202, 429), actor=alice, raw=data, + headers={"Content-Type": "image/png"}, schema="Attachment") + received = time.monotonic() + record = {"index": index, "status": response["status"], "upload_seconds": received - before} + if response["status"] == 202: + record.update(id=response["json"]["id"], accepted_at=received) + with lock: + records.append(record) + if response["status"] == 202: + pending[record["id"]] = record + except Failure: + with lock: + records.append({"index": index, "status": "transport_or_http_error"}) + try: + with ThreadPoolExecutor(max_workers=args.concurrency) as pool: + list(pool.map(upload, range(args.images))) + upload_finished = time.monotonic() + deadline = upload_finished + args.max_ready_seconds + while time.monotonic() < deadline: + with lock: + empty = not pending + if empty: + break + time.sleep(.1) + finally: + measured_end = time.monotonic() + finished.set() + observer.join(timeout=10) + sampler.join(timeout=45) + elapsed = measured_end - started + final = metrics(chat["id"]) + queue = broker_depth() + for _ in range(20): + if queue["messages_ready"] == 0 and queue["messages_unacknowledged"] == 0: + break + time.sleep(.25) + queue = broker_depth() + accepted = [r for r in records if r["status"] == 202] + ready = [r for r in accepted if r.get("terminal") == "ready"] + latencies = [r["terminal_at"] - r["accepted_at"] for r in ready] + waits = [r["queue_wait_seconds"] for r in final["jobs"] if r["queue_wait_seconds"] is not None] + drained = (bool(accepted) and len(ready) == len(accepted) and final["pending"] == 0 + and queue["messages_ready"] == 0 and queue["messages_unacknowledged"] == 0) + recovery = False + if drained: + try: + att, _ = client.attachment(alice, chat) + client.wait_ready(bob, att, wait_seconds=args.max_ready_seconds) + recovery = True + except Failure: + pass + rejected = sum(r["status"] == 429 for r in records) + report = { + "run_name": args.run_name, + "settings": {"workers": baseline["concurrency"], "prefetch": baseline["prefetch"], + "max_pending": args.max_pending, + "worker_limits": {key: resources[key] for key in ('Memory', 'NanoCpus')}}, + "workload": {"images": args.images, "concurrency": args.concurrency, + "size": args.size, "bytes": len(data), "type": "closed-loop burst"}, + "results": {"accepted": len(accepted), "rejected_429": rejected, "ready": len(ready), + "failed": sum(r.get("terminal") == "failed" for r in accepted), + "timed_out": len(pending), "elapsed_seconds": elapsed, + "ready_per_second": len(ready) / elapsed, + "ready_latency_ms": {str(p): percentile(latencies, p) for p in (.5, .95, .99)}, + "queue_wait_ms": {str(p): percentile(waits, p) for p in (.5, .95, .99)}, + "max_sampled_rss_bytes": max((s["rss_bytes"] for s in samples), default=None), + "max_sampled_pending": max((s["pending"] for s in samples), default=None), + "sample_count": len(samples), + "drained": drained, "recovery_upload_ready": recovery}, + "records": [{k: v for k, v in r.items() if k not in ("accepted_at", "terminal_at")} | + ({"ready_seconds": r["terminal_at"] - r["accepted_at"]} if "terminal_at" in r else {}) + for r in records], + "samples": samples, "metric_errors": errors, + "limitations": "Closed-loop; ready time is polling upper bound. RSS is sum of worker/container processes, shared pages may be counted twice. Sampling may miss peaks. Queue wait is first claim minus durable job creation.", + } + target = Path(args.output or ROOT / "evidence" / (args.run_name + ".json")) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(json.dumps(report, indent=2) + "\n") + print(json.dumps(report["results"], indent=2)) + check(not errors and bool(samples), "incomplete metrics; result saved but not valid evidence") + check(all(s['pending'] <= args.max_pending for s in samples), "admission limit exceeded") + check(all(r["status"] in (202, 429) for r in records), "unexpected HTTP/transport failures") + check(drained and recovery, "pipeline did not drain/recover; see saved result") + if args.require_backpressure: + check(rejected > 0, "backpressure was not observed") + + +if __name__ == "__main__": + run(main) diff --git a/scripts/verify-persistence.py b/scripts/verify-persistence.py new file mode 100644 index 0000000..5bfc5a4 --- /dev/null +++ b/scripts/verify-persistence.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python3 +"""Public persistence test plus proof of actual container restarts.""" +import json +from pathlib import Path +import subprocess +import sys + +ROOT = Path(__file__).resolve().parents[1] + + +def snapshot(): + result = {} + for service in ('postgres', 's3', 'api-a', 'api-b', 'load_balancer'): + container = subprocess.check_output(['docker', 'compose', 'ps', '-q', service], cwd=ROOT, text=True).strip() + started = subprocess.check_output(['docker', 'inspect', '--format', '{{.State.StartedAt}}', container], text=True).strip() + result[service] = {'container': container[:12], 'started_at': started} + return result + + +before = snapshot() +print('Before:', json.dumps(before, sort_keys=True), flush=True) +subprocess.run([sys.executable, 'tests/resilience.py', '--mode', 'persistence', + '--action-script', 'scripts/restart.sh'], cwd=ROOT, check=True) +after = snapshot() +print('After:', json.dumps(after, sort_keys=True), flush=True) +assert all(before[key]['started_at'] != after[key]['started_at'] for key in before) +print('PASS: PostgreSQL, S3, both API replicas and LB actually restarted; volumes preserved') diff --git a/scripts/verify-resilience.py b/scripts/verify-resilience.py new file mode 100644 index 0000000..02ccd00 --- /dev/null +++ b/scripts/verify-resilience.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +"""Sequential public tests with safe container-state evidence; no credentials.""" +import json +from pathlib import Path +import subprocess +import sys + +ROOT = Path(__file__).resolve().parents[1] + + +def states(): + ids = subprocess.check_output(['docker', 'compose', 'ps', '-aq'], text=True).split() + rows = json.loads(subprocess.check_output(['docker', 'inspect', *ids], text=True)) + return {r['Config']['Labels']['com.docker.compose.service']: + {'id': r['Id'][:12], 'started_at': r['State']['StartedAt'], 'status': r['State']['Status']} + for r in rows} + + +def main(): + results, stopped = [], set() + try: + for mode in ['persistence'] + ['failover'] * 6: + before = states() + hook = 'restart' if mode == 'persistence' else 'stop-one' + result = subprocess.run([sys.executable, 'tests/resilience.py', '--mode', mode, + '--action-script', 'scripts/' + hook + '.sh'], + text=True, capture_output=True, timeout=200) + after = states() + results.append(dict(mode=mode, returncode=result.returncode, + output=result.stdout + result.stderr, before=before, after=after)) + (ROOT / 'evidence/resilience.json').write_text(json.dumps(results, indent=2) + '\n') + print(result.stdout + result.stderr, flush=True) + assert result.returncode == 0, 'public resilience failed' + if mode == 'persistence': + for service in ('postgres', 's3', 'broker', 'worker', 'outbox', 'api-a', 'api-b'): + assert before[service]['started_at'] != after[service]['started_at'], service + else: + stopped.update(s for s in ('api-a', 'api-b') if after[s]['status'] == 'exited') + subprocess.run(['docker', 'compose', 'up', '-d', '--no-deps', '--wait', 'api-a', 'api-b'], check=True) + if len(stopped) == 2: + break + assert len(stopped) == 2, 'did not exercise both API replicas' + finally: + subprocess.run(['docker', 'compose', 'up', '-d', '--no-deps', '--wait', 'api-a', 'api-b'], check=True) + + +if __name__ == '__main__': + main() diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/src/__init__.py @@ -0,0 +1 @@ + diff --git a/src/attachments.py b/src/attachments.py new file mode 100644 index 0000000..ea496ee --- /dev/null +++ b/src/attachments.py @@ -0,0 +1,141 @@ +import asyncio +from hashlib import sha256 +import os +from tempfile import TemporaryFile +import unicodedata +from uuid import UUID, uuid4 +import warnings + +from fastapi import APIRouter, Depends, Query, Request, Response +from PIL import Image, UnidentifiedImageError +from starlette.concurrency import run_in_threadpool +from starlette.requests import ClientDisconnect + +from .errors import ApiError +from .models import Attachment +from .routes import get_current_user, get_store +from .storage.postgres import PostgresStore + +router = APIRouter() +Image.MAX_IMAGE_PIXELS = 40000000 +warnings.filterwarnings("error", category=Image.DecompressionBombWarning) + + +def inspect_image(file, content_type, max_pixels): + try: + file.seek(0) + with Image.open(file, formats=("PNG", "JPEG")) as image: + if image.width * image.height > max_pixels: + raise ApiError(413, "payload_too_large", "Слишком много пикселей") + actual = {"PNG": "image/png", "JPEG": "image/jpeg"}[image.format] + if actual != content_type or getattr(image, "n_frames", 1) != 1: + raise ApiError(415, "unsupported_media_type", "Нужен обычный PNG или JPEG") + image.verify() + # verify checks PNG CRC; load also catches truncated pixel streams/JPEGs. + file.seek(0) + with Image.open(file, formats=("PNG", "JPEG")) as image: + image.load() + except (Image.DecompressionBombError, Image.DecompressionBombWarning) as error: + raise ApiError(413, "payload_too_large", "Слишком много пикселей") from error + except (UnidentifiedImageError, OSError, ValueError, SyntaxError) as error: + raise ApiError(415, "unsupported_media_type", "Повреждённое изображение") from error + file.seek(0) + + +@router.post("/api/v1/chats/{chat_id}/attachments", response_model=Attachment, status_code=202) +async def upload(chat_id: UUID, request: Request, filename: str = Query(...), + user: dict = Depends(get_current_user), store: PostgresStore = Depends(get_store)): + await store.get_chat_for_member(chat_id, user["id"]) + if (not 1 <= len(filename) <= 255 or not filename.strip() or + any(c in "/\\" or unicodedata.category(c) in {"Cc", "Cs"} for c in filename)): + raise ApiError(400, "invalid_request", "Некорректное имя файла") + content_type = request.headers.get("content-type", "").lower() + if content_type not in {"image/png", "image/jpeg"}: + raise ApiError(415, "unsupported_media_type", "Нужен PNG или JPEG") + s3 = request.app.state.s3 + limit = s3.settings.max_bytes + if "content-length" in request.headers: + try: + length = int(request.headers["content-length"]) + if length < 0: + raise ValueError() + except ValueError: + raise ApiError(400, "invalid_request", "Некорректная длина") from None + if length > limit: + raise ApiError(413, "payload_too_large", "Файл больше лимита") + try: + timeout = int(os.environ.get("UPLOAD_TIMEOUT_SECONDS", "30")) + if not 1 <= timeout <= 120: + raise ApiError(503, "unavailable", "Некорректный лимит времени загрузки") + async with asyncio.timeout(timeout): + async with request.app.state.upload_slot: + # Temporary scratch space only; deleted on every exit, never used for downloads. + with TemporaryFile() as file: + size = 0 + digest = sha256() + async for chunk in request.stream(): + size += len(chunk) + if size > limit: + raise ApiError(413, "payload_too_large", "Файл больше лимита") + digest.update(chunk) + await run_in_threadpool(file.write, chunk) + await run_in_threadpool(inspect_image, file, content_type, s3.settings.max_pixels) + attachment_id, job_id = uuid4(), uuid4() + key = f"staging/{attachment_id}/original" + async with store.attachment_transaction() as connection: + try: + max_pending = int(os.environ.get("MAX_PENDING_JOBS", "100")) + except ValueError as error: + raise ApiError(503, "unavailable", "Некорректный лимит очереди") from error + if max_pending < 1: + raise ApiError(503, "unavailable", "Некорректный лимит очереди") + await store.ensure_attachment_queue_capacity(connection, max_pending) + await s3.put(key, file, size, content_type) + if os.environ.get("UPLOAD_FAULT_AFTER_S3_PUT") == "1": + raise ApiError(503, "unavailable", "Инъецирован отказ после S3 PUT") + attachment = await store.create_attachment( + attachment_id, chat_id, user["id"], filename, content_type, + size, digest.hexdigest(), key, job_id, connection=connection, + ) + # Fault-injection for level-4 proof: durable outbox must recover this commit. + if os.environ.get("OUTBOX_FAULT_AFTER_COMMIT") == "1": + os._exit(70) + attachment["variants"] = [] + return attachment + except ClientDisconnect: + raise ApiError(400, "invalid_request", "Загрузка прервана") from None + except TimeoutError: + raise ApiError(503, "unavailable", "Время загрузки истекло") from None + + +@router.get("/api/v1/attachments/{attachment_id}", response_model=Attachment) +async def metadata(attachment_id: UUID, user: dict = Depends(get_current_user), + store: PostgresStore = Depends(get_store)): + return await store.attachment_for_member(attachment_id, user["id"]) + + +@router.get("/api/v1/attachments/{attachment_id}/content") +async def download(attachment_id: UUID, request: Request, + variant: str = Query("original", pattern="^(original|thumbnail|crop|optimized)$"), + user: dict = Depends(get_current_user), store: PostgresStore = Depends(get_store)): + attachment = await store.attachment_for_member(attachment_id, user["id"]) + if attachment["status"] != "ready": + raise ApiError(409, "conflict", "Вложение ещё не готово") + if variant == "original": + data = await request.app.state.s3.download(attachment) + content_type = attachment["content_type"] + else: + matching = [item for item in attachment["variants"] if item["name"] == variant] + if not matching: + raise ApiError(404, "not_found", "Вариант не найден") + item = matching[0] + object_key = await store.pool.fetchval( + 'SELECT object_key FROM attachment_variants WHERE attachment_id=$1 AND pipeline_version=$2 AND name=$3', + attachment_id, attachment['pipeline_version'], variant, + ) + data = await request.app.state.s3.download_key( + object_key, item["size_bytes"], item["sha256"] + ) + content_type = item["content_type"] + return Response(data, media_type=content_type, + headers={"X-Content-Type-Options": "nosniff", "Cache-Control": "private, no-store"}) diff --git a/src/bootstrap_s3.py b/src/bootstrap_s3.py new file mode 100644 index 0000000..28f2c8b --- /dev/null +++ b/src/bootstrap_s3.py @@ -0,0 +1,51 @@ +"""Idempotent private bucket bootstrap; executed with separate admin credentials.""" +import time +from uuid import uuid4 +from botocore.exceptions import BotoCoreError, ClientError +from .storage.s3 import S3Settings, S3Store + + +def main(): + store = S3Store(S3Settings.load()) + try: + for attempt in range(60): + try: + store.client.list_buckets() + break + except (BotoCoreError, ClientError): + if attempt == 59: + raise RuntimeError("S3 startup timed out") from None + time.sleep(1) + bucket = store.settings.bucket + try: + store.client.head_bucket(Bucket=bucket) + except ClientError as error: + if str(error.response["Error"]["Code"]) not in {"404", "NoSuchBucket", "NotFound"}: + raise + store.client.create_bucket(Bucket=bucket) + # No anonymous identity; revoke any previously configured public grants/policy. + store.client.put_bucket_acl(Bucket=bucket, ACL="private") + try: + store.client.delete_bucket_policy(Bucket=bucket) + except ClientError as error: + if error.response["Error"]["Code"] != "NoSuchBucketPolicy": + raise + # The master health endpoint can become available before a volume accepts + # writes after a restart. Verify the actual data path before APIs start. + probe_key = f"_bootstrap/write-probe-{uuid4()}" + for attempt in range(30): + try: + store.client.put_object(Bucket=bucket, Key=probe_key, Body=b"ready") + store.client.delete_object(Bucket=bucket, Key=probe_key) + break + except (BotoCoreError, ClientError): + if attempt == 29: + raise RuntimeError("S3 write path startup timed out") from None + time.sleep(1) + print("PASS: private S3 bucket ready") + finally: + store.client.close() + + +if __name__ == "__main__": + main() diff --git a/src/broker.py b/src/broker.py new file mode 100644 index 0000000..4cd980d --- /dev/null +++ b/src/broker.py @@ -0,0 +1,83 @@ +"""Small durable RabbitMQ adapter; broker messages contain no attachment bytes.""" +import json +import os +from pathlib import Path + +import pika +from starlette.concurrency import run_in_threadpool + +from .errors import ApiError + +QUEUE = "attachment.process.v1" +DLQ = "attachment.process.v1.dlq" + + +def broker_url() -> str: + path = os.environ.get("BROKER_URL_FILE") + value = Path(path).read_text(encoding="utf-8").strip() if path else os.environ.get("BROKER_URL", "") + if not value.startswith(("amqp://", "amqps://")): + raise ValueError("BROKER_URL или BROKER_URL_FILE должен содержать AMQP URL") + return value + + +def _connection() -> pika.BlockingConnection: + connection = pika.BlockingConnection(pika.URLParameters(broker_url())) + channel = connection.channel() + channel.queue_declare(queue=QUEUE, durable=True) + channel.queue_declare(queue=DLQ, durable=True) + return connection + + +def _publish(payload: dict) -> None: + connection = _connection() + try: + channel = connection.channel() + channel.confirm_delivery() + channel.basic_publish( + exchange="", routing_key=QUEUE, + body=json.dumps(payload, separators=(",", ":")).encode("utf-8"), + properties=pika.BasicProperties(delivery_mode=pika.DeliveryMode.Persistent, content_type="application/json"), + mandatory=True, + ) + finally: + connection.close() + + +async def publish(payload: dict) -> None: + try: + await run_in_threadpool(_publish, payload) + except (pika.exceptions.AMQPError, OSError, ValueError) as error: + raise ApiError(503, "unavailable", "Брокер недоступен") from error + + +async def publish_dlq(payload: dict) -> None: + payload = {**payload, "failure": payload.get('failure', 'retry_exhausted')} + try: + await run_in_threadpool(_publish_to, DLQ, payload) + except (pika.exceptions.AMQPError, OSError, ValueError) as error: + raise RuntimeError("DLQ is unavailable") from error + + +def _publish_to(queue: str, payload: dict) -> None: + connection = _connection() + try: + channel = connection.channel() + channel.confirm_delivery() + channel.basic_publish( + exchange="", routing_key=queue, body=json.dumps(payload, separators=(",", ":")).encode("utf-8"), + properties=pika.BasicProperties(delivery_mode=pika.DeliveryMode.Persistent, content_type="application/json"), + mandatory=True, + ) + finally: + connection.close() + + +async def ready() -> None: + connection = None + try: + connection = await run_in_threadpool(_connection) + except (pika.exceptions.AMQPError, OSError, ValueError) as error: + raise ApiError(503, "unavailable", "Брокер недоступен") from error + finally: + if connection is not None: + await run_in_threadpool(connection.close) diff --git a/src/config.py b/src/config.py new file mode 100644 index 0000000..320063a --- /dev/null +++ b/src/config.py @@ -0,0 +1,125 @@ +from dataclasses import dataclass +import os +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +_CONFIGURATION_KEYS = { + "APP_HOST", + "APP_PORT", + "APP_ENV", + "DATABASE_URL", + "DATABASE_URL_FILE", + "SESSION_TTL_SECONDS", + "IDEMPOTENCY_TTL_SECONDS", + "INSTANCE_ID", + "CURSOR_SECRET", + "CURSOR_SECRET_FILE", + "S3_ENDPOINT", "S3_REGION", "S3_BUCKET", + "S3_ACCESS_KEY_ID", "S3_ACCESS_KEY_ID_FILE", + "S3_SECRET_ACCESS_KEY", "S3_SECRET_ACCESS_KEY_FILE", + "BROKER_URL", "BROKER_URL_FILE", + "MAX_UPLOAD_BYTES", "MAX_IMAGE_PIXELS", + "UPLOAD_CONCURRENCY", "UPLOAD_TIMEOUT_SECONDS", "UPLOAD_FAULT_AFTER_S3_PUT", + "WORKER_CONCURRENCY", "JOB_MAX_ATTEMPTS", "JOB_TIMEOUT_SECONDS", "STAGING_RETENTION_SECONDS", + "WORKER_PREFETCH", "MAX_PENDING_JOBS", "RENDER_MEMORY_MB", +} + + +@dataclass(frozen=True, slots=True) +class Settings: + """Проверенная конфигурация запуска HTTP-сервера.""" + + host: str + port: int + environment: str + database_url: str + session_ttl_seconds: int + idempotency_ttl_seconds: int + instance_id: str + cursor_secret: bytes + + +def load_settings(env_file: Path = ROOT / ".env") -> Settings: + """Читает разрешённые ключи .env, не заменяя явно переданное окружение.""" + + if env_file.is_file(): + for raw_line in env_file.read_text(encoding="utf-8").splitlines(): + line = raw_line.strip() + if not line or line.startswith("#"): + continue + key, separator, value = line.partition("=") + if not separator or key not in _CONFIGURATION_KEYS: + continue + value = value.strip() + if len(value) >= 2 and value[0] == value[-1] and value[0] in {"'", '"'}: + value = value[1:-1] + os.environ.setdefault(key, value) + + def value(name: str, default: str = "") -> str: + return os.environ.get(name, default).strip() + + def number(name: str, default: int) -> int: + try: + result = int(value(name, str(default))) + except ValueError as error: + raise ValueError(f"{name} должен быть целым числом") from error + if result < 1: + raise ValueError(f"{name} должен быть положительным") + return result + + def url(name: str, file_name: str, prefixes: tuple[str, ...]) -> str: + result = value(name) + path = value(file_name) + if path: + try: + result = Path(path).read_text(encoding="utf-8").strip() + except OSError as error: + raise ValueError(f"{file_name} недоступен") from error + if result and not result.startswith(prefixes): + raise ValueError(f"{name} содержит неверный URL") + return result + + def secret(name: str, file_name: str) -> bytes: + result = value(name) + path = value(file_name) + if path: + try: + result = Path(path).read_text(encoding="utf-8").strip() + except OSError as error: + raise ValueError(f"{file_name} недоступен") from error + encoded = result.encode("utf-8") + if len(encoded) < 32: + raise ValueError(f"{name} должен содержать не менее 32 байт") + return encoded + + host = value("APP_HOST", "0.0.0.0") + environment = value("APP_ENV", "lab") + if not host: + raise ValueError("APP_HOST не может быть пустым") + if not environment: + raise ValueError("APP_ENV не может быть пустым") + port = number("APP_PORT", 8080) + if port > 65535: + raise ValueError("APP_PORT должен находиться в диапазоне 1..65535") + database_url = url( + "DATABASE_URL", "DATABASE_URL_FILE", ("postgresql://", "postgres://") + ) + if not database_url: + raise ValueError("Нужен PostgreSQL DATABASE_URL или DATABASE_URL_FILE") + instance_id = value("INSTANCE_ID") or os.uname().nodename + if not instance_id: + raise ValueError("INSTANCE_ID не может быть пустым") + idempotency_ttl_seconds = number("IDEMPOTENCY_TTL_SECONDS", 86400) + if idempotency_ttl_seconds < 86400: + raise ValueError("IDEMPOTENCY_TTL_SECONDS должен быть не менее 86400") + return Settings( + host=host, + port=port, + environment=environment, + database_url=database_url, + session_ttl_seconds=number("SESSION_TTL_SECONDS", 3600), + idempotency_ttl_seconds=idempotency_ttl_seconds, + instance_id=instance_id, + cursor_secret=secret("CURSOR_SECRET", "CURSOR_SECRET_FILE"), + ) diff --git a/src/cursor.py b/src/cursor.py new file mode 100644 index 0000000..5dd8eab --- /dev/null +++ b/src/cursor.py @@ -0,0 +1,81 @@ +import base64 +from datetime import UTC, datetime +import hashlib +import hmac +import json +from uuid import UUID + + +_secret: bytes | None = None + + +class CursorError(ValueError): + """Недопустимый, повреждённый или применённый не к тому ресурсу курсор.""" + + +def configure_secret(secret: bytes) -> None: + """Устанавливает общий для всех реплик ключ подписи курсоров.""" + global _secret + if len(secret) < 32: + raise ValueError("cursor secret must contain at least 32 bytes") + _secret = secret + + +def signing_secret() -> bytes: + if _secret is None: + raise RuntimeError("cursor secret has not been configured") + return _secret + + +def _encode_bytes(value: bytes) -> str: + return base64.urlsafe_b64encode(value).decode().rstrip("=") + + +def _decode_bytes(value: str) -> bytes: + try: + return base64.urlsafe_b64decode(value + "=" * (-len(value) % 4)) + except (ValueError, UnicodeEncodeError) as error: + raise CursorError from error + + +def encode_cursor(scope: str, created_at: datetime, item_id: str) -> str: + payload = json.dumps( + { + "scope": scope, + "created_at": created_at.astimezone(UTC) + .isoformat() + .replace("+00:00", "Z"), + "id": item_id, + }, + separators=(",", ":"), + sort_keys=True, + ).encode() + signature = hmac.new(signing_secret(), payload, hashlib.sha256).digest() + return _encode_bytes(payload) + "." + _encode_bytes(signature) + + +def decode_cursor(cursor: str, expected_scope: str) -> tuple[datetime, str]: + try: + encoded_payload, encoded_signature = cursor.split(".") + payload = _decode_bytes(encoded_payload) + signature = _decode_bytes(encoded_signature) + except ValueError as error: + raise CursorError from error + + expected_signature = hmac.new(signing_secret(), payload, hashlib.sha256).digest() + if not hmac.compare_digest(signature, expected_signature): + raise CursorError + + try: + data = json.loads(payload) + if set(data) != {"scope", "created_at", "id"}: + raise CursorError + if data["scope"] != expected_scope: + raise CursorError + item_id = str(UUID(data["id"])) + created_at = datetime.fromisoformat(data["created_at"].replace("Z", "+00:00")) + if created_at.tzinfo is None: + raise CursorError + return created_at.astimezone(UTC), item_id + except (TypeError, ValueError, KeyError) as error: + raise CursorError from error diff --git a/src/errors.py b/src/errors.py new file mode 100644 index 0000000..258d699 --- /dev/null +++ b/src/errors.py @@ -0,0 +1,12 @@ +"""Ошибки, которые преобразуются в контрактные HTTP-ответы.""" + + +class ApiError(Exception): + + def __init__( + self, status_code: int, code: str, message: str, headers: dict[str, str] | None = None + ) -> None: + self.status_code = status_code + self.code = code + self.message = message + self.headers = headers or {} diff --git a/src/main.py b/src/main.py new file mode 100644 index 0000000..f355932 --- /dev/null +++ b/src/main.py @@ -0,0 +1,172 @@ +import asyncio +import logging +import os +import sys +import time +from contextlib import asynccontextmanager +from uuid import uuid4 + +from fastapi import FastAPI, Request +from fastapi.exceptions import RequestValidationError +import uvicorn +from starlette.responses import JSONResponse, Response +from starlette.exceptions import HTTPException + +from .errors import ApiError +from .cursor import configure_secret +from .models import ErrorDetails, ErrorResponse +from .config import load_settings +from .routes import router +from .attachments import router as attachment_router +from .storage.s3 import S3Settings, S3Store +from .storage.postgres import PostgresStore + + +def logger(name: str) -> logging.Logger: + result = logging.getLogger(name) + if result.handlers: + return result + handler = logging.StreamHandler(sys.stdout) + handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(name)s %(message)s")) + result.addHandler(handler) + result.setLevel(logging.INFO) + result.propagate = False + return result + + +access_logger = logger("messenger.access") + + +@asynccontextmanager +async def lifespan(application: FastAPI): + settings = load_settings() + configure_secret(settings.cursor_secret) + store = PostgresStore(settings.database_url) + await store.connect() + application.state.store = store + application.state.settings = settings + application.state.s3 = S3Store(S3Settings.load()) + concurrency = int(os.environ.get("UPLOAD_CONCURRENCY", "1")) + if not 1 <= concurrency <= 8: + raise ValueError("UPLOAD_CONCURRENCY must be 1..8") + application.state.upload_slot = asyncio.Semaphore(concurrency) + application.state.accepting_requests = True + try: + yield + finally: + application.state.accepting_requests = False + application.state.s3.client.close() + await store.close() + + +app = FastAPI( + title="MTUSI Messenger", + version="1.4.0", + redoc_url=None, + lifespan=lifespan, +) + + +@app.middleware("http") +async def add_request_id(request: Request, call_next) -> Response: + """Добавляет ID запроса и пишет безопасную строку access-лога.""" + + request.state.request_id = str(uuid4()) + started_at = time.monotonic() + status_code = 500 + try: + response = await call_next(request) + status_code = response.status_code + response.headers["X-Request-Id"] = request.state.request_id + response.headers["X-Instance-Id"] = request.app.state.settings.instance_id + return response + finally: + access_logger.info( + "request_id=%s method=%s path=%s status=%s elapsed_ms=%.3f", + request.state.request_id, + request.method, + request.url.path, + status_code, + (time.monotonic() - started_at) * 1000, + ) + + +def error_response( + request: Request, status_code: int, code: str, message: str, headers: dict[str, str] | None = None +) -> JSONResponse: + request_id = request.state.request_id + body = ErrorResponse( + error=ErrorDetails(code=code, message=message, request_id=request_id) + ) + # Неожиданная ошибка обрабатывается за пределами HTTP-middleware. + # Поэтому её ответ тоже должен сам получить заголовок из контракта. + return JSONResponse( + status_code=status_code, + content=body.model_dump(), + headers={ + "X-Request-Id": request_id, + "X-Instance-Id": request.app.state.settings.instance_id, + **(headers or {}), + }, + ) + + +@app.exception_handler(ApiError) +async def handle_api_error(request: Request, error: ApiError) -> JSONResponse: + """Преобразует контролируемую ошибку API в стандартный JSON-ответ.""" + + return error_response(request, error.status_code, error.code, error.message, error.headers) + + +@app.exception_handler(RequestValidationError) +async def handle_validation_error( + request: Request, error: RequestValidationError +) -> JSONResponse: + """Преобразует ошибки JSON и Pydantic в обязательный ответ 400.""" + + return error_response(request, 400, "invalid_request", "Некорректный запрос") + + +@app.exception_handler(HTTPException) +async def handle_http_error(request: Request, error: HTTPException) -> JSONResponse: + """Включает ошибки декодирования тела, возникающие до Pydantic.""" + codes = { + 400: "invalid_request", + 401: "unauthorized", + 403: "forbidden", + 404: "not_found", + 405: "method_not_allowed", + 503: "unavailable", + } + code = codes.get(error.status_code, "internal_error") + return error_response( + request, error.status_code, code, + "Некорректный запрос" if error.status_code < 500 else "Ошибка сервера", + error.headers, + ) + + +@app.exception_handler(Exception) +async def handle_unexpected_error(request: Request, error: Exception) -> JSONResponse: + """Не раскрывает внутренние детали при непредвиденной ошибке приложения.""" + + return error_response(request, 500, "internal_error", "Внутренняя ошибка сервера") + + +app.include_router(router) +app.include_router(attachment_router) + + +def main() -> None: + settings = load_settings() + uvicorn.run( + app, + host=settings.host, + port=settings.port, + access_log=False, + timeout_graceful_shutdown=8, + ) + + +if __name__ == "__main__": + main() diff --git a/src/migrate.py b/src/migrate.py new file mode 100644 index 0000000..4376e8d --- /dev/null +++ b/src/migrate.py @@ -0,0 +1,18 @@ +import asyncio + +from .config import load_settings +from .storage.postgres import PostgresStore + + +async def run() -> None: + settings = load_settings() + store = PostgresStore(settings.database_url) + await store.connect() + try: + await store.migrate() + finally: + await store.close() + + +if __name__ == "__main__": + asyncio.run(run()) diff --git a/src/models.py b/src/models.py new file mode 100644 index 0000000..15f3810 --- /dev/null +++ b/src/models.py @@ -0,0 +1,151 @@ +from datetime import datetime +from uuid import UUID + +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator + +class CreateUserRequest(BaseModel): + model_config = ConfigDict(extra="forbid", strict=True) + + username: str = Field(min_length=3, max_length=32, pattern=r"^[a-z0-9_]{3,32}$") + display_name: str = Field(min_length=1, max_length=100) + password: str = Field(min_length=12, max_length=128) + + @field_validator("display_name") + @classmethod + def display_name_must_not_be_whitespace(cls, value: str) -> str: + """Не допускает обязательное имя, состоящее только из пробелов.""" + + if not value.strip(): + raise ValueError("display_name не может состоять только из пробелов") + return value + + +class CreateChatRequest(BaseModel): + model_config = ConfigDict(extra="forbid") + + title: str = Field(min_length=1, max_length=100) + member_ids: list[UUID] = Field(min_length=1, max_length=100) + + @field_validator("title") + @classmethod + def title_must_not_be_whitespace(cls, value: str) -> str: + """Не допускает обязательное название, состоящее только из пробелов.""" + + if not value.strip(): + raise ValueError("title не может состоять только из пробелов") + return value + + +class ChatPageResponse(BaseModel): + model_config = ConfigDict(extra="forbid") + + items: list["Chat"] + next_cursor: str | None + + +class CreateMessageRequest(BaseModel): + model_config = ConfigDict(extra="forbid", strict=True) + + text: str = Field(max_length=4000) + attachment_ids: list[str] = Field(default_factory=list, max_length=10) + + @field_validator("attachment_ids") + @classmethod + def valid_ids(cls, values: list[str]) -> list[str]: + normalized = [str(UUID(value)) for value in values] + if len(normalized) != len(set(normalized)): + raise ValueError("Повторяющиеся attachment_ids") + return normalized + + @model_validator(mode="after") + def has_content(self): + if not self.text.strip() and not self.attachment_ids: + raise ValueError("Сообщение без содержимого") + return self + + +class MessagePageResponse(BaseModel): + model_config = ConfigDict(extra="forbid") + + items: list["Message"] + next_cursor: str | None + + +class ErrorDetails(BaseModel): + model_config = ConfigDict(extra="forbid") + + code: str + message: str + request_id: str + + +class ErrorResponse(BaseModel): + model_config = ConfigDict(extra="forbid") + + error: ErrorDetails + + +class User(BaseModel): + model_config = ConfigDict(extra="forbid") + + id: UUID + username: str + display_name: str + created_at: datetime + + +class Chat(BaseModel): + model_config = ConfigDict(extra="forbid") + + id: UUID + title: str + member_ids: list[UUID] + created_at: datetime + + +class Message(BaseModel): + model_config = ConfigDict(extra="forbid") + + id: UUID + chat_id: UUID + sender_id: UUID + text: str + attachment_ids: list[UUID] = Field(default_factory=list) + created_at: datetime + + +class Session(BaseModel): + model_config = ConfigDict(extra="forbid") + + token: str = Field(min_length=32) + token_type: str = "Bearer" + expires_at: datetime + user: User + + +class Attachment(BaseModel): + model_config = ConfigDict(extra="ignore") + + id: UUID + chat_id: UUID + owner_id: UUID + filename: str + content_type: str + size_bytes: int + sha256: str + status: str + variants: list["Variant"] = Field(default_factory=list) + error_code: str | None = None + created_at: datetime + updated_at: datetime + + +class Variant(BaseModel): + model_config = ConfigDict(extra="forbid") + + name: str + content_type: str + size_bytes: int + width: int + height: int + sha256: str diff --git a/src/outbox.py b/src/outbox.py new file mode 100644 index 0000000..60f0099 --- /dev/null +++ b/src/outbox.py @@ -0,0 +1,33 @@ +"""Transactional-outbox dispatcher; safe to run as one or more replicas.""" +import asyncio +import logging +import os + +from .broker import publish +from .config import load_settings +from .storage.postgres import PostgresStore + +logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s outbox %(message)s") + + +async def run() -> None: + settings = load_settings() + store = PostgresStore(settings.database_url) + await store.connect() + interval = max(0.1, float(os.environ.get("OUTBOX_DISPATCH_INTERVAL_SECONDS", "1"))) + try: + while True: + try: + await store.schedule_due_jobs(int(os.environ.get('JOB_MAX_ATTEMPTS', '5'))) + for event in await store.due_outbox_events(): + await publish(event["payload"]) + await store.mark_outbox_published(event["id"]) + except Exception: + logging.exception("outbox dispatch failed; durable event remains pending") + await asyncio.sleep(interval) + finally: + await store.close() + + +if __name__ == "__main__": + asyncio.run(run()) diff --git a/src/pipeline_metrics.py b/src/pipeline_metrics.py new file mode 100644 index 0000000..93cde0b --- /dev/null +++ b/src/pipeline_metrics.py @@ -0,0 +1,37 @@ +"""Operator-only metrics for one synthetic chat; never exposes credentials.""" +import asyncio +import json +import os +from pathlib import Path +import sys +from uuid import UUID +from .config import load_settings +from .storage.postgres import PostgresStore + + +async def main(chat): + store = PostgresStore(load_settings().database_url) + await store.connect() + try: + rows = await store.pool.fetch('SELECT a.id, a.status, extract(epoch from (j.first_started_at-j.created_at))::float8 AS queue_wait_seconds FROM attachments a JOIN attachment_jobs j ON j.attachment_id=a.id WHERE a.chat_id=$1', chat) + jobs = [dict(row, id=str(row['id'])) for row in rows] + pending = await store.pool.fetchval("SELECT count(*) FROM attachment_jobs WHERE status IN ('queued','processing')") + finally: + await store.close() + rss = 0 + for path in Path('/proc').glob('[0-9]*/status'): + try: + if int(path.parent.name) == os.getpid(): + continue + for line in path.read_text().splitlines(): + if line.startswith('VmRSS:'): + rss += int(line.split()[1]) * 1024 + except (OSError, ValueError): + pass + print(json.dumps({'jobs': jobs, 'pending': pending, 'rss_bytes': rss, + 'concurrency': int(os.environ['WORKER_CONCURRENCY']), + 'prefetch': int(os.environ['WORKER_PREFETCH'])})) + + +if __name__ == '__main__': + asyncio.run(main(UUID(sys.argv[1]))) diff --git a/src/reconcile.py b/src/reconcile.py new file mode 100644 index 0000000..1b60bb8 --- /dev/null +++ b/src/reconcile.py @@ -0,0 +1,104 @@ +"""Safely reconcile attachment metadata and original S3 objects. + +The command is dry-run by default. It only deletes unreferenced objects older +than a configured grace period; missing objects are reported and never cause a +metadata delete. That keeps a delayed PostgreSQL commit and active attachment +records safe from a maintenance run. +""" +import argparse +import asyncio +from datetime import datetime, timedelta, timezone +import json +import os +from time import perf_counter + +from .config import load_settings +from .storage.postgres import PostgresStore +from .storage.s3 import S3Settings, S3Store + + +async def run(args) -> dict: + settings = load_settings() + db = PostgresStore(settings.database_url) + s3 = S3Store(S3Settings.load()) + try: + await db.connect() + async with db.attachment_transaction(maintenance=True): + return await scan(args, db, s3) + finally: + s3.client.close() + await db.close() + + +async def scan(args, db, s3) -> dict: + # Age is measured at scan start, not after a potentially slow inventory. + cutoff = datetime.now(timezone.utc) - timedelta(seconds=args.grace_seconds) + started = perf_counter() + objects, s3_pages = {}, 0 + for prefix in ('staging/', 'originals/', 'variants/'): + entries, pages = await s3.list_with_pages(prefix) + objects.update(entries) + s3_pages += pages + listed_at = perf_counter() + metadata = await db.attachment_keys() + metadata_at = perf_counter() + orphan = sorted(key for key, modified in objects.items() + if key not in metadata and modified <= cutoff) + fresh = sorted(key for key, modified in objects.items() + if key not in metadata and modified > cutoff) + missing = sorted({str(attachment_id): key for key, attachment_id in metadata.items() + if key not in objects}.items()) + deleted = [] + selected = set(args.object_key) if args.object_key else None + if args.apply: + for key in orphan: + if selected is not None and key not in selected: + continue + await s3.delete(key) + deleted.append(key) + finished = perf_counter() + return { + "mode": "apply" if args.apply else "dry-run", + "grace_seconds": args.grace_seconds, + "s3_objects": len(objects), + "metadata_rows": len(metadata), + "orphan_candidates": orphan, + "fresh_unreferenced": fresh, + "missing_objects": [{"attachment_id": item[0], "object_key": item[1]} for item in missing], + "deleted": deleted, + "selected_keys": sorted(selected) if selected is not None else None, + "scan_metrics": { + "s3_list_pages": s3_pages, + "s3_list_seconds": round(listed_at - started, 6), + "postgres_metadata_seconds": round(metadata_at - listed_at, 6), + "total_seconds": round(finished - started, 6), + }, + } + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--apply", action="store_true", help="delete only aged orphan objects") + parser.add_argument("--grace-seconds", type=int, default=int(os.environ.get('STAGING_RETENTION_SECONDS', '86400'))) + parser.add_argument('--loop', action='store_true') + parser.add_argument("--object-key", action="append", default=[], + help="restrict apply to these exact keys; repeatable, inventory stays complete") + args = parser.parse_args() + if not 0 <= args.grace_seconds <= 7 * 24 * 3600: + parser.error("--grace-seconds must be 0..604800") + if any(not key.startswith(('originals/', 'staging/', 'variants/')) for key in args.object_key): + parser.error('unsupported object prefix') + if args.loop: + import time + while True: + try: + print(json.dumps(asyncio.run(run(args)), ensure_ascii=False, sort_keys=True)) + except Exception: + print('cleanup deferred: dependency unavailable', flush=True) + time.sleep(60) + else: + print(json.dumps(asyncio.run(run(args)), ensure_ascii=False, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/src/render.py b/src/render.py new file mode 100644 index 0000000..dbb1d1d --- /dev/null +++ b/src/render.py @@ -0,0 +1,27 @@ +"""Isolated image renderer. Parent enforces wall time and kills this process.""" +import os +import pickle +import sys + + +def main(): + if sys.platform == 'linux': + import resource + limit = int(os.environ.get('RENDER_MEMORY_MB', '768')) * 1024 * 1024 + resource.setrlimit(resource.RLIMIT_AS, (limit, limit)) + from .worker import variants + from PIL import UnidentifiedImageError, Image + try: + data = sys.stdin.buffer.read(10485761) + if len(data) > 10485760: + raise ValueError('bytes limit') + result = variants(data, int(os.environ.get('MAX_IMAGE_PIXELS', '40000000'))) + pickle.dump(result, sys.stdout.buffer) + except MemoryError: + sys.exit(21) + except (ValueError, OSError, SyntaxError, UnidentifiedImageError, Image.DecompressionBombError): + sys.exit(20) + + +if __name__ == '__main__': + main() diff --git a/src/replay.py b/src/replay.py new file mode 100644 index 0000000..2360cb5 --- /dev/null +++ b/src/replay.py @@ -0,0 +1,24 @@ +"""Operator-only safe replay for a failed attachment job.""" +import argparse +import asyncio +from uuid import UUID + +from .config import load_settings +from .storage.postgres import PostgresStore + + +async def run(attachment_id: UUID) -> None: + store = PostgresStore(load_settings().database_url) + await store.connect() + try: + if not await store.replay_failed_attachment(attachment_id): + raise SystemExit("attachment is not in failed state") + finally: + await store.close() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("attachment_id", type=UUID) + args = parser.parse_args() + asyncio.run(run(args.attachment_id)) diff --git a/src/routes.py b/src/routes.py new file mode 100644 index 0000000..fcd2e74 --- /dev/null +++ b/src/routes.py @@ -0,0 +1,185 @@ +import base64 +import re +from datetime import datetime +from uuid import UUID + +from fastapi import APIRouter, Depends, Header, Query, Request, Response + +from .cursor import CursorError, decode_cursor, encode_cursor +from .errors import ApiError +from .storage.postgres import PostgresStore +from .broker import ready as broker_ready +from .models import ( + Chat, + ChatPageResponse, + CreateChatRequest, + CreateMessageRequest, + CreateUserRequest, + Message, + MessagePageResponse, + Session, + User, +) + +router = APIRouter() +BEARER_CHALLENGE = {"WWW-Authenticate": 'Bearer realm="messenger"'} +BASIC_CHALLENGE = {"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'} +IDEMPOTENCY_KEY = re.compile(r"^[A-Za-z0-9._:-]{1,128}$") + +def get_store(request: Request) -> PostgresStore: + return request.app.state.store + + +async def get_current_user( + authorization: str | None = Header(default=None), + store: PostgresStore = Depends(get_store), +) -> dict: + if authorization is None or not authorization.startswith("Bearer "): + raise ApiError(401, "unauthorized", "Требуется Bearer-сессия", BEARER_CHALLENGE) + token = authorization[7:] + if not token or not token.isascii() or token.strip() != token: + raise ApiError(401, "unauthorized", "Требуется Bearer-сессия", BEARER_CHALLENGE) + user = await store.user_for_token(token) + if user is None: + raise ApiError(401, "unauthorized", "Недействительная сессия", BEARER_CHALLENGE) + return user + +def parse_basic(authorization: str | None) -> tuple[str, str]: + if authorization is None or not authorization.startswith("Basic "): + raise ApiError(401, "unauthorized", "Неверные учётные данные", BASIC_CHALLENGE) + try: + decoded = base64.b64decode(authorization[6:], validate=True) + username_bytes, password_bytes = decoded.split(b":", 1) + username = username_bytes.decode("ascii") + password = password_bytes.decode("utf-8") + except (UnicodeDecodeError, ValueError) as error: + raise ApiError(401, "unauthorized", "Неверные учётные данные", BASIC_CHALLENGE) from error + if not username or not (12 <= len(password) <= 128): + raise ApiError(401, "unauthorized", "Неверные учётные данные", BASIC_CHALLENGE) + return username, password + + +def cursor_position(cursor: str | None, scope: str) -> tuple[datetime, UUID] | None: + if cursor is None: + return None + try: + created_at, item_id = decode_cursor(cursor, scope) + return created_at, UUID(item_id) + except (CursorError, ValueError) as error: + raise ApiError(400, "invalid_request", "Некорректный cursor") from error + + +def next_page_cursor(rows: list[dict], limit: int, scope: str) -> str | None: + if len(rows) <= limit: + return None + last = rows[limit - 1] + return encode_cursor(scope, last["created_at"], str(last["id"])) + +@router.get("/health/live") +async def liveness() -> dict[str, str]: + return {"status": "ok"} + +@router.get("/health/ready") +async def readiness(request: Request, store: PostgresStore = Depends(get_store)) -> dict[str, str]: + if not request.app.state.accepting_requests: + raise ApiError(503, "unavailable", "API завершает работу") + await store.ready() + await request.app.state.s3.ready() + await broker_ready() + return {"status": "ok"} + +@router.post("/api/v1/users", response_model=User, status_code=201) +async def register_user( + payload: CreateUserRequest, store: PostgresStore = Depends(get_store) +) -> dict: + return await store.create_user(payload.username, payload.display_name, payload.password) + +@router.post("/api/v1/auth/sessions", response_model=Session, status_code=201) +async def create_session( + request: Request, + authorization: str | None = Header(default=None), + store: PostgresStore = Depends(get_store), +) -> dict: + if await request.body(): + raise ApiError(400, "invalid_request", "Тело запроса не требуется") + username, password = parse_basic(authorization) + user = await store.authenticate(username, password) + if user is None: + raise ApiError(401, "unauthorized", "Неверные учётные данные", BASIC_CHALLENGE) + return await store.create_session(user, request.app.state.settings.session_ttl_seconds) + +@router.delete("/api/v1/auth/sessions/current", status_code=204) +async def revoke_session( + authorization: str | None = Header(default=None), + store: PostgresStore = Depends(get_store), + user: dict = Depends(get_current_user), +) -> Response: + del user + await store.revoke_session(authorization[7:]) + return Response(status_code=204) + +@router.get("/api/v1/users/me", response_model=User) +async def current_user(user: dict = Depends(get_current_user)) -> dict: + return user + +@router.post("/api/v1/chats", response_model=Chat, status_code=201) +async def create_chat( + payload: CreateChatRequest, + user: dict = Depends(get_current_user), + store: PostgresStore = Depends(get_store), +) -> dict: + return await store.create_chat(user["id"], payload.title, payload.member_ids) + +@router.get("/api/v1/chats", response_model=ChatPageResponse) +async def list_chats( + user: dict = Depends(get_current_user), + store: PostgresStore = Depends(get_store), + limit: int = Query(default=50, ge=1, le=100), + cursor: str | None = Query(default=None, min_length=1), +) -> ChatPageResponse: + scope = "chats:" + str(user["id"]) + rows = await store.list_chats_for_member(user["id"], cursor_position(cursor, scope), limit) + page = rows[:limit] + return ChatPageResponse(items=page, next_cursor=next_page_cursor(rows, limit, scope)) + +@router.get("/api/v1/chats/{chat_id}", response_model=Chat) +async def get_chat( + chat_id: UUID, + user: dict = Depends(get_current_user), + store: PostgresStore = Depends(get_store), +) -> dict: + return await store.get_chat_for_member(chat_id, user["id"]) + +@router.post("/api/v1/chats/{chat_id}/messages", response_model=Message, status_code=201) +async def create_message( + chat_id: UUID, + payload: CreateMessageRequest, + request: Request, + idempotency_key: str | None = Header(default=None, alias="Idempotency-Key"), + user: dict = Depends(get_current_user), + store: PostgresStore = Depends(get_store), +) -> dict: + if idempotency_key is None or IDEMPOTENCY_KEY.fullmatch(idempotency_key) is None: + raise ApiError(400, "invalid_request", "Некорректный Idempotency-Key") + return await store.create_message_idempotent( + chat_id, + user["id"], + payload.text, + idempotency_key, + request.app.state.settings.idempotency_ttl_seconds, + [UUID(value) for value in payload.attachment_ids], + ) + +@router.get("/api/v1/chats/{chat_id}/messages", response_model=MessagePageResponse) +async def list_messages( + chat_id: UUID, + user: dict = Depends(get_current_user), + store: PostgresStore = Depends(get_store), + limit: int = Query(default=50, ge=1, le=100), + cursor: str | None = Query(default=None, min_length=1), +) -> MessagePageResponse: + scope = "messages:" + str(chat_id) + after = cursor_position(cursor, scope) + rows = await store.list_messages_for_member(chat_id, user["id"], after, limit) + page = rows[:limit] + return MessagePageResponse(items=page, next_cursor=next_page_cursor(rows, limit, scope)) diff --git a/src/storage/__init__.py b/src/storage/__init__.py new file mode 100644 index 0000000..e2e0419 --- /dev/null +++ b/src/storage/__init__.py @@ -0,0 +1,2 @@ +"""Слой in-memory хранения для приложения первой лабораторной работы.""" +"""Persistent storage adapters.""" diff --git a/src/storage/postgres.py b/src/storage/postgres.py new file mode 100644 index 0000000..d83f93d --- /dev/null +++ b/src/storage/postgres.py @@ -0,0 +1,531 @@ +from datetime import datetime, timedelta, timezone +from contextlib import asynccontextmanager +from hashlib import sha256 +import json +from pathlib import Path +from secrets import token_urlsafe +from uuid import UUID, uuid4 + +import asyncpg +from argon2 import PasswordHasher +from argon2.exceptions import InvalidHashError, VerificationError + +from ..errors import ApiError + + +ROOT = Path(__file__).resolve().parents[2] +MIGRATIONS = ROOT / "migrations" +PASSWORD_HASHER = PasswordHasher() +DB_ERRORS = (asyncpg.PostgresError, asyncpg.InterfaceError, OSError, TimeoutError) + + +class PostgresStore: + def __init__(self, database_url: str) -> None: + self._database_url = database_url + self._pool: asyncpg.Pool | None = None + + async def connect(self) -> None: + try: + self._pool = await asyncpg.create_pool( + self._database_url, min_size=1, max_size=10, command_timeout=5 + ) + except DB_ERRORS as error: + raise RuntimeError("PostgreSQL is unavailable") from error + + async def close(self) -> None: + if self._pool is not None: + await self._pool.close() + + @property + def pool(self) -> asyncpg.Pool: + if self._pool is None: + raise RuntimeError("PostgreSQL pool has not been initialized") + return self._pool + + async def migrate(self) -> None: + async with self.pool.acquire() as connection: + await connection.execute( + "CREATE TABLE IF NOT EXISTS schema_migrations (version text PRIMARY KEY, applied_at timestamptz NOT NULL DEFAULT now())" + ) + for migration in sorted(MIGRATIONS.glob("*.sql")): + applied = await connection.fetchval( + "SELECT 1 FROM schema_migrations WHERE version = $1", migration.name + ) + if applied: + continue + async with connection.transaction(): + await connection.execute(migration.read_text(encoding="utf-8")) + await connection.execute( + "INSERT INTO schema_migrations (version) VALUES ($1)", migration.name + ) + + @staticmethod + def _user(row: asyncpg.Record) -> dict: + return { + "id": row["id"], + "username": row["username"], + "display_name": row["display_name"], + "created_at": row["created_at"], + } + + @staticmethod + def _chat(row: asyncpg.Record) -> dict: + return { + "id": row["id"], + "title": row["title"], + "member_ids": row["member_ids"], + "created_at": row["created_at"], + } + + @staticmethod + def _message(row: asyncpg.Record) -> dict: + return { + "id": row["id"], + "chat_id": row["chat_id"], + "sender_id": row["sender_id"], + "text": row["text"], + "attachment_ids": row["attachment_ids"], + "created_at": row["created_at"], + } + + async def ready(self) -> None: + try: + await self.pool.fetchval("SELECT 1") + except DB_ERRORS as error: + raise ApiError(503, "unavailable", "PostgreSQL недоступен") from error + + async def create_user(self, username: str, display_name: str, password: str) -> dict: + try: + row = await self.pool.fetchrow( + "INSERT INTO users (id, username, display_name, password_hash) VALUES ($1, $2, $3, $4) RETURNING id, username, display_name, created_at", + uuid4(), + username, + display_name, + PASSWORD_HASHER.hash(password), + ) + except asyncpg.UniqueViolationError as error: + raise ApiError(409, "conflict", "Username уже занят") from error + except DB_ERRORS as error: + raise ApiError(503, "unavailable", "PostgreSQL недоступен") from error + return self._user(row) + + async def authenticate(self, username: str, password: str) -> dict | None: + try: + row = await self.pool.fetchrow( + "SELECT id, username, display_name, created_at, password_hash FROM users WHERE username = $1", + username, + ) + except DB_ERRORS as error: + raise ApiError(503, "unavailable", "PostgreSQL недоступен") from error + if row is None: + return None + try: + valid = PASSWORD_HASHER.verify(row["password_hash"], password) + except (InvalidHashError, VerificationError): + valid = False + return self._user(row) if valid else None + + async def create_session(self, user: dict, ttl_seconds: int) -> dict: + token = token_urlsafe(32) + token_hash = sha256(token.encode("ascii")).hexdigest() + try: + row = await self.pool.fetchrow( + "INSERT INTO sessions (token_hash, user_id, expires_at) VALUES ($1, $2, now() + $3 * interval '1 second') RETURNING expires_at", + token_hash, + user["id"], + ttl_seconds, + ) + except DB_ERRORS as error: + raise ApiError(503, "unavailable", "PostgreSQL недоступен") from error + return {"token": token, "token_type": "Bearer", "expires_at": row["expires_at"], "user": user} + + async def user_for_token(self, token: str) -> dict | None: + if not token.isascii(): + return None + token_hash = sha256(token.encode("ascii")).hexdigest() + try: + row = await self.pool.fetchrow( + "SELECT u.id, u.username, u.display_name, u.created_at FROM sessions s JOIN users u ON u.id = s.user_id WHERE s.token_hash = $1 AND s.expires_at > now()", + token_hash, + ) + except DB_ERRORS as error: + raise ApiError(503, "unavailable", "PostgreSQL недоступен") from error + return self._user(row) if row else None + + async def revoke_session(self, token: str) -> None: + try: + await self.pool.execute( + "DELETE FROM sessions WHERE token_hash = $1", sha256(token.encode("ascii")).hexdigest() + ) + except DB_ERRORS as error: + raise ApiError(503, "unavailable", "PostgreSQL недоступен") from error + + async def create_chat(self, creator_id: UUID, title: str, member_ids: list[UUID]) -> dict: + if len(member_ids) != len(set(member_ids)) or creator_id not in member_ids: + raise ApiError(400, "invalid_request", "Некорректный список участников") + try: + async with self.pool.acquire() as connection, connection.transaction(): + existing = await connection.fetchval( + "SELECT count(*) FROM users WHERE id = ANY($1::uuid[])", member_ids + ) + if existing != len(member_ids): + raise ApiError(400, "invalid_request", "Один или несколько участников не существуют") + row = await connection.fetchrow( + "INSERT INTO chats (id, title) VALUES ($1, $2) RETURNING id, title, created_at", + uuid4(), + title, + ) + await connection.executemany( + "INSERT INTO chat_members (chat_id, user_id) VALUES ($1, $2)", + [(row["id"], member_id) for member_id in member_ids], + ) + except DB_ERRORS as error: + raise ApiError(503, "unavailable", "PostgreSQL недоступен") from error + return { + "id": row["id"], + "title": row["title"], + "member_ids": member_ids, + "created_at": row["created_at"], + } + + async def get_chat_for_member(self, chat_id: UUID, user_id: UUID) -> dict: + try: + row = await self.pool.fetchrow( + "SELECT c.id, c.title, c.created_at, array_agg(cm_all.user_id ORDER BY cm_all.user_id) AS member_ids FROM chats c JOIN chat_members mine ON mine.chat_id = c.id AND mine.user_id = $2 JOIN chat_members cm_all ON cm_all.chat_id = c.id WHERE c.id = $1 GROUP BY c.id", + chat_id, + user_id, + ) + except DB_ERRORS as error: + raise ApiError(503, "unavailable", "PostgreSQL недоступен") from error + if row is None: + raise ApiError(404, "not_found", "Чат не найден") + return self._chat(row) + + async def list_chats_for_member( + self, user_id: UUID, after: tuple[datetime, UUID] | None, limit: int + ) -> list[dict]: + query = "SELECT c.id, c.title, c.created_at, array_agg(cm_all.user_id ORDER BY cm_all.user_id) AS member_ids FROM chats c JOIN chat_members mine ON mine.chat_id = c.id AND mine.user_id = $1 JOIN chat_members cm_all ON cm_all.chat_id = c.id" + args: list = [user_id] + if after is not None: + query += " WHERE (c.created_at, c.id) > ($2, $3)" + args.extend(after) + query += " GROUP BY c.id ORDER BY c.created_at, c.id LIMIT $" + str(len(args) + 1) + args.append(limit + 1) + try: + rows = await self.pool.fetch(query, *args) + except DB_ERRORS as error: + raise ApiError(503, "unavailable", "PostgreSQL недоступен") from error + return [self._chat(row) for row in rows] + + async def create_message_idempotent( + self, chat_id: UUID, sender_id: UUID, text: str, idempotency_key: str, ttl_seconds: int, attachment_ids: list[UUID] | None = None + ) -> dict: + """Создаёт сообщение и фиксирует результат retry в одной транзакции.""" + path = f"/api/v1/chats/{chat_id}/messages" + attachment_ids = attachment_ids or [] + # Preserve hashes of text-only requests already accepted by lab3. + canonical = {"text": text} + if attachment_ids: + canonical["attachment_ids"] = [str(value) for value in attachment_ids] + request_hash = sha256( + json.dumps(canonical, ensure_ascii=False, separators=(",", ":"), sort_keys=True).encode("utf-8") + ).hexdigest() + try: + async with self.pool.acquire() as connection, connection.transaction(): + member = await connection.fetchval( + "SELECT 1 FROM chat_members WHERE chat_id = $1 AND user_id = $2", chat_id, sender_id + ) + if member is None: + raise ApiError(404, "not_found", "Чат не найден") + await connection.execute( + "DELETE FROM idempotency_records WHERE created_at <= now() - $1 * interval '1 second'", ttl_seconds + ) + if attachment_ids: + count = await connection.fetchval( + "SELECT count(*) FROM attachments WHERE chat_id = $1 AND id = ANY($2::uuid[])", + chat_id, attachment_ids, + ) + if count != len(attachment_ids): + raise ApiError(404, "not_found", "Вложение не найдено") + failed = await connection.fetchval( + "SELECT 1 FROM attachments WHERE id=ANY($1::uuid[]) AND status='failed' LIMIT 1", attachment_ids + ) + if failed: + raise ApiError(409, 'conflict', 'Вложение завершилось ошибкой') + claimed = await connection.fetchrow( + "INSERT INTO idempotency_records (user_id, method, path, idempotency_key, request_hash) " + "VALUES ($1, 'POST', $2, $3, $4) ON CONFLICT DO NOTHING RETURNING message_id", + sender_id, path, idempotency_key, request_hash, + ) + if claimed is None: + recorded = await connection.fetchrow( + "SELECT request_hash, message_id FROM idempotency_records " + "WHERE user_id = $1 AND method = 'POST' AND path = $2 AND idempotency_key = $3 FOR UPDATE", + sender_id, path, idempotency_key, + ) + if recorded is None: + raise ApiError(503, "unavailable", "Не удалось получить результат идемпотентного запроса") + if recorded["request_hash"] != request_hash: + raise ApiError(409, "conflict", "Idempotency-Key использован с другим телом") + if recorded["message_id"] is None: + raise ApiError(503, "unavailable", "Не удалось получить результат идемпотентного запроса") + row = await connection.fetchrow( + "SELECT id, chat_id, sender_id, text, attachment_ids, created_at FROM messages WHERE id = $1", recorded["message_id"] + ) + if row is None: + raise ApiError(503, "unavailable", "Не удалось получить сохранённое сообщение") + return self._message(row) + row = await connection.fetchrow( + "INSERT INTO messages (id, chat_id, sender_id, text, attachment_ids) VALUES ($1, $2, $3, $4, $5) " + "RETURNING id, chat_id, sender_id, text, attachment_ids, created_at", + uuid4(), chat_id, sender_id, text, attachment_ids, + ) + await connection.execute( + "UPDATE idempotency_records SET message_id = $1 WHERE user_id = $2 AND method = 'POST' " + "AND path = $3 AND idempotency_key = $4", + row["id"], sender_id, path, idempotency_key, + ) + except DB_ERRORS as error: + raise ApiError(503, "unavailable", "PostgreSQL недоступен") from error + return self._message(row) + + async def list_messages_for_member( + self, chat_id: UUID, user_id: UUID, after: tuple[datetime, UUID] | None, limit: int + ) -> list[dict]: + await self.get_chat_for_member(chat_id, user_id) + query = "SELECT id, chat_id, sender_id, text, attachment_ids, created_at FROM messages WHERE chat_id = $1" + args: list = [chat_id] + if after is not None: + query += " AND (created_at, id) > ($2, $3)" + args.extend(after) + query += " ORDER BY created_at, id LIMIT $" + str(len(args) + 1) + args.append(limit + 1) + try: + rows = await self.pool.fetch(query, *args) + except DB_ERRORS as error: + raise ApiError(503, "unavailable", "PostgreSQL недоступен") from error + return [self._message(row) for row in rows] + + + @asynccontextmanager + async def attachment_transaction(self, *, maintenance=False): + """Shared writer / exclusive reconciler lock across all API replicas. + + Writers hold the transaction from before PUT through metadata COMMIT. + Maintenance holds the exclusive lock through inventory and deletion. + All attachment writers must participate in this protocol. + """ + try: + async with self.pool.acquire() as connection, connection.transaction(): + await connection.execute("SET LOCAL lock_timeout = '5s'") + lock = "pg_advisory_xact_lock" if maintenance else "pg_advisory_xact_lock_shared" + await connection.execute(f"SELECT {lock}(41004, 1)") + yield connection + except DB_ERRORS as error: + raise ApiError(503, "unavailable", "PostgreSQL недоступен") from error + + async def create_attachment(self, attachment_id, chat_id, owner_id, filename, + content_type, size_bytes, digest, staging_key, job_id, *, connection=None) -> dict: + try: + row = await (connection or self.pool).fetchrow( + "INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, staging_key) " + "VALUES ($1,$2,$3,$4,$5,$6,$7,$8) RETURNING *", + attachment_id, chat_id, owner_id, filename, content_type, size_bytes, digest, staging_key, + ) + await (connection or self.pool).execute( + "INSERT INTO attachment_jobs (job_id, attachment_id) VALUES ($1, $2)", job_id, attachment_id + ) + payload = { + "schema_version": 1, "job_id": str(job_id), "type": "attachment.process.v1", + "attachment_id": str(attachment_id), "pipeline_version": 1, "attempt": 1, + "created_at": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"), + } + await (connection or self.pool).execute( + "INSERT INTO attachment_outbox (id, job_id, payload) VALUES ($1, $2, $3::jsonb)", + uuid4(), job_id, json.dumps(payload, separators=(",", ":")), + ) + await (connection or self.pool).execute( + "UPDATE attachment_jobs SET next_attempt_at = 'infinity' WHERE job_id = $1", job_id + ) + except DB_ERRORS as error: + raise ApiError(503, "unavailable", "PostgreSQL недоступен") from error + return dict(row) + + async def ensure_attachment_queue_capacity(self, connection: asyncpg.Connection, maximum: int) -> None: + """Globally serialize admission so replicas cannot over-accept queued work.""" + try: + await connection.execute("SELECT pg_advisory_xact_lock(41005, 1)") + pending = await connection.fetchval( + "SELECT count(*) FROM attachment_jobs WHERE status IN ('queued', 'processing')" + ) + except DB_ERRORS as error: + raise ApiError(503, "unavailable", "PostgreSQL недоступен") from error + if pending >= maximum: + raise ApiError(429, "rate_limited", "Очередь обработки заполнена") + + async def attachment_for_member(self, attachment_id: UUID, user_id: UUID) -> dict: + try: + row = await self.pool.fetchrow( + "SELECT a.* FROM attachments a JOIN chat_members m ON m.chat_id = a.chat_id " + "WHERE a.id = $1 AND m.user_id = $2", attachment_id, user_id, + ) + except DB_ERRORS as error: + raise ApiError(503, "unavailable", "PostgreSQL недоступен") from error + if row is None: + raise ApiError(404, "not_found", "Вложение не найдено") + return await self._attachment_with_variants(row) + + async def _attachment_with_variants(self, row: asyncpg.Record | dict) -> dict: + result = dict(row) + variants = await self.pool.fetch( + "SELECT name, content_type, size_bytes, width, height, sha256 FROM attachment_variants " + "WHERE attachment_id = $1 AND pipeline_version = $2 ORDER BY name", + result["id"], result["pipeline_version"], + ) + result["variants"] = [dict(item) for item in variants] + return result + + async def claim_attachment_job(self, attachment_id: UUID, lease_seconds: int) -> dict | None: + try: + async with self.pool.acquire() as connection, connection.transaction(): + row = await connection.fetchrow( + "SELECT a.*, j.job_id, j.attempt, j.lease_until, " + "(j.next_attempt_at <> 'infinity'::timestamptz AND j.next_attempt_at > now()) AS retry_waiting " + "FROM attachments a JOIN attachment_jobs j ON j.attachment_id = a.id " + "WHERE a.id = $1 FOR UPDATE", attachment_id + ) + # Only the dispatcher recovers an expired lease and increments + # its attempt. Redelivery must not renew it or bypass backoff. + if row is None or row["status"] != "queued" or row["retry_waiting"]: + return None + await connection.execute( + "UPDATE attachments SET status = 'processing', processing_started_at = now(), updated_at = now() WHERE id = $1", + attachment_id, + ) + await connection.execute( + "UPDATE attachment_jobs SET status = 'processing', first_started_at=COALESCE(first_started_at, now()), lease_until = now() + $2 * interval '1 second', updated_at = now() WHERE attachment_id = $1", + attachment_id, lease_seconds, + ) + result = dict(row) + result['claim_time'] = await connection.fetchval('SELECT processing_started_at FROM attachments WHERE id=$1', attachment_id) + return result + except DB_ERRORS as error: + raise RuntimeError("PostgreSQL is unavailable") from error + + async def complete_attachment_job(self, attachment_id: UUID, original_key: str, variants: list[dict], claim_time) -> bool: + try: + async with self.pool.acquire() as connection, connection.transaction(): + valid = await connection.fetchval( + "SELECT 1 FROM attachments a JOIN attachment_jobs j ON j.attachment_id=a.id WHERE a.id=$1 AND a.status='processing' AND a.processing_started_at=$2 AND j.lease_until > now() FOR UPDATE", attachment_id, claim_time + ) + if not valid: + return False + for variant in variants: + await connection.execute( + "INSERT INTO attachment_variants (attachment_id, pipeline_version, name, object_key, content_type, size_bytes, width, height, sha256) " + "VALUES ($1, 1, $2, $3, $4, $5, $6, $7, $8) " + "ON CONFLICT (attachment_id, pipeline_version, name) DO UPDATE SET object_key = EXCLUDED.object_key, content_type = EXCLUDED.content_type, size_bytes = EXCLUDED.size_bytes, width = EXCLUDED.width, height = EXCLUDED.height, sha256 = EXCLUDED.sha256", + attachment_id, variant["name"], variant["object_key"], variant["content_type"], variant["size_bytes"], variant["width"], variant["height"], variant["sha256"], + ) + await connection.execute( + "UPDATE attachments SET status = 'ready', original_key = $2, error_code = NULL, updated_at = now() WHERE id = $1", + attachment_id, original_key, + ) + await connection.execute("UPDATE attachment_jobs SET status = 'ready', lease_until = NULL, updated_at = now() WHERE attachment_id = $1", attachment_id) + return True + except DB_ERRORS as error: + raise RuntimeError("PostgreSQL is unavailable") from error + + async def retry_attachment_job(self, attachment_id: UUID, max_attempts: int, delay_seconds: float, claim_time, permanent_code=None) -> bool: + """Return true only when a bounded at-least-once retry was recorded.""" + try: + async with self.pool.acquire() as connection, connection.transaction(): + job = await connection.fetchrow( + "SELECT j.attempt FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id WHERE a.id=$1 AND a.status='processing' AND a.processing_started_at=$2 FOR UPDATE", attachment_id, claim_time + ) + if job is None: + return False + if permanent_code is None and job["attempt"] < max_attempts: + await connection.execute( + "UPDATE attachment_jobs SET attempt = attempt + 1, status = 'queued', last_error = 'processing_failed', lease_until = NULL, next_attempt_at = now() + $2 * interval '1 second', updated_at = now() WHERE attachment_id = $1", + attachment_id, delay_seconds, + ) + await connection.execute( + "UPDATE attachments SET status = 'queued', updated_at = now() WHERE id = $1", attachment_id + ) + return True + code = permanent_code or 'retry_exhausted' + await connection.execute("UPDATE attachments SET status='failed', error_code=$2, updated_at=now() WHERE id=$1", attachment_id, code) + await connection.execute("UPDATE attachment_jobs SET status='failed', last_error=$2, lease_until=NULL, updated_at=now() WHERE attachment_id=$1", attachment_id, code) + return False + except DB_ERRORS as error: + raise RuntimeError("PostgreSQL is unavailable") from error + + async def due_outbox_events(self, limit: int = 20) -> list[dict]: + try: + return [{**dict(row), 'payload': json.loads(row['payload']) if isinstance(row['payload'], str) else row['payload']} for row in await self.pool.fetch( + "SELECT id, payload FROM attachment_outbox WHERE published_at IS NULL ORDER BY created_at LIMIT $1", limit + )] + except DB_ERRORS as error: + raise RuntimeError("PostgreSQL is unavailable") from error + + async def mark_outbox_published(self, event_id: UUID) -> None: + await self.pool.execute("UPDATE attachment_outbox SET published_at = now(), attempts = attempts + 1 WHERE id = $1", event_id) + + async def schedule_due_jobs(self, max_attempts: int = 5) -> None: + """Outbox dispatcher also restores expired leases and scheduled retries.""" + try: + async with self.pool.acquire() as connection, connection.transaction(): + rows = await connection.fetch( + "SELECT job_id, attachment_id, pipeline_version, attempt, status FROM attachment_jobs " + "WHERE (status = 'queued' AND (next_attempt_at <= now() OR " + "(next_attempt_at = 'infinity'::timestamptz AND updated_at < now() - interval '60 seconds'))) " + "OR (status = 'processing' AND lease_until < now()) " + "FOR UPDATE SKIP LOCKED" + ) + for row in rows: + row = dict(row) + if row["status"] == "processing": + if row['attempt'] >= max_attempts: + await connection.execute("UPDATE attachment_jobs SET status='failed', lease_until=NULL, last_error='retry_exhausted', updated_at=now() WHERE job_id=$1", row['job_id']) + await connection.execute("UPDATE attachments SET status='failed', error_code='retry_exhausted', updated_at=now() WHERE id=$1", row['attachment_id']) + continue + row['attempt'] += 1 + await connection.execute( + "UPDATE attachment_jobs SET status = 'queued', attempt=$2, lease_until = NULL, next_attempt_at = now(), last_error = 'lease_expired', updated_at = now() WHERE job_id = $1", row["job_id"], row['attempt'] + ) + await connection.execute("UPDATE attachments SET status = 'queued', updated_at = now() WHERE id = $1", row["attachment_id"]) + payload = {"schema_version": 1, "job_id": str(row["job_id"]), "type": "attachment.process.v1", "attachment_id": str(row["attachment_id"]), "pipeline_version": row["pipeline_version"], "attempt": row["attempt"], "created_at": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")} + await connection.execute("INSERT INTO attachment_outbox (id, job_id, payload) VALUES ($1, $2, $3::jsonb)", uuid4(), row["job_id"], json.dumps(payload, separators=(",", ":"))) + # Refresh the watchdog on publication intent, not on claim. + # Unclaimed jobs are republished after 60s. At-least-once + # duplicates are safe, and a published-but-lost delivery + # can no longer strand durable accepted work forever. + await connection.execute("UPDATE attachment_jobs SET next_attempt_at = 'infinity', updated_at=now() WHERE job_id = $1", row["job_id"]) + except DB_ERRORS as error: + raise RuntimeError("PostgreSQL is unavailable") from error + + async def replay_failed_attachment(self, attachment_id: UUID) -> bool: + """Explicit operator-only replay; never requeues an already successful job.""" + try: + async with self.pool.acquire() as connection, connection.transaction(): + job = await connection.fetchrow( + "SELECT job_id, status FROM attachment_jobs WHERE attachment_id = $1 FOR UPDATE", attachment_id + ) + if job is None or job["status"] != "failed": + return False + await connection.execute( + "UPDATE attachment_jobs SET status = 'queued', attempt = 1, next_attempt_at = now(), lease_until = NULL, replayed_at = now(), last_error = NULL, updated_at = now() WHERE attachment_id = $1", attachment_id + ) + await connection.execute("UPDATE attachments SET status = 'queued', error_code = NULL, updated_at = now() WHERE id = $1", attachment_id) + return True + except DB_ERRORS as error: + raise RuntimeError("PostgreSQL is unavailable") from error + + async def attachment_keys(self) -> dict[str, UUID]: + """Inventory used only by the maintenance reconciler.""" + try: + rows = await self.pool.fetch("SELECT staging_key AS object_key, id FROM attachments WHERE status <> 'ready' UNION ALL SELECT original_key, id FROM attachments WHERE original_key IS NOT NULL UNION ALL SELECT object_key, attachment_id AS id FROM attachment_variants") + except DB_ERRORS as error: + raise RuntimeError("PostgreSQL is unavailable") from error + return {row["object_key"]: row["id"] for row in rows} diff --git a/src/storage/s3.py b/src/storage/s3.py new file mode 100644 index 0000000..f154df9 --- /dev/null +++ b/src/storage/s3.py @@ -0,0 +1,120 @@ +"""Bounded S3 operations; synchronous SDK calls run outside the event loop.""" +from dataclasses import dataclass +from hashlib import sha256 +import os +from pathlib import Path + +import boto3 +from botocore.config import Config +from botocore.exceptions import BotoCoreError, ClientError +from starlette.concurrency import run_in_threadpool + +from ..errors import ApiError + + +@dataclass(frozen=True) +class S3Settings: + endpoint: str + region: str + bucket: str + access_key: str + secret_key: str + max_bytes: int + max_pixels: int + + @classmethod + def load(cls): + def secret(name): + path = os.environ.get(name + "_FILE") + result = Path(path).read_text().strip() if path else os.environ.get(name, "") + if not result: + raise ValueError(f"Missing {name}") + return result + + max_bytes = int(os.environ.get("MAX_UPLOAD_BYTES", "10485760")) + max_pixels = int(os.environ.get("MAX_IMAGE_PIXELS", "40000000")) + if not 1 <= max_bytes <= 10485760 or not 1 <= max_pixels <= 40000000: + raise ValueError("Image limits exceed contract") + return cls(os.environ.get("S3_ENDPOINT", "http://s3:8333"), + os.environ.get("S3_REGION", "us-east-1"), + os.environ.get("S3_BUCKET", "messenger-attachments"), + secret("S3_ACCESS_KEY_ID"), secret("S3_SECRET_ACCESS_KEY"), + max_bytes, max_pixels) + + +class S3Store: + def __init__(self, settings): + self.settings = settings + self.client = boto3.client( + "s3", endpoint_url=settings.endpoint, region_name=settings.region, + aws_access_key_id=settings.access_key, aws_secret_access_key=settings.secret_key, + config=Config(signature_version="s3v4", connect_timeout=2, read_timeout=5, + retries={"total_max_attempts": 1}, + s3={"addressing_style": "path"}, + request_checksum_calculation="when_required", + response_checksum_validation="when_required"), + ) + + async def call(self, method, **kwargs): + try: + return await run_in_threadpool(getattr(self.client, method), **kwargs) + except (BotoCoreError, ClientError, OSError) as error: + raise ApiError(503, "unavailable", "S3 недоступно") from error + + async def ready(self): + await self.call("head_bucket", Bucket=self.settings.bucket) + + async def put(self, key, file, size, content_type): + file.seek(0) + await self.call("put_object", Bucket=self.settings.bucket, Key=key, + Body=file, ContentLength=size, ContentType=content_type) + + async def copy(self, source_key: str, destination_key: str, content_type: str) -> None: + await self.call( + "copy_object", Bucket=self.settings.bucket, Key=destination_key, + CopySource={"Bucket": self.settings.bucket, "Key": source_key}, + ContentType=content_type, MetadataDirective="REPLACE", + ) + + async def list(self, prefix: str) -> dict[str, object]: + """Return every object under a controlled prefix, handling S3 pagination.""" + objects, _ = await self.list_with_pages(prefix) + return objects + + async def list_with_pages(self, prefix: str) -> tuple[dict[str, object], int]: + """Return inventory and number of S3 ListObjectsV2 pages consumed.""" + result: dict[str, object] = {} + token = None + pages = 0 + while True: + args = {"Bucket": self.settings.bucket, "Prefix": prefix} + if token: + args["ContinuationToken"] = token + page = await self.call("list_objects_v2", **args) + pages += 1 + for item in page.get("Contents", []): + result[item["Key"]] = item["LastModified"] + if not page.get("IsTruncated"): + return result, pages + token = page["NextContinuationToken"] + + async def delete(self, key: str) -> None: + await self.call("delete_object", Bucket=self.settings.bucket, Key=key) + + async def download(self, attachment): + return await self.download_key(attachment["original_key"], attachment["size_bytes"], attachment["sha256"]) + + async def download_key(self, key: str, expected_size: int | None = None, + expected_sha256: str | None = None) -> bytes: + response = await self.call("get_object", Bucket=self.settings.bucket, Key=key) + body = response["Body"] + try: + data = await run_in_threadpool(body.read, self.settings.max_bytes + 1) + except (BotoCoreError, OSError) as error: + raise ApiError(503, "unavailable", "S3 недоступно") from error + finally: + body.close() + if ((expected_size is not None and len(data) != expected_size) or + (expected_sha256 is not None and sha256(data).hexdigest() != expected_sha256)): + raise ApiError(503, "unavailable", "Оригинал недоступен") + return data diff --git a/src/worker.py b/src/worker.py new file mode 100644 index 0000000..d355b88 --- /dev/null +++ b/src/worker.py @@ -0,0 +1,234 @@ +"""RabbitMQ consumer for the version-1 attachment pipeline.""" +import asyncio +from hashlib import sha256 +from io import BytesIO +import json +import logging +import os +import random +import pickle +import subprocess +import sys +import time +from pathlib import Path +from concurrent.futures import ThreadPoolExecutor +from uuid import UUID, uuid4 + +from PIL import Image, ImageOps + +from .broker import QUEUE, _connection, publish_dlq +from .config import load_settings +from .storage.postgres import PostgresStore +from .storage.s3 import S3Settings, S3Store + +logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s worker %(message)s") + + +class PermanentFailure(Exception): + pass + + +def fit(image, bound): + longest = max(image.size) + if longest <= bound: + return image.copy() + return image.resize((max(1, image.width * bound // longest), + max(1, image.height * bound // longest)), Image.Resampling.LANCZOS) + + +def render_isolated(data, timeout): + try: + result = subprocess.run([sys.executable, '-m', 'src.render'], input=data, + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, timeout=timeout) + except subprocess.TimeoutExpired as error: + raise TimeoutError('render deadline exceeded') from error + if result.returncode in (20, 21): + raise PermanentFailure('invalid_image' if result.returncode == 20 else 'image_resource_limit') + if result.returncode: + raise RuntimeError('renderer exited') + # Only our own child process produces this payload; no external pickle input. + return pickle.loads(result.stdout) + + +def thumbnail(data: bytes) -> tuple[bytes, int, int]: + with Image.open(BytesIO(data), formats=("PNG", "JPEG")) as source: + image = ImageOps.exif_transpose(source) + if image.mode in {"RGBA", "LA"} or (image.mode == "P" and "transparency" in image.info): + background = Image.new("RGBA", image.size, "white") + background.alpha_composite(image.convert("RGBA")) + image = background.convert("RGB") + else: + image = image.convert("RGB") + image = fit(image, 256) + output = BytesIO() + image.save(output, format="JPEG", quality=85, optimize=True) + return output.getvalue(), image.width, image.height + + +def variants(data: bytes, max_pixels: int = 40_000_000) -> list[tuple[str, bytes, str, int, int]]: + """Render required variants without preserving EXIF or other input metadata.""" + with Image.open(BytesIO(data), formats=("PNG", "JPEG")) as source: + if source.width * source.height > max_pixels: + raise ValueError("image exceeds configured pixel limit") + image = ImageOps.exif_transpose(source).convert("RGBA") + white = Image.new("RGBA", image.size, "white") + white.alpha_composite(image) + rgb = white.convert("RGB") + + thumb = fit(rgb, 256) + thumbnail_out = BytesIO(); thumb.save(thumbnail_out, "JPEG", quality=85, optimize=True) + + side = min(rgb.width, rgb.height) + left, top = (rgb.width - side) // 2, (rgb.height - side) // 2 + crop = rgb.crop((left, top, left + side, top + side)).resize((128, 128), Image.Resampling.LANCZOS) + crop_out = BytesIO(); crop.save(crop_out, "JPEG", quality=85, optimize=True) + + optimized = fit(rgb, 1280) + optimized_out = BytesIO(); optimized.save(optimized_out, "WEBP", quality=80, method=4) + return [ + ("thumbnail", thumbnail_out.getvalue(), "image/jpeg", thumb.width, thumb.height), + ("crop", crop_out.getvalue(), "image/jpeg", 128, 128), + ("optimized", optimized_out.getvalue(), "image/webp", optimized.width, optimized.height), + ] + + +async def process(store: PostgresStore, s3: S3Store, payload: dict, lease_seconds: int) -> None: + # Excludes cleanup for the whole lifetime of S3 effects, including stale attempts. + async with store.attachment_transaction(): + await process_locked(store, s3, payload, lease_seconds) + + +async def process_locked(store: PostgresStore, s3: S3Store, payload: dict, lease_seconds: int) -> None: + attachment_id = UUID(payload["attachment_id"]) + attachment = await store.claim_attachment_job(attachment_id, lease_seconds) + if attachment is None: + return + payload['_claim_time'] = attachment['claim_time'] + payload['attempt'] = attachment.get('attempt', payload.get('attempt', 1)) + attempt_key = uuid4().hex + original_key = f"originals/{attachment_id}/{attempt_key}" + await s3.copy(attachment["staging_key"], original_key, attachment["content_type"]) + original = await s3.download_key(attachment["staging_key"], attachment["size_bytes"], attachment["sha256"]) + rendered = await asyncio.to_thread(render_isolated, original, lease_seconds) + metadata = [] + for name, content, content_type, width, height in rendered: + extension = "webp" if name == "optimized" else "jpg" + object_key = f"variants/{attachment_id}/{attempt_key}/{name}.{extension}" + await s3.put(object_key, BytesIO(content), len(content), content_type) + metadata.append({"name": name, "object_key": object_key, "content_type": content_type, + "size_bytes": len(content), "width": width, "height": height, + "sha256": sha256(content).hexdigest()}) + # Explicit, opt-in integration fault. Marker survives container restart, + # so one crash exercises lease recovery without a permanent crash loop. + if os.environ.get('WORKER_FAULT_AFTER_S3_PUT') == '1': + marker = Path('/tmp/worker-fault-after-s3') + try: + with marker.open('x') as stream: + stream.write(str(attachment_id)) + except FileExistsError: + pass + else: + os._exit(70) + committed = await store.complete_attachment_job(attachment_id, original_key, metadata, attachment['claim_time']) + if committed: + try: + await s3.delete(attachment["staging_key"]) + except Exception: + logging.warning('staging cleanup deferred attachment_id=%s', attachment_id) + + +def settle_delivery(channel, delivery_tag, future): + """Only a successful durable disposition permits acknowledgement.""" + try: + success = future.result() is True + except Exception: + success = False + if channel.is_open: + if success: + channel.basic_ack(delivery_tag) + else: + channel.basic_nack(delivery_tag, requeue=True) + + +def main() -> None: + settings = load_settings() + connection = _connection() + channel = connection.channel() + workers = max(1, int(os.environ.get("WORKER_CONCURRENCY", "2"))) + channel.basic_qos(prefetch_count=max(1, int(os.environ.get("WORKER_PREFETCH", str(workers))))) + + async def handle_async(payload): + """Each executor thread owns its event loop and its DB/S3 clients.""" + store = PostgresStore(settings.database_url) + s3 = S3Store(S3Settings.load()) + await store.connect() + try: + await process(store, s3, payload, max(1, int(os.environ.get("JOB_TIMEOUT_SECONDS", "60")))) + finally: + s3.client.close() + await store.close() + + async def schedule_retry(attachment_id, delay, claim_time, permanent_code=None): + retry_store = PostgresStore(settings.database_url) + await retry_store.connect() + try: + return await retry_store.retry_attachment_job( + attachment_id, max(1, int(os.environ.get("JOB_MAX_ATTEMPTS", "5"))), delay, claim_time, permanent_code + ) + finally: + await retry_store.close() + + def handle(body): + payload = None + try: + payload = json.loads(body) + if not isinstance(payload, dict): + raise ValueError('job must be an object') + if payload["type"] != "attachment.process.v1" or payload["schema_version"] != 1: + raise ValueError("unexpected broker payload") + UUID(payload['attachment_id']) + UUID(payload['job_id']) + except (ValueError, KeyError, TypeError): + try: + asyncio.run(publish_dlq({'failure': 'invalid_job'})) + return True + except Exception: + return False + try: + asyncio.run(handle_async(payload)) + except Exception as error: + logging.warning("job failed attachment_id=%s", payload['attachment_id']) + if '_claim_time' not in payload: + return False + try: + attachment_id = UUID(payload["attachment_id"]) + attempt = int(payload.get("attempt", 1)) + delay = min(60.0, float(2 ** min(attempt, 6))) * random.uniform(0.5, 1.5) + asyncio.run(schedule_retry(attachment_id, delay, payload['_claim_time'], + str(error) if isinstance(error, PermanentFailure) else None)) + except Exception: + return False + return True + + def consume(ch, method, properties, body): + del properties + future = executor.submit(handle, body) + def acknowledged(_future): + if connection.is_open: + connection.add_callback_threadsafe(lambda: settle_delivery(ch, method.delivery_tag, _future)) + future.add_done_callback(acknowledged) + + try: + def heartbeat(): + Path('/tmp/worker-heartbeat').write_text(str(time.time())) + connection.call_later(2, heartbeat) + heartbeat() + with ThreadPoolExecutor(max_workers=workers, thread_name_prefix="attachment-worker") as executor: + channel.basic_consume(queue=QUEUE, on_message_callback=consume, auto_ack=False) + channel.start_consuming() + finally: + connection.close() + + +if __name__ == "__main__": + main() diff --git a/student-tests/attachment_resilience.py b/student-tests/attachment_resilience.py new file mode 100644 index 0000000..4f72d9f --- /dev/null +++ b/student-tests/attachment_resilience.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 +"""Explicit local Docker tests: private bucket, uploader failover, S3 outage.""" +import json +from pathlib import Path +import subprocess +import sys +import time + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client, Failure, arguments, check, run + + +def compose(*args, input=None): + return subprocess.check_output(['docker', 'compose', *args], cwd=ROOT, input=input, + text=True, stderr=subprocess.DEVNULL) + + +def wait_ready(c): + deadline = time.monotonic() + 90 + while True: + try: + c.call('GET', '/health/ready', schema='Health') + return + except Failure: + check(time.monotonic() < deadline, 'stack did not recover') + time.sleep(1) + + +def main(): + def configure(parser): + parser.add_argument('--mode', required=True, choices=['private', 'failover', 's3-outage']) + args = arguments(__doc__, configure) + c = Client(args.base_url, args.ca_file) + alice, bob, eve, chat = c.bootstrap() + data = (ROOT / 'tests/fixtures/sample.png').read_bytes() + uploaded = c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=sample.png', + 201, actor=alice, raw=data, headers={'Content-Type': 'image/png'}, schema='Attachment') + att = uploaded['json'] + content_path = '/api/v1/attachments/' + att['id'] + '/content' + if args.mode == 'private': + script = '''import sys, urllib.request, urllib.error +url = "http://s3:8333/messenger-attachments/originals/" + sys.stdin.read().strip() +try: + urllib.request.urlopen(url, timeout=5) +except urllib.error.HTTPError as error: + assert error.code == 403, error.code + print("PASS: anonymous S3 GET returns 403 for an existing object") +else: + raise AssertionError("private object leaked") +''' + print(compose('exec', '-T', 'api-a', '/app/.venv/bin/python', '-c', script, input=att['id']).strip()) + elif args.mode == 'failover': + target = uploaded['headers']['X-Instance-Id'] + service = None + for candidate in ('api-a', 'api-b'): + container = compose('ps', '-q', candidate).strip() + hostname = subprocess.check_output(['docker', 'inspect', '--format', '{{.Config.Hostname}}', container], text=True).strip() + if hostname == target: + service = candidate + check(service is not None, 'uploading replica not found') + try: + compose('stop', service) + deadline = time.monotonic() + 15 + while True: + try: + download = c.call('GET', content_path, actor=bob) + break + except Failure: + check(time.monotonic() < deadline, 'download did not recover') + time.sleep(0.5) + check(download['headers']['X-Instance-Id'] != target, 'download used stopped uploader') + check(download['bytes'] == data, 'original changed after stopping uploader') + print('PASS: uploaded via ' + service + '; stopped that container; original downloaded through another replica') + finally: + compose('start', service) + wait_ready(c) + else: + try: + compose('stop', 's3') + # Direct API request inside the network: LB excludes unready backends. + script = '''import json, sys, urllib.request, urllib.error +payload = json.load(sys.stdin) +for method, path, body, headers in [ + ("GET", "/health/ready", None, {}), + ("GET", payload["path"], None, {"Authorization": "Bearer " + payload["token"]}), + ("POST", payload["upload_path"], bytes.fromhex(payload["bytes"]), + {"Authorization": "Bearer " + payload["token"], "Content-Type": "image/png"}), +]: + req = urllib.request.Request("http://localhost:8080" + path, data=body, headers=headers, method=method) + try: + urllib.request.urlopen(req, timeout=15) + except urllib.error.HTTPError as error: + assert error.code == 503, error.code + document = json.load(error) + assert document["error"]["code"] == "unavailable" + assert document["error"]["request_id"] == error.headers["X-Request-Id"] + else: + raise AssertionError("S3 outage was not reported") +assert urllib.request.urlopen("http://localhost:8080/health/live").status == 200 +print("PASS: S3 outage gives 503 for readiness, download and upload; liveness stays 200") +''' + payload = {'path': content_path, 'token': bob['token'], 'bytes': data.hex(), + 'upload_path': '/api/v1/chats/' + chat['id'] + '/attachments?filename=outage.png'} + print(compose('exec', '-T', 'api-a', '/app/.venv/bin/python', '-c', script, + input=json.dumps(payload)).strip()) + finally: + compose('start', 's3') + # HeadBucket/read recovery alone does not prove the write path is ready. + # Use the same write-probing bootstrap as the documented restart hook. + compose('run', '--rm', '--no-deps', 's3-init') + wait_ready(c) + check(c.call('GET', content_path, actor=bob)['bytes'] == data, 'original lost after S3 outage') + c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=after-recovery.png', + 201, actor=alice, raw=data, headers={'Content-Type': 'image/png'}, schema='Attachment') + print('PASS: original available and new upload accepted after S3 restart/bootstrap') + + +if __name__ == '__main__': + run(main) diff --git a/student-tests/attachments.py b/student-tests/attachments.py new file mode 100644 index 0000000..f9f7531 --- /dev/null +++ b/student-tests/attachments.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +"""Attachment contract checks against the running lab5 stack.""" +import hashlib +import http.client +from io import BytesIO +from pathlib import Path +import struct +import sys +import uuid +import zlib +from urllib.parse import quote, urlsplit + +from PIL import Image + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests")) +from client import Client, arguments, check, run + + +def main(): + args = arguments(__doc__) + c = Client(args.base_url, args.ca_file, timeout=30) + alice, bob, eve, chat = c.bootstrap() + path = '/api/v1/chats/' + chat['id'] + '/attachments?filename=' + png = (Path(__file__).resolve().parents[1] / 'tests/fixtures/sample.png').read_bytes() + + def upload(data, mime='image/png', filename='photo.png', status=202, actor=alice): + return c.call('POST', path + quote(filename, safe=''), status, actor=actor, raw=data, + headers={'Content-Type': mime}, schema='Attachment' if status == 202 else None) + + for fmt, mime in [('PNG', 'image/png'), ('JPEG', 'image/jpeg')]: + buf = BytesIO() + Image.new('RGB', (32, 20), 'blue').save(buf, fmt) + data = buf.getvalue() + att = upload(data, mime)['json'] + check(att['variants'] == [] and att['status'] == 'queued', 'wrong attachment state') + att = c.wait_ready(bob, att) + check(att['sha256'] == hashlib.sha256(data).hexdigest(), 'wrong checksum') + content_path = '/api/v1/attachments/' + att['id'] + check(c.call('GET', content_path, actor=bob, schema='Attachment')['json'] == att, + 'metadata changed') + check(c.call('GET', content_path + '/content', actor=bob)['bytes'] == data, 'bytes changed') + c.call('GET', content_path, 404, actor=eve) + c.call('GET', content_path + '/content', 404, actor=eve) + c.call('GET', content_path + '/content', 401) + c.call('GET', content_path + '/content?variant=thumbnail', actor=bob) + message = c.message(bob, chat, '', attachments=[att['id']])['json'] + history = c.call('GET', '/api/v1/chats/' + chat['id'] + '/messages', + actor=alice, schema='MessagePage')['json']['items'] + check(message in history, 'attachment message missing in history') + print('PASS: PNG/JPEG originals, metadata, attachment-only messages and ACL') + + for name in ('../x.png', 'x/y.png', 'x\\y.png', 'x\x00.png', 'x\n.png', 'x\x7f.png', ' ', 'x' * 256): + upload(png, filename=name, status=400) + upload(png, mime='image/jpeg', status=415) + upload(b'not an image', status=415) + upload(png[:30], status=415) + upload(png, mime='application/octet-stream', status=415) + upload(png, status=404, actor=eve) + upload(b'x' * (10485760 + 1), status=413) + + # Valid PNG header with an oversized canvas: reject before attempting pixel decoding. + def chunk(kind, data): + return struct.pack('>I', len(data)) + kind + data + struct.pack('>I', zlib.crc32(kind + data)) + huge = b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('>IIBBBBB', 8001, 5000, 8, 2, 0, 0, 0)) + huge += chunk(b'IDAT', zlib.compress(b'')) + chunk(b'IEND', b'') + upload(huge, status=413) + + # No Content-Length: limits must also work for chunked requests. + url = urlsplit(c.base) + connection = http.client.HTTPConnection(url.hostname, url.port, timeout=30) + connection.request('POST', path + 'chunked.png', body=iter([b'x' * 65536] * 161), + headers={'Authorization': 'Bearer ' + alice['token'], 'Content-Type': 'image/png'}, + encode_chunked=True) + response = connection.getresponse() + check(response.status == 413, 'chunked upload bypassed byte limit') + response.read() + connection.close() + print('PASS: filename, MIME, malformed image, byte/pixel and chunked limits') + + att = upload(png)['json'] + other_chat = c.call('POST', '/api/v1/chats', 201, actor=alice, + body={'title': 'other', 'member_ids': [alice['id']]}, schema='Chat')['json'] + for target, ids, code in ((chat, [str(uuid.uuid4())], 404), + (other_chat, [att['id']], 404), + (chat, [att['id'], att['id']], 400)): + c.call('POST', '/api/v1/chats/' + target['id'] + '/messages', code, actor=alice, + body={'text': '', 'attachment_ids': ids}, headers={'Idempotency-Key': uuid.uuid4().hex}) + key = uuid.uuid4().hex + first = c.message(alice, chat, 'same', key=key)['json'] + check(c.message(alice, chat, 'same', key=key, attachments=[])['json'] == first, + 'omitted and empty attachment_ids differ') + key = uuid.uuid4().hex + first = c.message(alice, chat, '', key=key, attachments=[att['id']])['json'] + check(c.message(alice, chat, '', key=key, attachments=[att['id']])['json'] == first, 'replay differs') + c.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', 409, actor=alice, + body={'text': 'changed', 'attachment_ids': [att['id']]}, headers={'Idempotency-Key': key}) + print('PASS: attachment scope, duplicates and idempotency normalization') + + # Exact limits are accepted; original bytes (including PNG trailing data) survive. + exact = png + b'\0' * (10485760 - len(png)) + att = upload(exact)['json'] + c.wait_ready(bob, att) + check(c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=bob)['bytes'] == exact, + '10 MiB original changed') + buf = BytesIO() + Image.new('RGB', (8000, 5000)).save(buf, 'PNG') + att = upload(buf.getvalue())['json'] + check(att['size_bytes'] == len(buf.getvalue()), '40M pixel upload failed') + print('PASS: inclusive 10 MiB and 40 million pixel boundaries') + + +if __name__ == '__main__': + run(main) diff --git a/student-tests/idempotency_concurrency.py b/student-tests/idempotency_concurrency.py new file mode 100755 index 0000000..fcf958f --- /dev/null +++ b/student-tests/idempotency_concurrency.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 +"""Один ключ одновременно через LB создаёт ровно одно сообщение.""" +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +import sys +import uuid + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests")) +from client import Client, check, run # noqa: E402 + + +def main() -> None: + client = Client() + alice, _bob, _eve, chat = client.bootstrap() + key = uuid.uuid4().hex + text = "concurrent idempotency " + key + + def send(_number: int): + return Client().message(alice, chat, text, key=key)["json"] + + with ThreadPoolExecutor(max_workers=16) as pool: + replies = list(pool.map(send, range(16))) + check(all(reply == replies[0] for reply in replies), "replays returned different messages") + + path = "/api/v1/chats/" + chat["id"] + "/messages?limit=100" + history = client.call("GET", path, actor=alice, schema="MessagePage")["json"]["items"] + check(sum(message["id"] == replies[0]["id"] for message in history) == 1, "concurrent retries created duplicates") + print("PASS: concurrent idempotency through load balancer") + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/job_contract.py b/student-tests/job_contract.py new file mode 100644 index 0000000..961c0f9 --- /dev/null +++ b/student-tests/job_contract.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python3 +"""Check real producer payloads and redeliver a complete persisted message.""" +import json +import time +from jsonschema import Draft202012Validator, FormatChecker +from pipeline_integration import ROOT, Client, remote, duplicate, snapshot, wait, check + + +def main(): + c = Client('http://localhost:8080', timeout=30) + actor, _, _, chat = c.bootstrap() + att, _ = c.attachment(actor, chat) + wait(c, actor, att) + payloads = remote('''rows = await db.pool.fetch('SELECT payload FROM attachment_outbox') +print(json.dumps([json.loads(row['payload']) if isinstance(row['payload'], str) else row['payload'] for row in rows]))''') + validator = Draft202012Validator(json.loads((ROOT / 'contracts/jobs.schema.json').read_text()), format_checker=FormatChecker()) + for payload in payloads: + validator.validate(payload) + before = snapshot(att) + duplicate(att) + duplicate(att) + time.sleep(3) + check(snapshot(att) == before, 'persisted redelivery changed result') + result = dict(status='PASS', validated_outbox_payloads=len(payloads), + extra_payload_fields=False, persisted_redelivery_noop=True) + (ROOT / 'evidence/job-contract.json').write_text(json.dumps(result, indent=2) + '\n') + print(json.dumps(result)) + + +if __name__ == '__main__': + main() diff --git a/student-tests/pipeline_critical.py b/student-tests/pipeline_critical.py new file mode 100644 index 0000000..dfb716f --- /dev/null +++ b/student-tests/pipeline_critical.py @@ -0,0 +1,107 @@ +"""Offline regression checks for delivery and terminal-state safety.""" +import asyncio +from contextlib import asynccontextmanager +from datetime import datetime, timezone +from io import BytesIO +import json +from pathlib import Path +import sys +from types import SimpleNamespace +import unittest +from unittest.mock import AsyncMock, MagicMock, patch +from uuid import uuid4 + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from src.storage.postgres import PostgresStore +from src.worker import process_locked as process, settle_delivery +from concurrent.futures import Future +from src.broker import _publish_to + + +class CriticalTests(unittest.IsolatedAsyncioTestCase): + def test_failed_or_unrecorded_work_is_requeued(self): + for result in (False, None, RuntimeError('database unavailable')): + future, channel = Future(), MagicMock() + if isinstance(result, Exception): + future.set_exception(result) + else: + future.set_result(result) + settle_delivery(channel, 7, future) + channel.basic_ack.assert_not_called() + channel.basic_nack.assert_called_once_with(7, requeue=True) + + def test_durable_success_is_acked(self): + future, channel = Future(), MagicMock() + future.set_result(True) + settle_delivery(channel, 7, future) + channel.basic_ack.assert_called_once_with(7) + channel.basic_nack.assert_not_called() + + async def test_outbox_decodes_jsonb(self): + store = PostgresStore('unused') + store._pool = AsyncMock() + payload = {'type': 'attachment.process.v1', 'attachment_id': str(uuid4())} + store.pool.fetch.return_value = [{'id': uuid4(), 'payload': json.dumps(payload)}] + event, = await store.due_outbox_events() + self.assertEqual(json.loads(json.dumps(event['payload'])), payload) + + async def run_processing(self, committed): + store, s3 = AsyncMock(), AsyncMock() + stamp = datetime.now(timezone.utc) + store.claim_attachment_job.return_value = { + 'claim_time': stamp, 'staging_key': 'staging/test', + 'content_type': 'image/png', 'size_bytes': 1, 'sha256': 'unused', + } + store.complete_attachment_job.return_value = committed + s3.settings = SimpleNamespace(max_pixels=40000000) + s3.download_key.return_value = b'fixture' + s3.delete.side_effect = RuntimeError('storage temporarily unavailable') + payload = {'attachment_id': str(uuid4())} + with patch('src.worker.render_isolated', return_value=[('thumbnail', b'jpeg', 'image/jpeg', 1, 1)]): + await process(store, s3, payload, 60) + self.assertEqual(store.complete_attachment_job.call_args.args[-1], stamp) + return store, s3 + + async def test_cleanup_error_does_not_retry_ready(self): + store, s3 = await self.run_processing(True) + s3.delete.assert_awaited_once() + store.retry_attachment_job.assert_not_awaited() + + async def test_stale_completion_keeps_staging(self): + _, s3 = await self.run_processing(False) + s3.delete.assert_not_awaited() + + async def test_stale_claim_cannot_publish_metadata(self): + store = PostgresStore('unused') + connection = AsyncMock() + connection.fetchval.return_value = None + @asynccontextmanager + async def context(): + yield connection + connection.transaction = context + store._pool = SimpleNamespace(acquire=context) + self.assertFalse(await store.complete_attachment_job(uuid4(), 'original', [], datetime.now(timezone.utc))) + connection.execute.assert_not_awaited() + + async def test_retry_of_terminal_or_stale_attempt_is_noop(self): + store = PostgresStore('unused') + connection = AsyncMock() + connection.fetchrow.return_value = None + @asynccontextmanager + async def context(): + yield connection + connection.transaction = context + store._pool = SimpleNamespace(acquire=context) + self.assertFalse(await store.retry_attachment_job(uuid4(), 5, 1, datetime.now(timezone.utc))) + connection.execute.assert_not_awaited() + + def test_dlq_waits_for_publisher_confirm(self): + connection = MagicMock() + with patch('src.broker._connection', return_value=connection): + _publish_to('dlq', {'failure': 'invalid_job'}) + connection.channel.return_value.confirm_delivery.assert_called_once() + self.assertTrue(connection.channel.return_value.basic_publish.call_args.kwargs['mandatory']) + + +if __name__ == '__main__': + unittest.main() diff --git a/student-tests/pipeline_integration.py b/student-tests/pipeline_integration.py new file mode 100644 index 0000000..c230440 --- /dev/null +++ b/student-tests/pipeline_integration.py @@ -0,0 +1,205 @@ +#!/usr/bin/env python3 +"""Destructive-to-availability tests on the local lab stack; volumes preserved. + +Only synthetic attachments created here are mutated. Tokens stay in memory. +Run sequentially, never alongside load or another fault test. +""" +import json +import os +from pathlib import Path +import subprocess +import sys +import time + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client, Failure, check + +RESULTS = [] + + +def compose(*args, env=None, source=None): + result = subprocess.run(['docker', 'compose', *args], cwd=ROOT, + env=dict(os.environ, **(env or {})), input=source, + text=True, capture_output=True, timeout=150) + check(result.returncode == 0, 'compose failed: ' + ' '.join(args[:4]) + '\n' + result.stderr[-1000:]) + return result.stdout.strip() + + +def remote(body, **values): + source = '''import asyncio, json, os +from uuid import UUID +from src.config import load_settings +from src.storage.postgres import PostgresStore +from src.storage.s3 import S3Store, S3Settings +from src.broker import publish +V = json.loads(%r) +async def main(): + db = PostgresStore(load_settings().database_url) + await db.connect() + s3 = S3Store(S3Settings.load()) + try: +%s + finally: + s3.client.close() + await db.close() +asyncio.run(main()) +''' % (json.dumps(values), '\n'.join(' ' + line for line in body.splitlines())) + return json.loads(compose('exec', '-T', 'api-a', '/app/.venv/bin/python', '-', source=source)) + + +def record(name, **facts): + RESULTS.append(dict(test=name, status='PASS', **facts)) + print('PASS: ' + name, flush=True) + (ROOT / 'evidence/pipeline-integration.json').write_text(json.dumps(RESULTS, indent=2) + '\n') + + +def ready(c): + deadline = time.monotonic() + 90 + successes = 0 + while time.monotonic() < deadline: + try: + c.call('GET', '/health/ready') + successes += 1 + if successes >= 5: + return + time.sleep(.5) + except Failure: + successes = 0 + time.sleep(.5) + raise Failure('API did not recover') + + +def wait(c, actor, att, status='ready', timeout=120): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + try: + value = c.call('GET', '/api/v1/attachments/' + att['id'], actor=actor, schema='Attachment')['json'] + except Failure: + time.sleep(.25) + continue + if value['status'] == status: + return value + check(value['status'] != 'failed' or status == 'failed', 'unexpected failed attachment') + time.sleep(.25) + raise Failure('attachment did not reach ' + status) + + +def snapshot(att): + return remote('''row = await db.pool.fetchrow("SELECT a.status, j.attempt, j.job_id, a.original_key, a.staging_key FROM attachments a JOIN attachment_jobs j ON j.attachment_id=a.id WHERE a.id=$1", UUID(V['id'])) +objects = sorted(list(await s3.list('originals/' + V['id'] + '/')) + list(await s3.list('variants/' + V['id'] + '/'))) +print(json.dumps(dict(row) | {'objects': objects}, default=str))''', id=att['id']) + + +def duplicate(att): + return remote('''payload = await db.pool.fetchval("SELECT o.payload FROM attachment_outbox o JOIN attachment_jobs j ON j.job_id=o.job_id WHERE j.attachment_id=$1 ORDER BY o.created_at LIMIT 1", UUID(V['id'])) +await publish(json.loads(payload) if isinstance(payload, str) else payload) +print('true')''', id=att['id']) + + +def main(): + c = Client('http://localhost:8080', timeout=30) + alice, bob, _, chat = c.bootstrap() + + # No worker: HTTP admission and message creation must remain asynchronous. + compose('stop', '-t', '2', 'worker') + att, data = c.attachment(alice, chat) + check(att['status'] == 'queued', 'upload is not queued with stopped worker') + c.message(alice, chat, 'queued attachment', attachments=[att['id']]) + time.sleep(2) + compose('restart', 'broker') + compose('up', '-d', '--no-deps', '--wait', 'broker') + compose('restart', 'api-a', 'api-b') + ready(c) + compose('start', 'worker') + completed = wait(c, bob, att) + check(c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=bob)['bytes'] == data, 'original changed') + check({v['name'] for v in completed['variants']} == {'thumbnail', 'crop', 'optimized'}, 'variants missing') + record('stopped worker, queued message, broker/API restart, ready', attachment_id=att['id']) + + before = snapshot(att) + duplicate(att) + duplicate(att) + time.sleep(3) + check(snapshot(att) == before, 'duplicate changed result or created S3 objects') + refused = remote("print(json.dumps(await db.replay_failed_attachment(UUID(V['id']))))", id=att['id']) + check(refused is False, 'ready replay accepted') + record('duplicate delivery and replay of ready are no-ops', object_count=len(before['objects'])) + + # Pause publication so the API crash is demonstrably before any publish. + compose('stop', '-t', '2', 'outbox') + compose('up', '-d', '--no-deps', '--wait', 'api-a', 'api-b', env={'OUTBOX_FAULT_AFTER_COMMIT': '1'}) + ready(c) + fault_actor, _, _, fault_chat = c.bootstrap() + try: + c.attachment(fault_actor, fault_chat) + raise AssertionError('crash hook did not interrupt upload') + except Failure: + pass + compose('up', '-d', '--no-deps', '--wait', 'api-a', 'api-b') + ready(c) + row = remote('''rows = await db.pool.fetch("SELECT a.id, o.published_at FROM attachments a JOIN attachment_jobs j ON j.attachment_id=a.id JOIN attachment_outbox o ON o.job_id=j.job_id WHERE a.chat_id=$1", UUID(V['chat'])) +assert len(rows) == 1 and rows[0]['published_at'] is None +print(json.dumps({'id': str(rows[0]['id'])}))''', chat=fault_chat['id']) + compose('start', 'outbox') + wait(c, fault_actor, row) + record('crash after COMMIT before publish recovered from existing outbox', attachment_id=row['id']) + + compose('up', '-d', '--no-deps', '--wait', 'worker', env={'WORKER_FAULT_AFTER_S3_PUT': '1', 'JOB_TIMEOUT_SECONDS': '10'}) + att, _ = c.attachment(alice, chat) + wait(c, alice, att) + snap = snapshot(att) + check(snap['attempt'] >= 2 and len(snap['objects']) >= 8, 'side effect crash/lease retry not observed') + marker = compose('exec', '-T', 'worker', 'cat', '/tmp/worker-fault-after-s3') + check(marker == att['id'], 'wrong fault target') + record('worker crash after S3 before commit/ack, lease recovery', attempt=snap['attempt'], objects_before_cleanup=len(snap['objects'])) + compose('up', '-d', '--no-deps', '--wait', 'worker') + + # Select only this test's unreferenced attempt keys for cleanup. + cleaned = remote('''from argparse import Namespace +from src.reconcile import run +keys = V['keys'] +result = await run(Namespace(apply=True, grace_seconds=0, object_key=keys)) +print(json.dumps(result['deleted']))''', keys=snap['objects']) + check(len(cleaned) == 4, 'stale attempt objects not cleaned safely') + check(len(snapshot(att)['objects']) == 4, 'live attempt affected by cleanup') + wait(c, alice, att) + record('targeted orphan cleanup preserves committed attempt', deleted=len(cleaned)) + + # Lose only this synthetic staging object to exercise transient retry exhaustion. + compose('stop', '-t', '2', 'worker') + att, data = c.attachment(alice, chat) + remote("await s3.delete('staging/' + V['id'] + '/original')\nprint('true')", id=att['id']) + compose('up', '-d', '--no-deps', '--wait', 'worker', env={'JOB_MAX_ATTEMPTS': '2'}) + failed = wait(c, alice, att, 'failed') + snap = snapshot(att) + check(snap['attempt'] == 2 and failed['error_code'] == 'retry_exhausted', 'unbounded/wrong retry') + # Restore exactly the bytes originally accepted, then explicit failed-only replay. + import base64 + replayed = remote('''from io import BytesIO +import base64 +content = base64.b64decode(V['data']) +await s3.put('staging/' + V['id'] + '/original', BytesIO(content), len(content), 'image/png') +print(json.dumps(await db.replay_failed_attachment(UUID(V['id']))))''', id=att['id'], data=base64.b64encode(data).decode()) + check(replayed, 'failed replay refused') + wait(c, alice, att) + record('bounded transient retry, failed registry and explicit replay', attempts_before_replay=snap['attempt']) + compose('up', '-d', '--no-deps', '--wait', 'worker') + + # Real Linux child with deliberately insufficient address space. + memory = remote('''import subprocess +from io import BytesIO +from PIL import Image +buf = BytesIO(); Image.new('RGB', (2400, 2400)).save(buf, 'PNG') +env = dict(os.environ, RENDER_MEMORY_MB='120') +p = subprocess.run(['/app/.venv/bin/python', '-m', 'src.render'], input=buf.getvalue(), capture_output=True, env=env, timeout=30) +print(json.dumps({'returncode': p.returncode}))''') + check(memory['returncode'] == 21, 'Linux memory limit did not produce bounded resource failure') + record('Linux renderer memory limit', **memory) + + +if __name__ == '__main__': + try: + main() + finally: + compose('up', '-d', '--no-deps', '--wait', 'api-a', 'api-b', 'worker', 'outbox') diff --git a/student-tests/pipeline_invariants.py b/student-tests/pipeline_invariants.py new file mode 100644 index 0000000..e571551 --- /dev/null +++ b/student-tests/pipeline_invariants.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python3 +"""Real PostgreSQL/S3/RabbitMQ invariant and security checks, synthetic data only.""" +import json +from pathlib import Path +import time +from pipeline_integration import Client, check, compose, remote, wait, ROOT + + +def main(): + results = [] + c = Client('http://localhost:8080', timeout=30) + alice, bob, eve, chat = c.bootstrap() + att, _ = c.attachment(alice, chat) + wait(c, bob, att) + from io import BytesIO + from PIL import Image + for name, fmt, bound in (('thumbnail', 'JPEG', 256), ('crop', 'JPEG', 128), ('optimized', 'WEBP', 1280)): + content = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=' + name, actor=bob)['bytes'] + with Image.open(BytesIO(content)) as image: + check(image.format == fmt and max(image.size) <= bound and not image.getexif(), 'variant format/size/EXIF') + if name == 'crop': + check(image.size == (128, 128), 'crop dimensions') + results.append('actual thumbnail/crop/WebP formats, dimensions and removed EXIF') + # Verify actual private object key, not a nonexistent legacy key. + private = remote('''import urllib.request, urllib.error +key = await db.pool.fetchval('SELECT original_key FROM attachments WHERE id=$1', UUID(V['id'])) +try: + urllib.request.urlopen('http://s3:8333/' + s3.settings.bucket + '/' + key, timeout=5) +except urllib.error.HTTPError as error: + assert error.code == 403, error.code +else: + raise AssertionError('anonymous object leaked') +print('true')''', id=att['id']) + check(private, 'private S3 check failed') + for variant in ('thumbnail', 'crop', 'optimized'): + c.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=' + variant, 404, actor=eve) + results.append('anonymous S3 403 on existing object; variant ACL 404') + + lock = remote('''import time +entered = asyncio.Event() +async def writer(): + async with db.attachment_transaction(): + entered.set() + await asyncio.sleep(1) +task = asyncio.create_task(writer()) +await entered.wait() +started = time.monotonic() +async with db.attachment_transaction(maintenance=True): + waited = time.monotonic() - started +await task +assert waited >= .9, waited +print(json.dumps({'maintenance_wait_seconds': waited}))''') + results.append(lock) + + # Poison staging only for a newly created synthetic attachment and update its + # digest so invalid-image classification (not transport checksum) is exercised. + compose('stop', '-t', '2', 'worker') + guarded, _ = c.attachment(alice, chat) + fence = remote('''key = UUID(V['id']) +job_id = await db.pool.fetchval('SELECT job_id FROM attachment_jobs WHERE attachment_id=$1', key) +before = await db.pool.fetchval('SELECT count(*) FROM attachment_outbox WHERE job_id=$1', job_id) +await db.pool.execute("UPDATE attachment_jobs SET updated_at=now()-interval '61 seconds' WHERE attachment_id=$1", key) +await db.schedule_due_jobs() +after = await db.pool.fetchval('SELECT count(*) FROM attachment_outbox WHERE job_id=$1', job_id) +assert after == before + 1, 'unclaimed published job stranded' +assert await db.pool.fetchval('SELECT attempt FROM attachment_jobs WHERE attachment_id=$1', key) == 1 +await db.schedule_due_jobs() +assert await db.pool.fetchval('SELECT count(*) FROM attachment_outbox WHERE job_id=$1', job_id) == after, 'watchdog floods outbox' +first = await db.claim_attachment_job(key, 30) +assert first is not None +assert await db.retry_attachment_job(key, 5, 30, first['claim_time']) +assert await db.claim_attachment_job(key, 30) is None, 'duplicate bypassed retry delay' +await db.pool.execute("UPDATE attachment_jobs SET next_attempt_at=now() WHERE attachment_id=$1", key) +await db.schedule_due_jobs() +second = await db.claim_attachment_job(key, 30) +assert second is not None and second['attempt'] == 2 +assert not await db.complete_attachment_job(key, 'invalid-stale-key', [], first['claim_time']) +assert not await db.retry_attachment_job(key, 5, 0, first['claim_time']) +await db.pool.execute("UPDATE attachment_jobs SET lease_until=now()-interval '1 second' WHERE attachment_id=$1", key) +assert await db.claim_attachment_job(key, 30) is None, 'duplicate bypassed lease attempt accounting' +await db.schedule_due_jobs() +third = await db.claim_attachment_job(key, 30) +assert third['attempt'] == 3 +assert await db.retry_attachment_job(key, 5, 0, third['claim_time']) +print('true')''', id=guarded['id']) + check(fence, 'SQL fencing/backoff invariant') + results.append('real SQL: queued watchdog republishes without attempt increment/flood; duplicate respects backoff; expired lease counted; stale completion/retry fenced') + att, _ = c.attachment(alice, chat) + remote('''from io import BytesIO +from hashlib import sha256 +data = b'invalid image payload' +await s3.put('staging/' + V['id'] + '/original', BytesIO(data), len(data), 'image/png') +await db.pool.execute('UPDATE attachments SET size_bytes=$2, sha256=$3 WHERE id=$1', UUID(V['id']), len(data), sha256(data).hexdigest()) +print('true')''', id=att['id']) + compose('start', 'worker') + wait(c, alice, guarded) + failed = wait(c, alice, att, 'failed') + check(failed['error_code'] == 'invalid_image', 'permanent classification failed') + attempt = remote("print(await db.pool.fetchval('SELECT attempt FROM attachment_jobs WHERE attachment_id=$1', UUID(V['id'])))", id=att['id']) + check(attempt == 1, 'permanent error retried') + results.append('permanent invalid_image reaches failed on attempt 1') + + # DLQ uses confirmed persistent publication; count change avoids consuming + # messages belonging to a different test. + dlq = remote('''from src.broker import _connection, _publish, DLQ +connection = _connection(); channel = connection.channel() +before = channel.queue_declare(queue=DLQ, durable=True).method.message_count +connection.close() +_publish({'malformed': True}) +print(json.dumps({'before': before}))''') + deadline = time.monotonic() + 20 + while time.monotonic() < deadline: + count = remote('''from src.broker import _connection, DLQ +connection = _connection() +print(connection.channel().queue_declare(queue=DLQ, durable=True).method.message_count) +connection.close()''') + if count > dlq['before']: + break + time.sleep(.5) + check(count == dlq['before'] + 1, 'invalid job not delivered to DLQ') + results.append('malformed broker message delivered to DLQ') + + # Actual Linux child timeout: no renderer can finish within this deadline. + timeout = remote('''from src.worker import render_isolated +try: + render_isolated(b'x', .000001) +except TimeoutError: + print('true') +else: + raise AssertionError('timeout not enforced')''') + check(timeout, 'timeout missing') + results.append('Linux render subprocess killed/reaped on deadline') + (ROOT / 'evidence/pipeline-invariants.json').write_text(json.dumps(results, indent=2) + '\n') + print(json.dumps(results, indent=2), flush=True) + + +if __name__ == '__main__': + try: + main() + finally: + compose('up', '-d', '--no-deps', '--wait', 'worker') diff --git a/student-tests/readiness_failover.py b/student-tests/readiness_failover.py new file mode 100644 index 0000000..cdb6c0a --- /dev/null +++ b/student-tests/readiness_failover.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +"""Запрет БД только для api-a через pg_hba; восстановление в finally.""" +import ipaddress +import json +from pathlib import Path +import subprocess +import sys +import time +import uuid + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, check, run + + +def compose(*args): + return subprocess.check_output(["docker", "compose", *args], cwd=ROOT, text=True).strip() + + +def sql(statement): + return compose("exec", "-T", "postgres", "psql", "-U", "messenger", "-d", + "messenger", "-v", "ON_ERROR_STOP=1", "-Atc", statement) + + +def ready(service): + code = """import urllib.request, urllib.error +try: + print(urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=7).status) +except urllib.error.HTTPError as error: + print(error.code) +""" + return compose("exec", "-T", service, "/app/.venv/bin/python", "-c", code) + + +def main(): + client = Client() + container = json.loads(subprocess.check_output( + ["docker", "inspect", compose("ps", "-q", "api-a")], text=True))[0] + addresses = list(container["NetworkSettings"]["Networks"].values()) + check(len(addresses) == 1, "test expects one API network") + address = str(ipaddress.IPv4Address(addresses[0]["IPAddress"])) + bad_id = container["Config"]["Hostname"] + before = {client.call("GET", "/health/live")["headers"]["X-Instance-Id"] for _ in range(40)} + check(len(before) == 2 and bad_id in before, "both API replicas must be reachable") + check(ready("api-a") == ready("api-b") == "200", "both APIs must be ready") + # Сохраняем исходный pg_hba; чужую оставшуюся резервную копию не перезаписываем. + compose("exec", "-T", "postgres", "sh", "-ec", + 'test ! -e "$PGDATA/pg_hba.conf.readiness-backup"; ' + 'cp -p "$PGDATA/pg_hba.conf" "$PGDATA/pg_hba.conf.readiness-backup"') + try: + compose("exec", "-T", "postgres", "sh", "-ec", + '{ printf "host all all %s/32 reject\\n" "$1"; ' + 'cat "$PGDATA/pg_hba.conf.readiness-backup"; } > "$PGDATA/pg_hba.conf"', + "sh", address) + sql("SELECT pg_reload_conf()") + time.sleep(0.5) + sql(f"SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE client_addr = '{address}'::inet") + check(ready("api-a") == "503", "api-a must be unready") + check(ready("api-b") == "200", "api-b must remain ready") + # 2s interval + 2s check timeout, с запасом на планировщик Docker Desktop. + time.sleep(6) + for _ in range(40): + result = client.call("GET", "/health/live", 200) + check(result["headers"]["X-Instance-Id"] != bad_id, "LB selected unready api-a") + for _ in range(20): + result = client.call("POST", "/api/v1/users", 201, body={ + "username": "r_" + uuid.uuid4().hex[:16], + "display_name": "Readiness regression", + "password": "Test-password-" + uuid.uuid4().hex, + }, schema="User") + check(result["headers"]["X-Instance-Id"] != bad_id, "registration reached api-a") + print("PASS: api-a ready=503, api-b ready=200; 40/40 live and 20/20 registrations use api-b") + finally: + compose("exec", "-T", "postgres", "sh", "-ec", + 'mv "$PGDATA/pg_hba.conf.readiness-backup" "$PGDATA/pg_hba.conf"') + sql("SELECT pg_reload_conf()") + deadline = time.monotonic() + 20 + while time.monotonic() < deadline: + observed = {client.call("GET", "/health/live")["headers"]["X-Instance-Id"] for _ in range(10)} + if observed == before: + check(ready("api-a") == "200", "restored api-a must be ready") + print("PASS: database access restored; both API replicas returned to LB") + return + time.sleep(0.5) + raise AssertionError("api-a did not return to LB") + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/readiness_unavailable.py b/student-tests/readiness_unavailable.py new file mode 100644 index 0000000..cf4bffa --- /dev/null +++ b/student-tests/readiness_unavailable.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python3 +"""Проверка 503 readiness при остановленном PostgreSQL.""" +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests")) +from client import Client, run # noqa: E402 + + +def main() -> None: + Client().call("GET", "/health/ready", 503) + print("PASS: readiness rejects unavailable PostgreSQL") + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/reconcile.py b/student-tests/reconcile.py new file mode 100644 index 0000000..04a8fd7 --- /dev/null +++ b/student-tests/reconcile.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python3 +"""Inject S3/DB disagreement and prove dry-run, grace period and safe repair.""" +import json +import base64 +from pathlib import Path +import subprocess +import sys +import time +import uuid + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, Failure, arguments, check, run + + +def compose(*args, input=None, env=None) -> str: + return subprocess.check_output(["docker", "compose", *args], cwd=ROOT, text=True, + input=input, env=env) + + +def reconcile(*args) -> dict: + output = compose("--profile", "maintenance", "run", "--rm", "--no-deps", "reconcile", *args) + return json.loads(next(line for line in reversed(output.splitlines()) if line.startswith("{"))) + + +def s3_code(action: str, key: str, data: bytes | None = None) -> None: + code = """from src.storage.s3 import S3Settings, S3Store +import base64, json, sys +s3 = S3Store(S3Settings.load()) +payload = json.load(sys.stdin) +key = payload['key'] +try: + if sys.argv[1] == 'put': + s3.client.put_object(Bucket=s3.settings.bucket, Key=key, + Body=base64.b64decode(payload.get('data', 'b3JwaGFu'))) + else: + s3.client.delete_object(Bucket=s3.settings.bucket, Key=key) +finally: + s3.client.close() +""" + payload = {"key": key} + if data is not None: + payload["data"] = base64.b64encode(data).decode() + compose("run", "--rm", "--no-deps", "s3-init", "/app/.venv/bin/python", "-c", code, action, + input=json.dumps(payload)) + + +def wait_ready(client: Client, env=None) -> None: + subprocess.run(["docker", "compose", "up", "-d", "--wait", "--no-deps", "api-a"], + cwd=ROOT, env=env, check=True, stdout=subprocess.DEVNULL) + deadline = time.monotonic() + 15 + while True: + try: + client.call("GET", "/health/ready", schema="Health") + return + except Failure: + check(time.monotonic() < deadline, "stack did not recover") + time.sleep(.5) + + +def main(): + args = arguments(__doc__) + client = Client(args.base_url, args.ca_file, timeout=30) + alice, bob, _eve, chat = client.bootstrap() + image = (ROOT / "tests/fixtures/sample.png").read_bytes() + path = "/api/v1/chats/" + chat["id"] + "/attachments?filename=" + + def upload(name, expected=201): + return client.call("POST", path + name, expected, actor=alice, raw=image, + headers={"Content-Type": "image/png"}, + schema="Attachment" if expected == 201 else None) + + # A reference object must survive an apply sweep, even with zero grace. + protected = upload("protected.png")["json"] + message = client.message(alice, chat, "protected", attachments=[protected["id"]])["json"] + orphan = "originals/test-orphan-" + uuid.uuid4().hex + s3_code("put", orphan) + + report = reconcile("--grace-seconds", "3600") + check(orphan in report["fresh_unreferenced"], "fresh orphan missing from dry-run") + check(not report["deleted"], "dry-run deleted an object") + report = reconcile("--apply", "--grace-seconds", "3600", "--object-key", orphan) + check(orphan in report["fresh_unreferenced"] and not report["deleted"], "grace period ignored") + report = reconcile("--apply", "--grace-seconds", "0", "--object-key", orphan) + check(report["deleted"] == [orphan], "aged orphan was not deleted safely") + report = reconcile("--apply", "--grace-seconds", "0", "--object-key", orphan) + check(orphan not in report["deleted"] and orphan not in report["orphan_candidates"], + "second repair run was not idempotent") + check(client.call("GET", "/api/v1/attachments/" + protected["id"] + "/content", actor=bob)["bytes"] == image, + "reconciler deleted a referenced object") + + # This models loss after metadata has committed: report it, do not delete metadata. + missing = upload("missing.png")["json"] + missing_key = "originals/" + missing["id"] + s3_code("delete", missing_key) + try: + report = reconcile("--apply", "--grace-seconds", "0", "--object-key", missing_key) + expected = {"attachment_id": missing["id"], "object_key": missing_key} + check(expected in report["missing_objects"], "missing metadata/object pair not reported") + client.call("GET", "/api/v1/attachments/" + missing["id"], actor=bob, schema="Attachment") + client.call("GET", "/api/v1/attachments/" + missing["id"] + "/content", 503, actor=bob) + finally: + s3_code("put", missing_key, image) + check(client.call("GET", "/api/v1/attachments/" + missing["id"] + "/content", actor=bob)["bytes"] == image, + "test object was not restored") + report = reconcile("--grace-seconds", "0") + check(expected not in report["missing_objects"], "restored object remained missing on second scan") + print("PASS: S3-side missing object, dry-run, grace period and idempotent orphan repair", flush=True) + + # Inject a process-equivalent failure after PUT directly into the configured + # replica. The public load balancer may correctly select the healthy peer, + # which would not exercise this boundary. + environment = dict(__import__("os").environ, UPLOAD_FAULT_AFTER_S3_PUT="1") + before_fault = set(reconcile("--grace-seconds", "0")["orphan_candidates"]) + try: + subprocess.run(["docker", "compose", "up", "-d", "--force-recreate", "--no-deps", "api-a"], + cwd=ROOT, env=environment, check=True, stdout=subprocess.DEVNULL) + wait_ready(client, environment) + fault = """import base64, json, sys, urllib.error, urllib.request +payload = json.load(sys.stdin) +request = urllib.request.Request('http://127.0.0.1:8080/api/v1/chats/' + payload['chat_id'] + + '/attachments?filename=fault-after-put.png', data=base64.b64decode(payload['image']), + method='POST', headers={'Authorization': 'Bearer ' + payload['token'], 'Content-Type': 'image/png'}) +try: + urllib.request.urlopen(request, timeout=30) +except urllib.error.HTTPError as error: + assert error.code == 503, error.code + assert json.load(error)['error']['code'] == 'unavailable' +else: + raise AssertionError('fault injection unexpectedly returned success') +""" + compose("exec", "-T", "api-a", "/app/.venv/bin/python", "-c", fault, + input=json.dumps({"chat_id": chat["id"], "token": alice["token"], + "image": base64.b64encode(image).decode()})) + report = reconcile("--grace-seconds", "0") + injected = sorted(set(report["orphan_candidates"]) - before_fault) + check(len(injected) == 1, "fault must create exactly one new orphan; run without other writers") + report = reconcile("--apply", "--grace-seconds", "0", "--object-key", injected[0]) + check(set(injected).issubset(report["deleted"]), "reconciler did not clean injected orphan") + history = client.call("GET", "/api/v1/chats/" + chat["id"] + "/messages", actor=bob, schema="MessagePage")["json"] + check(message in history["items"], "message reference was lost") + check(client.call("GET", "/api/v1/attachments/" + protected["id"] + "/content", actor=bob)["bytes"] == image, + "used original was lost") + print("PASS: injected pre-INSERT failure repaired; message and used original survive", flush=True) + finally: + environment["UPLOAD_FAULT_AFTER_S3_PUT"] = "0" + subprocess.run(["docker", "compose", "up", "-d", "--wait", "--force-recreate", "--no-deps", "api-a"], + cwd=ROOT, env=environment, check=True, stdout=subprocess.DEVNULL) + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/reconcile_audit.py b/student-tests/reconcile_audit.py new file mode 100644 index 0000000..8c1e046 --- /dev/null +++ b/student-tests/reconcile_audit.py @@ -0,0 +1,159 @@ +#!/usr/bin/env python3 +"""Real database rejection, concurrent commit, interrupted upload and pagination.""" +import base64 +import json +from pathlib import Path +import subprocess +import sys + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, arguments, run + +PROBE = r''' +import asyncio, base64, hashlib, http.client, json, sys, urllib.request, urllib.error +from types import SimpleNamespace +from uuid import UUID, uuid4 +from src.config import load_settings +from src.storage.postgres import PostgresStore +from src.storage.s3 import S3Store, S3Settings +from src.reconcile import run as reconcile, scan + +payload = json.load(sys.stdin) +image = base64.b64decode(payload['image']) +chat_id, user_id = UUID(payload['chat_id']), UUID(payload['user_id']) + +def upload(filename): + req = urllib.request.Request('http://api-a:8080/api/v1/chats/' + str(chat_id) + + '/attachments?filename=' + filename, data=image, method='POST', + headers={'Authorization': 'Bearer ' + payload['token'], 'Content-Type': 'image/png'}) + try: + with urllib.request.urlopen(req, timeout=30) as response: + return response.status, json.load(response) + except urllib.error.HTTPError as error: + return error.code, json.load(error) + +async def main(): + db = PostgresStore(load_settings().database_url) + s3 = S3Store(S3Settings.load()) + await db.connect() + def args(key): + return SimpleNamespace(apply=True, grace_seconds=0, object_key=[key]) + try: + # A failed metadata inventory must not authorize any S3 deletion. + key = 'originals/audit-inventory-' + uuid4().hex + await s3.call('put_object', Bucket=s3.settings.bucket, Key=key, Body=image) + class BrokenInventory: + async def attachment_keys(self): + # Real SQL failure, not an empty inventory on error. + return await db.pool.fetch('SELECT * FROM audit_missing_table_' + uuid4().hex) + try: + async with db.attachment_transaction(maintenance=True): + try: + await scan(args(key), BrokenInventory(), s3) + except Exception: + pass + else: + raise AssertionError('failed inventory did not abort') + assert key in await s3.list('originals/') + finally: + await s3.delete(key) + print('PASS: failed PostgreSQL inventory aborts apply without deleting the orphan', flush=True) + + # An actual PostgreSQL trigger rejects only this test chat's INSERT. + suffix = uuid4().hex + name = 'audit_reject_' + suffix + before = await s3.list('originals/') + await db.pool.execute(f"""CREATE FUNCTION {name}() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN IF NEW.chat_id = '{chat_id}'::uuid THEN + RAISE EXCEPTION 'synthetic audit rejection' USING ERRCODE = '23514'; + END IF; RETURN NEW; END $$""") + try: + await db.pool.execute(f'CREATE TRIGGER {name} BEFORE INSERT ON attachments FOR EACH ROW EXECUTE FUNCTION {name}()') + status, body = await asyncio.to_thread(upload, 'db-failure.png') + assert status == 503 and body['error']['code'] == 'unavailable', (status, body) + assert await db.pool.fetchval('SELECT count(*) FROM attachments WHERE chat_id=$1', chat_id) == 0 + finally: + await db.pool.execute(f'DROP TRIGGER IF EXISTS {name} ON attachments') + await db.pool.execute(f'DROP FUNCTION {name}()') + created = set(await s3.list('originals/')) - set(before) + assert len(created) == 1, created + key = created.pop() + report = await reconcile(SimpleNamespace(apply=False, grace_seconds=0, object_key=[key])) + assert key in report['orphan_candidates'] and not report['deleted'] + assert (await reconcile(args(key)))['deleted'] == [key] + assert not (await reconcile(args(key)))['deleted'] + print('PASS: actual PostgreSQL INSERT rejection -> 503, dry-run detection, scoped repair, repeat no-op', flush=True) + + # Writer holds a shared lock after PUT; cleanup must wait for COMMIT. + identifier = uuid4() + key = 'originals/' + str(identifier) + async with db.attachment_transaction() as connection: + await s3.call('put_object', Bucket=s3.settings.bucket, Key=key, Body=image) + sweep = asyncio.create_task(reconcile(args(key))) + for _ in range(40): + waiting = await db.pool.fetchval("SELECT count(*) FROM pg_locks WHERE locktype='advisory' AND classid=41004 AND objid=1 AND NOT granted") + if waiting: + break + await asyncio.sleep(.05) + assert waiting and not sweep.done(), 'maintenance failed to wait for active writer' + await db.create_attachment(identifier, chat_id, user_id, 'concurrent.png', 'image/png', + len(image), hashlib.sha256(image).hexdigest(), key, connection=connection) + report = await sweep + assert key not in report['deleted'] and key not in report['orphan_candidates'] + row = await db.attachment_for_member(identifier, user_id) + assert await s3.download(row) == image + print('PASS: apply with zero grace waits for in-flight PUT/COMMIT; committed original survives', flush=True) + + # Abrupt client close before all bytes arrive must not persist a file. + before = await s3.list('originals/') + rows = await db.attachment_keys() + def interrupt(): + conn = http.client.HTTPConnection('api-a', 8080, timeout=10) + conn.putrequest('POST', '/api/v1/chats/' + str(chat_id) + '/attachments?filename=interrupted.png') + conn.putheader('Authorization', 'Bearer ' + payload['token']) + conn.putheader('Content-Type', 'image/png') + conn.putheader('Content-Length', str(len(image) + 100000)) + conn.endheaders() + conn.send(image[:32]) + conn.close() + await asyncio.to_thread(interrupt) + await asyncio.sleep(1) + assert await db.attachment_keys() == rows + assert await s3.list('originals/') == before + status, body = await asyncio.to_thread(upload, 'after-disconnect.png') + assert status == 201, (status, body) + print('PASS: interrupted upload leaves no metadata/object and next upload succeeds', flush=True) + + # Exercise pagination on real S3 without adding thousands of fixtures. + original_call = s3.call + async def small_pages(method, **kwargs): + if method == 'list_objects_v2': + kwargs['MaxKeys'] = 7 + return await original_call(method, **kwargs) + expected = await s3.list('originals/') + s3.call = small_pages + actual, pages = await s3.list_with_pages('originals/') + assert actual == expected and pages > 1 + print(f'PASS: S3 pagination: {len(actual)} keys across {pages} pages, no missing/duplicate keys', flush=True) + finally: + s3.client.close() + await db.close() + +asyncio.run(main()) +''' + + +def main(): + args = arguments(__doc__) + client = Client(args.base_url, args.ca_file, timeout=30) + alice, _, _, chat = client.bootstrap() + payload = {"chat_id": chat["id"], "user_id": alice["id"], "token": alice["token"], + "image": base64.b64encode((ROOT / "tests/fixtures/sample.png").read_bytes()).decode()} + subprocess.run(["docker", "compose", "--profile", "maintenance", "run", "--rm", "--no-deps", + "--entrypoint", "/app/.venv/bin/python", "reconcile", "-c", PROBE], + cwd=ROOT, input=json.dumps(payload), text=True, check=True) + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/reconcile_benchmark.py b/student-tests/reconcile_benchmark.py new file mode 100644 index 0000000..358ba40 --- /dev/null +++ b/student-tests/reconcile_benchmark.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +"""Measure repeatable reconciler inventory cost for two controlled S3 sizes.""" +import json +from pathlib import Path +import statistics +import subprocess +import sys +import uuid + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import arguments, check, run + + +def compose(*args, input=None) -> str: + return subprocess.check_output(["docker", "compose", *args], cwd=ROOT, text=True, input=input) + + +def reconcile() -> dict: + output = compose("--profile", "maintenance", "run", "--rm", "--no-deps", "reconcile", + "--grace-seconds", "3600") + return json.loads(next(line for line in reversed(output.splitlines()) if line.startswith("{"))) + + +def mutate(action: str, keys: list[str]) -> None: + code = """from src.storage.s3 import S3Settings, S3Store +import json, sys +s3 = S3Store(S3Settings.load()) +try: + for key in json.load(sys.stdin): + if sys.argv[1] == 'put': + s3.client.put_object(Bucket=s3.settings.bucket, Key=key, Body=b'track-2-benchmark') + else: + s3.client.delete_object(Bucket=s3.settings.bucket, Key=key) +finally: + s3.client.close() +""" + compose("run", "--rm", "--no-deps", "s3-init", "/app/.venv/bin/python", "-c", code, action, + input=json.dumps(keys)) + + +def main() -> None: + arguments(__doc__) # Keeps the common --base-url/--ca-file command interface. + results = [] + for requested in (32, 128): + prefix = f"originals/track2-benchmark-{uuid.uuid4().hex}-" + keys = [prefix + str(index) for index in range(requested)] + try: + mutate("put", keys) + reports = [reconcile() for _ in range(3)] + for report in reports: + check(set(keys).issubset(report["fresh_unreferenced"]), + "benchmark objects were not included in the inventory") + check(not set(keys) & set(report["orphan_candidates"]), + "grace period classified fresh benchmark objects as deletable") + seconds = [report["scan_metrics"]["total_seconds"] for report in reports] + results.append({ + "added_unreferenced_objects": requested, + "runs": 3, + "s3_objects_scanned": reports[-1]["s3_objects"], + "metadata_rows_scanned": reports[-1]["metadata_rows"], + "s3_list_pages": reports[-1]["scan_metrics"]["s3_list_pages"], + "seconds": seconds, + "median_seconds": round(statistics.median(seconds), 6), + }) + finally: + mutate("delete", keys) + print(json.dumps({"method": "three dry-run inventories per controlled size", "results": results}, + sort_keys=True)) + print("PASS: reconciler cost measured for 32 and 128 added S3 objects; fixtures removed") + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/remaining_regressions.py b/student-tests/remaining_regressions.py new file mode 100644 index 0000000..af79f76 --- /dev/null +++ b/student-tests/remaining_regressions.py @@ -0,0 +1,127 @@ +"""Contract, isolated-render and benchmark regressions without Docker.""" +import asyncio +from contextlib import asynccontextmanager +from datetime import datetime, timedelta, timezone +import importlib.util +from io import BytesIO +import json +from pathlib import Path +import sys +import tempfile +import threading +import time +from types import SimpleNamespace +import unittest +from unittest.mock import AsyncMock, patch +from uuid import uuid4 + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) +from PIL import Image +from src.worker import variants, render_isolated, PermanentFailure +from src.storage.postgres import PostgresStore +from src.reconcile import scan +from src.errors import ApiError + + +def png(width, height): + buffer = BytesIO() + Image.new('RGBA', (width, height), (255, 0, 0, 0)).save(buffer, 'PNG') + return buffer.getvalue() + + +class Regressions(unittest.IsolatedAsyncioTestCase): + async def test_failed_attachment_is_rejected(self): + store, connection = PostgresStore('unused'), AsyncMock() + connection.fetchval.side_effect = [1, 1, 1] + @asynccontextmanager + async def context(): + yield connection + connection.transaction = context + store._pool = SimpleNamespace(acquire=context) + with self.assertRaises(ApiError) as error: + await store.create_message_idempotent(uuid4(), uuid4(), '', 'key', 86400, [uuid4()]) + self.assertEqual(error.exception.status_code, 409) + connection.fetchrow.assert_not_awaited() + + def test_floor_and_small_images(self): + self.assertEqual(variants(png(300, 200))[0][3:], (256, 170)) + self.assertEqual(variants(png(200, 300))[0][3:], (170, 256)) + self.assertEqual(variants(png(10, 8))[0][3:], (10, 8)) + for name, data, mime, width, height in variants(png(300, 200)): + with Image.open(BytesIO(data)) as image: + self.assertEqual(image.size, (width, height)) + self.assertTrue(all(value > 240 for value in image.convert('RGB').getpixel((0, 0)))) + + def test_renderer_success_and_permanent_error(self): + self.assertEqual(render_isolated(png(300, 200), 10)[0][3:], (256, 170)) + with self.assertRaises(PermanentFailure): + render_isolated(b'not an image', 10) + + def test_renderer_is_killed_on_deadline(self): + started = time.monotonic() + with self.assertRaises(TimeoutError): + render_isolated(png(300, 200), .0001) + self.assertLess(time.monotonic() - started, 2) + + async def test_cleanup_retains_active_and_failed_staging(self): + old = datetime.now(timezone.utc) - timedelta(days=2) + db, s3 = AsyncMock(), AsyncMock() + db.attachment_keys.return_value = {'staging/active': uuid4(), 'staging/failed': uuid4(), 'originals/ready': uuid4()} + s3.list_with_pages.side_effect = [({'staging/active': old, 'staging/failed': old, 'staging/orphan': old}, 1), + ({'originals/ready': old}, 1), ({'variants/orphan': old}, 1)] + report = await scan(SimpleNamespace(grace_seconds=60, apply=True, object_key=[]), db, s3) + self.assertEqual(report['deleted'], ['staging/orphan', 'variants/orphan']) + + async def test_permanent_failure_is_atomic_and_not_retried(self): + store = PostgresStore('unused') + connection = AsyncMock() + connection.fetchrow.return_value = {'attempt': 1} + @asynccontextmanager + async def context(): + yield connection + connection.transaction = context + store._pool = SimpleNamespace(acquire=context) + result = await store.retry_attachment_job(uuid4(), 5, 1, datetime.now(timezone.utc), 'invalid_image') + self.assertFalse(result) + self.assertEqual(connection.execute.await_count, 2) + for call in connection.execute.call_args_list: + self.assertIn("status='failed'", call.args[0]) + self.assertEqual(call.args[2], 'invalid_image') + + def test_benchmark_partial_batch_and_recovery(self): + spec = importlib.util.spec_from_file_location('track1', ROOT / 'scripts/track1.py') + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + jobs, lock = {}, threading.Lock() + class FakeClient: + def __init__(self, *args, **kwargs): pass + def bootstrap(self): return {}, {}, {}, {'id': str(uuid4())} + def call(self, method, path, *args, **kwargs): + if method == 'POST': + key = str(uuid4()) + with lock: jobs[key] = {'id': key, 'status': 'ready', 'queue_wait_seconds': .01} + return {'status': 202, 'json': {'id': key}} + return {'json': {'status': 'ready'}} + def attachment(self, *args): return {}, b'' + def wait_ready(self, *args, **kwargs): return {} + def metrics(_): + with lock: + return {'jobs': list(jobs.values()), 'pending': 0, 'rss_bytes': 1000, 'concurrency': 2, 'prefetch': 2} + with tempfile.TemporaryDirectory() as directory: + destination = str(Path(directory) / 'result.json') + args = SimpleNamespace(run_name='test', workers=2, prefetch=2, max_pending=20, images=10, + concurrency=8, size=8, max_ready_seconds=2, require_backpressure=False, + output=destination, base_url=None, ca_file=None) + info = {'Config': {'Env': ['MAX_PENDING_JOBS=20']}, 'HostConfig': {'Memory': 4096, 'NanoCpus': 2}} + with patch.object(module, 'arguments', return_value=args), patch.object(module, 'Client', FakeClient), \ + patch.object(module, 'metrics', side_effect=metrics), patch.object(module, 'container_info', return_value=info), \ + patch.object(module, 'broker_depth', return_value={'messages_ready': 0, 'messages_unacknowledged': 0}): + module.main() + report = json.loads(Path(destination).read_text()) + self.assertEqual(report['results']['ready'], 10) + self.assertTrue(report['results']['recovery_upload_ready']) + + +if __name__ == '__main__': + unittest.main() diff --git a/student-tests/review_regressions.py b/student-tests/review_regressions.py new file mode 100644 index 0000000..7e2bba7 --- /dev/null +++ b/student-tests/review_regressions.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 +"""Регрессии ревью: не-ASCII Bearer и невалидная кодировка JSON через LB.""" +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests")) +from client import Client, arguments, check, run + + +def main(): + args = arguments(__doc__) + client = Client(args.base_url, args.ca_file) + for path in ("/api/v1/users/me", "/api/v1/auth/sessions/current"): + method = "DELETE" if path.endswith("/current") else "GET" + result = client.call(method, path, 401, + headers={"Authorization": "Bearer é"}, schema="Error") + check(result["headers"].get("WWW-Authenticate", "").startswith("Bearer"), + "missing Bearer challenge") + check(result["json"]["error"]["code"] == "unauthorized", "wrong error code") + check(result["json"]["error"]["request_id"] == result["headers"]["X-Request-Id"], + "request IDs differ") + for raw in (b"\xff", b"{", b'{"username":'): + result = client.call("POST", "/api/v1/users", 400, raw=raw, + headers={"Content-Type": "application/json"}, schema="Error") + check(result["json"]["error"]["code"] == "invalid_request", "wrong JSON error code") + check(result["json"]["error"]["request_id"] == result["headers"]["X-Request-Id"], + "request IDs differ") + print("PASS: non-ASCII Bearer returns 401/challenge; malformed JSON returns contract Error/400") + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/s3_permissions.py b/student-tests/s3_permissions.py new file mode 100644 index 0000000..d67e7be --- /dev/null +++ b/student-tests/s3_permissions.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +"""Prove that the API identity is bucket-scoped and lacks admin operations.""" +import subprocess +from pathlib import Path +import sys +import uuid + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, arguments, check, run + + +def compose(*args, input=None): + return subprocess.check_output(["docker", "compose", *args], cwd=ROOT, text=True, + input=input, stderr=subprocess.DEVNULL) + + +def main(): + args = arguments(__doc__) + client = Client(args.base_url, args.ca_file) + alice, _bob, _eve, chat = client.bootstrap() + data = (ROOT / "tests/fixtures/sample.png").read_bytes() + client.call("POST", "/api/v1/chats/" + chat["id"] + "/attachments?filename=permission.png", 202, + actor=alice, raw=data, headers={"Content-Type": "image/png"}, schema="Attachment") + bucket = "permission-probe-" + uuid.uuid4().hex + code = """from botocore.exceptions import ClientError +from src.storage.s3 import S3Settings, S3Store +import sys +s3 = S3Store(S3Settings.load()) +try: + try: + s3.client.create_bucket(Bucket=sys.stdin.read().strip()) + except ClientError as error: + assert error.response['Error']['Code'] in {'AccessDenied', 'Unauthorized'}, error.response + else: + raise AssertionError('API S3 identity unexpectedly has admin bucket creation') +finally: + s3.client.close() +""" + compose("exec", "-T", "api-a", "/app/.venv/bin/python", "-c", code, input=bucket) + admin = """from src.storage.s3 import S3Settings, S3Store +import sys +s3 = S3Store(S3Settings.load()) +try: + s3.client.list_buckets() +finally: + s3.client.close() +""" + compose("run", "--rm", "--no-deps", "s3-init", "/app/.venv/bin/python", "-c", admin) + print("PASS: API identity cannot create buckets; maintenance identity has separate admin access") + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/session_expiry.py b/student-tests/session_expiry.py new file mode 100644 index 0000000..37e29fb --- /dev/null +++ b/student-tests/session_expiry.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 +"""Проверка expiry; запускается при SESSION_TTL_SECONDS=2.""" +from pathlib import Path +import sys +import time + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests")) +from client import Client, check, run # noqa: E402 + + +def main() -> None: + client = Client() + actor = client.user("expiry") + time.sleep(3) + client.call("GET", "/api/v1/users/me", 401, actor=actor) + print("PASS: expired Bearer session was rejected") + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/upload_memory.py b/student-tests/upload_memory.py new file mode 100644 index 0000000..1075a4e --- /dev/null +++ b/student-tests/upload_memory.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 +"""Ten concurrent bounded uploads, with API-container peak memory sampling.""" +from concurrent.futures import ThreadPoolExecutor +import json +import http.client +from pathlib import Path +import subprocess +import sys +import threading +import time +from urllib.parse import quote +from urllib.parse import urlsplit + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, arguments, check, run + + +def api_memory() -> dict[str, int]: + ids = subprocess.check_output(["docker", "compose", "ps", "-q", "api-a", "api-b"], cwd=ROOT, text=True).split() + return {container[:12]: int(subprocess.check_output( + ["docker", "exec", container, "cat", "/sys/fs/cgroup/memory.current"], text=True)) + for container in ids} + + +def main(): + def configure(parser): + parser.add_argument("--output", default="evidence/upload-memory.json") + parser.add_argument("--size-mib", type=int, default=5) + args = arguments(__doc__, configure) + check(1 <= args.size_mib <= 10, "--size-mib must be 1..10") + client = Client(args.base_url, args.ca_file, timeout=60) + alice, _bob, _eve, chat = client.bootstrap() + seed = (ROOT / "tests/fixtures/sample.png").read_bytes() + payload = seed + b"\0" * (args.size_mib * 1024 * 1024 - len(seed)) + baseline, peak, samples, stop = api_memory(), {}, [], threading.Event() + check(len(baseline) == 2, "memory experiment requires two running APIs") + sampling_errors = [] + timestamps = [] + barrier = threading.Barrier(10) + + def sampler(): + while not stop.is_set(): + try: + values = api_memory() + check(set(values) == set(baseline), "API containers changed during measurement") + except Exception as error: + sampling_errors.append(type(error).__name__) + return + timestamps.append(time.monotonic()) + samples.append(values) + for name, value in values.items(): + peak[name] = max(peak.get(name, 0), value) + time.sleep(.1) + + def upload(index): + barrier.wait(timeout=15) + parsed = urlsplit(client.base) + connection = http.client.HTTPConnection(parsed.hostname, parsed.port, timeout=60) + + def chunks(): + for offset in range(0, len(payload), 512 * 1024): + yield payload[offset:offset + 512 * 1024] + # Keep all ten client streams open long enough for multiple memory samples. + time.sleep(.5) + + connection.request("POST", "/api/v1/chats/" + chat["id"] + "/attachments?filename=" + quote(f"load-{index}.png"), + body=chunks(), encode_chunked=True, + headers={"Authorization": "Bearer " + alice["token"], "Content-Type": "image/png"}) + response = connection.getresponse() + body = response.read() + connection.close() + check(response.status == 201, "upload %s returned %s: %s" % (index, response.status, body[:200])) + return json.loads(body) + + started = time.monotonic() + thread = threading.Thread(target=sampler, daemon=True) + thread.start() + try: + with ThreadPoolExecutor(max_workers=10) as pool: + attachments = list(pool.map(upload, range(10))) + finally: + stop.set() + thread.join(timeout=5) + elapsed = time.monotonic() - started + check(len({item["id"] for item in attachments}) == 10, "uploads were not distinct") + check(not sampling_errors and len(samples) >= 2 and not thread.is_alive(), + "memory sampler failed or did not collect enough samples: " + repr(sampling_errors)) + report = { + "uploads": 10, "file_bytes": len(payload), "concurrency": 10, + "api_upload_concurrency_limit": 1, "elapsed_seconds": round(elapsed, 3), + "baseline_memory_bytes": baseline, "peak_memory_bytes": peak, + "peak_growth_bytes": {name: peak.get(name, 0) - baseline.get(name, 0) for name in baseline}, + "samples": len(samples), + "sample_intervals_seconds": [round(b-a, 3) for a, b in zip(timestamps, timestamps[1:])], + "method": "cgroup memory.current via Docker; 0.1s pause AFTER each poll, actual intervals include Docker latency; ten clients start at a barrier and send chunked 512 KiB blocks every 0.5s. Peak is sampled, not continuous.", + } + destination = ROOT / args.output + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n") + print(json.dumps(report, ensure_ascii=False)) + check(all(value <= 512 * 1024 * 1024 for value in peak.values()), "API exceeded configured memory limit") + + +if __name__ == "__main__": + run(main) diff --git a/uv.lock b/uv.lock new file mode 100644 index 0000000..6f5df98 --- /dev/null +++ b/uv.lock @@ -0,0 +1,596 @@ +version = 1 +revision = 3 +requires-python = ">=3.12" +resolution-markers = [ + "python_full_version >= '3.14'", + "python_full_version < '3.14'", +] + +[[package]] +name = "annotated-doc" +version = "0.0.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5a/8e/38aa427ed5402449e226975b649c5dc73ccadfefeb95e6aecb8f8ea4b6b6/annotated_doc-0.0.5.tar.gz", hash = "sha256:c7e58ce09192557605d8bbd92836d7e1d520ac9580096042c0bfd197efacf1bb", size = 10758, upload-time = "2026-07-28T13:50:58.129Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3e/30/e900b21425a860e195f32e37657aa1f7c7f2b1bfb26f03ca209b90933c06/annotated_doc-0.0.5-py3-none-any.whl", hash = "sha256:117bac03a25ede5df5440e855b32d556049ca169ead221505badf432fed4b101", size = 5302, upload-time = "2026-07-28T13:50:57.239Z" }, +] + +[[package]] +name = "annotated-types" +version = "0.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5f/56/a8120250d128bed162cd73c76d45f6ef9991f3e068f62a8ee060afa3104a/annotated_types-0.8.0.tar.gz", hash = "sha256:13b2beaad985e05e2d6407ee4c4f35590b11f8d693a258a561055cac8f64cab7", size = 15893, upload-time = "2026-07-23T20:16:13.995Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/99/91/8acff4f5e50511b911bbccb72b8628a49c68ce14148cd9f6431094859a90/annotated_types-0.8.0-py3-none-any.whl", hash = "sha256:f072f4d804ea359e4eaf198b1af7a8b0943881a87f31bb764f8bf219bb9419e0", size = 13427, upload-time = "2026-07-23T20:16:12.938Z" }, +] + +[[package]] +name = "anyio" +version = "4.15.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "idna" }, + { name = "typing-extensions", marker = "python_full_version < '3.15'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a9/d2/f4d173e22df740bc37b1db102b386ba719b66e95b0f0d751f556b387e6d2/anyio-4.15.1.tar.gz", hash = "sha256:9f28306018cbd6d329e64a36d58256edff76dd996fe423bc957326e578b82a94", size = 276966, upload-time = "2026-09-05T10:42:39.44Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/12/b8/4bd346e22b28902df4d651910f5242c28d84e4a5c2435ca5c3f797ed7e2e/anyio-4.15.1-py3-none-any.whl", hash = "sha256:6152fdbbf9a77fdec97731721bebf7c4c44f7c29b424b0065826173efc7ed101", size = 132079, upload-time = "2026-09-05T10:42:37.923Z" }, +] + +[[package]] +name = "argon2-cffi" +version = "25.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "argon2-cffi-bindings" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0e/89/ce5af8a7d472a67cc819d5d998aa8c82c5d860608c4db9f46f1162d7dab9/argon2_cffi-25.1.0.tar.gz", hash = "sha256:694ae5cc8a42f4c4e2bf2ca0e64e51e23a040c6a517a85074683d3959e1346c1", size = 45706, upload-time = "2025-06-03T06:55:32.073Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4f/d3/a8b22fa575b297cd6e3e3b0155c7e25db170edf1c74783d6a31a2490b8d9/argon2_cffi-25.1.0-py3-none-any.whl", hash = "sha256:fdc8b074db390fccb6eb4a3604ae7231f219aa669a2652e0f20e16ba513d5741", size = 14657, upload-time = "2025-06-03T06:55:30.804Z" }, +] + +[[package]] +name = "argon2-cffi-bindings" +version = "26.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0b/43/bb8b6e8708d49a5ab36781333af092d9f483b198a2710d01281204640055/argon2_cffi_bindings-26.1.0.tar.gz", hash = "sha256:63505c71542a44b68b1e38060450fb006404170da375feb31af153e7f9c6205d", size = 1790807, upload-time = "2026-08-20T07:44:22.492Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e7/d2/0ae991f1b2181e5be49007c574710a800ad36c2978683addb3e67c474e55/argon2_cffi_bindings-26.1.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:21ca0396fe5ec995dd54431c32698189666f9224810acfa752e50d2bd94d9df2", size = 25521, upload-time = "2026-08-20T07:32:43.019Z" }, + { url = "https://files.pythonhosted.org/packages/7e/e4/ad91d8297638aa2258aad4501c306aca99480dfe76ccd638173fa3702db9/argon2_cffi_bindings-26.1.0-cp310-abi3-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:78de2d65e0b9ea7ce9d1b1c3e87297b2d7305a02c266ee2a2d6910daddd7ee69", size = 27177, upload-time = "2026-08-20T07:32:44.158Z" }, + { url = "https://files.pythonhosted.org/packages/6f/86/5363df11b86d02cf3662208e7406496327649cc90eb365bf6f4e8a54a41f/argon2_cffi_bindings-26.1.0-cp310-abi3-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:27f1821903e2ceadcb88ec2b45ef190897b7682449c772f4d9b53e42c520cf29", size = 26597, upload-time = "2026-08-20T07:32:45.172Z" }, + { url = "https://files.pythonhosted.org/packages/f4/b5/a14dcc592652347dad23ee93b278a4da5d2a25c9ed3ebd10d68eea823a4f/argon2_cffi_bindings-26.1.0-cp310-abi3-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d88e5f7e60f28ae0b0cc6b2f16c43e87cd642a196a86f85e0d8bb6fe016fc16d", size = 27403, upload-time = "2026-08-20T07:32:46.13Z" }, + { url = "https://files.pythonhosted.org/packages/b3/81/b4a20d4902af7f796390bf9245ff83c5217dfa7367efa1d14986956c482b/argon2_cffi_bindings-26.1.0-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:34b7d9c24a4165a2c61cc8ae11d44d48c9ce2830fb536cb7914e11fdd9962728", size = 27132, upload-time = "2026-08-20T07:32:47.13Z" }, + { url = "https://files.pythonhosted.org/packages/7e/1b/c8de358af07b1c490e0fcb863ef98e46ddb486e45567aca5a60bd68d9daa/argon2_cffi_bindings-26.1.0-cp310-abi3-musllinux_1_2_riscv64.whl", hash = "sha256:224865cbbcb7a2bd1356741dff12b0134df726b6d44bb7b500df8e303cbd9e81", size = 27588, upload-time = "2026-08-20T07:32:48.087Z" }, + { url = "https://files.pythonhosted.org/packages/48/2f/7ee62a6e79f9309f9d9982d301b22a00010adb580c05c8109b94d7b33de0/argon2_cffi_bindings-26.1.0-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:ffff613aaa9ce6236766e2fc6dc560bb5abde7a2e2416e3db1f9ae395a2b4dd4", size = 26785, upload-time = "2026-08-20T07:32:48.977Z" }, + { url = "https://files.pythonhosted.org/packages/e9/10/960d0ee93d4897741bcaf4799c697dae2d81499f66fd1ed042a7dd54c1f4/argon2_cffi_bindings-26.1.0-cp310-abi3-win32.whl", hash = "sha256:a86c069c91a747a2c4e5c51473590aeb48172fff9b2130d23729a42d98665ecb", size = 23898, upload-time = "2026-08-20T07:32:50.114Z" }, + { url = "https://files.pythonhosted.org/packages/6d/3a/0cc14a05810e6add9bce5e87693334baa2222de5f647fa31781885b6573f/argon2_cffi_bindings-26.1.0-cp310-abi3-win_amd64.whl", hash = "sha256:2c36ff87b5dfaa477d0bd51e9d7f6abdae7c8955d2983c97419085d842154b3e", size = 25730, upload-time = "2026-08-20T07:32:51.091Z" }, + { url = "https://files.pythonhosted.org/packages/4e/db/d83cf2af140547f0b9cdaece05b2dc2dcbf991be4667331d073eff771435/argon2_cffi_bindings-26.1.0-cp310-abi3-win_arm64.whl", hash = "sha256:f9c4420a7a864fe1b86ce35befc95b8e39fb852493b81cf798671ddc265de638", size = 24478, upload-time = "2026-08-20T07:32:52.111Z" }, + { url = "https://files.pythonhosted.org/packages/bb/5f/f652055e18d2627e2eed94c7f31a792127cfe38df786635395d742321674/argon2_cffi_bindings-26.1.0-cp313-cp313-pyemscripten_2025_0_wasm32.whl", hash = "sha256:af11ac37a7c53dc16cb7950a6190851b0870fe218b6c60c0bb7ac355234e3083", size = 15434, upload-time = "2026-08-20T07:32:53.143Z" }, + { url = "https://files.pythonhosted.org/packages/76/38/de696045960f5b846d428c0fb6c130ed3da87aac2af209b05c193815404c/argon2_cffi_bindings-26.1.0-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:db0fcd827ca61622a01b220aadfbece01939acf53888f2cb98cd93e9b1e2c97e", size = 15449, upload-time = "2026-08-20T07:32:54.075Z" }, + { url = "https://files.pythonhosted.org/packages/91/0a/c25af768f6b75a5a71e31207f87c540656b2808c015260444a22763221ad/argon2_cffi_bindings-26.1.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:28524438cd3e723f25412f63d4fd516ff5bae9ae5aa56acbe2a1404398a0cf31", size = 25683, upload-time = "2026-08-20T07:32:55.05Z" }, + { url = "https://files.pythonhosted.org/packages/a8/7e/be212c751ab0bcea7f646615f933bf262e8e50b3f7bef32f861d0a2d066b/argon2_cffi_bindings-26.1.0-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ac82fc756a446b6ccd7139ce70efa9d8bbe541e7ad579a12dcb52764b7175c5f", size = 27311, upload-time = "2026-08-20T07:32:56.166Z" }, + { url = "https://files.pythonhosted.org/packages/a6/ee/f84b28e4afd13d3cac36c1d8fa8c239d2dc2c51cd978d02ee5d5ad98d9bb/argon2_cffi_bindings-26.1.0-cp314-cp314t-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6a4e68eed961a8de6928d1c17ff3dc2a547e0e923c17f8f1cd79fb7bc9502f98", size = 26771, upload-time = "2026-08-20T07:32:57.206Z" }, + { url = "https://files.pythonhosted.org/packages/21/c3/95c07a023691ecd529da9cb6a8f0779e13ebc1bdfaa86d145fdc1c6e7e79/argon2_cffi_bindings-26.1.0-cp314-cp314t-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:151dfaad9de753f4af2a7854e707e4784f2acc434340ade64239c5b104b2d605", size = 27568, upload-time = "2026-08-20T07:32:58.361Z" }, + { url = "https://files.pythonhosted.org/packages/e6/31/3a18e31406d8694b4d6a31573c3e572fff6bed318bb744453eb653766d22/argon2_cffi_bindings-26.1.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:061a6919145bbf282ebf1f9c59d3135d4833c25313c8595c0d68cf7712ddfce2", size = 27280, upload-time = "2026-08-20T07:32:59.343Z" }, + { url = "https://files.pythonhosted.org/packages/0b/39/d4be4577e178b2397aa5b5575c8a309bf0da2afe05fe0c72c8f398662d63/argon2_cffi_bindings-26.1.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:62ff20cd130c956c7c9144d5fe35228f98b51c579b2439e988b27ef93e16c02a", size = 27776, upload-time = "2026-08-20T07:33:00.325Z" }, + { url = "https://files.pythonhosted.org/packages/71/47/78f4dd96f7411339f723b96fe24039c1bd5835102b8a5ba71ac4ec712ac7/argon2_cffi_bindings-26.1.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:19423e5d7ac1cc354baab59eaabf18db2ec04ef6593b5abe5a34f323c4a8f87a", size = 26932, upload-time = "2026-08-20T07:33:01.272Z" }, + { url = "https://files.pythonhosted.org/packages/3b/cd/96bfd37434cc0a848a9066c291d84b28846c4c9ea289ed9866b1164d622b/argon2_cffi_bindings-26.1.0-cp314-cp314t-win32.whl", hash = "sha256:4f84cdd868978d7b7350a566c254042d44216d9e37f241f3a6d3b1dfebeede35", size = 24878, upload-time = "2026-08-20T07:33:02.189Z" }, + { url = "https://files.pythonhosted.org/packages/f1/42/d8b6810abd9b1bd2f47ebbccf460da59c9f32e94888bea4f7b137d998797/argon2_cffi_bindings-26.1.0-cp314-cp314t-win_amd64.whl", hash = "sha256:2b741888c93147444fdfc851abd81cc207f37f7f7da42062a00deb3888e57da8", size = 26656, upload-time = "2026-08-20T07:33:03.222Z" }, + { url = "https://files.pythonhosted.org/packages/a9/d1/095d95eaf2ed1d9f77268cf3291bde148c6cd56121f8db2c74c1ba618a0e/argon2_cffi_bindings-26.1.0-cp314-cp314t-win_arm64.whl", hash = "sha256:6ab674f668d5962a3a4136ae0812519b0f1586874263723a32181d60d64137e1", size = 25378, upload-time = "2026-08-20T07:33:04.332Z" }, + { url = "https://files.pythonhosted.org/packages/66/cb/214092c39c4dbcb72cf98b12234ddac2221f8fe2c0acf29c6a70fa83be53/argon2_cffi_bindings-26.1.0-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:1d98e33bd8bd67d7206c124e200bf2229c4cfa8c9c19f7b44a897f0fc71837eb", size = 25683, upload-time = "2026-08-20T07:33:05.337Z" }, + { url = "https://files.pythonhosted.org/packages/83/e5/02015b83e9b05ccb85ff2ced424cf6e83a12d3810bc7f66d679a92b69ffb/argon2_cffi_bindings-26.1.0-cp315-cp315t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ccaf0a46cbb380f1fd102a874e32aa629fd3cb0c0e94f4943fa1f6d5edc5dac6", size = 27310, upload-time = "2026-08-20T07:33:06.344Z" }, + { url = "https://files.pythonhosted.org/packages/c3/4a/85e612787d0796878b3b4f6bd53dcd5484b6fe7b64cc6fc7b6e6a04cf835/argon2_cffi_bindings-26.1.0-cp315-cp315t-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f0c3103fcff20183e593459cfea6e012281c0e76ae3ed8b5565ad1b92eac3990", size = 26771, upload-time = "2026-08-20T07:33:07.429Z" }, + { url = "https://files.pythonhosted.org/packages/f6/84/ccb003b6f9969820e87656398f4d49c857def71a85ca1588a0e809afd7ce/argon2_cffi_bindings-26.1.0-cp315-cp315t-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c49e853a3bef9dd10329f31f702e7fa9b5c58229ff9c2ff6d069efaf09177c08", size = 27569, upload-time = "2026-08-20T07:33:08.598Z" }, + { url = "https://files.pythonhosted.org/packages/88/07/c26b76debf0998ee08fbe947ab2058ac5de37d4b9d46b06c17abaa6c4ce9/argon2_cffi_bindings-26.1.0-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:6376d4b3aca039375ca8bf92f770da0ec424a1ce3a37077a8d3c557411aa56ca", size = 27279, upload-time = "2026-08-20T07:33:09.518Z" }, + { url = "https://files.pythonhosted.org/packages/ee/0d/ead6ddc029f91bc9b9390686dad3c808ab08100d348f6266b5f93f8970ee/argon2_cffi_bindings-26.1.0-cp315-cp315t-musllinux_1_2_riscv64.whl", hash = "sha256:9bacedc04b0402837586a17f0919e3dfdd95291f441f1f56bd80ec274c2840a1", size = 27774, upload-time = "2026-08-20T07:33:10.728Z" }, + { url = "https://files.pythonhosted.org/packages/7d/47/c108530d9eb86036b78d3af4de28b83b4a2d9a70512bd10ff8e59966aab4/argon2_cffi_bindings-26.1.0-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:76ae29acace5d33355344612844d588e19deaaba4639d8bb01601e4b1418ef36", size = 26933, upload-time = "2026-08-20T07:33:11.661Z" }, + { url = "https://files.pythonhosted.org/packages/a9/02/0bfc59e781c89acf64c31c388aade9d9d1c1ea38aa1ba1292fe07f607fe9/argon2_cffi_bindings-26.1.0-cp315-cp315t-win32.whl", hash = "sha256:df612391feca41c44d20118f3b88d1b86419465cd1f5496859f715ca60ec2210", size = 24875, upload-time = "2026-08-20T07:33:12.616Z" }, + { url = "https://files.pythonhosted.org/packages/61/c7/c3e46068cddffccecb8ad94d71135e9bf62bbc789589e7dfadc7c6f59214/argon2_cffi_bindings-26.1.0-cp315-cp315t-win_amd64.whl", hash = "sha256:1a0a29ed86960e44eaace7e081bdfab4f08b012fd96ec8edba71e2ad020939e4", size = 26655, upload-time = "2026-08-20T07:33:13.521Z" }, + { url = "https://files.pythonhosted.org/packages/f4/ca/18b9c8c45fecf34b9100ec6d7946057f14a158f2eaa20ea123a3e82351cb/argon2_cffi_bindings-26.1.0-cp315-cp315t-win_arm64.whl", hash = "sha256:d157ddfab1e8b21f2f1dedda9c09645d98b5ed0b667b0626be600a345d426440", size = 25376, upload-time = "2026-08-20T07:33:14.491Z" }, +] + +[[package]] +name = "asyncpg" +version = "0.31.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fe/cc/d18065ce2380d80b1bcce927c24a2642efd38918e33fd724bc4bca904877/asyncpg-0.31.0.tar.gz", hash = "sha256:c989386c83940bfbd787180f2b1519415e2d3d6277a70d9d0f0145ac73500735", size = 993667, upload-time = "2025-11-24T23:27:00.812Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/a6/59d0a146e61d20e18db7396583242e32e0f120693b67a8de43f1557033e2/asyncpg-0.31.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b44c31e1efc1c15188ef183f287c728e2046abb1d26af4d20858215d50d91fad", size = 662042, upload-time = "2025-11-24T23:25:49.578Z" }, + { url = "https://files.pythonhosted.org/packages/36/01/ffaa189dcb63a2471720615e60185c3f6327716fdc0fc04334436fbb7c65/asyncpg-0.31.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0c89ccf741c067614c9b5fc7f1fc6f3b61ab05ae4aaa966e6fd6b93097c7d20d", size = 638504, upload-time = "2025-11-24T23:25:51.501Z" }, + { url = "https://files.pythonhosted.org/packages/9f/62/3f699ba45d8bd24c5d65392190d19656d74ff0185f42e19d0bbd973bb371/asyncpg-0.31.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:12b3b2e39dc5470abd5e98c8d3373e4b1d1234d9fbdedf538798b2c13c64460a", size = 3426241, upload-time = "2025-11-24T23:25:53.278Z" }, + { url = "https://files.pythonhosted.org/packages/8c/d1/a867c2150f9c6e7af6462637f613ba67f78a314b00db220cd26ff559d532/asyncpg-0.31.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:aad7a33913fb8bcb5454313377cc330fbb19a0cd5faa7272407d8a0c4257b671", size = 3520321, upload-time = "2025-11-24T23:25:54.982Z" }, + { url = "https://files.pythonhosted.org/packages/7a/1a/cce4c3f246805ecd285a3591222a2611141f1669d002163abef999b60f98/asyncpg-0.31.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3df118d94f46d85b2e434fd62c84cb66d5834d5a890725fe625f498e72e4d5ec", size = 3316685, upload-time = "2025-11-24T23:25:57.43Z" }, + { url = "https://files.pythonhosted.org/packages/40/ae/0fc961179e78cc579e138fad6eb580448ecae64908f95b8cb8ee2f241f67/asyncpg-0.31.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:bd5b6efff3c17c3202d4b37189969acf8927438a238c6257f66be3c426beba20", size = 3471858, upload-time = "2025-11-24T23:25:59.636Z" }, + { url = "https://files.pythonhosted.org/packages/52/b2/b20e09670be031afa4cbfabd645caece7f85ec62d69c312239de568e058e/asyncpg-0.31.0-cp312-cp312-win32.whl", hash = "sha256:027eaa61361ec735926566f995d959ade4796f6a49d3bde17e5134b9964f9ba8", size = 527852, upload-time = "2025-11-24T23:26:01.084Z" }, + { url = "https://files.pythonhosted.org/packages/b5/f0/f2ed1de154e15b107dc692262395b3c17fc34eafe2a78fc2115931561730/asyncpg-0.31.0-cp312-cp312-win_amd64.whl", hash = "sha256:72d6bdcbc93d608a1158f17932de2321f68b1a967a13e014998db87a72ed3186", size = 597175, upload-time = "2025-11-24T23:26:02.564Z" }, + { url = "https://files.pythonhosted.org/packages/95/11/97b5c2af72a5d0b9bc3fa30cd4b9ce22284a9a943a150fdc768763caf035/asyncpg-0.31.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c204fab1b91e08b0f47e90a75d1b3c62174dab21f670ad6c5d0f243a228f015b", size = 661111, upload-time = "2025-11-24T23:26:04.467Z" }, + { url = "https://files.pythonhosted.org/packages/1b/71/157d611c791a5e2d0423f09f027bd499935f0906e0c2a416ce712ba51ef3/asyncpg-0.31.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:54a64f91839ba59008eccf7aad2e93d6e3de688d796f35803235ea1c4898ae1e", size = 636928, upload-time = "2025-11-24T23:26:05.944Z" }, + { url = "https://files.pythonhosted.org/packages/2e/fc/9e3486fb2bbe69d4a867c0b76d68542650a7ff1574ca40e84c3111bb0c6e/asyncpg-0.31.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c0e0822b1038dc7253b337b0f3f676cadc4ac31b126c5d42691c39691962e403", size = 3424067, upload-time = "2025-11-24T23:26:07.957Z" }, + { url = "https://files.pythonhosted.org/packages/12/c6/8c9d076f73f07f995013c791e018a1cd5f31823c2a3187fc8581706aa00f/asyncpg-0.31.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bef056aa502ee34204c161c72ca1f3c274917596877f825968368b2c33f585f4", size = 3518156, upload-time = "2025-11-24T23:26:09.591Z" }, + { url = "https://files.pythonhosted.org/packages/ae/3b/60683a0baf50fbc546499cfb53132cb6835b92b529a05f6a81471ab60d0c/asyncpg-0.31.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:0bfbcc5b7ffcd9b75ab1558f00db2ae07db9c80637ad1b2469c43df79d7a5ae2", size = 3319636, upload-time = "2025-11-24T23:26:11.168Z" }, + { url = "https://files.pythonhosted.org/packages/50/dc/8487df0f69bd398a61e1792b3cba0e47477f214eff085ba0efa7eac9ce87/asyncpg-0.31.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:22bc525ebbdc24d1261ecbf6f504998244d4e3be1721784b5f64664d61fbe602", size = 3472079, upload-time = "2025-11-24T23:26:13.164Z" }, + { url = "https://files.pythonhosted.org/packages/13/a1/c5bbeeb8531c05c89135cb8b28575ac2fac618bcb60119ee9696c3faf71c/asyncpg-0.31.0-cp313-cp313-win32.whl", hash = "sha256:f890de5e1e4f7e14023619399a471ce4b71f5418cd67a51853b9910fdfa73696", size = 527606, upload-time = "2025-11-24T23:26:14.78Z" }, + { url = "https://files.pythonhosted.org/packages/91/66/b25ccb84a246b470eb943b0107c07edcae51804912b824054b3413995a10/asyncpg-0.31.0-cp313-cp313-win_amd64.whl", hash = "sha256:dc5f2fa9916f292e5c5c8b2ac2813763bcd7f58e130055b4ad8a0531314201ab", size = 596569, upload-time = "2025-11-24T23:26:16.189Z" }, + { url = "https://files.pythonhosted.org/packages/3c/36/e9450d62e84a13aea6580c83a47a437f26c7ca6fa0f0fd40b6670793ea30/asyncpg-0.31.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:f6b56b91bb0ffc328c4e3ed113136cddd9deefdf5f79ab448598b9772831df44", size = 660867, upload-time = "2025-11-24T23:26:17.631Z" }, + { url = "https://files.pythonhosted.org/packages/82/4b/1d0a2b33b3102d210439338e1beea616a6122267c0df459ff0265cd5807a/asyncpg-0.31.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:334dec28cf20d7f5bb9e45b39546ddf247f8042a690bff9b9573d00086e69cb5", size = 638349, upload-time = "2025-11-24T23:26:19.689Z" }, + { url = "https://files.pythonhosted.org/packages/41/aa/e7f7ac9a7974f08eff9183e392b2d62516f90412686532d27e196c0f0eeb/asyncpg-0.31.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:98cc158c53f46de7bb677fd20c417e264fc02b36d901cc2a43bd6cb0dc6dbfd2", size = 3410428, upload-time = "2025-11-24T23:26:21.275Z" }, + { url = "https://files.pythonhosted.org/packages/6f/de/bf1b60de3dede5c2731e6788617a512bc0ebd9693eac297ee74086f101d7/asyncpg-0.31.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9322b563e2661a52e3cdbc93eed3be7748b289f792e0011cb2720d278b366ce2", size = 3471678, upload-time = "2025-11-24T23:26:23.627Z" }, + { url = "https://files.pythonhosted.org/packages/46/78/fc3ade003e22d8bd53aaf8f75f4be48f0b460fa73738f0391b9c856a9147/asyncpg-0.31.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:19857a358fc811d82227449b7ca40afb46e75b33eb8897240c3839dd8b744218", size = 3313505, upload-time = "2025-11-24T23:26:25.235Z" }, + { url = "https://files.pythonhosted.org/packages/bf/e9/73eb8a6789e927816f4705291be21f2225687bfa97321e40cd23055e903a/asyncpg-0.31.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ba5f8886e850882ff2c2ace5732300e99193823e8107e2c53ef01c1ebfa1e85d", size = 3434744, upload-time = "2025-11-24T23:26:26.944Z" }, + { url = "https://files.pythonhosted.org/packages/08/4b/f10b880534413c65c5b5862f79b8e81553a8f364e5238832ad4c0af71b7f/asyncpg-0.31.0-cp314-cp314-win32.whl", hash = "sha256:cea3a0b2a14f95834cee29432e4ddc399b95700eb1d51bbc5bfee8f31fa07b2b", size = 532251, upload-time = "2025-11-24T23:26:28.404Z" }, + { url = "https://files.pythonhosted.org/packages/d3/2d/7aa40750b7a19efa5d66e67fc06008ca0f27ba1bd082e457ad82f59aba49/asyncpg-0.31.0-cp314-cp314-win_amd64.whl", hash = "sha256:04d19392716af6b029411a0264d92093b6e5e8285ae97a39957b9a9c14ea72be", size = 604901, upload-time = "2025-11-24T23:26:30.34Z" }, + { url = "https://files.pythonhosted.org/packages/ce/fe/b9dfe349b83b9dee28cc42360d2c86b2cdce4cb551a2c2d27e156bcac84d/asyncpg-0.31.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:bdb957706da132e982cc6856bb2f7b740603472b54c3ebc77fe60ea3e57e1bd2", size = 702280, upload-time = "2025-11-24T23:26:32Z" }, + { url = "https://files.pythonhosted.org/packages/6a/81/e6be6e37e560bd91e6c23ea8a6138a04fd057b08cf63d3c5055c98e81c1d/asyncpg-0.31.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6d11b198111a72f47154fa03b85799f9be63701e068b43f84ac25da0bda9cb31", size = 682931, upload-time = "2025-11-24T23:26:33.572Z" }, + { url = "https://files.pythonhosted.org/packages/a6/45/6009040da85a1648dd5bc75b3b0a062081c483e75a1a29041ae63a0bf0dc/asyncpg-0.31.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:18c83b03bc0d1b23e6230f5bf8d4f217dc9bc08644ce0502a9d91dc9e634a9c7", size = 3581608, upload-time = "2025-11-24T23:26:35.638Z" }, + { url = "https://files.pythonhosted.org/packages/7e/06/2e3d4d7608b0b2b3adbee0d0bd6a2d29ca0fc4d8a78f8277df04e2d1fd7b/asyncpg-0.31.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e009abc333464ff18b8f6fd146addffd9aaf63e79aa3bb40ab7a4c332d0c5e9e", size = 3498738, upload-time = "2025-11-24T23:26:37.275Z" }, + { url = "https://files.pythonhosted.org/packages/7d/aa/7d75ede780033141c51d83577ea23236ba7d3a23593929b32b49db8ed36e/asyncpg-0.31.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:3b1fbcb0e396a5ca435a8826a87e5c2c2cc0c8c68eb6fadf82168056b0e53a8c", size = 3401026, upload-time = "2025-11-24T23:26:39.423Z" }, + { url = "https://files.pythonhosted.org/packages/ba/7a/15e37d45e7f7c94facc1e9148c0e455e8f33c08f0b8a0b1deb2c5171771b/asyncpg-0.31.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:8df714dba348efcc162d2adf02d213e5fab1bd9f557e1305633e851a61814a7a", size = 3429426, upload-time = "2025-11-24T23:26:41.032Z" }, + { url = "https://files.pythonhosted.org/packages/13/d5/71437c5f6ae5f307828710efbe62163974e71237d5d46ebd2869ea052d10/asyncpg-0.31.0-cp314-cp314t-win32.whl", hash = "sha256:1b41f1afb1033f2b44f3234993b15096ddc9cd71b21a42dbd87fc6a57b43d65d", size = 614495, upload-time = "2025-11-24T23:26:42.659Z" }, + { url = "https://files.pythonhosted.org/packages/3c/d7/8fb3044eaef08a310acfe23dae9a8e2e07d305edc29a53497e52bc76eca7/asyncpg-0.31.0-cp314-cp314t-win_amd64.whl", hash = "sha256:bd4107bb7cdd0e9e65fae66a62afd3a249663b844fa34d479f6d5b3bef9c04c3", size = 706062, upload-time = "2025-11-24T23:26:44.086Z" }, +] + +[[package]] +name = "boto3" +version = "1.43.115" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore" }, + { name = "jmespath" }, + { name = "s3transfer" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0e/4a/b0ce191492a1724be73d8aacaf70d508bc8cbb9674c046d2ff7181dc51fb/boto3-1.43.115.tar.gz", hash = "sha256:c0f35e5f9cdf0edb365195e306d5586d3970b224660a11a87a4ec361ec3d7168", size = 112677, upload-time = "2026-10-19T19:19:53.093Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/be/bc/40dffae3238ef8d3415393132751fe2c88e8663ef84a31f0950e439ed2d5/boto3-1.43.115-py3-none-any.whl", hash = "sha256:6667a0c21d48dc125dbba88444bc8a601ce881a99341a0e6169f9333898e620d", size = 140044, upload-time = "2026-10-19T19:19:51.444Z" }, +] + +[[package]] +name = "botocore" +version = "1.43.115" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jmespath" }, + { name = "python-dateutil" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6f/10/ca525e88db50f5153ddad2a3d55908b5ace915d10f443d5e5135ba808d8d/botocore-1.43.115.tar.gz", hash = "sha256:aa4d9c4e7ef99988b7347e0cd276ba84e1eb1625510557b0e36cd57ecb19bbf2", size = 16383796, upload-time = "2026-10-19T19:19:45.382Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/77/2d/4b6b6f5704d1ec9b412c1dd9700bda0c315619dbac4a21418180a81a2780/botocore-1.43.115-py3-none-any.whl", hash = "sha256:9ceda640c868bc15f5399129e8cfdef8cbb4b350e8a99547c9b5f477cfb9619a", size = 16085101, upload-time = "2026-10-19T19:19:42.203Z" }, +] + +[[package]] +name = "cffi" +version = "2.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pycparser", marker = "implementation_name != 'PyPy'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9e/ef/008a1939e372c06329a3fce4279c02f328488f3526744906eeec3da7ad5f/cffi-2.1.1.tar.gz", hash = "sha256:dd31f52ea1086513bb9df30f8fcee9b8918323ae067a3d5b78bc826a000712be", size = 530807, upload-time = "2026-08-03T21:21:18.939Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/10/69/43965eccfdead3b9220015fd1320e117be8c6ed01a62ffab76eeb752f5d5/cffi-2.1.1-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:c8c69575568085ba0b1b10c0249d779a214aea6f6522e949a0fc9fb0fcb449d0", size = 184821, upload-time = "2026-08-03T21:19:44.887Z" }, + { url = "https://files.pythonhosted.org/packages/54/7d/16e5a096677b5e313ca80cd5e5170efa3ea44624a82bb111925522da64b1/cffi-2.1.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f81b3b8f3d4e343550fa4baa0e479bba9f2d29ce9c2e9b51d1ce1718d7442fcf", size = 184719, upload-time = "2026-08-03T21:19:46.129Z" }, + { url = "https://files.pythonhosted.org/packages/56/e6/8941622732edec876dd17d0453dce07317ae96db34f2ec1436c9d3785986/cffi-2.1.1-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:811bd1e21d32de12efca32393a0ab3f5133b54fce9bd44b8bd77ab07da14bf6a", size = 214799, upload-time = "2026-08-03T21:19:47.218Z" }, + { url = "https://files.pythonhosted.org/packages/44/de/f98430906df1545ffde0d543dd124a7a439bc2cd32b36b9c53f805df7333/cffi-2.1.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:68e62fe11f30d5ca8289242866f0a5291402d8529ca2178ab8afc5c9694ae890", size = 222389, upload-time = "2026-08-03T21:19:48.331Z" }, + { url = "https://files.pythonhosted.org/packages/6a/5b/717f1526b9957b34456313c31645c5b82b8fb5c3fe9e4752999be7128bfc/cffi-2.1.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:4a7c934f7360e8cd64fe9efadcbd10c7c6364f531e432b9a4bf5ccbc9e0e8b50", size = 210249, upload-time = "2026-08-03T21:19:49.543Z" }, + { url = "https://files.pythonhosted.org/packages/64/b3/f8aa4f3e34986c7e4ec45072d1b1b9dd295b6b18007b45518d79726dd725/cffi-2.1.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:3143d81e29e1e20a9ce10901ec369012947876596f75a222235965f2b7ae832e", size = 208775, upload-time = "2026-08-03T21:19:50.918Z" }, + { url = "https://files.pythonhosted.org/packages/b1/db/dceb9dd5b231e1da801793f8acc9f3c52a7e1afe40bb1aae37e02b0faad5/cffi-2.1.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c1453022f490d2459a11819d83ad1d586e9ff65a12ac3e705ffebd46d3685dcf", size = 221822, upload-time = "2026-08-03T21:19:52.054Z" }, + { url = "https://files.pythonhosted.org/packages/a0/d2/6cd24ae3be000a634109c247d1475d62e5616d0dc78c82770942ec384248/cffi-2.1.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:208f941bb9d18e768138677f0a6d2ce01f590df56043dda1df1535ac57c88517", size = 225232, upload-time = "2026-08-03T21:19:53.109Z" }, + { url = "https://files.pythonhosted.org/packages/cb/52/3fa190537004dd7f0ab860a6dc7c0175b8667f68d1e618a46f5498d30250/cffi-2.1.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:210019b6c7cf07f081b4c54635c8cf744377001350e29cc0f81c4377b4797735", size = 223597, upload-time = "2026-08-03T21:19:54.515Z" }, + { url = "https://files.pythonhosted.org/packages/80/fb/0bb75b7039588c074b37ae99f40d9bfddf990ecb2fbc346ebccd2e56b9be/cffi-2.1.1-cp312-cp312-win32.whl", hash = "sha256:046bfc24911b37851ee1b51aab8bffe713d89c68c6a057b09484ce9fd5f69b4e", size = 175292, upload-time = "2026-08-03T21:19:55.566Z" }, + { url = "https://files.pythonhosted.org/packages/d9/79/615cc094e2fb508cade7de88d3b4f6c4ec2bab695c97bce9153dc65aadf5/cffi-2.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:f53e442b08449d42821fa4a4fba000095af9f62742a500f978a9f557ec44339a", size = 185919, upload-time = "2026-08-03T21:19:56.89Z" }, + { url = "https://files.pythonhosted.org/packages/70/c6/d0ea84713fe46b243a436a18fcd47d639732747e21635c8a27191b06dc30/cffi-2.1.1-cp312-cp312-win_arm64.whl", hash = "sha256:7bde5e4cc5c10140859842b9d383af292b22639a4dffb725314baf45968cef80", size = 180093, upload-time = "2026-08-03T21:19:58.155Z" }, + { url = "https://files.pythonhosted.org/packages/9d/f4/035513d4117049066b4779dc3b7c0c0fdad175fa13731c9f4003f1cd1478/cffi-2.1.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:b5bdfd1c873d4e093aabc0ca84c4ca6dbc4f752afb5c86f146d9742580c9da2e", size = 194248, upload-time = "2026-08-03T21:19:59.399Z" }, + { url = "https://files.pythonhosted.org/packages/76/af/2aeb4dbb5fc41a04161ae9ff1518de7cec08e164f44a8ce6a4cf7fd2cd1d/cffi-2.1.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:31348097ff5bbe827ccc41795d4dd099d9f0625e7def00ee653c137a490c2a6c", size = 196908, upload-time = "2026-08-03T21:20:00.746Z" }, + { url = "https://files.pythonhosted.org/packages/a7/46/2e5fdde8555706dd98139a910ca11be02809f3f605ce956f655d0214e100/cffi-2.1.1-cp313-cp313-macosx_10_15_x86_64.whl", hash = "sha256:9d2055050ea716bd38b7f7f1579c275386646b4894c155a3e2f3cd62ed41b7c6", size = 184805, upload-time = "2026-08-03T21:20:02.02Z" }, + { url = "https://files.pythonhosted.org/packages/55/41/4c7042f317b9217502988f0873af87e16ad606dc20f84e546e3e6ce9764c/cffi-2.1.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:19ee6127ee34de7d83ce3d371ebc5ed91addbdcc39f9ab15ce4eb35a4e534971", size = 184764, upload-time = "2026-08-03T21:20:03.141Z" }, + { url = "https://files.pythonhosted.org/packages/43/1f/1c3d90d91811c8f86ced9ed637956c54bfe5b79ca98fe976d7f8c8979f6b/cffi-2.1.1-cp313-cp313-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:6a8dddef476fab96d066d578fc88526767b836ab5ab21754e1d5bf3879c31c7c", size = 214722, upload-time = "2026-08-03T21:20:04.377Z" }, + { url = "https://files.pythonhosted.org/packages/37/6f/3b5ce4c3b2192d250f04908f2bfd91ef34552ec8f7716a5d4abdb8d67bb2/cffi-2.1.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f16c709686a78c727bbbf059f92b0bf41c6fc60deec706d2dc19f529175a6125", size = 222369, upload-time = "2026-08-03T21:20:05.544Z" }, + { url = "https://files.pythonhosted.org/packages/02/10/4b3c75dde3d9663c9e02ba05c2668b954f671d4bbe346413ca8c696b295a/cffi-2.1.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:fcd22650c908d7b7da162bbfaab594a1227a15d1643a98c68b122ac642fa2264", size = 210175, upload-time = "2026-08-03T21:20:06.75Z" }, + { url = "https://files.pythonhosted.org/packages/df/62/14f74b9543e605d17701dc797b815958b8bb70b7624ce1b832ddad48ed6c/cffi-2.1.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:aa9511c62d14da7aacc9b4bf51f3f697a621e83b2d6919008243c3aad168eea3", size = 208670, upload-time = "2026-08-03T21:20:08.04Z" }, + { url = "https://files.pythonhosted.org/packages/95/95/86342356ff5953b3fb06f7ef7c5bee212d45e770abc7218d451b9148313c/cffi-2.1.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a931079504ecc49efed7744c476a5c343a92fabf66dec2db95edb1b2fdc770e2", size = 221824, upload-time = "2026-08-03T21:20:09.274Z" }, + { url = "https://files.pythonhosted.org/packages/eb/ff/7b3429ff53aafe931ed8a5fc69f481bbef7ba6de87ddcbb63d08f483f613/cffi-2.1.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a2d7755bef5a12ed488f4ef1f1b69ee9191d7396083b755a5d2295f6edb4768b", size = 225148, upload-time = "2026-08-03T21:20:10.7Z" }, + { url = "https://files.pythonhosted.org/packages/34/34/a95870b9221e09cf4f2ce3178b1a210abdfe63a1bd357da940418d7b8d15/cffi-2.1.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e0bcb7e0f677f543555d2adff3bf19c05f66cdb4796e5ff602442ab2fe3c4ef7", size = 223564, upload-time = "2026-08-03T21:20:12.165Z" }, + { url = "https://files.pythonhosted.org/packages/70/ea/839b50531021a647fb5e929f72cf97bc1ff702b5472166164b5b6e76b851/cffi-2.1.1-cp313-cp313-win32.whl", hash = "sha256:334644fbac4eff73d985a17a91226df55d0f394160c4cfb880e084c8f7161cac", size = 175263, upload-time = "2026-08-03T21:20:13.559Z" }, + { url = "https://files.pythonhosted.org/packages/60/a6/8b149b2c3f2e11aaa1618ef64500b45f50f22c57a977a4dff1aff1f91042/cffi-2.1.1-cp313-cp313-win_amd64.whl", hash = "sha256:1aa5645c30469b09530c4ebca77ebf8f17618293c58f8549cb1a543a50236e7d", size = 185688, upload-time = "2026-08-03T21:20:14.69Z" }, + { url = "https://files.pythonhosted.org/packages/01/9a/11f687cb39d6a3504060d5242f04f48c735afb4d3d533958a20594890cb2/cffi-2.1.1-cp313-cp313-win_arm64.whl", hash = "sha256:63bbfd5ded17c4840ac07cd8f1c21ba9d9708141f840b324f422f41b207e3973", size = 180078, upload-time = "2026-08-03T21:20:15.917Z" }, + { url = "https://files.pythonhosted.org/packages/d3/7b/d6bbf82b8b96e7391438898c42f5bd96dd02030fd5b64937d248220003e2/cffi-2.1.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:7dbb61fe3a7699468030f71bbe5f8a0e326a151daa91beb11a6fc1f980c55e1c", size = 194064, upload-time = "2026-08-03T21:20:17.148Z" }, + { url = "https://files.pythonhosted.org/packages/94/e6/bcc91b283be94735e268487a054004f0aa19947b6348fa367db53230abc8/cffi-2.1.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:f24fb43132a4c6b4cb4eb029492919b2db645be6808d738f244fd146c03c32cb", size = 196720, upload-time = "2026-08-03T21:20:18.268Z" }, + { url = "https://files.pythonhosted.org/packages/d9/99/c4b0c17cacdc9c3b8f280026286a9826d6a208c0f047591a3c3ce99b91fd/cffi-2.1.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:d28630f5854ab07ab1fd4aba756de52326c82e6be15d414b12793f1975048b54", size = 184964, upload-time = "2026-08-03T21:20:19.708Z" }, + { url = "https://files.pythonhosted.org/packages/b3/a9/9db617d05d7367c1ad0ab00b3aa6e6f9281edd689b4ee9ea0e5a84e89c97/cffi-2.1.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:661c298b4821edebead0c91edd2b00374d67ad7c5a1f7a91d4442633b79d6a72", size = 184962, upload-time = "2026-08-03T21:20:20.833Z" }, + { url = "https://files.pythonhosted.org/packages/67/b8/b42132ca113dc567d37684437b46ca1dafc885902b02a110a02d5b511857/cffi-2.1.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:58acb8ab8e295e6c5ea12f888cbb13cf21511ef2a3303a23f4325c29d17fe5c1", size = 222328, upload-time = "2026-08-03T21:20:22.118Z" }, + { url = "https://files.pythonhosted.org/packages/80/10/c5c0cbf0a657aecf59ef511409734230bf556f05a0d6c9eed7aa5c0a0166/cffi-2.1.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:456a61fa52d579ebf9df2e9552ead5129855dbaff6c1e5a9b1bc408809bdc062", size = 209985, upload-time = "2026-08-03T21:20:23.401Z" }, + { url = "https://files.pythonhosted.org/packages/d5/6c/bfa0b87b03b9238148beca990292843c9396ba069b54496596594173de7b/cffi-2.1.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:a4f00aa42f75d6e4595e8866e748cc1705adc0cddfeb2ca86d0d03993d63ba03", size = 208530, upload-time = "2026-08-03T21:20:24.628Z" }, + { url = "https://files.pythonhosted.org/packages/e9/02/4e7d553a7ac4b4238b38b3c1b80d486e9d4436f8d2acbf87a0997fe3f402/cffi-2.1.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b0431303acaea1089ad4b3e9ce4e6518193def1118d4073ca848635ee4ea2e96", size = 221525, upload-time = "2026-08-03T21:20:25.758Z" }, + { url = "https://files.pythonhosted.org/packages/82/1d/a4aaf9babd75acb4d5f223bff71533bee748dd770a382619a798960ee9ba/cffi-2.1.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:64faea20f4e2613363a1a9b9c7dd73058f3ecd00133a511e72ad7c511658f527", size = 225053, upload-time = "2026-08-03T21:20:26.985Z" }, + { url = "https://files.pythonhosted.org/packages/81/10/5dc0e7bdd18e22107054288283380fc97a06ae3f1656a106908d666a3c88/cffi-2.1.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5c58fe613dc5e5336357eff555824a314d8e43282600435c8d1cb6a7a2fedd13", size = 223213, upload-time = "2026-08-03T21:20:28.277Z" }, + { url = "https://files.pythonhosted.org/packages/0b/e9/d0061c364cde06ee43168a0d076ac1da512cbc380d44767b844ba34fe2b6/cffi-2.1.1-cp314-cp314-win32.whl", hash = "sha256:1a18a57b58cfb21fc28d72e876acf10eaed67a1ed96226f92af4df681d571c4c", size = 177682, upload-time = "2026-08-03T21:20:44.288Z" }, + { url = "https://files.pythonhosted.org/packages/a7/06/1c3e01e3ba14c39f6d10bfbac52753b7e22259e38088e5cfe1d704918690/cffi-2.1.1-cp314-cp314-win_amd64.whl", hash = "sha256:3222ba5d678f80a030e6afbcc33dc1ae5cb45facabb61cee2c7016b8432fde48", size = 187949, upload-time = "2026-08-03T21:20:45.623Z" }, + { url = "https://files.pythonhosted.org/packages/87/5b/da4e39efe18eeb89cf580ea9cfc66b6a7c3eadb808fc0cc1d3a295cb5a5d/cffi-2.1.1-cp314-cp314-win_arm64.whl", hash = "sha256:ab36d55f9ed2d067327667c2fea18dda018eb628dd6347aa01dda6cf1f5d3836", size = 182947, upload-time = "2026-08-03T21:20:46.955Z" }, + { url = "https://files.pythonhosted.org/packages/23/59/40338bf421c5accea1d45158170c87006ef1cd371b05c077e76476949728/cffi-2.1.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:7750c6449dff7864bb9bb27ddfb0267756189201a3afc911d82b3caacd70dfc3", size = 188504, upload-time = "2026-08-03T21:20:29.495Z" }, + { url = "https://files.pythonhosted.org/packages/7d/47/5ecf1023850036e674c77ec4de86182d309ae344e39e7cba984b7df5d647/cffi-2.1.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:0beceaabe56af686895136a2de78db54ecd8e4046b236b8fd6d6cb61389e9bf2", size = 188259, upload-time = "2026-08-03T21:20:31.291Z" }, + { url = "https://files.pythonhosted.org/packages/2a/9c/92934c3bea9f785b23eba304538c0b4d37a2a96d2431eb3a1bc87a11aa19/cffi-2.1.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:49cbc70e6542d4ccccb936558d1064a8012541e78f821f955cff24e357776c94", size = 223864, upload-time = "2026-08-03T21:20:32.571Z" }, + { url = "https://files.pythonhosted.org/packages/4d/45/ba4c93527bc38616a8bd36488acb69a2212d60486794f0c1f318949bbb76/cffi-2.1.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:e2d65b31f36619cda3999b78b2aa9632e76b78448e7a56fc4240824200e7c4fc", size = 211538, upload-time = "2026-08-03T21:20:33.808Z" }, + { url = "https://files.pythonhosted.org/packages/80/e9/b6ef565e452acb932fb0cb5443f44a78efbd1233e566f02b5a83855e9115/cffi-2.1.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:28907ab9bfb6aa13184cfc17c6b8e1023c5ab6fd7076d8c20a35e59fe04f8f29", size = 210688, upload-time = "2026-08-03T21:20:34.974Z" }, + { url = "https://files.pythonhosted.org/packages/9a/95/eff5f0cee78d2eabc7eebffec40d3fc1876b5f3c95582e018bb4b99601f2/cffi-2.1.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:51b31d1c98274844cfd7838ce00bfc27c7423a4dc00fc0772fc3331c2cc90676", size = 223803, upload-time = "2026-08-03T21:20:36.564Z" }, + { url = "https://files.pythonhosted.org/packages/fa/01/579d39fb8bef00a335a23d83757b44feb24cd6345a2c451b64cb67b9c362/cffi-2.1.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:5e7cecbaadb83884793e05828cee59b210b24583b9c7425d0ba6a754fe22eb4e", size = 226763, upload-time = "2026-08-03T21:20:37.816Z" }, + { url = "https://files.pythonhosted.org/packages/8d/b0/0b44f47c60b01b57b6e2bbd92343f13a85a1d93bc46ccf6e47e244acd99c/cffi-2.1.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:25792eac27877609e7bb06d42ff88278a6624fff2ba9bbb523c09616b117e80f", size = 225688, upload-time = "2026-08-03T21:20:38.959Z" }, + { url = "https://files.pythonhosted.org/packages/eb/d2/3b7176cb570a1d3e27faf67b72f591af508036e0d8b2be2ef9af9e8c84bb/cffi-2.1.1-cp314-cp314t-win32.whl", hash = "sha256:8ef53b2de9bcb9197d31854256575d59dbac0cba72ac627bb291ef5eceb74be4", size = 182868, upload-time = "2026-08-03T21:20:40.388Z" }, + { url = "https://files.pythonhosted.org/packages/56/78/31f00c1bcd97c9bbf55f1bfdf5bc809a5de8887473e90bb9960dca825e80/cffi-2.1.1-cp314-cp314t-win_amd64.whl", hash = "sha256:616f097f2fe415bc92a247f02e11f634e1f9e9a83d327e3c915c15089c87869e", size = 194104, upload-time = "2026-08-03T21:20:41.725Z" }, + { url = "https://files.pythonhosted.org/packages/7b/1b/58496f2ed0a35de575250c02a43ab3cc2c04d494a88fed31c1cabc0fd176/cffi-2.1.1-cp314-cp314t-win_arm64.whl", hash = "sha256:ad2c86c495b899d862ea0f4b42891b8713a3bd45dd4105c7fd51c2a72f39f3a5", size = 186402, upload-time = "2026-08-03T21:20:43.042Z" }, + { url = "https://files.pythonhosted.org/packages/c1/8f/9ebe220eab48a093d1a5a5e339ab0dc7316eef3bb04d63c42f0251b61f50/cffi-2.1.1-cp315-cp315-ios_13_0_arm64_iphoneos.whl", hash = "sha256:dddad92b554513a31f272570678ba307fb9f618f05e3d4a5eacafff9eae03e1d", size = 194043, upload-time = "2026-08-03T21:20:48.179Z" }, + { url = "https://files.pythonhosted.org/packages/ff/69/844bad3ece306c4782c2ecb93597035b6690d48704b803914c199da1e8b3/cffi-2.1.1-cp315-cp315-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:da0e573f9f97159390c89d9f1a9e41908b66d408cc5b58d08cf3847d844c531b", size = 196737, upload-time = "2026-08-03T21:20:49.457Z" }, + { url = "https://files.pythonhosted.org/packages/1b/8a/af668013284634733f02d683458a0728739c7d6ddb5e14cb0c20832266fe/cffi-2.1.1-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:fb92203a88b3d3053034db775110081c49d28be6551923805e039924093761e4", size = 184933, upload-time = "2026-08-03T21:20:50.639Z" }, + { url = "https://files.pythonhosted.org/packages/0c/75/2f5207ff6d1a613133b23a5203cc0c2a628313b5eb3974d7956ae3c57950/cffi-2.1.1-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:2ae64be792b8966f2c69538199728b290e34726562896df1e5dc8ffd8d8188e8", size = 185002, upload-time = "2026-08-03T21:20:52.173Z" }, + { url = "https://files.pythonhosted.org/packages/e2/31/9e1313b0a6e30e91b3b3d3fff51ae99c857c07738e3afcce1f7334e1b7ab/cffi-2.1.1-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:507a24c282e0f42f8ed737cf048572cbf580468da5555764a8331735e9c736b6", size = 222271, upload-time = "2026-08-03T21:20:53.462Z" }, + { url = "https://files.pythonhosted.org/packages/50/e3/f6234a833e6e08c7007003074723c406559eecf9b48dfc97471e5a8eb7a0/cffi-2.1.1-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:246fa40ce8645a614ff682e0b70f37134e460eaf93a775e0cbe3cca585a67a80", size = 209919, upload-time = "2026-08-03T21:20:54.783Z" }, + { url = "https://files.pythonhosted.org/packages/0d/fc/5f74e293fced6edb51af3a46c4ccf6c23c9943774ecb375ddbd522c76add/cffi-2.1.1-cp315-cp315-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:471cee653ae88de62096552e6d24ccb4a5adb8c8c9f10b5054d0122c15bf2779", size = 208529, upload-time = "2026-08-03T21:20:56.066Z" }, + { url = "https://files.pythonhosted.org/packages/44/16/29e6d01b388bef055ecd6ca8244b3f4d336bd09e92d5d892187b9601084e/cffi-2.1.1-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:aeae0e330c9f6acd681f647d46cefd30c29f93e3392882e792e82080c9691399", size = 221630, upload-time = "2026-08-03T21:20:57.336Z" }, + { url = "https://files.pythonhosted.org/packages/a4/18/fa7f1f6857d5eb88a4ca99ffcbfb7c387a287ccc154c64a73e86314745d7/cffi-2.1.1-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:42a494cee34437f05546455144f2b5d9ac09b1face62bcfce597d2e521066688", size = 225134, upload-time = "2026-08-03T21:20:58.675Z" }, + { url = "https://files.pythonhosted.org/packages/e0/9f/e8e3dfa04a1b4c241f8c91faacad872b4d4efd051d49764ad4e2fd4b9fea/cffi-2.1.1-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:cc572dace3f60ef98d7b12ff411d20f5362feb31a0439eab0085bbfd349982d7", size = 223197, upload-time = "2026-08-03T21:20:59.968Z" }, + { url = "https://files.pythonhosted.org/packages/f8/7e/8debeb04f1ab9fe2a6963964cd6f1aaf7192627b83926586a6a4e089c9fa/cffi-2.1.1-cp315-cp315-win32.whl", hash = "sha256:4f42141fc14250de6dde5ee7ea4432be017252d91f19c5ad043c084cea629cac", size = 177683, upload-time = "2026-08-03T21:21:14.901Z" }, + { url = "https://files.pythonhosted.org/packages/e0/31/5158704cc474ab65c1647932e88be78dc0873f47130e253be38bcaf13d01/cffi-2.1.1-cp315-cp315-win_amd64.whl", hash = "sha256:e6e8cff14d6fb0be70a09c0bdc58096f501952d04624ebf867e0e56da2df8960", size = 187897, upload-time = "2026-08-03T21:21:16.108Z" }, + { url = "https://files.pythonhosted.org/packages/cc/4b/b3a2da8570c704ffc0f9762cdc3ec0f02c8573798e0b5cf7f11c82bbb70f/cffi-2.1.1-cp315-cp315-win_arm64.whl", hash = "sha256:27350daa11d4f10c540e6e89dada4c54feb7256ad03e9a4dc075ebad7ba360d1", size = 182935, upload-time = "2026-08-03T21:21:17.271Z" }, + { url = "https://files.pythonhosted.org/packages/d0/ef/5443574510a1207e6f6bc38ba6e1f1de36cb48fef07b2728bb896a21f430/cffi-2.1.1-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:c26608d2222fb1e94487e4a387d85f13eb55d5ed725cb25a0c589ac4ee60e7bc", size = 188464, upload-time = "2026-08-03T21:21:01.163Z" }, + { url = "https://files.pythonhosted.org/packages/7e/ae/a56fa8c4686ad50e148fcbc8d3ae0d03915ff5c30d795058988c24118cef/cffi-2.1.1-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:4be96343e422f2dfcd12ab5c9f5aebe03f82f737c6bffeca6830b3875cb44aab", size = 188262, upload-time = "2026-08-03T21:21:02.382Z" }, + { url = "https://files.pythonhosted.org/packages/53/b2/6187f46f2912276a3ae284076109cc5c8680482f11f766ccf26db4a86427/cffi-2.1.1-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:937c0052c05a31ca1daf18de3158eed4dbfcb9cc107adbea227728d647be701e", size = 223779, upload-time = "2026-08-03T21:21:03.553Z" }, + { url = "https://files.pythonhosted.org/packages/8a/f6/c3ad28bd19f77047a03084424fbd4cbe997303267c14423737324be0385d/cffi-2.1.1-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:df423d40ee8654634421812bc3b196da3f9bd7d32929da813f8394c4348a5358", size = 211520, upload-time = "2026-08-03T21:21:04.863Z" }, + { url = "https://files.pythonhosted.org/packages/a0/cd/ccac9013a5bd9fd764de118674ab9c805b5ca10c19270d90ee273f8b2240/cffi-2.1.1-cp315-cp315t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:a730a083190634c65cca36ba5f489531576ebd79bcd5c8e172130f6453127231", size = 210673, upload-time = "2026-08-03T21:21:06.223Z" }, + { url = "https://files.pythonhosted.org/packages/52/86/2976131c639aead931c5bee5aba67e4b09fbeb8018b6f282f70803f923a7/cffi-2.1.1-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:363e05fa78e15116c3c32c210ee36884fd6b9afa6d440e47112c3bd511d64cb6", size = 223835, upload-time = "2026-08-03T21:21:07.539Z" }, + { url = "https://files.pythonhosted.org/packages/ac/0c/33a7aeab2f9c76918c52e084beb39c570db3588133412929e8ec06fab90b/cffi-2.1.1-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:770de9db11e84213beec501cfcaa013b019820ca881e03344dea5844f7876d94", size = 226705, upload-time = "2026-08-03T21:21:08.774Z" }, + { url = "https://files.pythonhosted.org/packages/e3/26/2cde30fdde421130bfc18f70395731a6e6b2053c6a1978a5258ff04e72fa/cffi-2.1.1-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:7da0c5eff80f0197f3b3d1232ec5a682a9325f4ae9016a78f5f5ca35f9ced1f5", size = 225539, upload-time = "2026-08-03T21:21:09.911Z" }, + { url = "https://files.pythonhosted.org/packages/6d/cd/a361394c94b2129d604bb846f624a8e88255a3ee33129c434a00d715e64f/cffi-2.1.1-cp315-cp315t-win32.whl", hash = "sha256:06c72bb76605a4b0cd0aad6930b69d4baf7dd5d806cfc409b824191099700e66", size = 182707, upload-time = "2026-08-03T21:21:11.226Z" }, + { url = "https://files.pythonhosted.org/packages/9b/b5/ba2b299993c26577d529b6ae29841f9e15b9fcf004d65f423f4fcf94ade9/cffi-2.1.1-cp315-cp315t-win_amd64.whl", hash = "sha256:d9c275eaacd24aa73f94ffd6de08fc3f932424d8b6c376f4bed7cde376fe7bc3", size = 193772, upload-time = "2026-08-03T21:21:12.39Z" }, + { url = "https://files.pythonhosted.org/packages/aa/29/35e016098c814cd93de9cd320c66b5bfba14dc6ecedd3cb518fa7c408c69/cffi-2.1.1-cp315-cp315t-win_arm64.whl", hash = "sha256:d18e5ac0f2f03f4f518d3e23db0f0cad7faa1da8620e9c09461d443bbf6e6692", size = 186360, upload-time = "2026-08-03T21:21:13.636Z" }, +] + +[[package]] +name = "click" +version = "8.5.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c7/0e/7fa0ef50764b67090eca4114772a2abf8b6148198475e54c660b97caeee6/click-8.5.0.tar.gz", hash = "sha256:ba0d2089de75ea0310e2dde03160e6ca10009947fb95a182f9b54021bb272e34", size = 382235, upload-time = "2026-08-26T13:33:14.56Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/58/50/6c0d534c5f134586a8e1ba4e330569e32f057e33372ae556463212fb4cd3/click-8.5.0-py3-none-any.whl", hash = "sha256:255bc9599cf7748b4b1a446ccc735421bd08a2ae529a8b88597d3de5664ee360", size = 125251, upload-time = "2026-08-26T13:33:12.928Z" }, +] + +[[package]] +name = "fastapi" +version = "0.141.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-doc" }, + { name = "pydantic" }, + { name = "starlette" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8a/02/91e3416a8fdd715abb903a952a6bec7cdd8d14eed55d415fc8595524c319/fastapi-0.141.1.tar.gz", hash = "sha256:e8822fc40db1e1858054d7a949a888695bc9bdce70139178e33bd2871a453ca1", size = 425799, upload-time = "2026-07-29T17:18:05.568Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/03/10388a42375ee7e4ac9b94eb2c5c569c8b5795e377e701c9ac3ad63de890/fastapi-0.141.1-py3-none-any.whl", hash = "sha256:bfb91aa2d334c61cb35ba9a116fc123b3d3df31640b801cf57a7a78ec3f603b3", size = 131954, upload-time = "2026-07-29T17:18:04.364Z" }, +] + +[[package]] +name = "h11" +version = "0.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/ee/02a2c011bdab74c6fb3c75474d40b3052059d95df7e73351460c8588d963/h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1", size = 101250, upload-time = "2025-04-24T03:35:25.427Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, +] + +[[package]] +name = "idna" +version = "3.20" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f5/08/8eea9d4b8302028f3abb2c0813953f7aec26d33b7a8960ed760e65ff29fa/idna-3.20.tar.gz", hash = "sha256:a7db850025b95ded1eae8a46181a1a6c56c92c96f0e2b005d9ff8dc0210cab44", size = 216463, upload-time = "2026-09-17T14:11:04.752Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/58/a2/bb081bab032533a855d44de1d56f8e8426114ff1ba5d1f07a438a0a654f8/idna-3.20-py3-none-any.whl", hash = "sha256:ab7ae7122974553370f0bdb919e1a960b2cd1bc1ef0276416d896db81c14582c", size = 69583, upload-time = "2026-09-17T14:11:03.168Z" }, +] + +[[package]] +name = "jmespath" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d3/59/322338183ecda247fb5d1763a6cbe46eff7222eaeebafd9fa65d4bf5cb11/jmespath-1.1.0.tar.gz", hash = "sha256:472c87d80f36026ae83c6ddd0f1d05d4e510134ed462851fd5f754c8c3cbb88d", size = 27377, upload-time = "2026-01-22T16:35:26.279Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/14/2f/967ba146e6d58cf6a652da73885f52fc68001525b4197effc174321d70b4/jmespath-1.1.0-py3-none-any.whl", hash = "sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64", size = 20419, upload-time = "2026-01-22T16:35:24.919Z" }, +] + +[[package]] +name = "mtusi2026lab4" +version = "0.1.0" +source = { virtual = "." } +dependencies = [ + { name = "argon2-cffi" }, + { name = "asyncpg" }, + { name = "boto3" }, + { name = "fastapi" }, + { name = "pika" }, + { name = "pillow" }, + { name = "uvicorn" }, +] + +[package.metadata] +requires-dist = [ + { name = "argon2-cffi", specifier = "==25.1.0" }, + { name = "asyncpg", specifier = "==0.31.0" }, + { name = "boto3", specifier = "==1.43.115" }, + { name = "fastapi", specifier = "==0.141.1" }, + { name = "pika", specifier = "==1.3.2" }, + { name = "pillow", specifier = "==13.0.0" }, + { name = "uvicorn", specifier = "==0.40.0" }, +] + +[[package]] +name = "pika" +version = "1.3.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/db/db/d4102f356af18f316c67f2cead8ece307f731dd63140e2c71f170ddacf9b/pika-1.3.2.tar.gz", hash = "sha256:b2a327ddddf8570b4965b3576ac77091b850262d34ce8c1d8cb4e4146aa4145f", size = 145029, upload-time = "2023-05-05T14:25:43.368Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f9/f3/f412836ec714d36f0f4ab581b84c491e3f42c6b5b97a6c6ed1817f3c16d0/pika-1.3.2-py3-none-any.whl", hash = "sha256:0779a7c1fafd805672796085560d290213a465e4f6f76a6fb19e378d8041a14f", size = 155415, upload-time = "2023-05-05T14:25:41.484Z" }, +] + +[[package]] +name = "pillow" +version = "13.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ce/a4/9cc30690a394a8305e8be4c9036aa16d0dfddadd09079860b582f562496d/pillow-13.0.0.tar.gz", hash = "sha256:cdcd1ba258178f7356d74170c9a81ba4eab6a5d55748bc402a5a2a6fe6be3510", size = 47058631, upload-time = "2026-10-16T22:18:09.718Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7a/b8/58d79332229482eb0d27c82973850808c19d37e5bdc14443d3417bd373f5/pillow-13.0.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:67c8d44ad89fa7eb88de5d6ef6199b1e03dd24974f85a67639fc8badb801fe79", size = 5568643, upload-time = "2026-10-16T22:15:44.474Z" }, + { url = "https://files.pythonhosted.org/packages/39/4b/f2ed53d83aa44ac19599ac0a54897f867fafb95f406980d1e24083f8cbe3/pillow-13.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc481f742ccda20e9ed760b8f978dff50f2a4e9bc253ac167fb0a05b0783ef52", size = 4852329, upload-time = "2026-10-16T22:15:46.284Z" }, + { url = "https://files.pythonhosted.org/packages/3b/bb/9e30e8ed18c20800ee327a12107d5249ad8dc444867b3adfcc21654e3c71/pillow-13.0.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8bf295fe459343bc6093c8e5e0df60bfa189e0fcb775a22e2c2488fdeccc6c0d", size = 6519309, upload-time = "2026-10-16T22:15:48.14Z" }, + { url = "https://files.pythonhosted.org/packages/e6/99/fdf65cc539e266ab93546c261bdf98411fe165c6e4e13f9dbf597097192b/pillow-13.0.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:42d63be3af4f9b951b199b0a9d23365d37ab14687e6d77b74940432c8515c4ca", size = 7144542, upload-time = "2026-10-16T22:15:50.093Z" }, + { url = "https://files.pythonhosted.org/packages/26/9b/f38b7c4d8f599537e2d1a6d546cec9beef7536ff8f5c49a3f9c78179d8ac/pillow-13.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4bd024d567eca931b668978d672dba914d4b8e2628e2c3073a2ad178de8a2620", size = 6618042, upload-time = "2026-10-16T22:15:52.054Z" }, + { url = "https://files.pythonhosted.org/packages/a8/7c/45ee28531f89ee4582967d8a17f7a26d71ca64cd902b93c269590cd60a00/pillow-13.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:23edf4ec4b6f4f240e8df6e4985e9ee2ced86474632433e55fb1e2548744bf40", size = 7228137, upload-time = "2026-10-16T22:15:54.009Z" }, + { url = "https://files.pythonhosted.org/packages/01/bc/9c2a969cb1b70201d04fa17fd5aedf02a732870b8da86bceaaa30b0ca385/pillow-13.0.0-cp312-cp312-win32.whl", hash = "sha256:02464da86f561f410e13b061ec37c3068a9cb557a095494157742d07b3c24a27", size = 6515423, upload-time = "2026-10-16T22:15:56.25Z" }, + { url = "https://files.pythonhosted.org/packages/ff/cd/73a79ecdc822076002eb24b5e1e4a530b0bb5310fcb6d239abbec877110b/pillow-13.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:ff3fe9eb621fbb9db4c4be33ab946f6bac5f98ab50ca8d3260503653656c961b", size = 7272166, upload-time = "2026-10-16T22:15:58.597Z" }, + { url = "https://files.pythonhosted.org/packages/4a/a3/be5a1236efd085b3984e4305da8f179ae9dab80c991d2ad78f2bd5c4c843/pillow-13.0.0-cp312-cp312-win_arm64.whl", hash = "sha256:c3bc8ee0271387e0b1e3ad0ff893645990a656e79bf9c42f4f734a3642b11d51", size = 2698943, upload-time = "2026-10-16T22:16:00.637Z" }, + { url = "https://files.pythonhosted.org/packages/82/e2/9132d38df320e8099e2642d9fa538f18a7dfebf68a875a31725dc2a115aa/pillow-13.0.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:e98b4f91f99b24b1da72e10621ba181f7281922384a7403094debb4199de6c00", size = 4190620, upload-time = "2026-10-16T22:16:02.527Z" }, + { url = "https://files.pythonhosted.org/packages/cb/be/436dac0996876f65ef1916ffb711c5ef4af2ebc57414b6f16642e07f6b47/pillow-13.0.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:34fb3db1149c879343861831728c243c432d4a5afc8f1878ca7612ef8de2a033", size = 4290286, upload-time = "2026-10-16T22:16:04.861Z" }, + { url = "https://files.pythonhosted.org/packages/60/16/7c1007162e4169ea0d5bec47e2e0b2174f56c50cf5b8c02a1dc23e41037e/pillow-13.0.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:2d50500a4a3bc6469adcdb319cf05d688b08ca3bf0f22a05339f3b25b433a599", size = 3723169, upload-time = "2026-10-16T22:16:06.956Z" }, + { url = "https://files.pythonhosted.org/packages/08/bc/578ab2eaa04705a8978bd9be0fe15c3d43a8fc95097c4f7e39540e79e4fa/pillow-13.0.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:ef5fe6b7db3f501c56000ba8fe7c7e1a35ca1d45824fbeb95a67daa85cff4dbe", size = 5568469, upload-time = "2026-10-16T22:16:08.664Z" }, + { url = "https://files.pythonhosted.org/packages/98/c8/ccede3dcc4f274b80fb49199eed1ef34ba05566c8b71976b9d1f111da2e3/pillow-13.0.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f11a3c1358bb3c68012a7b1f41b223c3db9f7bfde90b481df028cfe1c3363313", size = 4852187, upload-time = "2026-10-16T22:16:10.693Z" }, + { url = "https://files.pythonhosted.org/packages/fb/3c/cd0770e96d20bfd990db024bcbd43897ff3e88da1f4782cf90ea231b1d4f/pillow-13.0.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:19b66ca0a4d25cc0200b381ce824c7ea090db61427315e73cfb05dbf1b35c369", size = 6515908, upload-time = "2026-10-16T22:16:12.896Z" }, + { url = "https://files.pythonhosted.org/packages/ef/fe/f5cdaa2d1c4086e757430f368c271a58ee29cfe7378e7ae7b9d964015c25/pillow-13.0.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5f56499c1a74d2b862beb0c33b8016f99d095858455df9d21583bbbc1c15b582", size = 7142544, upload-time = "2026-10-16T22:16:14.844Z" }, + { url = "https://files.pythonhosted.org/packages/fb/b5/a54f6d44adbcd23737c8c12953f42be107e4cca64e247c0ca3a5b563061c/pillow-13.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a2a14cd73d5e6c927f644b0110ee81f59c4ef506259ec9183904747f2caacc81", size = 6613097, upload-time = "2026-10-16T22:16:16.893Z" }, + { url = "https://files.pythonhosted.org/packages/e6/b4/9e6c1207c21b2e91db0d15c16d30f3701ce5add1bd4da0e787761e66cb6e/pillow-13.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:08b826a9bf1265c76de04a42061d61ea6b109418b7826f28ff1ecdfaa26c5502", size = 7224846, upload-time = "2026-10-16T22:16:19.195Z" }, + { url = "https://files.pythonhosted.org/packages/78/9f/d3c3cb0518f3dd1eee0ef501535f53d15597a53c5c4118d178086143397a/pillow-13.0.0-cp313-cp313-win32.whl", hash = "sha256:12b0fa15879fe5a9d9eebe66e4e2d74128a773b1a4448010c9c714f45e04b9d8", size = 6516013, upload-time = "2026-10-16T22:16:21.389Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ed/1b16fb66a37e1b9e41370b93b2c614c7da761a5b2a41dc333f7af4639962/pillow-13.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:9dda0788a9932776a75d8bcc020076edc308ce2508b51f6739a45e3158b3fc42", size = 7284777, upload-time = "2026-10-16T22:16:23.393Z" }, + { url = "https://files.pythonhosted.org/packages/87/7d/8d02eba9f7ecc87f8e90f731e032cc4773802f2a843763afd0c66a535ff4/pillow-13.0.0-cp313-cp313-win_arm64.whl", hash = "sha256:40a266371ece581e5de13b6635ef78d7487824af80561b6ca27694a88c19bc0f", size = 2698826, upload-time = "2026-10-16T22:16:25.382Z" }, + { url = "https://files.pythonhosted.org/packages/8e/45/fbad9c7f5e2f1745c373a434d46072bc84eddfefaa28b852425ac957fda6/pillow-13.0.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:44629d4498c9cb6b2051806155f22141df0d8a929c56134b166655232dfb2f8c", size = 4190721, upload-time = "2026-10-16T22:16:27.524Z" }, + { url = "https://files.pythonhosted.org/packages/a2/bf/2074902a259a90692c5dbf132885d21671644cbbd3f690047516243a55db/pillow-13.0.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:bf97ba1b5542050e9b0c74e632323b6d8136cc99d64e193abebee22fe6013c76", size = 4290236, upload-time = "2026-10-16T22:16:29.562Z" }, + { url = "https://files.pythonhosted.org/packages/ff/a5/3d1bb93b8ea38d15d21356c1b8e9f50971ac592aaa248f7fb71739094c0e/pillow-13.0.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:aa694c49787040943570d61c9b8fc01271bd2a2cfdd55f0d896f8aecdce1a981", size = 3723093, upload-time = "2026-10-16T22:16:31.249Z" }, + { url = "https://files.pythonhosted.org/packages/84/a1/be060135d85187b25fa8b2573786c0e45c62ae27527fff5257675eb97f4a/pillow-13.0.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:49c12af515073e9bf511e4ef736743bf70d531a643668722279314b937742ba4", size = 5572253, upload-time = "2026-10-16T22:16:33.103Z" }, + { url = "https://files.pythonhosted.org/packages/b5/f4/0bf6f1ac9ba7ee2ffe9914aeb410a1035a0c1a3c42f122388d7fc950bc4b/pillow-13.0.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:d9b0a2bee3f3e03e9f29057aedb807dbf1e1d96ee1b9d0f422d9db6c8583271d", size = 4852269, upload-time = "2026-10-16T22:16:35.102Z" }, + { url = "https://files.pythonhosted.org/packages/83/16/b603274617b3e6728da3a0133d4fba1ea8d7fbe0c4ea5fef9cbffa398b8b/pillow-13.0.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5933fcea5a5ba64c686f55c886a115cbfd92f2900460784a1e79909fdbafd7b6", size = 6516058, upload-time = "2026-10-16T22:16:37.228Z" }, + { url = "https://files.pythonhosted.org/packages/1d/f2/1371707ce438ba9a43f9d24089828494ec6f9396e28cd7903aa704dd6306/pillow-13.0.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e91c2b1cbeae768c98752e67e717e283d68dae45717b41f2d91eaee648e37b7d", size = 7142574, upload-time = "2026-10-16T22:16:39.172Z" }, + { url = "https://files.pythonhosted.org/packages/b9/a9/62cf68587f910bc87c5951bcb6e9430c7643a65076adf5f18b4bf350e2bd/pillow-13.0.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7f6299e494e46253d40c34653cac369412f44ca53b5f85d848d3f95b1ed0546c", size = 6614295, upload-time = "2026-10-16T22:16:41.117Z" }, + { url = "https://files.pythonhosted.org/packages/6c/c9/9e9b48178649730299be0bafbf5f810a5b6977538f5a92d47e4ba5f5d608/pillow-13.0.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f9bd7d260d986f3553518c5d723544346fa48031f8aef3265d786aeb59845bc0", size = 7224841, upload-time = "2026-10-16T22:16:43.118Z" }, + { url = "https://files.pythonhosted.org/packages/8f/48/c44e91384b1b6c072a714a63d00608acc16a6a968735e478e4be4384e5ac/pillow-13.0.0-cp314-cp314-win32.whl", hash = "sha256:2e31c4e68cffd7b1f116127626813b82fcba19edf6b940a7b0c266d6484d0055", size = 6508694, upload-time = "2026-10-16T22:16:45.883Z" }, + { url = "https://files.pythonhosted.org/packages/50/e2/7ad17747517e4c813dca78257d6bb593466f8231baf1384bc98d92745edb/pillow-13.0.0-cp314-cp314-win_amd64.whl", hash = "sha256:f7cbb7a64e7ec664775195e4f81923bc8a93d9451b963f844e59118f63147a7a", size = 7281830, upload-time = "2026-10-16T22:16:47.725Z" }, + { url = "https://files.pythonhosted.org/packages/41/9f/e5b51f2ce8ef4563215334035403ce61a244f93a2e65c24a940131618f87/pillow-13.0.0-cp314-cp314-win_arm64.whl", hash = "sha256:a6aaa7871aebca081eb5c45f83e845bbd9712ae99fa9286dbfdcee835e63f4b4", size = 2695468, upload-time = "2026-10-16T22:16:49.515Z" }, + { url = "https://files.pythonhosted.org/packages/97/73/a53502f731f6c84805a4ce87a8881e86d07a65aee64728b900d112dc703e/pillow-13.0.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:7b999816b5850a35ada08443e6e03945f1f6488144a83afe2d37b240a552aafa", size = 5578196, upload-time = "2026-10-16T22:16:51.291Z" }, + { url = "https://files.pythonhosted.org/packages/21/50/7e949959a56b5559c78ef172b70fc6025706acb289f3f648a15329e4fa6f/pillow-13.0.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:67871a13cf47275201553b536c025da777f997fa562a19eb798af4ad4b97c18f", size = 4857276, upload-time = "2026-10-16T22:16:53.399Z" }, + { url = "https://files.pythonhosted.org/packages/c9/52/43ded0ac08e77e39d17bd6d894162572201b5e8fdc4a022b14c55a1960a4/pillow-13.0.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e8ca8f69b7c77fa96dad99f8638c1fc4f1c0bed4acaa1baf7a3c31bdb80a0ab1", size = 6607008, upload-time = "2026-10-16T22:16:55.525Z" }, + { url = "https://files.pythonhosted.org/packages/0d/2e/4a57ed216d5bf877387cba4d44c02aa87abe51ad2e06404aa6b597c4da39/pillow-13.0.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6bce6a29c3898191f4c9998e4d4e8f69790caeabde46e54da4bc235ce5bde683", size = 7224281, upload-time = "2026-10-16T22:16:57.738Z" }, + { url = "https://files.pythonhosted.org/packages/5c/3d/2929b248e135752d63aaeabf53f6de797e905b7b6263f9c1e2adade20aba/pillow-13.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:816752a5ef76a80484c8c441c85593b7ea9304f6231fc5776ad1fbff7e773058", size = 6695600, upload-time = "2026-10-16T22:16:59.842Z" }, + { url = "https://files.pythonhosted.org/packages/bc/33/fb0da992cf084ed1ee6ee05e96b6191e35c351eb707ca2df874feb24ce47/pillow-13.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5bdb38ba220b61b4ec31dd1046dea6b016d3c6850a23f43ed8b58332b2fc668e", size = 7300589, upload-time = "2026-10-16T22:17:01.686Z" }, + { url = "https://files.pythonhosted.org/packages/bb/af/c30dd9ef6a3dce84ecb1b2393cac629047a32e66279056c6cb46c3ffd65c/pillow-13.0.0-cp314-cp314t-win32.whl", hash = "sha256:06903aac074d00c988e77a1fc435286dc498506d4ca8d6f69df9babddf4ecc15", size = 6517204, upload-time = "2026-10-16T22:17:03.722Z" }, + { url = "https://files.pythonhosted.org/packages/85/d5/ef3a8ae7609c8ab26570abf18194882534b480d245a9148b895a2262c004/pillow-13.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:9a635068d9793c7f0d929f39f18b58f1bd86c236e8e7b64167b86af9232ebca0", size = 7286695, upload-time = "2026-10-16T22:17:05.543Z" }, + { url = "https://files.pythonhosted.org/packages/84/3c/0861db0c69a03689ebe3e7599d35668a414deb5aaafd8255215e4e62e3d3/pillow-13.0.0-cp314-cp314t-win_arm64.whl", hash = "sha256:0d1a9cf7bc6917538b20f8d06b9ab2147e542f68849e96fc58eb66bdf6acc692", size = 2698098, upload-time = "2026-10-16T22:17:07.674Z" }, + { url = "https://files.pythonhosted.org/packages/6e/eb/e86a41962a3a31ad495a3d6a912d9002b01bba4a6fa71c28703abdca7f69/pillow-13.0.0-cp315-cp315-ios_13_0_arm64_iphoneos.whl", hash = "sha256:7d5e097eabd5d4915c27ac9f6e9325bb1e01a7c94d7348b5c4451d3fc30e8521", size = 4191213, upload-time = "2026-10-16T22:17:09.449Z" }, + { url = "https://files.pythonhosted.org/packages/dd/04/7e7445e706afec84ef06dc25b2c77f9cecff242c23730c62928ad1089d4a/pillow-13.0.0-cp315-cp315-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:51ebb529fc5e8abdd55302f6c3f8943037348ec2384329a3e55677f69ae2fdfc", size = 4290650, upload-time = "2026-10-16T22:17:11.196Z" }, + { url = "https://files.pythonhosted.org/packages/46/85/d534089068ac40c0f61f650389a296c812eab745a8a733f81feec39ae955/pillow-13.0.0-cp315-cp315-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:c87a04e7f152ee4db5dbe15f0bfa3fe628a1aeedbe66b81c43b87f52c813f9ac", size = 3723078, upload-time = "2026-10-16T22:17:12.772Z" }, + { url = "https://files.pythonhosted.org/packages/ce/2b/00b5c3b401df538b5dbe311d3899730fae5ba1e21cb8b80a128887ac6747/pillow-13.0.0-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:20722cd5375533ad754053363a80b8fb30c79d37930a6f7228030e237ce3bd5d", size = 5572292, upload-time = "2026-10-16T22:17:14.539Z" }, + { url = "https://files.pythonhosted.org/packages/72/f5/a6b0610f946e9b54ae888ba7ad29d6e2f8d9b11f69e212d70416b7186c09/pillow-13.0.0-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:04a63720f79e22fd54d897c233a68314c50ca0603360f54ce2bd5db8f0f43782", size = 4852222, upload-time = "2026-10-16T22:17:16.521Z" }, + { url = "https://files.pythonhosted.org/packages/1e/d0/c0ff25bb0427ec70f32aac995453321fc3c4494a46c7292ff02d9d260b04/pillow-13.0.0-cp315-cp315-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4f8db54177035a2b5f717f94475b74907e12a96dc74f6d487f3966b0a4694071", size = 6527349, upload-time = "2026-10-16T22:17:18.534Z" }, + { url = "https://files.pythonhosted.org/packages/85/11/e46fa0bcc0fc117f1eacc206b3d51f04d583fab328b4b740672d0438d83f/pillow-13.0.0-cp315-cp315-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8f36aa77f74b14a2103622b196402c71330e4e7cf4122ee45c854e9a01e0eecf", size = 7152873, upload-time = "2026-10-16T22:17:20.415Z" }, + { url = "https://files.pythonhosted.org/packages/7c/53/2bdeca705f3a58679fa9201233193c86a30238340ca29b3a152c3fcc719b/pillow-13.0.0-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:149a1c323ba366f90b48e6f1a6db6582eb50512df374588de56fc909abc0f0f3", size = 6626871, upload-time = "2026-10-16T22:17:22.361Z" }, + { url = "https://files.pythonhosted.org/packages/14/de/3829030f209722ade3753bb456b805a0e7f885cf885b483407365c82e9ca/pillow-13.0.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:e892354a424b1e82a78ba16785822bdd40de06f19446a3d4a28047df1a4f8932", size = 7236439, upload-time = "2026-10-16T22:17:24.352Z" }, + { url = "https://files.pythonhosted.org/packages/44/b1/67fc37388382bc036a74f04da5bb6fcbf90ec3ac1f45893e06176c373fd3/pillow-13.0.0-cp315-cp315-win32.whl", hash = "sha256:8e459b613cf31f8980f584387b4fe47bf08e076e288d61aa8eba1f7fd4ddbb81", size = 6508688, upload-time = "2026-10-16T22:17:26.314Z" }, + { url = "https://files.pythonhosted.org/packages/44/05/fdb6a79aee10861165526b41b7b440e36ee1ff58613b2fbefe3acbf06d4f/pillow-13.0.0-cp315-cp315-win_amd64.whl", hash = "sha256:d4737f50d23ddd11d6685faacbb9252887dde0bc7997127757247983d74399d6", size = 7281107, upload-time = "2026-10-16T22:17:28.198Z" }, + { url = "https://files.pythonhosted.org/packages/e6/9c/6c6f7dc2a566467baf121394e83de25ac287cffe34cc030726beb4be0224/pillow-13.0.0-cp315-cp315-win_arm64.whl", hash = "sha256:2310e2adfc13c6d457d7d0f2ffdd5c56a74a317650cf765887df8d515a647ac0", size = 2695513, upload-time = "2026-10-16T22:17:30.013Z" }, + { url = "https://files.pythonhosted.org/packages/23/16/e9120defdb8bff693b582d035312acc6f530cdc2bc5238a87f410efccd82/pillow-13.0.0-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:21c32061a5512dc13f87cc7303e120e4d77c2f70df89a35af0e3ce246e5ac57f", size = 5578129, upload-time = "2026-10-16T22:17:31.642Z" }, + { url = "https://files.pythonhosted.org/packages/47/22/76faf2bb2448f48a84cf894c1e2bb1090d7d2b1aa521ece131cb2f05d631/pillow-13.0.0-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:1c774b9c0ba0d0b356f71530677b39028878a2cae5a93e0a13779cfd17120f88", size = 4857428, upload-time = "2026-10-16T22:17:33.43Z" }, + { url = "https://files.pythonhosted.org/packages/b3/64/0861ea4e03c50b14af9ed9dd8b9f388ab42f445c623a9fcbab19aef7aa55/pillow-13.0.0-cp315-cp315t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:20822140c9a2ba9c5e42df342b0a892ab2e102fc0e3db967a133bb9ce3f898bb", size = 6612582, upload-time = "2026-10-16T22:17:35.415Z" }, + { url = "https://files.pythonhosted.org/packages/5c/20/cd1e9b3f60a9f3d5ebe46da36955c26004092a501f7f2c42f9fcc26611bc/pillow-13.0.0-cp315-cp315t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ffea6b2986a797c84b6d1d27d4f43e0b83c73fc16281a930891cd237cf0d08fe", size = 7230176, upload-time = "2026-10-16T22:17:37.362Z" }, + { url = "https://files.pythonhosted.org/packages/b6/ef/ffb0a2fc85690714fb7ac662956949645d57f15bf4426f520da361e7b6e3/pillow-13.0.0-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:05cd2cd39055d0b9650060e4513ef4a9a0aea870083950bd046a0d6490f93a22", size = 6703640, upload-time = "2026-10-16T22:17:39.313Z" }, + { url = "https://files.pythonhosted.org/packages/e0/c2/6402ee3ccb3363aeed8fdb3f9fe56703ac5377a539f0b81b53999d7e121f/pillow-13.0.0-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:c6fd031333dedcad7ce3e03d244093f43504f209e86925ebdb2b868fa4037630", size = 7309231, upload-time = "2026-10-16T22:17:41.021Z" }, + { url = "https://files.pythonhosted.org/packages/fc/96/d6b82e8b8c06ad143b66b12e0b31051e4bf489b4fabca1f390a14fe1ebde/pillow-13.0.0-cp315-cp315t-win32.whl", hash = "sha256:c50c80e3591e5cf9a8dfa6007d4f10732f02e32263850999286e56ccd650cfb2", size = 6517200, upload-time = "2026-10-16T22:17:42.886Z" }, + { url = "https://files.pythonhosted.org/packages/60/d4/0062e6005e9b270b165015aafd343364ac378ec5a685431ea42baad33357/pillow-13.0.0-cp315-cp315t-win_amd64.whl", hash = "sha256:adb3c2124c68e480f4632d06cb7caa3b48a51678475f483b4edb1c6dee29ea43", size = 7287488, upload-time = "2026-10-16T22:17:45.56Z" }, + { url = "https://files.pythonhosted.org/packages/e4/81/b8eb349746be7bce47dd95337c6634e84910af91b59aaa9159c034c614fa/pillow-13.0.0-cp315-cp315t-win_arm64.whl", hash = "sha256:4a25756244bdcce3b50580baadf320fa48297f0eeb8e114ba64139a40d4d0fc8", size = 2698073, upload-time = "2026-10-16T22:17:47.556Z" }, + { url = "https://files.pythonhosted.org/packages/60/da/f6d644e631c5b370c5d0b899f3ccadd2780aab070b29a110527cf4e1fd2c/pillow-13.0.0-pp312-pypy312_pp80-macosx_10_15_x86_64.whl", hash = "sha256:8619685cad8efc4c38c66d3b36aad854523297c43d748b3b6f0cc03c90ba3463", size = 5523188, upload-time = "2026-10-16T22:17:59.334Z" }, + { url = "https://files.pythonhosted.org/packages/e4/b0/2594f202d743a35d4fe80682bfd96f277e5ca7aecb27cf64ef5bfc1d000b/pillow-13.0.0-pp312-pypy312_pp80-macosx_11_0_arm64.whl", hash = "sha256:7e91e97a929232cdb828889ee6ad0e48d7f40f4ec38818f4e4a8968b2b1af22e", size = 4806808, upload-time = "2026-10-16T22:18:01.188Z" }, + { url = "https://files.pythonhosted.org/packages/8e/ed/d504165ef79af6fafe25c76726cf30f544cc236002f7ed054f1d51adc4ff/pillow-13.0.0-pp312-pypy312_pp80-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:93ff9c9c3663ce23aaa5d0b6b6abac55359e2a0c06c7e30f3771829af816be63", size = 5464358, upload-time = "2026-10-16T22:18:02.779Z" }, + { url = "https://files.pythonhosted.org/packages/7b/fe/a10aba5503ca13e9e009b8bd0a1a4ce5584bdce246245fd6f3a83701369c/pillow-13.0.0-pp312-pypy312_pp80-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e65dd6e75de67b2362a4fa3e16aa627595f65054dc4ec73d64680c84d592fa61", size = 6043514, upload-time = "2026-10-16T22:18:04.538Z" }, + { url = "https://files.pythonhosted.org/packages/c0/37/66b12dc476041fe523bca29c9d58fbf1ea92a4eaf73acfb9e993948f1191/pillow-13.0.0-pp312-pypy312_pp80-win_amd64.whl", hash = "sha256:567f20f1c75897d7663830da6ea8af719e03d712b1d9ee7f813319f9ece42e5a", size = 7366030, upload-time = "2026-10-16T22:18:06.638Z" }, +] + +[[package]] +name = "pycparser" +version = "3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ac/d3/eb1d3bc30dda12f7e69640ae2ac8cb10240b71fb73024ad528b7d2ae73da/pycparser-3.1.tar.gz", hash = "sha256:b3fc6dec06a8b2fefa0ed4ff92285306a5e3be9987bc5603c9edbdc4e492418f", size = 105946, upload-time = "2026-10-08T12:26:34.349Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/99/ce/b3ae9ee0324d991c860187be2a6ee436d27a3d02397eaddbb101ec901f3d/pycparser-3.1-py3-none-any.whl", hash = "sha256:f09d358c840bd147b79e55f2bc494f18ea869dc897f5852a8f5766b74f787882", size = 48709, upload-time = "2026-10-08T12:26:32.97Z" }, +] + +[[package]] +name = "pydantic" +version = "2.13.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-types" }, + { name = "pydantic-core" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/53/ef/fc4f868f4e2cee79f863883abffceff107875f569b848507319842d2a681/pydantic-2.13.5.tar.gz", hash = "sha256:51a9c5f7b2f8e636f04c6cada605d9b6a3bf1348fdf945a3d8869b19bba0ee08", size = 845750, upload-time = "2026-08-28T14:04:00.916Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/eb/47/c95ffc2009878c7aac0c5e08528022dcb885933252a88b5f170058014464/pydantic-2.13.5-py3-none-any.whl", hash = "sha256:346a034f080da3755d8e9cb5e00e8b07de1d39e4f6e2c87d8ab7cafa0b269a73", size = 472589, upload-time = "2026-08-28T14:03:59.136Z" }, +] + +[[package]] +name = "pydantic-core" +version = "2.46.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/af/f9/8a06bea35ef8daf588f707784c973a7046e0034c8d8cfb08828eeffb8b75/pydantic_core-2.46.5.tar.gz", hash = "sha256:10416c15b8839ecc4ef4d0885da76da6fd0f67333a0eb8aff6d93c4b8f2910fc", size = 472262, upload-time = "2026-08-28T10:01:31.677Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/82/3f/76358795aa7a8c6d4f36e2cb828ad1c90ee118e1393a9281664f5aade9d4/pydantic_core-2.46.5-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:b9fe6fb92520e3fd61f2e49000b6911b188824f089b75973ea06d6267f0b476d", size = 2076516, upload-time = "2026-08-28T09:58:21.576Z" }, + { url = "https://files.pythonhosted.org/packages/db/50/26b091836076ce4cb2fac264186936acc069e0595772cfd02a563bc4761a/pydantic_core-2.46.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a39ac25a9a2fa4072efdb429833c4a4c8009a51ff9eea3eeae131713cd27991e", size = 1922874, upload-time = "2026-08-28T09:58:23.766Z" }, + { url = "https://files.pythonhosted.org/packages/09/f0/2a8ce3849e299d44e2d2c196b6082643a3235565a735cb51db7a6261f614/pydantic_core-2.46.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4fdc8b93a41521988916eeaa271173fcca7fa0803d62f87675aac8dcec1c8e29", size = 1951772, upload-time = "2026-08-28T09:58:25.435Z" }, + { url = "https://files.pythonhosted.org/packages/87/46/ac0dc8bdd9e6048183a14eb127764e7ad9240021c17513074a4711b0e31e/pydantic_core-2.46.5-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b98134087d9de723658d17a42c7d0da8d6e2ef08015dee7dc93889047315f5e4", size = 2031832, upload-time = "2026-08-28T09:58:27.102Z" }, + { url = "https://files.pythonhosted.org/packages/c4/c2/339de5bef7be36301a2231eaa52e62163742c2281f11b5f4892bc79785cd/pydantic_core-2.46.5-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e652ab17569c94bff5475520f907b7148b8c24036a8ebbe5cf7cf7493d28579a", size = 2208645, upload-time = "2026-08-28T09:58:28.948Z" }, + { url = "https://files.pythonhosted.org/packages/7b/a0/9ff22b797724262da14427abaed4dd1d864a139693fc5e7809114376a716/pydantic_core-2.46.5-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d925f3d9afd05a8c0fb3a1031463a8d59ebe5e2afad297e29c78be19e13b4e62", size = 2265935, upload-time = "2026-08-28T09:58:30.625Z" }, + { url = "https://files.pythonhosted.org/packages/c0/a4/eb9409ec0736e50aa70a412f16c204ed149516846912f7e6724d4c73ee53/pydantic_core-2.46.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0fc5be0abd4a407e200d844b404e33639a554e7bd0d448e7b9ae181be4789ac2", size = 2066284, upload-time = "2026-08-28T09:58:32.289Z" }, + { url = "https://files.pythonhosted.org/packages/c0/02/7f6156ffc926857f1c37c07d9a388682865a81830ab6a1b637082c25e399/pydantic_core-2.46.5-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:816ff0a6550ffc06c098ccd2e0698600f9aa7da192a79eaa6f9af504a35db869", size = 2105889, upload-time = "2026-08-28T09:58:33.986Z" }, + { url = "https://files.pythonhosted.org/packages/92/b1/e781d357ebe09fc929f995700f1b3503e8897f1cece183ecb1300d4d67e9/pydantic_core-2.46.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c7ea57fc63aa7da93a1bd2d644e6577befae10c52c4e36377635eea1056a74f5", size = 2158006, upload-time = "2026-08-28T09:58:35.647Z" }, + { url = "https://files.pythonhosted.org/packages/70/0a/644597d84ab400e50609c192120b85c9681c22d3a20461b9060a79be0a7a/pydantic_core-2.46.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:efd62a42486f1bda5d24cb4f63d15a3c7768375fe83d36f9417b4ad7a2fb20b3", size = 2158408, upload-time = "2026-08-28T09:58:37.38Z" }, + { url = "https://files.pythonhosted.org/packages/1e/ee/ca3b7b3a4b3769ffe9ce9432a7c9be755de9593a46d3b0d54d0409323e44/pydantic_core-2.46.5-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:2bc9419666990c06d7397831f2126a1ecc3594aaa3ff7de5bf2d066802f4e07b", size = 2309609, upload-time = "2026-08-28T09:58:39.22Z" }, + { url = "https://files.pythonhosted.org/packages/ce/52/39fa1f451486019524ca685020390e7ca351832fd874530ba30c8628e6dc/pydantic_core-2.46.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:18a09e1e1011b462f2e32774f25859ef1223d5c2b0546a633cf56654710721e0", size = 2342618, upload-time = "2026-08-28T09:58:40.89Z" }, + { url = "https://files.pythonhosted.org/packages/81/5e/468fc630568c61dcef3cd47ad32ffbeed9af643f49208d1ea86ab4f890c4/pydantic_core-2.46.5-cp312-cp312-win32.whl", hash = "sha256:5cb482e9e84c851f4e623fe4acc1ced89168cf1fe18f7089db4548c8f5bbb65b", size = 1939475, upload-time = "2026-08-28T09:58:42.591Z" }, + { url = "https://files.pythonhosted.org/packages/cf/c9/4c19f41b84cf6b622a72fbeed7665b25d47a187d68d47d0d430c07f23268/pydantic_core-2.46.5-cp312-cp312-win_amd64.whl", hash = "sha256:5e81740c09e310f5aa5cbd3e434a01c154d4bef93241c7877b39f211d2b78ba8", size = 2043140, upload-time = "2026-08-28T09:58:44.272Z" }, + { url = "https://files.pythonhosted.org/packages/af/dd/0c1a050299147c746e5256db16d645ab5efd4f78c59937d581a0524e74a2/pydantic_core-2.46.5-cp312-cp312-win_arm64.whl", hash = "sha256:f7b0ec93a2893de856652154d73b7ba622f26fa97726487dcac373de5f4c6084", size = 1997729, upload-time = "2026-08-28T09:58:46.13Z" }, + { url = "https://files.pythonhosted.org/packages/f5/37/5abe39a8372a61d3dc3c1338fc504281c01b32fdb3169cd7187153b56d3e/pydantic_core-2.46.5-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:b7ca9034437b6022f941f4857459562ee00a560b97e7cce8a0ec5a74fc6766e0", size = 2075885, upload-time = "2026-08-28T09:58:47.856Z" }, + { url = "https://files.pythonhosted.org/packages/21/43/6323b1f8b217780454c61304bcd2b38ae4762f50754414124603ccc90bb2/pydantic_core-2.46.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f332f0e72a5a0400141f830744e141bf9f97917878dbe968669e8a7fefea78ff", size = 1922768, upload-time = "2026-08-28T09:58:49.58Z" }, + { url = "https://files.pythonhosted.org/packages/0f/a3/c05ca796e1197618a774b01e596aeedfefc2f7d8c01ae3054e910b120e8a/pydantic_core-2.46.5-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:193375f3548919d3f0b60936ca113ada3e38f264f91b9b8e0508efaad57be931", size = 1951241, upload-time = "2026-08-28T09:58:51.511Z" }, + { url = "https://files.pythonhosted.org/packages/68/32/33bc39ac705c52cffc908e8389f9754fdb208aea5c69cceddf4eb3ce99af/pydantic_core-2.46.5-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:79bdfa52f843137045b2d081cc05c120ba6665d29b7559c2c47690906f39279f", size = 2031975, upload-time = "2026-08-28T09:58:53.166Z" }, + { url = "https://files.pythonhosted.org/packages/b0/70/2333e885c0f6a67bc105c5916965dac9b57f2718ee20d81d1a06a4ebdc13/pydantic_core-2.46.5-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:24922243639cbdac66c75fcb6fd6495a9cb52b213d62f9a0d16f0310b1ff8038", size = 2208542, upload-time = "2026-08-28T09:58:55.017Z" }, + { url = "https://files.pythonhosted.org/packages/f7/ea/296debfb4264207bbda5936133892e027c0a58875ad53ebd512fba8ec3a2/pydantic_core-2.46.5-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c76fe65e607be28c7fd4d56fc3c42b1583aa058ce3408b7ad0fd540171d31f9f", size = 2264692, upload-time = "2026-08-28T09:58:56.767Z" }, + { url = "https://files.pythonhosted.org/packages/d3/f2/9e4de77a6271e07a76d2d58b11c091a979c191ed2939bf80067568b369d2/pydantic_core-2.46.5-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6f7b393a8b3da82f5c1fc0751e6d01ac6c55b93c18226a60bdfba4a724efafd1", size = 2066633, upload-time = "2026-08-28T09:58:58.531Z" }, + { url = "https://files.pythonhosted.org/packages/8d/db/f9e9d0c97445987b2084823d5c240de88087338f04fc2cfaa2df186b8049/pydantic_core-2.46.5-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:7ac031912d54f3d83ef3b3eb98dfabc1608802e2202263d25957eeed40b94761", size = 2105235, upload-time = "2026-08-28T09:59:00.421Z" }, + { url = "https://files.pythonhosted.org/packages/07/c5/79169b047b3b2c3e99e04bc76372af9637e0bf6db638274fa927df96369e/pydantic_core-2.46.5-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:837b396ca3d7b74091ca623f6cbd8351bd42d670a79c2683e79fb089f06a2de5", size = 2157367, upload-time = "2026-08-28T09:59:02.442Z" }, + { url = "https://files.pythonhosted.org/packages/26/b5/ba6057afb7c291bd449f51b867f95aef2072941c4ce4e5c31d6ffd132d3b/pydantic_core-2.46.5-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:5ee239d575f80b08eca11f6e20f90c4c695de7825c67eefe6091fbf20dda648e", size = 2158420, upload-time = "2026-08-28T09:59:04.2Z" }, + { url = "https://files.pythonhosted.org/packages/6e/28/2057abecaafdc22912afa819603a51f0a62d40643b7c4871c51721fea9be/pydantic_core-2.46.5-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:e80675d75ae2cd14372cb65cad5400d9347a3d3f6c13000183f22dfd027283ed", size = 2309588, upload-time = "2026-08-28T09:59:06.048Z" }, + { url = "https://files.pythonhosted.org/packages/71/9d/881156dc404e27479c4246128d73538464cab4a239bec61995e227644c30/pydantic_core-2.46.5-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:9c4b71f10dd532fb7a5cbc8f58707779e64f03a258c2bf8bfbaecfcd9970b519", size = 2341866, upload-time = "2026-08-28T09:59:08.539Z" }, + { url = "https://files.pythonhosted.org/packages/5a/38/d66f443a259f84d13babdceae568e572b0ed26da17ca5d0a649ebb110a67/pydantic_core-2.46.5-cp313-cp313-win32.whl", hash = "sha256:97bf8de4d541598c94a59344eeb988a94c08ff76b5723c41f6567ec18c7892ea", size = 1938580, upload-time = "2026-08-28T09:59:10.402Z" }, + { url = "https://files.pythonhosted.org/packages/2c/1e/1d5371213f4cc9a7ed70c0bfcc7911de22311ee99a662a56077d7292d2ac/pydantic_core-2.46.5-cp313-cp313-win_amd64.whl", hash = "sha256:15f4a94963c95accac15b7b657bb177d3ad82bb90b0d0526d9a9b85079925db5", size = 2041980, upload-time = "2026-08-28T09:59:12.396Z" }, + { url = "https://files.pythonhosted.org/packages/5a/48/4222d90b1c67568bace4dec6dca6271449c66de3595d72b6d098f5fde597/pydantic_core-2.46.5-cp313-cp313-win_arm64.whl", hash = "sha256:d22a945598fb91236b4dd793a6e42e4f3dd7740bb5aace5ebd7d4c08d13bb575", size = 1997213, upload-time = "2026-08-28T09:59:14.245Z" }, + { url = "https://files.pythonhosted.org/packages/8e/8a/14596f2a8367da50cf7cbac48169ee5d9c8e11d486a3b527082384630c72/pydantic_core-2.46.5-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:c1c43ad4339643d70ebb8124e1305a7dab423001eff58bb41a0f731adbc98355", size = 2074081, upload-time = "2026-08-28T09:59:16.141Z" }, + { url = "https://files.pythonhosted.org/packages/ae/d5/d8a4eb6d6c7f66b91dd37c576d76e9e60fba900caf5372c17bcf949febc2/pydantic_core-2.46.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1a353f84de772f423b5ffb11d7ae352fbbef0f446f3c0b0af0f8236d7233606e", size = 1920497, upload-time = "2026-08-28T09:59:18.065Z" }, + { url = "https://files.pythonhosted.org/packages/8e/26/092079428f86e927e030b2c0ced87df69dbb1c875cdeaa67bf42ea2be746/pydantic_core-2.46.5-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5086029a57366b8cf81b130a43908738095c270c21a8d7f0e8bdfdb89718e2f3", size = 1952130, upload-time = "2026-08-28T09:59:20.476Z" }, + { url = "https://files.pythonhosted.org/packages/08/c3/8ec0e290a9ebaebd64047bf5fda94be835c6b1551b02437e4b76778fbcd7/pydantic_core-2.46.5-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:46c25dda9d092a06c08db76ffe0a197107904d0dfac653f7d5306bbcd6d6119c", size = 2026371, upload-time = "2026-08-28T09:59:22.227Z" }, + { url = "https://files.pythonhosted.org/packages/01/72/4fd20ad520fb8da0157f95b27a7eb05a72790ef08138e7701ac972c342ea/pydantic_core-2.46.5-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:37ea7b83c935e5b0d68c9449b82651accf78a10828b2c02b2f2d9e9496446c21", size = 2202822, upload-time = "2026-08-28T09:59:24.277Z" }, + { url = "https://files.pythonhosted.org/packages/31/b0/d16e0771206b29314f0d52198b720be21e8a99ab2bf11e3bc0d7c9cebdff/pydantic_core-2.46.5-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e64e88d5585bea9ce95861079de72006c7fa6d3df4e3a3b65ba31eb979c15c9f", size = 2262756, upload-time = "2026-08-28T09:59:26.608Z" }, + { url = "https://files.pythonhosted.org/packages/2c/9b/59634b7ac631c63b2a37760eb6943af3e29573d6b59a4abc5e7f019d4cee/pydantic_core-2.46.5-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:54d510bac3ee52247af28ed4bb18a1e799f040ac60fd2bf5ccd4c92f1fbe786f", size = 2068352, upload-time = "2026-08-28T09:59:29.044Z" }, + { url = "https://files.pythonhosted.org/packages/08/7c/570abb1ad2155348dc754ea91be22e5aaa18eb6d69a6068f7c6f2679a6ed/pydantic_core-2.46.5-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:a2a5e1d0ff29adddc9f6d6821a66302e4493f8ca898b715b6b1182c2c201ea0a", size = 2104777, upload-time = "2026-08-28T09:59:30.95Z" }, + { url = "https://files.pythonhosted.org/packages/8e/25/5bf74adc65a1ac5b7be3f6cb0bcb5433615c1598a801c19d830d84c98ded/pydantic_core-2.46.5-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:03b9666e41e35d8909852ba191a0607520f81b74eaf12ccf8737005dbb313821", size = 2156312, upload-time = "2026-08-28T09:59:32.604Z" }, + { url = "https://files.pythonhosted.org/packages/90/6a/2ef38830675e050121040618135564ed56b860b45433b02d9b4ebece46f3/pydantic_core-2.46.5-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:a91c17edf6eea2402cb5457b4c89e99bc5ed1004aa34c4adf1d4258c1a5c22c2", size = 2150067, upload-time = "2026-08-28T09:59:34.453Z" }, + { url = "https://files.pythonhosted.org/packages/90/ef/a7dbb03a14a64c2a4621f989c615ed9a892535a6cad938fc27079f919d80/pydantic_core-2.46.5-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:b49924c73a235e969511bf2aabdff3beebf9820931f646c80274d5d780010c47", size = 2304516, upload-time = "2026-08-28T09:59:36.194Z" }, + { url = "https://files.pythonhosted.org/packages/68/f8/6bb4c4b80e8a6fde1904c64a51c62a1d04fcdfa3ea521a66b2ddefa1d885/pydantic_core-2.46.5-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:2cbd9a5eff05e51c447c34dfa4632145b26b09120cf04bd0c871e44c1a5e1c9a", size = 2335223, upload-time = "2026-08-28T09:59:37.931Z" }, + { url = "https://files.pythonhosted.org/packages/2a/80/f46b8c681195190b2c1f1c7c0a81abce60663e987613e09ef64d433dd96b/pydantic_core-2.46.5-cp314-cp314-win32.whl", hash = "sha256:2d5d76654becf5efd62c9e51c3756c67b49498b0c9a40884934c40807adbd074", size = 1934827, upload-time = "2026-08-28T09:59:39.836Z" }, + { url = "https://files.pythonhosted.org/packages/f7/3c/60674207246bc0a4009d2391b7c7251c7159f279c8d2ab8aae8ef46f3dee/pydantic_core-2.46.5-cp314-cp314-win_amd64.whl", hash = "sha256:fa10ef4112775900e7a0661068635eb67b2ab824fbde764de6e0e21982a93db0", size = 2042648, upload-time = "2026-08-28T09:59:41.792Z" }, + { url = "https://files.pythonhosted.org/packages/69/0c/117c562c7c1babdf44576b72a5e496906506c93690387ecfbca7c729ae2e/pydantic_core-2.46.5-cp314-cp314-win_arm64.whl", hash = "sha256:045ab3b6d308439e32b81cc173bba5b9018bc6ed896afd0c65b3b009b1699af5", size = 1989652, upload-time = "2026-08-28T09:59:43.702Z" }, + { url = "https://files.pythonhosted.org/packages/e8/66/9336ae58f9eb68c41d121894e52c4c89eccb07eb8f602a04ee9c3f37736a/pydantic_core-2.46.5-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:8816f3d218beb4b787de5c9759c259b8fa61f9dec42dc7811f320a33771778b7", size = 2065829, upload-time = "2026-08-28T09:59:45.364Z" }, + { url = "https://files.pythonhosted.org/packages/c5/02/bc19b47a96c2d3109760711acf22369e56bd7e405ca52f7ade164d2ead57/pydantic_core-2.46.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:bce57638e08ac148e5778cce7feb968307a727d66f8e2274a543d0cf0c9ad6a3", size = 1905716, upload-time = "2026-08-28T09:59:47.18Z" }, + { url = "https://files.pythonhosted.org/packages/52/a4/70b47c0509923dd98ccfed04fb3e32ea3849c82a0ff2205bb41009b43c00/pydantic_core-2.46.5-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:976e1128455aa595ea04c79ccfedff1aaeab96ee013fcc916bed120c4f0ad94f", size = 1934216, upload-time = "2026-08-28T09:59:49.241Z" }, + { url = "https://files.pythonhosted.org/packages/52/ab/aa03b65f7bb198585edf806b906c3223ecf1795543e39e23aec4cce27ad2/pydantic_core-2.46.5-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e7b891faeedeafba41b2983e5001a81b6a915b69544c7e7570d1989ce1c36ac7", size = 2010635, upload-time = "2026-08-28T09:59:51.692Z" }, + { url = "https://files.pythonhosted.org/packages/3c/8b/0da06343f30b84ec549aafd309c6456223d5dc8bd36af504c573faad561d/pydantic_core-2.46.5-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5f194189415698233dd1114a093a9b56e61e2c57e11b469be3b0506f46f0771c", size = 2209369, upload-time = "2026-08-28T09:59:53.582Z" }, + { url = "https://files.pythonhosted.org/packages/d6/5b/844c4defaa34a3df66eb9257087d121d70c201298b96abdf9f492fc2f1bf/pydantic_core-2.46.5-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:82a36973cf8a2ef5406f4fe2edbf8ed0c99629535d959e0b100c76a32535a111", size = 2253238, upload-time = "2026-08-28T09:59:55.484Z" }, + { url = "https://files.pythonhosted.org/packages/f4/64/a4e536cb16d7f61a7fd3120b46c577fc7fa7325992f69c4f52bc786d77d8/pydantic_core-2.46.5-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cdbb78909f52b981d3b2d56b97328d71eb0b974c36bd77c920123a7ebb192829", size = 2065740, upload-time = "2026-08-28T09:59:58.038Z" }, + { url = "https://files.pythonhosted.org/packages/5f/75/aaa38c6bc2d085f6605b34eabdc6a8a4e0b2e61fc9c8e6e52b28e97b3125/pydantic_core-2.46.5-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:52e24eacdb536cade636aa90fb851835222becff8484b7001fdc78cb0290f2aa", size = 2087425, upload-time = "2026-08-28T09:59:59.898Z" }, + { url = "https://files.pythonhosted.org/packages/55/ae/fcab4cfc39aba3689e1d20c8b5250ad280957022c09af2ed9cd585602a5e/pydantic_core-2.46.5-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:37ae34309d7bd8c0d61ab839668058f2a7962ea1fc51d105d2db228fe0618034", size = 2139306, upload-time = "2026-08-28T10:00:03.057Z" }, + { url = "https://files.pythonhosted.org/packages/2d/f4/f1d03a4bc9d9acbc62f4d742b8a319af52f71885079868b2ff8e48a651ee/pydantic_core-2.46.5-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:0cdbada856a1c69a7624a64d3d9aefe79300bd6ef827b43a4f265010b9b55184", size = 2144589, upload-time = "2026-08-28T10:00:05.645Z" }, + { url = "https://files.pythonhosted.org/packages/83/f3/7a53bb1356de514a4cd295f25b6ac39237895620c0462d2592b76c16e114/pydantic_core-2.46.5-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:545f26c504b27c3758439a5e6d9349931f0a04f855668d5fe323c89e82300a38", size = 2288882, upload-time = "2026-08-28T10:00:07.931Z" }, + { url = "https://files.pythonhosted.org/packages/cd/94/5a81583660c175c59d49ffb09f4b3a44debeaf86a19fca664ae1cdd9ee32/pydantic_core-2.46.5-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:ff218293c9c806138dca139765e3b067621be52bcd93cdc14c7711be7ddc90a9", size = 2335210, upload-time = "2026-08-28T10:00:10.177Z" }, + { url = "https://files.pythonhosted.org/packages/5a/9f/5d685c2693b972d1a59c998586e8823712b66603aeff47ee60a4bdaafd37/pydantic_core-2.46.5-cp314-cp314t-win32.whl", hash = "sha256:97cf3eb53a8cccacf9d46686a0926186c9bfb5574f2ed66d3639d5fe117cd3a9", size = 1921180, upload-time = "2026-08-28T10:00:12.35Z" }, + { url = "https://files.pythonhosted.org/packages/70/12/5c94ee16d65a37a15f9e869f5e6256df111154491173801a4c5e800ab548/pydantic_core-2.46.5-cp314-cp314t-win_amd64.whl", hash = "sha256:d2f9fc07a8042a8f95925b35c4f04f469707c981fc33245b6ca187cf5d2dd290", size = 2020515, upload-time = "2026-08-28T10:00:14.774Z" }, + { url = "https://files.pythonhosted.org/packages/63/19/67830dda664e6bdf9285ee2e40f355d0d7d6b92aa0c42e8d217bb8d33d36/pydantic_core-2.46.5-cp314-cp314t-win_arm64.whl", hash = "sha256:acf8a67ba51f4ca9ddbd0e6b3000a65ac51ab734661778b3e7ba64d99a710f2f", size = 1989276, upload-time = "2026-08-28T10:00:16.984Z" }, + { url = "https://files.pythonhosted.org/packages/df/dd/053c2e4303f791f3b8f8a14ab0b22008e8eb21d868c0c90b4f9be705b76a/pydantic_core-2.46.5-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:013d6f3483d81e02e7c328831808f336c8596ee33b4bd4026b9ffb1e960b8942", size = 2062540, upload-time = "2026-08-28T10:01:00.318Z" }, + { url = "https://files.pythonhosted.org/packages/d7/dd/a18df751a5e37dd51bfad7f68e766999125bebe68c9e1d10a493ad01bd63/pydantic_core-2.46.5-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:e9c134bb666dd54b778b9fc0d2b50cbb7f979b9e3716f26a88c9ab3b6fc1dd0f", size = 1902040, upload-time = "2026-08-28T10:01:02.529Z" }, + { url = "https://files.pythonhosted.org/packages/b7/13/01d40f9d07ce8a779fd6e0bd8ad4fba91309500dd67b869e2e219d261a6d/pydantic_core-2.46.5-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:347ec774390c87326a2e4929d58d3f7e8763a104d5d35f4cd595a4c952366433", size = 1967479, upload-time = "2026-08-28T10:01:05.004Z" }, + { url = "https://files.pythonhosted.org/packages/fa/04/c81d4841331c2178b6fb09ae225425e110ed72d990c9fe556c4ec03d1013/pydantic_core-2.46.5-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8e24d8f05fa2d28513d94e877e9c75ad66175376209b3977f916e240e623193c", size = 2111034, upload-time = "2026-08-28T10:01:07.345Z" }, +] + +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, +] + +[[package]] +name = "s3transfer" +version = "0.19.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/49/c5/0a5ce655109cb96b8dfad36908a4e108d15f5dc2eff8fbc3146c4312e704/s3transfer-0.19.3.tar.gz", hash = "sha256:8326acc3d056d7f7b8a37110c7d2aea41f67e83f6ae79cd3b5fbcca34c726406", size = 166087, upload-time = "2026-10-19T19:19:49.751Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e2/a4/88f9e10498759121060b8050a1554753dfc04df39bbbcc7aad9ecc8d9440/s3transfer-0.19.3-py3-none-any.whl", hash = "sha256:24eb7535804306355e31c952a90022bf5b2f9d760f9a11062c7493a217c08553", size = 90252, upload-time = "2026-10-19T19:19:48.309Z" }, +] + +[[package]] +name = "six" +version = "1.17.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, +] + +[[package]] +name = "starlette" +version = "1.7.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7b/2b/3850dc6bf7ef71b088962eba31dafc6cffd2f96e577ebb0bb316df96da3e/starlette-1.7.0.tar.gz", hash = "sha256:c79f74ea63cff761804fbbfb182f1e0b440c2d07b164d24700c5a1bab5d6ff5d", size = 2736246, upload-time = "2026-09-23T07:30:26.35Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4e/d6/1ec1b290f9e0fb067899b61e1d37a30c923068bad260b216dbe37a7d2967/starlette-1.7.0-py3-none-any.whl", hash = "sha256:67f8e99895493dd2911a03f11314af6ceebeae4e704bb9f43dfc6a9db151c93e", size = 78980, upload-time = "2026-09-23T07:30:24.567Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f6/cc/6253133b5bb138fc3306cebfbda2c520f545d36b5be2c7255cc528bb45d6/typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5", size = 113555, upload-time = "2026-07-02T08:40:05.92Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", size = 45571, upload-time = "2026-07-02T08:40:04.659Z" }, +] + +[[package]] +name = "typing-inspection" +version = "0.4.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a3/26/b09b8010994eccc3c09092e6b34058f36a460eea2d4c3e8b910c695975a0/typing_inspection-0.4.4.tar.gz", hash = "sha256:547274fa6b0a561ccf549cc9524b999a578e737d015d8709d021f9d0d13bea47", size = 76928, upload-time = "2026-08-12T12:37:25.997Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/67/81/4add07e5172b7ac40d8ed5ff580409a7801a4fe26d529bdd915401dabfbe/typing_inspection-0.4.4-py3-none-any.whl", hash = "sha256:65b8397ba37ccbce054456aaccddfc91e6e3083c92824df348d96ca832f3f147", size = 14750, upload-time = "2026-08-12T12:37:24.648Z" }, +] + +[[package]] +name = "urllib3" +version = "2.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e3/05/b17359e1cefb4f909b5e40b1b90a496d987258916dbbf88e842c729f510e/urllib3-2.8.0.tar.gz", hash = "sha256:63bf2ead4c879426ebf22ef2a781eeb4aa3b4ae798a0435506f8687fd5bb9b63", size = 458972, upload-time = "2026-09-15T19:29:36.253Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/92/9d/c4e665119135114480843e7ab388fa94d8480650450e6f8e26b70d323a4c/urllib3-2.8.0-py3-none-any.whl", hash = "sha256:0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3", size = 135717, upload-time = "2026-09-15T19:29:34.577Z" }, +] + +[[package]] +name = "uvicorn" +version = "0.40.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "h11" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c3/d1/8f3c683c9561a4e6689dd3b1d345c815f10f86acd044ee1fb9a4dcd0b8c5/uvicorn-0.40.0.tar.gz", hash = "sha256:839676675e87e73694518b5574fd0f24c9d97b46bea16df7b8c05ea1a51071ea", size = 81761, upload-time = "2025-12-21T14:16:22.45Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3d/d8/2083a1daa7439a66f3a48589a57d576aa117726762618f6bb09fe3798796/uvicorn-0.40.0-py3-none-any.whl", hash = "sha256:c6c8f55bc8bf13eb6fa9ff87ad62308bbbc33d0b67f84293151efe87e0d5f2ee", size = 68502, upload-time = "2025-12-21T14:16:21.041Z" }, +] -- 2.54.0 From 45c7aa0085513f991d14a7763efec87013bf6361 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9C=D0=B0=D1=82=D0=B2=D0=B5=D0=B9?= Date: Thu, 5 Nov 2026 22:23:52 +0300 Subject: [PATCH 2/4] add level 3 and 4 --- REPORT.md | 48 +- compose.yaml | 61 ++- docs/observability.md | 26 ++ evidence/db-incident-load-repeat.json | 17 + evidence/db-incident-load.json | 17 + evidence/observability.md | 29 ++ evidence/tempo-outage-load.json | 17 + infra/grafana/dashboards/messenger.json | 437 ++++++++++++++++++ .../provisioning/dashboards/dashboards.yaml | 8 + .../provisioning/datasources/datasources.yaml | 25 + infra/loki/loki.yaml | 17 + infra/otel/collector.yaml | 32 ++ infra/prometheus/alerts.yaml | 13 + infra/prometheus/prometheus.yaml | 7 + infra/tempo/tempo.yaml | 13 + migrations/008_trace_context.sql | 2 + pyproject.toml | 2 + scripts/init-secrets.py | 1 + src/broker.py | 11 +- src/main.py | 46 +- src/outbox.py | 8 +- src/storage/postgres.py | 40 +- src/storage/s3.py | 11 +- src/telemetry.py | 92 ++++ src/worker.py | 43 +- student-tests/alerts.test.yaml | 27 ++ student-tests/queue_incident.py | 11 + student-tests/telemetry_unit.py | 68 +++ student-tests/trace_retry.py | 20 + uv.lock | 289 ++++++++++++ 30 files changed, 1370 insertions(+), 68 deletions(-) create mode 100644 docs/observability.md create mode 100644 evidence/db-incident-load-repeat.json create mode 100644 evidence/db-incident-load.json create mode 100644 evidence/observability.md create mode 100644 evidence/tempo-outage-load.json create mode 100644 infra/grafana/dashboards/messenger.json create mode 100644 infra/grafana/provisioning/dashboards/dashboards.yaml create mode 100644 infra/grafana/provisioning/datasources/datasources.yaml create mode 100644 infra/loki/loki.yaml create mode 100644 infra/otel/collector.yaml create mode 100644 infra/prometheus/alerts.yaml create mode 100644 infra/prometheus/prometheus.yaml create mode 100644 infra/tempo/tempo.yaml create mode 100644 migrations/008_trace_context.sql create mode 100644 src/telemetry.py create mode 100644 student-tests/alerts.test.yaml create mode 100644 student-tests/queue_incident.py create mode 100644 student-tests/telemetry_unit.py create mode 100644 student-tests/trace_retry.py diff --git a/REPORT.md b/REPORT.md index 0f8f976..e23516b 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,13 +1,13 @@ # Отчёт — лабораторная 6 - Автор, группа: Kozyrev, BVT2301 -- Целевая оценка; выбранный трек 5, если нужен: TODO +- Целевая оценка: 4 - Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Источники: [контракт телеметрии](contracts/observability.md), [OTel Collector](https://opentelemetry.io/docs/collector/configuration/), [OTel Python exporters](https://opentelemetry.io/docs/languages/python/exporters/), [Loki OTLP](https://grafana.com/docs/loki/latest/send-data/otel/), [Grafana provisioning](https://grafana.com/docs/grafana/latest/administration/provisioning/), исходники lab5 commit `6bc0712`. При реализации использован Codex. -## Запуск из чистого clone +## Запуск -Перенесённый стенд lab5 требует Docker Engine с Compose v2 и Python 3 для генерации локальных секретов. Порт `127.0.0.1:8080` должен быть свободен. Версии образов указаны в `Dockerfile` и `compose.yaml`. +Нужны Python 3, Docker Engine и Compose v2. API слушает `127.0.0.1:8080`, Grafana — `127.0.0.1:3000`. Версии образов закреплены в [Dockerfile](Dockerfile) и [compose.yaml](compose.yaml). ```sh python3 scripts/init-secrets.py @@ -19,35 +19,31 @@ make check make test ``` -Миграции и создание S3 bucket выполняют сервисы `migrate` и `s3-init`. Секреты создаются в игнорируемом каталоге `secrets/`; `.env` для значений Compose по умолчанию не обязателен. `docker compose down` останавливает стенд с сохранением volumes. Команда `docker compose down -v` намеренно удаляет учебные данные PostgreSQL, S3 и RabbitMQ. - -После переноса `docker compose config --quiet` и `make check` выполнены успешно. Запуск стенда и `make test` ещё не проверены: Docker Engine был недоступен. Телеметрия lab6 пока не реализована. +Миграции и создание S3 bucket выполняют сервисы `migrate` и `s3-init`. Секреты создаются в игнорируемом `secrets/`; `.env` для значений Compose по умолчанию не обязателен. В Grafana логин `admin`, пароль — в локальном `secrets/grafana_password`. `docker compose down` сохраняет volumes; `docker compose down -v` намеренно удаляет все именованные volumes этого проекта, включая учебные данные и историю телеметрии. ## Архитектура и решения -Перенесены исходники, миграции, инфраструктура, собственные тесты и вспомогательные скрипты из lab5, commit `6bc0712`. Контракты и публичные тесты lab6 сохранены. Клиент обращается через HAProxy к двум API-репликам; PostgreSQL хранит данные и outbox, SeaweedFS — файлы, RabbitMQ передаёт задания worker. Dispatcher публикует задания из outbox; worker обрабатывает изображения. S3 и PostgreSQL не образуют общую транзакцию; сервис reconcile отвечает за сверку и очистку. Текущий Compose не содержит Collector, хранилища телеметрии и Grafana. +Клиент обращается через HAProxy к двум API-репликам. PostgreSQL хранит бизнес-данные, задания и outbox; SeaweedFS — байты вложений. Dispatcher публикует события из outbox в RabbitMQ, worker формирует варианты; reconcile выполняет сверку. S3 и PostgreSQL не образуют общую транзакцию. Из lab5 перенесены исходники, миграции, инфраструктура и собственные тесты; контракты и публичные тесты lab6 сохранены. -`.env.example` описывает логические настройки; `compose.yaml` использует только часть значений из окружения, остальные заданы в нём явно. Конфигурация OpenTelemetry в примере пока не подключена к приложению. +API, outbox и worker отправляют OTLP через Collector в Prometheus, Loki и Tempo; Grafana использует эти хранилища через provisioned datasources. [Dashboard](infra/grafana/dashboards/messenger.json), [правила alerts](infra/prometheus/alerts.yaml), конфигурации и миграция context находятся в репозитории. Схема сигналов, запросы, SLI/SLO, ограничения и сценарии — в [docs/observability.md](docs/observability.md). `.env.example` описывает логические настройки; Compose часть значений задаёт явно. OTLP включён в Compose для API, worker и outbox. -## Доказательства критериев +## Проверки и доказательства -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +| Критерий | Команда/сценарий | Результат | +| --- | --- | --- | +| Целостность и smoke | `make check`, `make validate PYTHON=.venv/bin/python`, `make test` | PASS; smoke повторён после восстановления. | +| Конфигурации | `docker compose config --quiet`; штатные проверки Collector, Loki, Tempo и `promtool` | PASS; Grafana API вернул три datasource и dashboard, запросы панелей выполнены. | +| Контекст и конфиденциальность | `.venv/bin/python student-tests/telemetry_unit.py` | 3 теста PASS: parent/child через AMQP, ограниченные поля логов и шаблон маршрута метрики. | +| Обработка вложений | `.venv/bin/python student-tests/pipeline_critical.py` | 8 тестов PASS. | +| Данные и реплики | `make test-persistence ACTION_SCRIPT=scripts/restart.sh`; `make test-failover ACTION_SCRIPT=scripts/stop-one.sh` | Оба PASS; две реплики восстановлены. Фактический состав hook описан в скриптах; отдельного снимка состояний до/после для persistence нет. | +| Связь API → worker, включая retry | upload с тестовой инъекцией crash worker | ready на второй попытке; 2 producer и 2 consumer spans в одном trace. | +| Alert очереди и alert ошибок API | остановка worker; блокировка таблицы users при нагрузке; `promtool test rules student-tests/alerts.test.yaml` в контейнере | Оба наблюдались в firing и отсутствовали после восстановления; синтетический тест финальных правил PASS. | +| Отказ хранилища telemetry | остановить Tempo, выполнить smoke и отдельную нагрузку 75 с | smoke PASS; очередь ограничена, зафиксировано переполнение и потери traces. | -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +Идентификаторы трасс, результаты, формулы и границы проверки приведены в [evidence/observability.md](evidence/observability.md). Raw summaries нагрузки: [инцидент БД](evidence/db-incident-load-repeat.json), [отказ Tempo](evidence/tempo-outage-load.json). Полный набор наблюдений живёт в локальных volumes, а не в Git. -## Эксперименты и отказы +## Эксперименты и ограничения -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +Docker Engine 29.8.0, `aarch64`; Docker VM сообщил 10 CPU и 10 416 644 096 байт RAM. Лимиты контейнеров указаны в Compose: например, Collector — 256 MiB, каждая API-реплика — 0,5 CPU и 512 MB. Ограниченная нагрузка была closed-loop; прогрев и статистические повторы для сравнения производительности не проводились. Числа не являются оценкой максимальной пропускной способности. В отчёте о первом опыте блокировки БД alert не успел перейти в firing; после изменения выдержки до 10 секунд повторный опыт показал firing и восстановление. -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | - -## Ограничения и следующие шаги - -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +`messenger_job_ready_duration_seconds` измеряет время от создания DB-записи о вложении до фиксации ready; оно включает время до commit принятия. Измерение SLO за полное 24-часовое окно ещё не проведено. Данные queue age обновляет outbox: при его остановке gauge устаревает. RSS/CPU worker не включают отдельный render-процесс; pool waiting зависит от внутреннего API закреплённой версии asyncpg. Один Docker-host и единственные экземпляры DB, broker и LB не обеспечивают HA при их потере. Три замера RAM при отказе Tempo не доказывают поведение для произвольной длительности; переполнение очереди приводит к потере spans. При необходимости переносимой защиты следует отдельно сохранить наблюдения и подтвердить состояние alerts и трасс после нового clone. diff --git a/compose.yaml b/compose.yaml index 8b0b05d..551c0cd 100644 --- a/compose.yaml +++ b/compose.yaml @@ -4,7 +4,9 @@ x-api: &api environment: APP_HOST: 0.0.0.0 APP_PORT: 8080 - APP_ENV: lab5 + APP_ENV: lab + OTEL_ENABLED: "1" + OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318 DATABASE_URL_FILE: /run/secrets/database_url SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400} @@ -206,7 +208,9 @@ services: context: . command: ["/app/.venv/bin/python", "-m", "src.worker"] environment: - APP_ENV: lab5 + APP_ENV: lab + OTEL_ENABLED: "1" + OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318 DATABASE_URL_FILE: /run/secrets/database_url CURSOR_SECRET_FILE: /run/secrets/cursor_secret S3_ENDPOINT: http://s3:8333 @@ -243,7 +247,9 @@ services: context: . command: ["/app/.venv/bin/python", "-m", "src.outbox"] environment: - APP_ENV: lab5 + APP_ENV: lab + OTEL_ENABLED: "1" + OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318 DATABASE_URL_FILE: /run/secrets/database_url CURSOR_SECRET_FILE: /run/secrets/cursor_secret BROKER_URL_FILE: /run/secrets/broker_url @@ -283,7 +289,52 @@ services: cpus: "0.25" memory: 128M + otel-collector: + image: otel/opentelemetry-collector-contrib:0.120.0 + command: ["--config=/etc/otelcol/config.yaml"] + volumes: + - ./infra/otel/collector.yaml:/etc/otelcol/config.yaml:ro + mem_limit: 256m + restart: unless-stopped + prometheus: + image: prom/prometheus:v3.2.1 + command: ["--config.file=/etc/prometheus/prometheus.yaml", "--storage.tsdb.path=/prometheus", "--storage.tsdb.retention.time=7d"] + volumes: + - ./infra/prometheus/prometheus.yaml:/etc/prometheus/prometheus.yaml:ro + - ./infra/prometheus/alerts.yaml:/etc/prometheus/alerts.yaml:ro + - prometheus_data:/prometheus + restart: unless-stopped + loki: + image: grafana/loki:3.4.2 + command: ["-config.file=/etc/loki/loki.yaml"] + volumes: + - ./infra/loki/loki.yaml:/etc/loki/loki.yaml:ro + - loki_data:/loki + restart: unless-stopped + tempo: + image: grafana/tempo:2.7.1 + command: ["-config.file=/etc/tempo/tempo.yaml"] + volumes: + - ./infra/tempo/tempo.yaml:/etc/tempo/tempo.yaml:ro + - tempo_data:/var/tempo + restart: unless-stopped + grafana: + image: grafana/grafana:11.5.2 + ports: ["127.0.0.1:3000:3000"] + environment: + GF_SECURITY_ADMIN_USER: admin + GF_SECURITY_ADMIN_PASSWORD__FILE: /run/secrets/grafana_password + GF_USERS_ALLOW_SIGN_UP: "false" + secrets: [grafana_password] + volumes: + - grafana_data:/var/lib/grafana + - ./infra/grafana/provisioning:/etc/grafana/provisioning:ro + - ./infra/grafana/dashboards:/var/lib/grafana/dashboards:ro + restart: unless-stopped + secrets: + grafana_password: + file: ./secrets/grafana_password s3_config: file: ./secrets/s3.json s3_admin_key: @@ -306,6 +357,10 @@ secrets: file: ./secrets/broker_url volumes: + prometheus_data: + loki_data: + tempo_data: + grafana_data: s3_data: postgres_data: rabbitmq_data: diff --git a/docs/observability.md b/docs/observability.md new file mode 100644 index 0000000..9caf9a3 --- /dev/null +++ b/docs/observability.md @@ -0,0 +1,26 @@ +# Наблюдаемость и проверка lab6 + +## Сигналы и назначение + +API, worker и outbox отправляют OTLP/HTTP в Collector. Prometheus хранит метрики, Loki — логи, Tempo — трассы, Grafana предоставляет запросы и dashboard. Данные хранятся в именованных volumes, retention для Prometheus/Loki/Tempo — 7 дней. Grafana доступна на `http://localhost:3000`, пользователь `admin`, пароль находится в локальном `secrets/grafana_password`. Provisioning импортирует источники и dashboard `Messenger — observability`. + +Сохраняются только выбранные поля логов; трассы не содержат SQL, аргументов запросов, заголовков авторизации, имён файлов и тел. Метрики не размечаются ID пользователей, чатов и вложений. В экспортируемых логах доступен `request_id`, а `trace_id`/`span_id` передаются полями OTLP. В Explore Loki запрос `{service_name="messenger-api"}` находит события API, `{service_name="messenger-worker"}` — worker; в метаданных события доступны ID трассы. По ним можно перейти в Tempo. Локальные stdout-логи содержат JSON с теми же trace/span ID для событий телеметрии. + +При upload текущий W3C trace context сохраняется в `attachment_jobs` одной транзакцией с outbox. Dispatcher извлекает его и публикует сообщение с дочерним producer span и AMQP headers; worker извлекает контекст и создаёт consumer span. Повторная публикация и retry используют сохранённый контекст исходной загрузки. Каждая попытка имеет собственные spans; они являются потомками исходной транзакции загрузки. Baggage не переносится. Контекст — технические данные, не содержимое изображения. + +Dashboard использует счётчики и гистограммы с единицами секунд: p50/p95/p99 HTTP вычисляются через `histogram_quantile` по сумме buckets, не по среднему перцентилей. `messenger_job_ready_duration_seconds` измеряет от времени создания записи о вложении в DB до фиксации ready; эта оценка включает время до commit принятия и не претендует на точную границу durable acceptance. Queue depth и age — число и возраст самого старого задания в состоянии queued из PostgreSQL, включая ожидающие retry; это не длина очереди RabbitMQ. Данные обновляет outbox. DB pool отражает пул каждой API-реплики; число ожидающих читается из внутренней очереди asyncpg 0.31.0 и зависит от этой версии. Worker busy/capacity — число занятых executor threads и их лимит; повторная доставка уже завершённой задачи учитывается как успешная обработка. CPU и RSS — процессы API/worker/outbox, без дочерних процессов render. При остановленном outbox queue gauge устаревает. + +SLI: доля ответов 5xx среди всех HTTP бизнес-запросов API, исключая health. Целевой SLO: не менее 99% запросов без 5xx за скользящие 24 часа; ошибки балансировщика, которые не дошли до API, SLI не учитывает. Целевое значение выбрано для учебного стенда, а не по измеренным требованиям пользователей. Alert `MessengerHighErrorRate` срабатывает при >5% за минуту в течение 10 секунд; это оперативный порог, не burn-rate alert. `MessengerQueueLag` срабатывает при возрасте queued >30 секунд в течение 15 секунд. Состояния доступны в Prometheus через Grafana Explore или локальный API Prometheus внутри контейнера. + +Collector использует memory limiter 192 MiB, batch до 1024 записей, очереди экспортёров по 128 batches и retry до 60 секунд; лимит контейнера 256 MiB. SDK использует ограниченные очереди spans/logs по 2048 и timeout экспорта 2 секунды. При долгом отказе хранилища возможна потеря телеметрии; это не должно блокировать бизнес-обработку. Наблюдайте внутренние метрики `otelcol_exporter_queue_size`, `otelcol_exporter_send_failed_spans_total`, `otelcol_exporter_enqueue_failed_spans_total`, `otelcol_exporter_send_failed_log_records_total` и `otelcol_receiver_refused_*`. Очереди не сохраняются после перезапуска Collector. + +## Проверки на локальном учебном стенде + +Приведённые сценарии воспроизведены на локальном стенде; зафиксированные результаты и ограничения — в `evidence/observability.md`. Повторный запуск создаёт новые синтетические данные. + +1. Выполните `python3 scripts/init-secrets.py`, `docker compose up -d --build`, дождитесь readiness и запустите `make test`. В Grafana найдите событие API и worker, раскройте trace_id, проверьте дерево upload → DB/outbox → producer → consumer → S3/DB и убедитесь, что содержимое/credentials не попали в telemetry. Для повторной попытки используйте имеющийся fault injection worker и проверьте новую попытку в том же trace. +2. Очередь: `docker compose stop worker`, затем `python3 student-tests/queue_incident.py`. Смотрите queue age и alert; после перехода в firing выполните `docker compose start worker`, дождитесь уменьшения backlog и resolved. Время и скриншоты/запросы сохраните в evidence. +3. DB: начните `python3 tests/load.py --duration 180 --concurrency 8 --output evidence/db-incident-load.json`; после завершения bootstrap в отдельном терминале выполните `docker compose exec -T postgres psql -U messenger -d messenger -c "BEGIN; LOCK TABLE users IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep(90); COMMIT;"`. Наблюдайте задержку `postgres.user_for_token`, ошибки и alert; после окончания блокировки проверьте восстановление и resolved. Тест работает с синтетическими данными и намеренно мешает запросам. +4. Отказ telemetry backend: `docker compose stop tempo`, затем `make test` и контролируемая нагрузка; сравните `docker stats --no-stream`, queue size и failed exports до/во время отказа. Восстановите `docker compose start tempo` и проверьте восстановление экспорта. Длительный отказ может привести к потере traces; измерьте её, а не предполагайте отсутствие потерь. + +Для каждого сценария фиксируйте UTC-время воздействия, исходный профиль нагрузки, запросы метрик, firing/resolved, trace/log и результат после восстановления. `docker compose down` сохраняет volumes; не используйте `down -v`, если нужны сохранённые наблюдения. diff --git a/evidence/db-incident-load-repeat.json b/evidence/db-incident-load-repeat.json new file mode 100644 index 0000000..1205c54 --- /dev/null +++ b/evidence/db-incident-load-repeat.json @@ -0,0 +1,17 @@ +{ + "lab": 6, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 150.0, + "elapsed_seconds": 150.009, + "concurrency": 8, + "successful_requests": 32056, + "failed_requests": 142, + "error_ratio": 0.00441, + "successful_rps": 213.694, + "successful_latency_ms": { + "p50": 6.773, + "p95": 57.056, + "p99": 90.041 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/db-incident-load.json b/evidence/db-incident-load.json new file mode 100644 index 0000000..35d57df --- /dev/null +++ b/evidence/db-incident-load.json @@ -0,0 +1,17 @@ +{ + "lab": 6, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 160.0, + "elapsed_seconds": 160.008, + "concurrency": 8, + "successful_requests": 42418, + "failed_requests": 128, + "error_ratio": 0.003009, + "successful_rps": 265.099, + "successful_latency_ms": { + "p50": 6.611, + "p95": 55.284, + "p99": 84.916 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/observability.md b/evidence/observability.md new file mode 100644 index 0000000..6c9760c --- /dev/null +++ b/evidence/observability.md @@ -0,0 +1,29 @@ +# Результаты проверки наблюдаемости + +Проверка 2026-11-05 на локальном Docker Compose; данные синтетические. Это отдельный учебный стенд, результаты не доказывают отказоустойчивость одного Docker-host. Конфигурация и команды повторения — [docs/observability.md](../docs/observability.md). Данные Prometheus/Loki/Tempo хранятся в локальных Docker volumes и не включены в Git. + +## Базовый сценарий и retry + +- `make test` — PASS. Проверены конфигурации Collector, Loki и Tempo штатными командами их образов. Grafana API вернул три datasource и dashboard `Messenger — observability`. +- Трасса `f16ae0c9583b3e1c68ad7e160af8f9ff`: Tempo вернул 17 spans, включая upload, producer, consumer, операции S3 и PostgreSQL; Loki вернул по одному или более потоку API и worker с этим trace ID. Метрики HTTP, jobs, очереди и pool присутствовали в Prometheus. +- С `WORKER_FAULT_AFTER_S3_PUT=1` команда `python3 student-tests/trace_retry.py` создала трассу `606a4c671b000b9da623c47ca04e9d15`: задача стала ready за 61,474 с. В БД `attempt=2`, Docker показал один рестарт worker. Tempo вернул 21 span, включая две публикации, две обработки и фиксацию результата. После проверки обычная конфигурация worker восстановлена. Процесс, завершённый через `os._exit`, может потерять ещё не экспортированные spans. + +## Инцидент 1: worker остановлен + +Команды: `docker compose stop worker`, `python3 student-tests/queue_incident.py`, затем `docker compose start worker`. Три задания приняты, в Prometheus наблюдались queue depth=3 и рост возраста; `MessengerQueueLag` перешёл в `firing`. После запуска worker depth стал 0 и активных alerts не осталось. Точная длительность обнаружения и восстановления не зафиксирована; замер не следует трактовать как SLA. + +## Инцидент 2: блокировка БД + +На фоне `python3 tests/load.py --duration 150 --concurrency 8 --output evidence/db-incident-load-repeat.json` таблица users была заблокирована транзакцией на 90 секунд. `MessengerHighErrorRate` перешёл в `firing`; после завершения транзакции активных alerts не осталось. В Loki найден API log со status 503 и trace ID `691ac053578c1fbfe02abeac1f222dfd`: в Tempo span `postgres.user_for_token` длился 5,007 с, весь POST — 5,009 с. Из этого конкретного trace видно ожидание на DB-операции, а не только длительный HTTP handler. + +Повторная нагрузка: 32 056 успешных и 142 ошибочных запроса; ошибка 142 / (32 056 + 142) = 0,004410, успешные RPS 32 056 / 150,009 с = 213,694. Полные результаты и ограничения closed-loop генератора — в [db-incident-load-repeat.json](db-incident-load-repeat.json). Клиентские timeout не обязательно соответствуют HTTP 5xx, поэтому это не значение SLI из Prometheus. + +## Недоступен Tempo + +На время smoke Tempo был остановлен: `make test` прошёл, затем Tempo восстановлен. В отдельном 75-секундном испытании без Tempo генератор с concurrency=4 получил 40 677 успешных запросов и 0 ошибок за 75,049 с: 40 677 / 75,049 = 542,003 успешных RPS. Полные данные — в [tempo-outage-load.json](tempo-outage-load.json). + +Память Collector по `docker stats --no-stream`: 83,43 MiB в начале, 97,18 MiB около 35-й секунды и 115,6 MiB после нагрузки при лимите 256 MiB. Очередь экспортёра Tempo достигла 128/128 batches. После восстановления Tempo очередь снизилась до 1. При последующем запросе накопленные с запуска Collector счётчики составили `otelcol_exporter_enqueue_failed_spans_total=56361` и `otelcol_exporter_send_failed_spans_total=639`; журналы содержали `sending queue is full`. До испытания значения счётчиков не были сохранены, поэтому эти числа нельзя приписать только данному запуску или суммировать как уникальные потерянные spans. Результат показывает ограничение очереди и реальную потерю телеметрии, но три замера не доказывают ограниченность памяти для произвольной длительности отказа. + +## Сохранность и отказ реплики + +`make test-persistence ACTION_SCRIPT=scripts/restart.sh` — PASS: тест подтвердил данные и прежнюю сессию. Hook перезапускает PostgreSQL, S3, broker, worker, outbox, reconcile, API и LB с сохранением volumes; отдельный снимок состояний до/после не сохранён. `make test-failover ACTION_SCRIPT=scripts/stop-one.sh` — PASS: до отказа тест видел 2 реплики, записал 113 подтверждённых сообщений, 1 попытка завершилась ошибкой, первое восстановление за 5,605 с. После сценария обе реплики запущены и healthy. diff --git a/evidence/tempo-outage-load.json b/evidence/tempo-outage-load.json new file mode 100644 index 0000000..a21ddb7 --- /dev/null +++ b/evidence/tempo-outage-load.json @@ -0,0 +1,17 @@ +{ + "lab": 6, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 75.0, + "elapsed_seconds": 75.049, + "concurrency": 4, + "successful_requests": 40677, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 542.003, + "successful_latency_ms": { + "p50": 4.117, + "p95": 30.147, + "p99": 52.281 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/infra/grafana/dashboards/messenger.json b/infra/grafana/dashboards/messenger.json new file mode 100644 index 0000000..05cb19a --- /dev/null +++ b/infra/grafana/dashboards/messenger.json @@ -0,0 +1,437 @@ +{ + "uid": "messenger-lab6", + "title": "Messenger — observability", + "schemaVersion": 39, + "version": 1, + "refresh": "10s", + "time": { + "from": "now-30m", + "to": "now" + }, + "panels": [ + { + "id": 1, + "title": "API RPS", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "fieldConfig": { + "defaults": { + "unit": "reqps" + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "expr": "sum(rate(messenger_http_requests_total{route!~\"/health/.*\"}[1m]))", + "legendFormat": "{{service_name}} {{outcome}}" + } + ] + }, + { + "id": 2, + "title": "API error ratio", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit" + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "expr": "(sum(rate(messenger_http_requests_total{status_class=\"5xx\",route!~\"/health/.*\"}[1m])) or vector(0)) / clamp_min(sum(rate(messenger_http_requests_total{route!~\"/health/.*\"}[1m])),0.001)", + "legendFormat": "{{service_name}} {{outcome}}" + } + ] + }, + { + "id": 3, + "title": "API latency p50 / p95 / p99", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "expr": "histogram_quantile(0.5, sum by (le) (rate(messenger_http_duration_seconds_bucket{route!~\"/health/.*\"}[5m])))", + "legendFormat": "p50" + }, + { + "refId": "B", + "expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_http_duration_seconds_bucket{route!~\"/health/.*\"}[5m])))", + "legendFormat": "p95" + }, + { + "refId": "C", + "expr": "histogram_quantile(0.99, sum by (le) (rate(messenger_http_duration_seconds_bucket{route!~\"/health/.*\"}[5m])))", + "legendFormat": "p99" + } + ] + }, + { + "id": 4, + "title": "Jobs by outcome", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "ops" + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "expr": "sum by (outcome) (rate(messenger_jobs_total[1m]))", + "legendFormat": "{{service_name}} {{outcome}}" + } + ] + }, + { + "id": 5, + "title": "Job duration p95", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_job_duration_seconds_bucket[5m])))", + "legendFormat": "{{service_name}} {{outcome}}" + } + ] + }, + { + "id": 6, + "title": "Acceptance to ready p95", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_job_ready_duration_seconds_bucket[5m])))", + "legendFormat": "{{service_name}} {{outcome}}" + } + ] + }, + { + "id": 7, + "title": "Queued jobs", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "expr": "max(messenger_queue_depth{service_name=\"messenger-outbox\"})", + "legendFormat": "{{service_name}} {{outcome}}" + } + ] + }, + { + "id": 8, + "title": "Oldest queued age", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "expr": "max(messenger_queue_age_seconds{service_name=\"messenger-outbox\"})", + "legendFormat": "{{service_name}} {{outcome}}" + } + ] + }, + { + "id": 9, + "title": "DB pool used / capacity / waiting", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 32 + }, + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "expr": "messenger_pool_used{service_name=\"messenger-api\"}", + "legendFormat": "used" + }, + { + "refId": "B", + "expr": "messenger_pool_capacity{service_name=\"messenger-api\"}", + "legendFormat": "capacity" + }, + { + "refId": "C", + "expr": "messenger_pool_waiting{service_name=\"messenger-api\"}", + "legendFormat": "waiting" + } + ] + }, + { + "id": 10, + "title": "Worker busy / capacity", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 32 + }, + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "expr": "messenger_worker_busy{service_name=\"messenger-worker\"}", + "legendFormat": "busy" + }, + { + "refId": "B", + "expr": "messenger_worker_capacity{service_name=\"messenger-worker\"}", + "legendFormat": "capacity" + } + ] + }, + { + "id": 11, + "title": "Process CPU", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 40 + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit" + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "expr": "sum by (service_name, service_instance_id) (rate(messenger_process_cpu_seconds_total[1m]))", + "legendFormat": "{{service_name}} {{outcome}}" + } + ] + }, + { + "id": 12, + "title": "Process resident memory", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 40 + }, + "fieldConfig": { + "defaults": { + "unit": "bytes" + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "expr": "messenger_process_memory_bytes", + "legendFormat": "{{service_name}} {{outcome}}" + } + ] + }, + { + "id": 13, + "title": "Collector failed / dropped spans per second", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 48 + }, + "fieldConfig": { + "defaults": { + "unit": "ops" + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "expr": "sum by (exporter) (rate(otelcol_exporter_send_failed_spans_total[1m]))", + "legendFormat": "send failed {{exporter}}" + }, + { + "refId": "B", + "expr": "sum by (exporter) (rate(otelcol_exporter_enqueue_failed_spans_total[1m]))", + "legendFormat": "queue full {{exporter}}" + } + ] + }, + { + "id": 14, + "title": "API SLI: successful ratio over 24h", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 48 + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "min": 0, + "max": 1 + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "expr": "1 - (sum(increase(messenger_http_requests_total{status_class=\"5xx\",route!~\"/health/.*\"}[24h])) or vector(0)) / clamp_min(sum(increase(messenger_http_requests_total{route!~\"/health/.*\"}[24h])),1)", + "legendFormat": "SLI" + } + ] + } + ] +} diff --git a/infra/grafana/provisioning/dashboards/dashboards.yaml b/infra/grafana/provisioning/dashboards/dashboards.yaml new file mode 100644 index 0000000..c278fc1 --- /dev/null +++ b/infra/grafana/provisioning/dashboards/dashboards.yaml @@ -0,0 +1,8 @@ +apiVersion: 1 +providers: + - name: messenger + orgId: 1 + folder: Messenger + type: file + disableDeletion: false + options: {path: /var/lib/grafana/dashboards} diff --git a/infra/grafana/provisioning/datasources/datasources.yaml b/infra/grafana/provisioning/datasources/datasources.yaml new file mode 100644 index 0000000..2264ab3 --- /dev/null +++ b/infra/grafana/provisioning/datasources/datasources.yaml @@ -0,0 +1,25 @@ +apiVersion: 1 +datasources: + - name: Prometheus + uid: prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + - name: Loki + uid: loki + type: loki + access: proxy + url: http://loki:3100 + jsonData: + derivedFields: + - name: TraceID + matcherType: label + matcherRegex: '^trace_?id$' + url: '$${__value.raw}' + datasourceUid: tempo + - name: Tempo + uid: tempo + type: tempo + access: proxy + url: http://tempo:3200 diff --git a/infra/loki/loki.yaml b/infra/loki/loki.yaml new file mode 100644 index 0000000..ec83888 --- /dev/null +++ b/infra/loki/loki.yaml @@ -0,0 +1,17 @@ +auth_enabled: false +server: {http_listen_port: 3100} +common: + path_prefix: /loki + replication_factor: 1 + ring: {kvstore: {store: inmemory}} +schema_config: + configs: + - from: '2024-01-01' + store: tsdb + object_store: filesystem + schema: v13 + index: {prefix: index_, period: 24h} +storage_config: + filesystem: {directory: /loki/chunks} +limits_config: {allow_structured_metadata: true, retention_period: 168h} +compactor: {working_directory: /loki/compactor, retention_enabled: true, delete_request_store: filesystem} diff --git a/infra/otel/collector.yaml b/infra/otel/collector.yaml new file mode 100644 index 0000000..d29d6fb --- /dev/null +++ b/infra/otel/collector.yaml @@ -0,0 +1,32 @@ +receivers: + otlp: + protocols: + grpc: {endpoint: "0.0.0.0:4317"} + http: {endpoint: "0.0.0.0:4318"} +processors: + memory_limiter: {check_interval: 1s, limit_mib: 192, spike_limit_mib: 48} + batch: {timeout: 2s, send_batch_size: 512, send_batch_max_size: 1024} +exporters: + prometheus: + endpoint: "0.0.0.0:8889" + resource_to_telemetry_conversion: {enabled: true} + otlp/tempo: + endpoint: tempo:4317 + tls: {insecure: true} + sending_queue: {enabled: true, queue_size: 128, num_consumers: 2} + retry_on_failure: {enabled: true, initial_interval: 1s, max_interval: 10s, max_elapsed_time: 60s} + otlphttp/loki: + endpoint: http://loki:3100/otlp + sending_queue: {enabled: true, queue_size: 128, num_consumers: 2} + retry_on_failure: {enabled: true, initial_interval: 1s, max_interval: 10s, max_elapsed_time: 60s} +service: + telemetry: + metrics: + readers: + - pull: + exporter: + prometheus: {host: 0.0.0.0, port: 8888} + pipelines: + metrics: {receivers: [otlp], processors: [memory_limiter, batch], exporters: [prometheus]} + traces: {receivers: [otlp], processors: [memory_limiter, batch], exporters: [otlp/tempo]} + logs: {receivers: [otlp], processors: [memory_limiter, batch], exporters: [otlphttp/loki]} diff --git a/infra/prometheus/alerts.yaml b/infra/prometheus/alerts.yaml new file mode 100644 index 0000000..3bcc613 --- /dev/null +++ b/infra/prometheus/alerts.yaml @@ -0,0 +1,13 @@ +groups: + - name: messenger + rules: + - alert: MessengerHighErrorRate + expr: (sum(rate(messenger_http_requests_total{status_class="5xx",route!~"/health/.*"}[1m])) or vector(0)) / clamp_min(sum(rate(messenger_http_requests_total{route!~"/health/.*"}[1m])), 0.001) > 0.05 + for: 10s + labels: {severity: warning} + annotations: {summary: 'API 5xx ratio exceeds 5%', runbook: 'Inspect API latency, traces and dependency spans; check DB readiness.'} + - alert: MessengerQueueLag + expr: max(messenger_queue_age_seconds{service_name="messenger-outbox"}) > 30 + for: 15s + labels: {severity: warning} + annotations: {summary: 'Oldest queued job exceeds 30 seconds', runbook: 'Check worker health, busy/capacity, broker and consumer traces.'} diff --git a/infra/prometheus/prometheus.yaml b/infra/prometheus/prometheus.yaml new file mode 100644 index 0000000..4db25df --- /dev/null +++ b/infra/prometheus/prometheus.yaml @@ -0,0 +1,7 @@ +global: {scrape_interval: 5s, evaluation_interval: 5s} +rule_files: ['/etc/prometheus/alerts.yaml'] +scrape_configs: + - job_name: collector + static_configs: [{targets: ['otel-collector:8889']}] + - job_name: collector-internal + static_configs: [{targets: ['otel-collector:8888']}] diff --git a/infra/tempo/tempo.yaml b/infra/tempo/tempo.yaml new file mode 100644 index 0000000..2e501ea --- /dev/null +++ b/infra/tempo/tempo.yaml @@ -0,0 +1,13 @@ +server: {http_listen_port: 3200} +distributor: + receivers: + otlp: + protocols: + grpc: {endpoint: "0.0.0.0:4317"} +ingester: {trace_idle_period: 10s} +compactor: {compaction: {block_retention: 168h}} +storage: + trace: + backend: local + wal: {path: /var/tempo/wal} + local: {path: /var/tempo/blocks} diff --git a/migrations/008_trace_context.sql b/migrations/008_trace_context.sql new file mode 100644 index 0000000..4eee3fc --- /dev/null +++ b/migrations/008_trace_context.sql @@ -0,0 +1,2 @@ +-- Context is technical metadata, never user content. Retained across retry/outbox recreation. +ALTER TABLE attachment_jobs ADD COLUMN trace_context jsonb NOT NULL DEFAULT '{}'::jsonb; diff --git a/pyproject.toml b/pyproject.toml index 20c78ef..6d3ce68 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,4 +12,6 @@ dependencies = [ "pillow==13.0.0", "pika==1.3.2", "uvicorn==0.40.0", + "opentelemetry-sdk==1.36.0", + "opentelemetry-exporter-otlp-proto-http==1.36.0", ] diff --git a/scripts/init-secrets.py b/scripts/init-secrets.py index a416049..616933d 100644 --- a/scripts/init-secrets.py +++ b/scripts/init-secrets.py @@ -18,6 +18,7 @@ def write(name, value): return value +write("grafana_password", secrets.token_urlsafe(24)) password = write("postgres_password", secrets.token_hex(32)) write("database_url", f"postgresql://messenger:{password}@postgres:5432/messenger") write("cursor_secret", secrets.token_hex(32)) diff --git a/src/broker.py b/src/broker.py index 4cd980d..fadd839 100644 --- a/src/broker.py +++ b/src/broker.py @@ -7,6 +7,7 @@ import pika from starlette.concurrency import run_in_threadpool from .errors import ApiError +from . import telemetry as tel QUEUE = "attachment.process.v1" DLQ = "attachment.process.v1.dlq" @@ -28,7 +29,7 @@ def _connection() -> pika.BlockingConnection: return connection -def _publish(payload: dict) -> None: +def _publish(payload: dict, headers: dict | None = None) -> None: connection = _connection() try: channel = connection.channel() @@ -36,16 +37,18 @@ def _publish(payload: dict) -> None: channel.basic_publish( exchange="", routing_key=QUEUE, body=json.dumps(payload, separators=(",", ":")).encode("utf-8"), - properties=pika.BasicProperties(delivery_mode=pika.DeliveryMode.Persistent, content_type="application/json"), + properties=pika.BasicProperties(delivery_mode=pika.DeliveryMode.Persistent, content_type="application/json", headers=headers), mandatory=True, ) finally: connection.close() -async def publish(payload: dict) -> None: +async def publish(payload: dict, context=None) -> None: try: - await run_in_threadpool(_publish, payload) + with tel.tracer.start_as_current_span("attachment.publish", context=tel.extract(context), kind=tel.SpanKind.PRODUCER, record_exception=False, set_status_on_exception=False) as span: + span.set_attribute("messaging.retry.attempt", int(payload.get("attempt", 1))) + await run_in_threadpool(_publish, payload, tel.carrier()) except (pika.exceptions.AMQPError, OSError, ValueError) as error: raise ApiError(503, "unavailable", "Брокер недоступен") from error diff --git a/src/main.py b/src/main.py index f355932..a8cb62d 100644 --- a/src/main.py +++ b/src/main.py @@ -20,6 +20,9 @@ from .routes import router from .attachments import router as attachment_router from .storage.s3 import S3Settings, S3Store from .storage.postgres import PostgresStore +from . import telemetry as tel + +tel.setup("messenger-api") def logger(name: str) -> logging.Logger: @@ -51,9 +54,20 @@ async def lifespan(application: FastAPI): raise ValueError("UPLOAD_CONCURRENCY must be 1..8") application.state.upload_slot = asyncio.Semaphore(concurrency) application.state.accepting_requests = True + async def sample_pool(): + while True: + pool = store.pool + tel.state.update(pool_used=pool.get_size()-pool.get_idle_size(), pool_capacity=pool.get_max_size(), pool_waiting=len(pool._queue._getters)) + await asyncio.sleep(1) + sampler = asyncio.create_task(sample_pool()) try: yield finally: + sampler.cancel() + try: + await sampler + except asyncio.CancelledError: + pass application.state.accepting_requests = False application.state.s3.client.close() await store.close() @@ -74,21 +88,23 @@ async def add_request_id(request: Request, call_next) -> Response: request.state.request_id = str(uuid4()) started_at = time.monotonic() status_code = 500 - try: - response = await call_next(request) - status_code = response.status_code - response.headers["X-Request-Id"] = request.state.request_id - response.headers["X-Instance-Id"] = request.app.state.settings.instance_id - return response - finally: - access_logger.info( - "request_id=%s method=%s path=%s status=%s elapsed_ms=%.3f", - request.state.request_id, - request.method, - request.url.path, - status_code, - (time.monotonic() - started_at) * 1000, - ) + with tel.tracer.start_as_current_span("HTTP request", context=tel.extract(dict(request.headers)), kind=tel.SpanKind.SERVER, record_exception=False, set_status_on_exception=False) as span: + try: + response = await call_next(request) + status_code = response.status_code + response.headers["X-Request-Id"] = request.state.request_id + response.headers["X-Instance-Id"] = request.app.state.settings.instance_id + return response + finally: + route = getattr(request.scope.get("route"), "path", "unmatched") + attrs = {"method": request.method, "route": route, "status_class": f"{status_code // 100}xx"} + span.update_name(f"{request.method} {route}") + span.set_attributes({"http.request.method": request.method, "http.route": route, "http.response.status_code": status_code}) + if status_code >= 500: + span.set_status(tel.Status(tel.StatusCode.ERROR)) + tel.http_requests.add(1, attrs) + tel.http_duration.record(time.monotonic() - started_at, attrs) + tel.log.info("http.request", extra={"request_id": request.state.request_id, "method": request.method, "route": route, "status": status_code}) def error_response( diff --git a/src/outbox.py b/src/outbox.py index 60f0099..6e8e3d0 100644 --- a/src/outbox.py +++ b/src/outbox.py @@ -2,15 +2,18 @@ import asyncio import logging import os +import json from .broker import publish from .config import load_settings from .storage.postgres import PostgresStore +from . import telemetry as tel logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s outbox %(message)s") async def run() -> None: + tel.setup("messenger-outbox") settings = load_settings() store = PostgresStore(settings.database_url) await store.connect() @@ -19,8 +22,11 @@ async def run() -> None: while True: try: await store.schedule_due_jobs(int(os.environ.get('JOB_MAX_ATTEMPTS', '5'))) + queue = await store.pool.fetchrow("SELECT count(*) AS depth, COALESCE(EXTRACT(EPOCH FROM now()-min(created_at)), 0)::float8 AS age FROM attachment_jobs WHERE status='queued'") + tel.state.update(queue_depth=queue['depth'], queue_age=queue['age']) for event in await store.due_outbox_events(): - await publish(event["payload"]) + context = event['trace_context'] + await publish(event["payload"], json.loads(context) if isinstance(context, str) else context) await store.mark_outbox_published(event["id"]) except Exception: logging.exception("outbox dispatch failed; durable event remains pending") diff --git a/src/storage/postgres.py b/src/storage/postgres.py index d83f93d..b4cedd9 100644 --- a/src/storage/postgres.py +++ b/src/storage/postgres.py @@ -11,6 +11,7 @@ from argon2 import PasswordHasher from argon2.exceptions import InvalidHashError, VerificationError from ..errors import ApiError +from .. import telemetry as tel ROOT = Path(__file__).resolve().parents[2] @@ -316,11 +317,16 @@ class PostgresStore: All attachment writers must participate in this protocol. """ try: - async with self.pool.acquire() as connection, connection.transaction(): - await connection.execute("SET LOCAL lock_timeout = '5s'") - lock = "pg_advisory_xact_lock" if maintenance else "pg_advisory_xact_lock_shared" - await connection.execute(f"SELECT {lock}(41004, 1)") - yield connection + with tel.tracer.start_as_current_span("postgres.attachment_transaction", kind=tel.SpanKind.CLIENT, record_exception=False, set_status_on_exception=False) as span: + try: + async with self.pool.acquire() as connection, connection.transaction(): + await connection.execute("SET LOCAL lock_timeout = '5s'") + lock = "pg_advisory_xact_lock" if maintenance else "pg_advisory_xact_lock_shared" + await connection.execute(f"SELECT {lock}(41004, 1)") + yield connection + except Exception: + span.set_status(tel.Status(tel.StatusCode.ERROR)) + raise except DB_ERRORS as error: raise ApiError(503, "unavailable", "PostgreSQL недоступен") from error @@ -333,7 +339,7 @@ class PostgresStore: attachment_id, chat_id, owner_id, filename, content_type, size_bytes, digest, staging_key, ) await (connection or self.pool).execute( - "INSERT INTO attachment_jobs (job_id, attachment_id) VALUES ($1, $2)", job_id, attachment_id + "INSERT INTO attachment_jobs (job_id, attachment_id, trace_context) VALUES ($1, $2, $3::jsonb)", job_id, attachment_id, json.dumps(tel.carrier()) ) payload = { "schema_version": 1, "job_id": str(job_id), "type": "attachment.process.v1", @@ -464,7 +470,7 @@ class PostgresStore: async def due_outbox_events(self, limit: int = 20) -> list[dict]: try: return [{**dict(row), 'payload': json.loads(row['payload']) if isinstance(row['payload'], str) else row['payload']} for row in await self.pool.fetch( - "SELECT id, payload FROM attachment_outbox WHERE published_at IS NULL ORDER BY created_at LIMIT $1", limit + "SELECT o.id, o.payload, j.trace_context FROM attachment_outbox o JOIN attachment_jobs j ON j.job_id=o.job_id WHERE o.published_at IS NULL ORDER BY o.created_at LIMIT $1", limit )] except DB_ERRORS as error: raise RuntimeError("PostgreSQL is unavailable") from error @@ -529,3 +535,23 @@ class PostgresStore: except DB_ERRORS as error: raise RuntimeError("PostgreSQL is unavailable") from error return {row["object_key"]: row["id"] for row in rows} + +# Span names describe operations only; never export SQL, parameters, credentials or IDs. +def _trace_db(method): + from functools import wraps + @wraps(method) + async def wrapped(*args, **kwargs): + with tel.tracer.start_as_current_span('postgres.' + method.__name__, kind=tel.SpanKind.CLIENT, + record_exception=False, set_status_on_exception=False) as span: + try: + return await method(*args, **kwargs) + except Exception: + span.set_status(tel.Status(tel.StatusCode.ERROR)) + raise + return wrapped + +for _name in ('ready', 'create_user', 'authenticate', 'create_session', 'user_for_token', 'revoke_session', + 'create_chat', 'get_chat_for_member', 'list_chats_for_member', 'create_message_idempotent', + 'list_messages_for_member', 'create_attachment', 'attachment_for_member', 'claim_attachment_job', + 'complete_attachment_job', 'retry_attachment_job'): + setattr(PostgresStore, _name, _trace_db(getattr(PostgresStore, _name))) diff --git a/src/storage/s3.py b/src/storage/s3.py index f154df9..7188fb2 100644 --- a/src/storage/s3.py +++ b/src/storage/s3.py @@ -10,6 +10,7 @@ from botocore.exceptions import BotoCoreError, ClientError from starlette.concurrency import run_in_threadpool from ..errors import ApiError +from .. import telemetry as tel @dataclass(frozen=True) @@ -56,10 +57,12 @@ class S3Store: ) async def call(self, method, **kwargs): - try: - return await run_in_threadpool(getattr(self.client, method), **kwargs) - except (BotoCoreError, ClientError, OSError) as error: - raise ApiError(503, "unavailable", "S3 недоступно") from error + with tel.tracer.start_as_current_span("s3." + method, kind=tel.SpanKind.CLIENT, record_exception=False, set_status_on_exception=False) as span: + try: + return await run_in_threadpool(getattr(self.client, method), **kwargs) + except (BotoCoreError, ClientError, OSError) as error: + span.set_status(tel.Status(tel.StatusCode.ERROR)) + raise ApiError(503, "unavailable", "S3 недоступно") from error async def ready(self): await self.call("head_bucket", Bucket=self.settings.bucket) diff --git a/src/telemetry.py b/src/telemetry.py new file mode 100644 index 0000000..01ca0d4 --- /dev/null +++ b/src/telemetry.py @@ -0,0 +1,92 @@ +"""Bounded OTLP telemetry; only explicitly selected, non-sensitive fields leave the process.""" +import json +import logging +import os +import socket +import time +import resource +from datetime import datetime, timezone + +from opentelemetry import metrics, trace +from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator +from opentelemetry.trace import SpanKind, Status, StatusCode + +tracer = trace.get_tracer("messenger") +meter = metrics.get_meter("messenger") +trace_context = TraceContextTextMapPropagator() +http_requests = meter.create_counter("messenger.http.requests", unit="{request}") +http_duration = meter.create_histogram("messenger.http.duration", unit="s") +jobs = meter.create_counter("messenger.jobs", unit="{job}") +job_duration = meter.create_histogram("messenger.job.duration", unit="s") +ready_duration = meter.create_histogram("messenger.job.ready.duration", unit="s") +busy = meter.create_up_down_counter("messenger.worker.busy", unit="{worker}") +capacity = meter.create_up_down_counter("messenger.worker.capacity", unit="{worker}") +state = {"queue_depth": 0, "queue_age": 0, "pool_used": 0, "pool_capacity": 0, "pool_waiting": 0} +for name, unit in (("queue_depth", "{job}"), ("queue_age", "s"), ("pool_used", "{connection}"), ("pool_capacity", "{connection}"), ("pool_waiting", "{request}")): + meter.create_observable_gauge("messenger." + name, callbacks=[lambda options, key=name: [metrics.Observation(state[key])]], unit=unit) + +def process_cpu(options): + usage = resource.getrusage(resource.RUSAGE_SELF) + return [metrics.Observation(usage.ru_utime + usage.ru_stime)] + +def process_rss(options): + try: + with open('/proc/self/statm', encoding='ascii') as stream: + return [metrics.Observation(int(stream.read().split()[1]) * os.sysconf('SC_PAGE_SIZE'))] + except (OSError, ValueError, IndexError): + return [] + +meter.create_observable_counter("messenger.process.cpu", callbacks=[process_cpu], unit="s") +meter.create_observable_gauge("messenger.process.memory", callbacks=[process_rss], unit="By") + +log = logging.getLogger("messenger.telemetry") +log.setLevel(logging.INFO) +log.propagate = False + +class JsonFormatter(logging.Formatter): + def format(self, record): + ctx = trace.get_current_span().get_span_context() + obj = {"timestamp": datetime.now(timezone.utc).isoformat(), "severity": record.levelname, + "service.name": os.environ.get("OTEL_SERVICE_NAME", "messenger"), "event": record.getMessage()} + if ctx.is_valid: + obj.update(trace_id=f"{ctx.trace_id:032x}", span_id=f"{ctx.span_id:016x}") + for field in ("request_id", "method", "route", "status", "outcome"): + if hasattr(record, field): obj[field] = getattr(record, field) + return json.dumps(obj, ensure_ascii=False) + +def setup(service): + if os.environ.get("OTEL_ENABLED", "0") != "1": + return + os.environ["OTEL_SERVICE_NAME"] = service + from opentelemetry.sdk.resources import Resource + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.export import BatchSpanProcessor + from opentelemetry.sdk.metrics import MeterProvider + from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader + from opentelemetry.sdk.metrics.view import View, ExplicitBucketHistogramAggregation + from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler + from opentelemetry.sdk._logs.export import BatchLogRecordProcessor + from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter + from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter + from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter + resource = Resource.create({"service.name": service, "service.version": "1.6.0", "service.instance.id": socket.gethostname(), "deployment.environment.name": "lab"}) + provider = TracerProvider(resource=resource) + provider.add_span_processor(BatchSpanProcessor(OTLPSpanExporter(timeout=2), max_queue_size=2048, max_export_batch_size=256, schedule_delay_millis=1000)) + trace.set_tracer_provider(provider) + buckets = [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 120, 300, 600] + views = [View(instrument_name=name, aggregation=ExplicitBucketHistogramAggregation(boundaries=buckets)) for name in ("messenger.http.duration", "messenger.job.duration", "messenger.job.ready.duration")] + metrics.set_meter_provider(MeterProvider(resource=resource, views=views, metric_readers=[PeriodicExportingMetricReader(OTLPMetricExporter(timeout=2), export_interval_millis=5000)])) + logs = LoggerProvider(resource=resource) + logs.add_log_record_processor(BatchLogRecordProcessor(OTLPLogExporter(timeout=2), max_queue_size=2048, max_export_batch_size=256, schedule_delay_millis=1000)) + log.addHandler(LoggingHandler(level=logging.INFO, logger_provider=logs)) + stream = logging.StreamHandler() + stream.setFormatter(JsonFormatter()) + log.addHandler(stream) + +def carrier(): + result = {} + trace_context.inject(result) + return result + +def extract(value): + return trace_context.extract(value or {}) diff --git a/src/worker.py b/src/worker.py index d355b88..ef82c21 100644 --- a/src/worker.py +++ b/src/worker.py @@ -20,6 +20,8 @@ from .broker import QUEUE, _connection, publish_dlq from .config import load_settings from .storage.postgres import PostgresStore from .storage.s3 import S3Settings, S3Store +from . import telemetry as tel +from datetime import datetime, timezone logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s worker %(message)s") @@ -131,6 +133,8 @@ async def process_locked(store: PostgresStore, s3: S3Store, payload: dict, lease os._exit(70) committed = await store.complete_attachment_job(attachment_id, original_key, metadata, attachment['claim_time']) if committed: + if attachment.get('created_at') is not None: + tel.ready_duration.record(max(0, (datetime.now(timezone.utc) - attachment['created_at']).total_seconds()), {"job_type": "attachment"}) try: await s3.delete(attachment["staging_key"]) except Exception: @@ -151,10 +155,12 @@ def settle_delivery(channel, delivery_tag, future): def main() -> None: + tel.setup("messenger-worker") settings = load_settings() connection = _connection() channel = connection.channel() workers = max(1, int(os.environ.get("WORKER_CONCURRENCY", "2"))) + tel.capacity.add(workers) channel.basic_qos(prefetch_count=max(1, int(os.environ.get("WORKER_PREFETCH", str(workers))))) async def handle_async(payload): @@ -178,7 +184,7 @@ def main() -> None: finally: await retry_store.close() - def handle(body): + def handle_job(body): payload = None try: payload = json.loads(body) @@ -188,18 +194,19 @@ def main() -> None: raise ValueError("unexpected broker payload") UUID(payload['attachment_id']) UUID(payload['job_id']) + tel.trace.get_current_span().set_attribute("messaging.retry.attempt", int(payload.get('attempt', 1))) except (ValueError, KeyError, TypeError): try: asyncio.run(publish_dlq({'failure': 'invalid_job'})) - return True + return True, False except Exception: - return False + return False, False try: asyncio.run(handle_async(payload)) except Exception as error: logging.warning("job failed attachment_id=%s", payload['attachment_id']) if '_claim_time' not in payload: - return False + return False, False try: attachment_id = UUID(payload["attachment_id"]) attempt = int(payload.get("attempt", 1)) @@ -207,12 +214,32 @@ def main() -> None: asyncio.run(schedule_retry(attachment_id, delay, payload['_claim_time'], str(error) if isinstance(error, PermanentFailure) else None)) except Exception: - return False - return True + return False, False + return True, False + return True, True + + def handle(body, headers): + started = time.monotonic() + tel.busy.add(1) + with tel.tracer.start_as_current_span("attachment.consume", context=tel.extract(headers), kind=tel.SpanKind.CONSUMER, record_exception=False, set_status_on_exception=False) as span: + try: + acknowledged, succeeded = handle_job(body) + outcome = "success" if succeeded else "failed" + return acknowledged + except Exception: + outcome = "failed" + raise + finally: + if 'outcome' not in locals(): outcome = "failed" + if outcome == "failed": span.set_status(tel.Status(tel.StatusCode.ERROR)) + attrs = {"job_type": "attachment", "outcome": outcome} + tel.jobs.add(1, attrs) + tel.job_duration.record(time.monotonic() - started, attrs) + tel.log.info("job.processed", extra={"outcome": outcome}) + tel.busy.add(-1) def consume(ch, method, properties, body): - del properties - future = executor.submit(handle, body) + future = executor.submit(handle, body, properties.headers or {}) def acknowledged(_future): if connection.is_open: connection.add_callback_threadsafe(lambda: settle_delivery(ch, method.delivery_tag, _future)) diff --git a/student-tests/alerts.test.yaml b/student-tests/alerts.test.yaml new file mode 100644 index 0000000..30c3b72 --- /dev/null +++ b/student-tests/alerts.test.yaml @@ -0,0 +1,27 @@ +rule_files: + - /workspace/infra/prometheus/alerts.yaml +evaluation_interval: 5s +tests: + - interval: 5s + input_series: + - series: 'messenger_http_requests_total{method="POST",route="/api/v1/chats/{chat_id}/messages",status_class="5xx"}' + values: '0+50x30' + - series: 'messenger_http_requests_total{method="POST",route="/api/v1/chats/{chat_id}/messages",status_class="2xx"}' + values: '0+450x30' + - series: 'messenger_queue_age_seconds{service_name="messenger-outbox"}' + values: '60x30' + alert_rule_test: + - eval_time: 2m + alertname: MessengerHighErrorRate + exp_alerts: + - exp_labels: {severity: warning} + exp_annotations: + summary: 'API 5xx ratio exceeds 5%' + runbook: 'Inspect API latency, traces and dependency spans; check DB readiness.' + - eval_time: 2m + alertname: MessengerQueueLag + exp_alerts: + - exp_labels: {severity: warning} + exp_annotations: + summary: 'Oldest queued job exceeds 30 seconds' + runbook: 'Check worker health, busy/capacity, broker and consumer traces.' diff --git a/student-tests/queue_incident.py b/student-tests/queue_incident.py new file mode 100644 index 0000000..6e07a5c --- /dev/null +++ b/student-tests/queue_incident.py @@ -0,0 +1,11 @@ +"""Submit synthetic attachments without waiting for worker; no credentials printed.""" +import sys +from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests')) +from client import Client + +client = Client() +actor, _, _, chat = client.bootstrap() +for _ in range(3): + attachment, _ = client.attachment(actor, chat) + print('accepted attachment:', attachment['id']) diff --git a/student-tests/telemetry_unit.py b/student-tests/telemetry_unit.py new file mode 100644 index 0000000..96f514f --- /dev/null +++ b/student-tests/telemetry_unit.py @@ -0,0 +1,68 @@ +"""Offline checks for trace propagation and telemetry redaction.""" +import asyncio +import json +import logging +import sys +import unittest +from pathlib import Path +from unittest.mock import patch + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from opentelemetry import trace, metrics +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics.export import InMemoryMetricReader +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from src import telemetry as tel +from src import broker + +class TelemetryTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(cls.exporter)) + trace.set_tracer_provider(provider) + cls.reader = InMemoryMetricReader() + metrics.set_meter_provider(MeterProvider(metric_readers=[cls.reader])) + + def test_upload_outbox_publish_consume_context(self): + self.exporter.clear() + recorded = {} + def publish(payload, headers): + recorded.update(headers) + with tel.tracer.start_as_current_span('upload') as upload: + saved = tel.carrier() + self.assertIn('traceparent', saved) + with patch.object(broker, '_publish', side_effect=publish): + asyncio.run(broker.publish({'type': 'attachment.process.v1', 'attempt': 2}, json.loads(json.dumps(saved)))) + with tel.tracer.start_as_current_span('consume', context=tel.extract(recorded), kind=tel.SpanKind.CONSUMER): + pass + spans = {s.name: s for s in self.exporter.get_finished_spans()} + self.assertEqual(spans['attachment.publish'].parent.span_id, spans['upload'].context.span_id) + self.assertEqual(spans['attachment.publish'].attributes['messaging.retry.attempt'], 2) + self.assertEqual(spans['consume'].parent.span_id, spans['attachment.publish'].context.span_id) + self.assertEqual(spans['consume'].context.trace_id, spans['upload'].context.trace_id) + self.assertNotIn('type', recorded) + + def test_log_formatter_only_explicit_fields(self): + rec = logging.LogRecord('test', logging.INFO, __file__, 1, 'http.request', (), None) + rec.authorization = 'secret'; rec.body = 'private message'; rec.route = '/api/v1/chats/{chat_id}' + output = tel.JsonFormatter().format(rec) + self.assertNotIn('secret', output) + self.assertNotIn('private message', output) + self.assertEqual(json.loads(output)['route'], '/api/v1/chats/{chat_id}') + + def test_metrics_keep_route_template(self): + attrs = {'method': 'POST', 'route': '/api/v1/chats/{chat_id}/attachments', 'status_class': '2xx'} + tel.http_requests.add(1, attrs) + tel.http_duration.record(0.125, attrs) + result = self.reader.get_metrics_data() + data = [item for resource in result.resource_metrics for scope in resource.scope_metrics for item in scope.metrics] + requests = next(item for item in data if item.name == 'messenger.http.requests') + point = next(point for point in requests.data.data_points if point.attributes == attrs) + self.assertEqual(point.value, 1) + self.assertEqual(set(point.attributes), {'method', 'route', 'status_class'}) + +if __name__ == '__main__': unittest.main() diff --git a/student-tests/trace_retry.py b/student-tests/trace_retry.py new file mode 100644 index 0000000..da858de --- /dev/null +++ b/student-tests/trace_retry.py @@ -0,0 +1,20 @@ +"""Synthetic upload with a known W3C trace ID; wait through a worker restart.""" +import secrets +import sys +import time +from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests')) +from client import Client + +client = Client() +actor, _, _, chat = client.bootstrap() +trace_id = secrets.token_hex(16) +parent_id = secrets.token_hex(8) +content = (Path(__file__).resolve().parents[1] / 'tests/fixtures/sample.png').read_bytes() +attachment = client.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=sample.png', 202, + actor=actor, raw=content, headers={'Content-Type':'image/png', + 'traceparent':f'00-{trace_id}-{parent_id}-01'}, schema='Attachment')['json'] +print('trace_id:', trace_id, 'attachment_id:', attachment['id'], flush=True) +started = time.monotonic() +result = client.wait_ready(actor, attachment, wait_seconds=150) +print('status:', result['status'], 'elapsed_seconds:', round(time.monotonic()-started, 3), flush=True) diff --git a/uv.lock b/uv.lock index 6f5df98..359ef2c 100644 --- a/uv.lock +++ b/uv.lock @@ -160,6 +160,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/77/2d/4b6b6f5704d1ec9b412c1dd9700bda0c315619dbac4a21418180a81a2780/botocore-1.43.115-py3-none-any.whl", hash = "sha256:9ceda640c868bc15f5399129e8cfdef8cbb4b350e8a99547c9b5f477cfb9619a", size = 16085101, upload-time = "2026-10-19T19:19:42.203Z" }, ] +[[package]] +name = "certifi" +version = "2026.10.23" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fd/11/b524cbd9bbdaca58608e4cf4053fd463417549e1959fa8d3aa18319f9555/certifi-2026.10.23.tar.gz", hash = "sha256:a9895e6e2758b371e0545e321a3a8fd1fcfd9829ada8824a936dfb16aaaf36cd", size = 136225, upload-time = "2026-10-23T11:36:16.478Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/32/d1/a3652fe2c19134d0d15922177821d863cbd7e842d53a21bbe7e8e3a1b5ac/certifi-2026.10.23-py3-none-any.whl", hash = "sha256:e9b54de76e530bb6cf0ccd036787103acb58fd42ba930ea8bf948ddbaa2f61d1", size = 135082, upload-time = "2026-10-23T11:36:14.763Z" }, +] + [[package]] name = "cffi" version = "2.1.1" @@ -245,6 +254,137 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/aa/29/35e016098c814cd93de9cd320c66b5bfba14dc6ecedd3cb518fa7c408c69/cffi-2.1.1-cp315-cp315t-win_arm64.whl", hash = "sha256:d18e5ac0f2f03f4f518d3e23db0f0cad7faa1da8620e9c09461d443bbf6e6692", size = 186360, upload-time = "2026-08-03T21:21:13.636Z" }, ] +[[package]] +name = "charset-normalizer" +version = "3.5.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/33/1c/f41d4e74c28ab327ff3acd36053f7ea506c55872d7a90b0fa71aa3ab0c89/charset_normalizer-3.5.2.tar.gz", hash = "sha256:39de2a259fc954455c57274dc94c79d5842774e1247a016aff30bc0efed0f4ef", size = 172659, upload-time = "2026-09-30T04:39:23.398Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e7/c8/693809898870237d82785a03f3b2b58fe4c9f14669f84a7d4e623c92a59e/charset_normalizer-3.5.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:ed2a239c0ea213acc1908150a3037257083c7c083128f1a4cec2ec4b97dca491", size = 367780, upload-time = "2026-09-30T04:35:30.888Z" }, + { url = "https://files.pythonhosted.org/packages/c9/87/2fea8c13dc24b3ca9c6f803a5b2dfdeae73eb4f9e12c7885ed908ff0433c/charset_normalizer-3.5.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b91363207bd9dc966a691e959bb47f64b30f7ac4b072be9968b366982f7db77c", size = 246730, upload-time = "2026-09-30T04:35:32.286Z" }, + { url = "https://files.pythonhosted.org/packages/a8/9e/09efac30b937722f46d3110ba30b875b24b2e3a266ed746cc4e376a94d80/charset_normalizer-3.5.2-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:38a873987f3be698494da8b2e3085e29da02da7b633dce73e79c699a113d7bf0", size = 237707, upload-time = "2026-09-30T04:35:33.709Z" }, + { url = "https://files.pythonhosted.org/packages/9e/18/70d76670b13686237863a379928d60bd10e021f17d243ab3d7014c4a5f4e/charset_normalizer-3.5.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:355ad8011081dec5412240c087a9a0c9d4d5039f3ed11a3f13e18c2b29b56c51", size = 273050, upload-time = "2026-09-30T04:35:35.138Z" }, + { url = "https://files.pythonhosted.org/packages/54/e2/77a8b09d5adc013ed07b95b01b8b8fa5441c4e810e83ee7e4aae2fa4d91a/charset_normalizer-3.5.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ee21e28f0430bd6dc9086c6e525d5e818a44a5ad19720c8a0ef766792f3eb5e5", size = 270345, upload-time = "2026-09-30T04:35:36.502Z" }, + { url = "https://files.pythonhosted.org/packages/7f/c5/38806a25ab5e65fc178f39affeda20858efafede2fce1ffc2556cfc9fe73/charset_normalizer-3.5.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3d31298449090ab8d47b7b1b2a555ff73cac7ed438a08b7ac160980c7ebed649", size = 257601, upload-time = "2026-09-30T04:35:37.919Z" }, + { url = "https://files.pythonhosted.org/packages/ae/8d/213565184708fdb263ae55e2c04ee1ff748129dd65d48ed0e3502da9c85a/charset_normalizer-3.5.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5cde776b7cc66e4f6c99612cea4aa7269aa65863f7a15841b2c264f103822f4e", size = 252222, upload-time = "2026-09-30T04:35:39.544Z" }, + { url = "https://files.pythonhosted.org/packages/7e/24/76d2cefc25472531e4c5c7dfff68865eb1c39b78482f0fdc15b46f047830/charset_normalizer-3.5.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ae4f5fea5b8b8ccff88238cc8569303e5ee95efae67fa62922a311397a71f346", size = 248482, upload-time = "2026-09-30T04:35:41.088Z" }, + { url = "https://files.pythonhosted.org/packages/7d/dc/65a801b66ab4c197e22c433ab25e7ac24324ac6f45a2269aca42cce309bf/charset_normalizer-3.5.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:f7d486c83842422badd511868fd8a9a20e9407ace71564b6af47ce7e60a336c1", size = 241206, upload-time = "2026-09-30T04:35:42.59Z" }, + { url = "https://files.pythonhosted.org/packages/a7/95/ca9b5eabde673002c6f1e7ada1b223916fe18f6d661da7aabd4d643718f1/charset_normalizer-3.5.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:11a4d68a6ecda3292cb1e50239e111543ba5d709bb62a6b4ea1afcfa729d8875", size = 273190, upload-time = "2026-09-30T04:35:44.347Z" }, + { url = "https://files.pythonhosted.org/packages/2d/8b/803b4d2a3f6e1740f63f1e87b04d14b42f3d4fdfe6ed7d4db2d34102b14f/charset_normalizer-3.5.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:d6734d2ef8a50fbf8445c139477da401f50d62a0606bf00e20ec6d87773fefb1", size = 253527, upload-time = "2026-09-30T04:35:45.915Z" }, + { url = "https://files.pythonhosted.org/packages/a9/55/93c0e5dbd085ae0471346026abbe7e0db9ea2d6fea74e51f0b5a46f233a7/charset_normalizer-3.5.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:a815775b6c38d4e0ff7bcffbeba67feded90202bb6a226b8dd35f1c855217413", size = 271285, upload-time = "2026-09-30T04:35:47.49Z" }, + { url = "https://files.pythonhosted.org/packages/95/69/0dbd0e0b9b16cfa816cdfcb3e2e3854a1f680dc07fb1245ea125e7448060/charset_normalizer-3.5.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:23851fb4e1b85ed3f6c2a27b777cdfe2e19fb5b38429a8faf38c7542b7665869", size = 260010, upload-time = "2026-09-30T04:35:48.996Z" }, + { url = "https://files.pythonhosted.org/packages/58/9d/e7b88e7b1bf403590c3b573277b5e1e488c68c7a6fbacca310a2c324e90c/charset_normalizer-3.5.2-cp312-cp312-win32.whl", hash = "sha256:db19d07e2e0129e974a0e65d0064fc222a446cd5122c2fd4184d2af9fc734a9e", size = 184125, upload-time = "2026-09-30T04:35:50.777Z" }, + { url = "https://files.pythonhosted.org/packages/eb/e6/e6e083884cbcfd49c64865af05027fe7011be7b2d9179524f099a1b611f3/charset_normalizer-3.5.2-cp312-cp312-win_amd64.whl", hash = "sha256:780fbe7cab297b81dad9fb8dc5eb003c0468ffb0d9e5f65068c53a34661a96bc", size = 207486, upload-time = "2026-09-30T04:35:52.194Z" }, + { url = "https://files.pythonhosted.org/packages/c4/e3/017aea0911ada7405a825c7d937eb3a13009664e2f5b38e8c4bbf2abf894/charset_normalizer-3.5.2-cp312-cp312-win_arm64.whl", hash = "sha256:e2af3aad578aa6bd1384bcf4750fc285e5a9de53f40b7d41e5a0bf748edeb2b3", size = 196734, upload-time = "2026-09-30T04:35:53.636Z" }, + { url = "https://files.pythonhosted.org/packages/c5/34/68292d68512768591aaff07c59bb53ee31341c87759433a859c4641a50c2/charset_normalizer-3.5.2-cp313-cp313-android_24_arm64_v8a.whl", hash = "sha256:ed905975ab14056a2e5eb1c376cb2e1ebc5396baf84163939c518556fccde9f5", size = 225946, upload-time = "2026-09-30T04:35:55.313Z" }, + { url = "https://files.pythonhosted.org/packages/e3/80/bee0b01b90ccd5322ae1d0abb33fab1bd95b7c2eadaf02aeccf22e04ee83/charset_normalizer-3.5.2-cp313-cp313-android_24_x86_64.whl", hash = "sha256:a66c3bc5ab1f0ff2164fc9965ddd611ff0802173f4b9d24554c563f6ab7e1d6e", size = 238619, upload-time = "2026-09-30T04:35:56.863Z" }, + { url = "https://files.pythonhosted.org/packages/78/6e/60ce52a85a7fd631ae8482ae6d74521014ca2f255892679484dc04d7ef56/charset_normalizer-3.5.2-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:d2374b62878abb00cd8309b32af6c0b715cd02dec0ca74ef12e5069bdc64144a", size = 206701, upload-time = "2026-09-30T04:35:58.639Z" }, + { url = "https://files.pythonhosted.org/packages/36/8c/71aafad23f971afc84c2b295bc0c560739ce1dac558aad9fec22e39f3639/charset_normalizer-3.5.2-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:d376bbd28b3a8999db1a103b3b388aee6f1ddeb3e51bc2172993efdcd86e064d", size = 210506, upload-time = "2026-09-30T04:36:00.147Z" }, + { url = "https://files.pythonhosted.org/packages/91/da/3c5a7798c046df7d2d68ad653cf5b6c5a8bfee225055a843c6f2f42aac1a/charset_normalizer-3.5.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:6045373d5a89a5ec71afde535db987ca28e76dfa276c2d4c818265b375d4b055", size = 366626, upload-time = "2026-09-30T04:36:01.77Z" }, + { url = "https://files.pythonhosted.org/packages/e1/16/710ac3de2ee354e2bd1a9c94efe45a2d27b5c6ad39b2d6a905be2c094b6c/charset_normalizer-3.5.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:849df64e889b2e17230d58410a03dba311a65b163508fd33679b2b737d4b7858", size = 244624, upload-time = "2026-09-30T04:36:03.389Z" }, + { url = "https://files.pythonhosted.org/packages/d6/39/45c7439f5b63d24f7d5b2a1d760f34af7628782d7144b4cc8ded45c2d4bc/charset_normalizer-3.5.2-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:15c44f7edfd477b06f517a5cc317fc1707edb9de2c865f43d4b6513907473234", size = 235731, upload-time = "2026-09-30T04:36:04.987Z" }, + { url = "https://files.pythonhosted.org/packages/4d/34/38f3154785ce92e9f56eb226f4d35bdfae6b008480dd055f58837a89c810/charset_normalizer-3.5.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a89012d6d5476ee112d20d998570ed58df2260a852afb1758809cd6900411d21", size = 270352, upload-time = "2026-09-30T04:36:06.412Z" }, + { url = "https://files.pythonhosted.org/packages/04/f3/859f74e7babc977705026b30593b3be04049632a522fb7000f83c033d747/charset_normalizer-3.5.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:0c951d5e6dd9c2ff60609476752bee49da4206adde960ebc247766937f72e718", size = 267393, upload-time = "2026-09-30T04:36:07.865Z" }, + { url = "https://files.pythonhosted.org/packages/4b/85/41d27f234b82e47c167a5f6c0f62501dc0c640585ff4aba79e08a390336a/charset_normalizer-3.5.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7218e8f32b0956cfcd048fd42d9d5779809745ca1d86113ca56f66e7ae1549c4", size = 254660, upload-time = "2026-09-30T04:36:09.248Z" }, + { url = "https://files.pythonhosted.org/packages/58/ca/5d1a997587febe5b26d8daffe363b5c1a091cece19828eec6502fd09c5ef/charset_normalizer-3.5.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a19a731138fc27d5682277d3b9df22855cea1239bce7fcec5f78f42ef2d1f3c3", size = 249996, upload-time = "2026-09-30T04:36:10.73Z" }, + { url = "https://files.pythonhosted.org/packages/b3/1f/d1e78246f7ed60c8c8d606b4ac27f66ce49cc3e95f24893ccbeba9f77302/charset_normalizer-3.5.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:62603db9a7caa0802eaa28c1c46fecd7b3a263a774069c24c3c28c302448721c", size = 247028, upload-time = "2026-09-30T04:36:12.294Z" }, + { url = "https://files.pythonhosted.org/packages/8e/37/eba316edd4f0c4d3a5d945924c4eeeae59abac4056aa815d8a4268f863a2/charset_normalizer-3.5.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:b6856554c4f44d79fc2307d5768854310a8f0096e501c75637542c82292b0429", size = 240562, upload-time = "2026-09-30T04:36:13.887Z" }, + { url = "https://files.pythonhosted.org/packages/c8/8e/aaa037d40ca9ef045977f1a661048b1aa33f223adfce3452fe9be9f79d14/charset_normalizer-3.5.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:1bc0baf5ef96b6ede57d47f4b8fe4d9d84019c3bfcbeb20a41edc6a6ee341f1f", size = 271699, upload-time = "2026-09-30T04:36:15.41Z" }, + { url = "https://files.pythonhosted.org/packages/26/19/1c1c9f75974adf523b87f34b8a2adc5a435cd65916812bcbd0dfa45f9a29/charset_normalizer-3.5.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:56bc200a365efb37383b7852e4cc5898d3b2da5987289b543956cf8cad71018a", size = 252030, upload-time = "2026-09-30T04:36:16.839Z" }, + { url = "https://files.pythonhosted.org/packages/bc/90/0660ef18e18df0a4d2a1a0edff7dfbba42d4e50ef2425557a5bb7051f77b/charset_normalizer-3.5.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:2c9ad19a6cfcd5ea5c0d41161d22f9df1dcc277e9bef2751391334546a314c00", size = 268565, upload-time = "2026-09-30T04:36:18.468Z" }, + { url = "https://files.pythonhosted.org/packages/79/ba/57adc269824e8658f1a0f97a9e514c247445a9632b3419b97e0ba37f16dc/charset_normalizer-3.5.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e243bd13217235fc7290c621941c3f5cc8b66e4872495be821d7436ba2fb838d", size = 257627, upload-time = "2026-09-30T04:36:19.938Z" }, + { url = "https://files.pythonhosted.org/packages/9a/85/33abd4315c052d3d4f54c92b1ee49bfbc0dc7115a981e462a793b6d2ab87/charset_normalizer-3.5.2-cp313-cp313-pyemscripten_2025_0_wasm32.whl", hash = "sha256:a090bb2c68df85450502e3e20d665e3a5af9c65a84d6508ed477badd49166fd3", size = 143407, upload-time = "2026-09-30T04:36:21.376Z" }, + { url = "https://files.pythonhosted.org/packages/4f/de/6435e18d1aaa5d910b896d551411c96af1f42a0c56c29afc2016c61ccc2e/charset_normalizer-3.5.2-cp313-cp313-win32.whl", hash = "sha256:2b7b3bbfb4fe8ef40600792d762fbaa9057559f9d3fad209525b7a22b99e91fd", size = 183207, upload-time = "2026-09-30T04:36:22.776Z" }, + { url = "https://files.pythonhosted.org/packages/9c/76/b8ec57f4e9ee3253541abf95e4a462c0175fe8032dcd070f1f2421240942/charset_normalizer-3.5.2-cp313-cp313-win_amd64.whl", hash = "sha256:78456a747de8dc58360ffa581f30a002baf5aa28cb262536545e91f113ed7639", size = 206489, upload-time = "2026-09-30T04:36:24.306Z" }, + { url = "https://files.pythonhosted.org/packages/3e/60/c647c6ae47480221e875ea5d743ff94946f7416e3c69415ab772928e8d32/charset_normalizer-3.5.2-cp313-cp313-win_arm64.whl", hash = "sha256:11912e4bb14baae7c5d8791aa55ba0a3a03ec6729073307b0f57270abaa713d3", size = 195501, upload-time = "2026-09-30T04:36:25.846Z" }, + { url = "https://files.pythonhosted.org/packages/58/ca/7aa91362a2f77ac8e9e28a9b902a74f7d0e11a851ef0d27a74308da8cd90/charset_normalizer-3.5.2-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:1afb975bd5d68d5ce9f6b6d44fdf2f7e34b895a35e95708a7a91b20a3b51d187", size = 226010, upload-time = "2026-09-30T04:36:27.669Z" }, + { url = "https://files.pythonhosted.org/packages/a8/cf/ac8878d0322cf88a1aad4c7b147db32ca0bd806eb0060957b2e31486dbe6/charset_normalizer-3.5.2-cp314-cp314-android_24_x86_64.whl", hash = "sha256:bbbfc8e28816f19d7c0f1816664980c0a9875d01b27cdf8eedddb639d9e108ad", size = 239266, upload-time = "2026-09-30T04:36:29.434Z" }, + { url = "https://files.pythonhosted.org/packages/c9/6d/9a08d7e0b29b7208e2c6c01dc56c8e0520e7c7beadbbfb024b58fd69c8a5/charset_normalizer-3.5.2-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:7967d08cf06dee78443b874f98c98036f624f3a4e73e11f9f64f5be4d25393cf", size = 206783, upload-time = "2026-09-30T04:36:30.872Z" }, + { url = "https://files.pythonhosted.org/packages/82/44/b0aa350280e6ff5a5492d17cf10460dd39d5ee848f872f7ba2df10607f60/charset_normalizer-3.5.2-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:4c2b5031f63e331e3839b40aed2dd6f191e9c07edbde303e7876846ea1946995", size = 210449, upload-time = "2026-09-30T04:36:32.625Z" }, + { url = "https://files.pythonhosted.org/packages/7c/8a/40db9aa9f5907bb0e6f8b6d64064bf8852fb33d4b813ff9414911df7647c/charset_normalizer-3.5.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:fcff63213e8e6e47770541a4607175404f47cbb3ebea7b6058cc82d524a0e424", size = 367815, upload-time = "2026-09-30T04:36:34.197Z" }, + { url = "https://files.pythonhosted.org/packages/7f/72/9c5e7707b57c8ddfa9ddf7b0b1d009d7fbab9e9e887d5b721060f37e307d/charset_normalizer-3.5.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8d86d6fc60743dc916eb79e2eb1ec4818e21e427731543af40a3021851174a13", size = 247095, upload-time = "2026-09-30T04:36:35.803Z" }, + { url = "https://files.pythonhosted.org/packages/83/09/71e453691e927de4ddf792770cfaab3f49d494e222f66ea5e404bbd5e39c/charset_normalizer-3.5.2-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:7a881931aa470808df94a8c380eed2bbbc76cd9dc622310f99665658c821eb6d", size = 233942, upload-time = "2026-09-30T04:36:37.407Z" }, + { url = "https://files.pythonhosted.org/packages/9f/86/85c84e4da8b27dd409577d9437926ff581c5f9d3c66038dc68c1a526de51/charset_normalizer-3.5.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8024d00c3faf3fc0c16e07a69f4405e8eac7cc0ab15f65fe6cf43827c4cf72b4", size = 273762, upload-time = "2026-09-30T04:36:38.904Z" }, + { url = "https://files.pythonhosted.org/packages/92/08/564955a4b5f2ccb410ab480bbe8c6a18063ff27f2d35458731c4a5335df9/charset_normalizer-3.5.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4d48f2d08b9de5864e2c8744d4461b862fb149a18274abc8b698c45975573438", size = 269514, upload-time = "2026-09-30T04:36:40.469Z" }, + { url = "https://files.pythonhosted.org/packages/18/24/bad3ac4271589df29cf5ce2f5ae490518a5739358052bd0d61209e6fea54/charset_normalizer-3.5.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:34276fd796040bf0993ab33a369aa572e6979c7aab225a88893667ad8eac8f7a", size = 255056, upload-time = "2026-09-30T04:36:42.02Z" }, + { url = "https://files.pythonhosted.org/packages/d6/3e/350d89ad49916b86554d6f5f2d03ec1152148f87e5ff735106c6a03b1a36/charset_normalizer-3.5.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0521c5665880b33d603717defa76c094048900010897909952397feb3039da56", size = 251311, upload-time = "2026-09-30T04:36:43.577Z" }, + { url = "https://files.pythonhosted.org/packages/56/5b/4970a2d154df502e133402906dd04e3ae7cada7b3011283c88d0479a2585/charset_normalizer-3.5.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:eff0ac9dbe711a4aee69bf04a83896aa9b85f19641264053a9f6d48573abb7dd", size = 250190, upload-time = "2026-09-30T04:36:45.185Z" }, + { url = "https://files.pythonhosted.org/packages/88/8c/f1a91bddc8fb47c2889e29ea7ea49a194eb0d9868675d786806519c00d76/charset_normalizer-3.5.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:1503bccbeb36d5527790c3930327704c39af22de3112f1b1666a9f3ce15ee204", size = 237098, upload-time = "2026-09-30T04:36:46.689Z" }, + { url = "https://files.pythonhosted.org/packages/24/0e/bb5dace3cc7e79068425386a6589c19b5a2ab5fefc2a46abea6919683332/charset_normalizer-3.5.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:52aa6992700996af31f375de0c6bacd402b0097fe40b53c426b9f51a90ebabc7", size = 274228, upload-time = "2026-09-30T04:36:48.31Z" }, + { url = "https://files.pythonhosted.org/packages/9d/79/b849ad523017ea9f5a45581bbebed91439e0cf42fd2860a6f64e358eb5a6/charset_normalizer-3.5.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:e09a3942ecbdee5cce73ea9d42da82b81b72ac1bf031ce069b93b5adf4eac8cd", size = 252401, upload-time = "2026-09-30T04:36:50.091Z" }, + { url = "https://files.pythonhosted.org/packages/89/8c/75469d690cf47200bce8f6cad7655724fc23148e147abfc5ce78b5f65863/charset_normalizer-3.5.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:c7c9ab723cde841fefb34efbad91e87f00a674b1fe1cd0784fde742bf2c154dc", size = 271846, upload-time = "2026-09-30T04:36:51.719Z" }, + { url = "https://files.pythonhosted.org/packages/26/cd/6d52d3c7437cdcf2e310ce9f28f282e733d4ef60ed19105d1819c356255f/charset_normalizer-3.5.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ddc7dacc8ece3a182e7f15cb862d1fd616b46d076cb1ae9dd232b2c38b655874", size = 258151, upload-time = "2026-09-30T04:36:53.234Z" }, + { url = "https://files.pythonhosted.org/packages/f7/4c/070b38bdb5f49a70199fce923ec0726a49536a63ab262abbfcaaf351110b/charset_normalizer-3.5.2-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:ee43c17b173d46a3212baa6ead3ae258eeabdae48c263a01ccf0218c366dd655", size = 143570, upload-time = "2026-09-30T04:36:54.816Z" }, + { url = "https://files.pythonhosted.org/packages/81/84/9ebfc8ed6c8c4fcd8e726ff6bf220cc8deb3966e31dce9be8dd8aa017e64/charset_normalizer-3.5.2-cp314-cp314-win32.whl", hash = "sha256:4f87960d57feabfb618e4e0af6e7371645fa26a277860739d6e5d6e0012c92f0", size = 186412, upload-time = "2026-09-30T04:36:56.643Z" }, + { url = "https://files.pythonhosted.org/packages/d1/78/5ed86f743d4bc350db307e7636419a0a5ee1d91806d30c7f667bd5c80dae/charset_normalizer-3.5.2-cp314-cp314-win_amd64.whl", hash = "sha256:e4e81e09c1578b8df602e3db08b0b3ea0a6947ad612f52bf8dc5ea8d47691f0c", size = 210965, upload-time = "2026-09-30T04:36:58.205Z" }, + { url = "https://files.pythonhosted.org/packages/53/94/a3a7698e9b1a395e1eb99ccd9a324be9347973bff4e72db2a06496d7cd27/charset_normalizer-3.5.2-cp314-cp314-win_arm64.whl", hash = "sha256:80d02b6f04e92601a081dd97b23d3128033098bff5d35d392ddcc0476ea11253", size = 200406, upload-time = "2026-09-30T04:36:59.764Z" }, + { url = "https://files.pythonhosted.org/packages/c1/48/c5dd00d5ef7791f02666de250a5bb6071e29b7e133cf4b835800b6d3bc27/charset_normalizer-3.5.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:dca9ab98072a5a54ebacebdc45f53e645336b320c667410b061be1ca588ae709", size = 396871, upload-time = "2026-09-30T04:37:01.543Z" }, + { url = "https://files.pythonhosted.org/packages/12/c8/8379554b42e8368161d898476686947a0fdbd3e8865170d7909dcabfdee8/charset_normalizer-3.5.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f0aa869112ef88429ae17820d99c3dd9504c9e9c671d3c246f3d7442cb051084", size = 249711, upload-time = "2026-09-30T04:37:03.111Z" }, + { url = "https://files.pythonhosted.org/packages/4a/eb/2ddb1035d17320caa9f41682935123a9a250277b261c3efc86b2d2a21343/charset_normalizer-3.5.2-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c0afc6800ba57ccc350374c5bd6150419915d95ce93cdbab2d783d75eaf30ecb", size = 231146, upload-time = "2026-09-30T04:37:04.721Z" }, + { url = "https://files.pythonhosted.org/packages/4a/24/2ecb4bde104322cd7859d6594fcfa74649f8d90b3221c9feecbef149875b/charset_normalizer-3.5.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7dcd882da75ef9adf94903b1e3b9419e8aa8fb4c7396822b834b9ef7fb96954f", size = 271552, upload-time = "2026-09-30T04:37:06.295Z" }, + { url = "https://files.pythonhosted.org/packages/3f/98/9d5f6ebc3aee9fef5d30b4aff11fb2ab7a1222b4064f8ef2c7c87cde217a/charset_normalizer-3.5.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2e06a3a98f916dd41d27f3105e02e7a40181c98c94b9158733d03a6f80506c09", size = 269810, upload-time = "2026-09-30T04:37:07.905Z" }, + { url = "https://files.pythonhosted.org/packages/09/e1/a3b06a10461b1b7628853c934c644e03bc28e42767116afb52f19a56519b/charset_normalizer-3.5.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6bd128f206a7752ae1f2ab6c61bf8a24ba28913a10df8b14c2637b973ff97a80", size = 259845, upload-time = "2026-09-30T04:37:09.554Z" }, + { url = "https://files.pythonhosted.org/packages/fd/d3/6f561f74a296cf27d61775a1dc665ad13f3bff6a798810ca05907f37a7c4/charset_normalizer-3.5.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c8f3d67aeaf55f017982b73683f0e7342ba2f6635a78f69ce89ebb26aa411e5c", size = 252825, upload-time = "2026-09-30T04:37:11.274Z" }, + { url = "https://files.pythonhosted.org/packages/26/9f/69e13ca3b18f43e0eafcd34c04a45b732ae22a43b54a5fc9e119103356eb/charset_normalizer-3.5.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:fe9753dfee015c570d73df76f899f18444d41388bffcde097deba51c4fadbb9f", size = 252659, upload-time = "2026-09-30T04:37:12.941Z" }, + { url = "https://files.pythonhosted.org/packages/73/a9/ace29806a0dae18939919c76ba526472d83214afa101105fabff2cf30625/charset_normalizer-3.5.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:92888bb3187c5ba50500b00b3b310c9f2c651709d28036077680cb5255450a03", size = 238819, upload-time = "2026-09-30T04:37:14.659Z" }, + { url = "https://files.pythonhosted.org/packages/f8/c1/6116d52a2e3311ec80f21f5fb5e17b27405f10b9608af8f6e69516841a1b/charset_normalizer-3.5.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:d008d90a7f2471519aef0c90dfbe73b3e6e4d5e66ac48e19154c17e89e98b604", size = 272973, upload-time = "2026-09-30T04:37:16.346Z" }, + { url = "https://files.pythonhosted.org/packages/19/aa/9955c7e93bba10a9c7e8f7a5031b7ced66f3a1883a55c00712b8d5850ff3/charset_normalizer-3.5.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:31f3930700408d211f13378ccbe1c40845d8da54bd0681fac3a9b5aae81c7aa8", size = 253692, upload-time = "2026-09-30T04:37:18.212Z" }, + { url = "https://files.pythonhosted.org/packages/bb/33/2a6ae7fdc1b10cb581cef91addd8cdfc5f40d50abb5702309369d5834579/charset_normalizer-3.5.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:2a925889534b3748302dae5dead07cc13480de1dac3aea80a941b729b471ef93", size = 268757, upload-time = "2026-09-30T04:37:19.877Z" }, + { url = "https://files.pythonhosted.org/packages/a2/22/80992720a0282cd39bba1db35868e6b9c22f41281160143a836544bc1d8a/charset_normalizer-3.5.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:f5ec61164adcec446f8969a3358ec3f9b26bbda3b9213e5586d219afa8df2915", size = 262220, upload-time = "2026-09-30T04:37:21.583Z" }, + { url = "https://files.pythonhosted.org/packages/92/9f/181fd07e1bffea1d95cd80c84ac537354f50699c22cfc4d3c02b6fc16208/charset_normalizer-3.5.2-cp314-cp314t-win32.whl", hash = "sha256:598a11a2c7ebaa5334bf698bf29568c9c390abac6a154d8170fedecd1cea38c5", size = 197797, upload-time = "2026-09-30T04:37:23.235Z" }, + { url = "https://files.pythonhosted.org/packages/49/1c/25d8415ec1c4f2f41f1680435e4c87cfb378ff2f677d950946f2a45d0632/charset_normalizer-3.5.2-cp314-cp314t-win_amd64.whl", hash = "sha256:7fdde2c9fd9e3eca40631e024664cf2584272cc8f96308cbe5fdfc930f51d8bc", size = 222765, upload-time = "2026-09-30T04:37:24.891Z" }, + { url = "https://files.pythonhosted.org/packages/3e/b4/46b48f013dadfc0d0d33b375438e31bdf5a989dc68389c6bf627054d4df9/charset_normalizer-3.5.2-cp314-cp314t-win_arm64.whl", hash = "sha256:d1befeed746d247c81127bb14de9dc3d30edb6e5976d34f83f86ed262b1d9105", size = 210985, upload-time = "2026-09-30T04:37:26.634Z" }, + { url = "https://files.pythonhosted.org/packages/ca/e9/34e597dee616d0b8ee4b34d29399e85c2204ade174157a48505d42baa4ff/charset_normalizer-3.5.2-cp315-cp315-macosx_10_15_universal2.whl", hash = "sha256:87475fabc8d9996fd9c27debb395e642e8c838d78a00b6e932227a0e06b81e26", size = 367676, upload-time = "2026-09-30T04:37:28.329Z" }, + { url = "https://files.pythonhosted.org/packages/60/9f/a5d1c91c0263745e2cd344c5a4415d787c575501ab1d449f1148ac6b495d/charset_normalizer-3.5.2-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9409a8bf35cf78353942504b24a57de3d75b708997a1e4bd8db71ac8633ce364", size = 248414, upload-time = "2026-09-30T04:37:30.167Z" }, + { url = "https://files.pythonhosted.org/packages/26/79/e697f77464748a3ee3cf490c83d592459400d4898380d66c38366b03080c/charset_normalizer-3.5.2-cp315-cp315-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:498dc3188ca05a68231ac3fdbfc7f57eb67e1343c30e0fea17f8218c1599b253", size = 244775, upload-time = "2026-09-30T04:37:31.964Z" }, + { url = "https://files.pythonhosted.org/packages/ca/87/3d42a42e18ea066e2513936fd678a00696e77878b5ae04528976abdbcb83/charset_normalizer-3.5.2-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e242bb1c5e76e97dfa9e7f209a71e93a01d7f19ffdd5cfbb2e2d55b4f08f8ab0", size = 273877, upload-time = "2026-09-30T04:37:33.661Z" }, + { url = "https://files.pythonhosted.org/packages/c3/76/8a28136f3938ba9836f84280ce0c4d61ed1cf15a036b2034900c62634162/charset_normalizer-3.5.2-cp315-cp315-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:def79fa35ef0cef8d2accec024f4fdc7ead3012ff02f5215c783f39f03ef8cfc", size = 269455, upload-time = "2026-09-30T04:37:35.573Z" }, + { url = "https://files.pythonhosted.org/packages/a0/a1/4fbf5d0f0f1b2a080474c1cf9a2f12c4c6531bb0e8ba591055e846d2b4e9/charset_normalizer-3.5.2-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3df041de8887954562c9b261cba85ca0e9ded74048daf125f45edcfaa4832229", size = 257336, upload-time = "2026-09-30T04:37:37.397Z" }, + { url = "https://files.pythonhosted.org/packages/ba/a2/8b50aa320adb880ad579518e6f718f24944804b42a88b83d267d5d444125/charset_normalizer-3.5.2-cp315-cp315-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:04851f73ae72b8413dddadb16a49dfee95263553741fd42d546f7d66907e6be5", size = 257584, upload-time = "2026-09-30T04:37:39.522Z" }, + { url = "https://files.pythonhosted.org/packages/a5/57/50e3fed84e175f40349bd0da7a4fce94c87f0378f52d74f511d89e0bdc20/charset_normalizer-3.5.2-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:183b88127acdb4fabe59d951ab424faf1af7b63cdbb5f776186c1ea2ffcaed98", size = 251854, upload-time = "2026-09-30T04:37:41.23Z" }, + { url = "https://files.pythonhosted.org/packages/d6/54/f7fbb3493c9f49091213b9c2d6dd65800696f1ce1a3f196a4205f50417b1/charset_normalizer-3.5.2-cp315-cp315-musllinux_1_2_armv7l.whl", hash = "sha256:16fa0eccf81304b79c5cd87f9271c3b85dd9dd99245e4422ae9c0dd45e0f99d3", size = 248767, upload-time = "2026-09-30T04:37:42.883Z" }, + { url = "https://files.pythonhosted.org/packages/d9/37/b3a6385acc5a1e45b39ae9c90bfb9cf838a09b9dd37ef2740ab4c6b4a2eb/charset_normalizer-3.5.2-cp315-cp315-musllinux_1_2_ppc64le.whl", hash = "sha256:7441d755b7ab94f8d4eb3e43ec05482d760842fd263d003a99102d742cd835e2", size = 274479, upload-time = "2026-09-30T04:37:44.658Z" }, + { url = "https://files.pythonhosted.org/packages/89/44/809913e2cfd279e635a9294fdbbfb1b1dc62a8189d473d561f649fce98d8/charset_normalizer-3.5.2-cp315-cp315-musllinux_1_2_riscv64.whl", hash = "sha256:ca403d7e4798f525fdfc78e258820419cbbd0f0ecbab9de7840e3c017cf6b8cf", size = 258310, upload-time = "2026-09-30T04:37:46.529Z" }, + { url = "https://files.pythonhosted.org/packages/af/a2/f28400ab13359d91bd39179df8e149376b9bf36588e739a3a4f9de2b84b2/charset_normalizer-3.5.2-cp315-cp315-musllinux_1_2_s390x.whl", hash = "sha256:df29a0a7107f7011e77f4eebdddec4c7331e24d787a0b21a46d63bdf7445da95", size = 271100, upload-time = "2026-09-30T04:37:48.399Z" }, + { url = "https://files.pythonhosted.org/packages/e9/89/9bab37955edf0adb3b66f8a3a6617d9f2f487e0d56f295a6a286cb640aa6/charset_normalizer-3.5.2-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:f3c96f633825733f735c5a9cf21d21a257d8e1edf0b1cee0a064b9c424ca0f7d", size = 259936, upload-time = "2026-09-30T04:37:50.023Z" }, + { url = "https://files.pythonhosted.org/packages/23/b5/4459e08d45a679f903d50fea08bc52cfa728cca4d7bd02c757b5e5abda2e/charset_normalizer-3.5.2-cp315-cp315-win32.whl", hash = "sha256:281cb91036248400f4cc957495cccd44c275c2e0c5854f7e45ac5cf7dc193847", size = 186301, upload-time = "2026-09-30T04:37:51.722Z" }, + { url = "https://files.pythonhosted.org/packages/98/e8/55d5fd3935b4bce6da4fe0df61898e8c82653e317e677bd58aceb9c60f13/charset_normalizer-3.5.2-cp315-cp315-win_amd64.whl", hash = "sha256:89b53f3cda69831909888e0494f4fa0bcd3537e3e138dabeb620bd6ad946bae8", size = 210858, upload-time = "2026-09-30T04:37:53.427Z" }, + { url = "https://files.pythonhosted.org/packages/a9/5b/974423c2fd8e524c7a7f64318c1e02240ef954912fa2b4d70344107b9c68/charset_normalizer-3.5.2-cp315-cp315-win_arm64.whl", hash = "sha256:6be488a102b8cf28d0391d8c4ba7748938ae28b78ad901f8585520fca33ead1a", size = 200309, upload-time = "2026-09-30T04:37:55.015Z" }, + { url = "https://files.pythonhosted.org/packages/ee/f9/00ee0195db1013d8f7c416fd770fbeb560bb46eb2e36b054d05cb56f6cfa/charset_normalizer-3.5.2-cp315-cp315t-macosx_10_15_universal2.whl", hash = "sha256:915563965d418f986e7e145accc592eae9e1a1be3566ff98a05d7a9ec42a76e1", size = 395343, upload-time = "2026-09-30T04:37:56.743Z" }, + { url = "https://files.pythonhosted.org/packages/04/3a/c00b50e94c964cf934c7899cd47c97952fc11dad71cc5884b3c61795b09b/charset_normalizer-3.5.2-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:65cd72beeeca9d3aaea1201e5923859f308f952f9c71de93f06063c79f0f7a3b", size = 250114, upload-time = "2026-09-30T04:37:58.607Z" }, + { url = "https://files.pythonhosted.org/packages/50/27/d102dc880bbcffd0479ab64dfc1fb96777a854355a55e2bda72a71efadcb/charset_normalizer-3.5.2-cp315-cp315t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b7fd005a73d9e657273b7a10dc71a9e03c8fb9ee6999798d6918ce095b81ac7f", size = 239997, upload-time = "2026-09-30T04:38:00.511Z" }, + { url = "https://files.pythonhosted.org/packages/a5/4a/bf7ef45794dd293fab5f98a9309817977fbb845b9998f171b8cc5d8437a3/charset_normalizer-3.5.2-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e54da4baf05720032d527874d40b65fa4d7e5c6c6a43d0c3adbeffcaf275a2b3", size = 272313, upload-time = "2026-09-30T04:38:02.509Z" }, + { url = "https://files.pythonhosted.org/packages/e8/ee/008a2837737991474c5754bb3191010007663860979701990982a502cbaf/charset_normalizer-3.5.2-cp315-cp315t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:124fbf1a8ff966d87ae05bb8bd45a71f966055ed8bba320d0c7cf450bc5f4d0e", size = 269986, upload-time = "2026-09-30T04:38:04.435Z" }, + { url = "https://files.pythonhosted.org/packages/93/ad/bd74a283940dc910c5b14f8e4f80a248082bc9c0fcbe1f54530cb6d9cc5e/charset_normalizer-3.5.2-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:28b4f0d66fb834ff90f28209ac7bce77868c45d8c93e26f906709d9b7c2e1af9", size = 260202, upload-time = "2026-09-30T04:38:06.549Z" }, + { url = "https://files.pythonhosted.org/packages/8a/7b/ed341c66f69f688723501fac752be3d63c7159ca0d0d4174fc611e5710bb/charset_normalizer-3.5.2-cp315-cp315t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:58ca3755ee7ff7f59b57789ec9833c9de9ea275405cdd240eda1f193112e398a", size = 259083, upload-time = "2026-09-30T04:38:08.311Z" }, + { url = "https://files.pythonhosted.org/packages/cc/9d/e41588b777965e5031a43128a1e96173ebb35ac75fc53ec3b517e7c21cd4/charset_normalizer-3.5.2-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:443eae2bf318abeaf6f15d785138f71fd6de770e99a92158b8b814265e079115", size = 253081, upload-time = "2026-09-30T04:38:10.402Z" }, + { url = "https://files.pythonhosted.org/packages/81/35/b761eb6d8c1eb218b9b42b9b4d5ac902afdc399fb6dac6f9a9aac7bda589/charset_normalizer-3.5.2-cp315-cp315t-musllinux_1_2_armv7l.whl", hash = "sha256:58f361dcbab699cf8f42db3f47c8e7fd1036f138c23a5d08de9fde5f425a730c", size = 248712, upload-time = "2026-09-30T04:38:12.317Z" }, + { url = "https://files.pythonhosted.org/packages/4d/2c/147169a041b747759f37405c0a97157e8e92de967968373101ff14915cba/charset_normalizer-3.5.2-cp315-cp315t-musllinux_1_2_ppc64le.whl", hash = "sha256:1b4cbc7c3491ccb4aa17fcd8165649d01cf39f76de1696da8631b5f71b85401d", size = 274176, upload-time = "2026-09-30T04:38:14.138Z" }, + { url = "https://files.pythonhosted.org/packages/f0/2d/0ff8db0d373ba8538db686db11cd7e8912031490b9e4f383b41912e8d594/charset_normalizer-3.5.2-cp315-cp315t-musllinux_1_2_riscv64.whl", hash = "sha256:ba0b1d2620edf869789c3879223f52bf2afc5d31b3cb47cc57b3a12c05e2aa9d", size = 259413, upload-time = "2026-09-30T04:38:15.841Z" }, + { url = "https://files.pythonhosted.org/packages/8a/8e/b4a085fb47c9d3a7e43576a4784fdd8fe23f907514a972de8086edaf7a48/charset_normalizer-3.5.2-cp315-cp315t-musllinux_1_2_s390x.whl", hash = "sha256:5e2b6b57e9733d39f0c9fd3185efa6b8e29652c4cd8fe94180272cf6ed9a78c4", size = 269434, upload-time = "2026-09-30T04:38:17.626Z" }, + { url = "https://files.pythonhosted.org/packages/83/1c/d8d8d7322a7c3eecdf3237a4a419cf41d2eaad8e006ce7dfdd9d4c8fa2eb/charset_normalizer-3.5.2-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:51cf45226a9b588d0d2b4880c62d686934b63ab0bd79ca23ab0e9762eb27441b", size = 262716, upload-time = "2026-09-30T04:38:19.214Z" }, + { url = "https://files.pythonhosted.org/packages/a0/16/0e4c6ba9b44e97a2da150e52d331e8f9c968b21b358fbffa6c856cebcd89/charset_normalizer-3.5.2-cp315-cp315t-win32.whl", hash = "sha256:5fb29fb8cd1a46c27a1bf9613ad5ec2599310d46b4025d9556404a6b6a292800", size = 197159, upload-time = "2026-09-30T04:38:21.037Z" }, + { url = "https://files.pythonhosted.org/packages/be/33/e90bc2b1374f7f36ef106f56620de5a783907e19ca857efe2277e31cac3e/charset_normalizer-3.5.2-cp315-cp315t-win_amd64.whl", hash = "sha256:a192e2c40070d92c3ccf777e3a5c4ff515573cd2bb7ed0c537fdadbbec5bbf21", size = 221322, upload-time = "2026-09-30T04:38:22.886Z" }, + { url = "https://files.pythonhosted.org/packages/66/89/dfa6dcb08c200b7830ab56439e8c1890f2971d51aafbb3937894a2e7fcfc/charset_normalizer-3.5.2-cp315-cp315t-win_arm64.whl", hash = "sha256:749e97e1b32313717a565abbe321bc2190bc8b35f1a67e4cdbc7c56c8d8ffe58", size = 209192, upload-time = "2026-09-30T04:38:24.648Z" }, + { url = "https://files.pythonhosted.org/packages/8c/ab/176fbfd5b64939c55d652366aa5b9ef1d767af207a3aa6ebeb0d226c484d/charset_normalizer-3.5.2-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:4275811936e2f06feff5e598fb42a1b7ae852da8e39605211892b56b81a34efd", size = 331815, upload-time = "2026-09-30T04:38:26.216Z" }, + { url = "https://files.pythonhosted.org/packages/7e/84/371eac6b30bdbcbf2d632a1a01809103459216fcaae61b8b8d922c1bfb8a/charset_normalizer-3.5.2-cp37-abi3-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:1c50fe28bbc2ced33386f298650d91218076c05420e6cbd790b913adc41659e7", size = 253276, upload-time = "2026-09-30T04:38:28.032Z" }, + { url = "https://files.pythonhosted.org/packages/43/6f/c4fbae58febff71709c51bc7e18fdfa55341dc382704740f9f0cbf03817b/charset_normalizer-3.5.2-cp37-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d19fbd981a488e22cd04883659ca6b08f50b5974f9fd7c95655ef6a043e5893f", size = 241239, upload-time = "2026-09-30T04:38:29.732Z" }, + { url = "https://files.pythonhosted.org/packages/61/71/458c3f42164a07d0c5210798e9e704b39e540a6793b05aba67f3a35243a9/charset_normalizer-3.5.2-cp37-abi3-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:0fed1d06615f022ee3b13caf5e8b180cfea32bb2c5aded8a9d44277afc040f93", size = 231121, upload-time = "2026-09-30T04:38:31.462Z" }, + { url = "https://files.pythonhosted.org/packages/09/54/ab9e89367076f6331bb6c65c4bf14a5361fa5191cb6561bf534f18504e1b/charset_normalizer-3.5.2-cp37-abi3-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:838dcc90063569a0448120554591a1d6c4a4ffe11babf048908793154ab86ade", size = 260350, upload-time = "2026-09-30T04:38:33.239Z" }, + { url = "https://files.pythonhosted.org/packages/7c/c1/061431ecc688d9d76602502cb57cc01e691e682c18f1beb45f9673b5bbd2/charset_normalizer-3.5.2-cp37-abi3-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2ce45c6627b22c47e390bc91a41c3d13032192e699fa0bea96e9671b373d69b0", size = 255430, upload-time = "2026-09-30T04:38:34.865Z" }, + { url = "https://files.pythonhosted.org/packages/8d/1f/20c8949f0676f7ab811abdeb7f4d7f1cbc6e61ff20bef08b44edeb092bc8/charset_normalizer-3.5.2-cp37-abi3-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0774bf9bf620249fee3e0b8b9fd3065de213be30f3aa94ce2494b3b638949e26", size = 250612, upload-time = "2026-09-30T04:38:36.649Z" }, + { url = "https://files.pythonhosted.org/packages/2b/9e/46f2fa4c431fc98c4ae76a8cb5bdca54e0341e3cfc3fcfd8e82740250818/charset_normalizer-3.5.2-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:1db38f4c5496827c1a501846d64d14c3b80c7e6714e406cd7dc36a9899fa1011", size = 242083, upload-time = "2026-09-30T04:38:38.26Z" }, + { url = "https://files.pythonhosted.org/packages/bd/39/559be29a0c0f086e0bba6922babd38916cc5e0b58ced4de13ee01ea05508/charset_normalizer-3.5.2-cp37-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:304d8e4d493af723536393eee0c689eb7813f4a474c8b479dee63f1fdd98f621", size = 232738, upload-time = "2026-09-30T04:38:39.81Z" }, + { url = "https://files.pythonhosted.org/packages/ff/6c/387b0e4f756a282831c1d9fc6aeb6c51ca4507ca202767c8de15ce9b12e2/charset_normalizer-3.5.2-cp37-abi3-musllinux_1_2_ppc64le.whl", hash = "sha256:9b7f416ff0978e2f2249330527f0ad6fa02f4932e6199692d3b52da2048c19e4", size = 260703, upload-time = "2026-09-30T04:38:41.346Z" }, + { url = "https://files.pythonhosted.org/packages/96/92/1fdf015f09ef449f50d3ac4b67c90887c9c318b727daa95cc4f866e6521d/charset_normalizer-3.5.2-cp37-abi3-musllinux_1_2_riscv64.whl", hash = "sha256:01077390b03f7988f11d700a2194e69b119741a86b1a638b1db88891e3eced8e", size = 247622, upload-time = "2026-09-30T04:38:42.937Z" }, + { url = "https://files.pythonhosted.org/packages/dc/3c/8e7b8a5671ad5d433669fb2a76f1a0164df2d9b1718b0206bc2a16d840cc/charset_normalizer-3.5.2-cp37-abi3-musllinux_1_2_s390x.whl", hash = "sha256:7e841fb9010836c992c9f12fcbd43a831de93a5f726fc1ccd8ca1d0268c5014c", size = 257500, upload-time = "2026-09-30T04:38:44.604Z" }, + { url = "https://files.pythonhosted.org/packages/b4/f0/45b579df5cabc1d5d53ea1cc35e8437d3ca768c0acccc7041517cb6fbb32/charset_normalizer-3.5.2-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:9cae88599c7219005d879f98e5ed53341e9a122af585e1091200358a3003d2a0", size = 255100, upload-time = "2026-09-30T04:38:46.289Z" }, + { url = "https://files.pythonhosted.org/packages/31/68/fdec18a343f5fb3f310588dd478b09ac4799e0b187dbade3a8cd776f03ef/charset_normalizer-3.5.2-cp37-abi3-win32.whl", hash = "sha256:01b0c0d2262a9e28e8484a278c7e1b5d650e3ac8cf2683d2967e25899f208bdf", size = 174499, upload-time = "2026-09-30T04:38:47.999Z" }, + { url = "https://files.pythonhosted.org/packages/9d/8a/b618149cc5207943a0242068d7a27897f56a62947b5a039085f2a22029f8/charset_normalizer-3.5.2-cp37-abi3-win_amd64.whl", hash = "sha256:9f56f72050826f63dcee7a7f55b0a77168cb3bfc553fd405e7f8f9ece75a4036", size = 200092, upload-time = "2026-09-30T04:38:49.707Z" }, + { url = "https://files.pythonhosted.org/packages/03/cf/4c66866fa9e2b1c78e3c911516d1de497a677b7ac60f1eceda74ce777ca3/charset_normalizer-3.5.2-cp37-abi3-win_arm64.whl", hash = "sha256:40ab6bffa02ae10a0581e6c198be7d2d8ca5c2a0c64e4ed3465d766df457573e", size = 294363, upload-time = "2026-09-30T04:38:51.312Z" }, + { url = "https://files.pythonhosted.org/packages/fc/ad/d07d7862a62ffa6d79d68074d14823243dd235a77c45262acbf6adeb28bf/charset_normalizer-3.5.2-py3-none-any.whl", hash = "sha256:b6b751274acb69d77b3323d6b7dbaa3c7fdfc1eb829b7eb61d262f32e1af9685", size = 68872, upload-time = "2026-09-30T04:39:21.828Z" }, +] + [[package]] name = "click" version = "8.5.0" @@ -270,6 +410,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cb/03/10388a42375ee7e4ac9b94eb2c5c569c8b5795e377e701c9ac3ad63de890/fastapi-0.141.1-py3-none-any.whl", hash = "sha256:bfb91aa2d334c61cb35ba9a116fc123b3d3df31640b801cf57a7a78ec3f603b3", size = 131954, upload-time = "2026-07-29T17:18:04.364Z" }, ] +[[package]] +name = "googleapis-common-protos" +version = "1.76.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/62/b6/bc095b23a3e0d978e75c8c18cabe4c11f791837410670e4486f0d539bfba/googleapis_common_protos-1.76.0.tar.gz", hash = "sha256:71db0e44a1d4c2dd024ebad44ca67129cc1e2892e4861df466764749d5ef8642", size = 156250, upload-time = "2026-10-22T23:43:36.263Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/05/6d/78dc67a9e779e594b49bf62604034d3a6aa428a55afc729cafda4b7333e1/googleapis_common_protos-1.76.0-py3-none-any.whl", hash = "sha256:59e13344a9f3651aa201a03e2bee815a3bf43a4baf1f9ff8a9b62a840ce25fac", size = 307707, upload-time = "2026-10-22T23:43:10.445Z" }, +] + [[package]] name = "h11" version = "0.16.0" @@ -288,6 +440,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/58/a2/bb081bab032533a855d44de1d56f8e8426114ff1ba5d1f07a438a0a654f8/idna-3.20-py3-none-any.whl", hash = "sha256:ab7ae7122974553370f0bdb919e1a960b2cd1bc1ef0276416d896db81c14582c", size = 69583, upload-time = "2026-09-17T14:11:03.168Z" }, ] +[[package]] +name = "importlib-metadata" +version = "8.7.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "zipp" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f3/49/3b30cad09e7771a4982d9975a8cbf64f00d4a1ececb53297f1d9a7be1b10/importlib_metadata-8.7.1.tar.gz", hash = "sha256:49fef1ae6440c182052f407c8d34a68f72efc36db9ca90dc0113398f2fdde8bb", size = 57107, upload-time = "2025-12-21T10:00:19.278Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fa/5e/f8e9a1d23b9c20a551a8a02ea3637b4642e22c2626e3a13a9a29cdea99eb/importlib_metadata-8.7.1-py3-none-any.whl", hash = "sha256:5a1f80bf1daa489495071efbb095d75a634cf28a8bc299581244063b53176151", size = 27865, upload-time = "2025-12-21T10:00:18.329Z" }, +] + [[package]] name = "jmespath" version = "1.1.0" @@ -306,6 +470,8 @@ dependencies = [ { name = "asyncpg" }, { name = "boto3" }, { name = "fastapi" }, + { name = "opentelemetry-exporter-otlp-proto-http" }, + { name = "opentelemetry-sdk" }, { name = "pika" }, { name = "pillow" }, { name = "uvicorn" }, @@ -317,11 +483,95 @@ requires-dist = [ { name = "asyncpg", specifier = "==0.31.0" }, { name = "boto3", specifier = "==1.43.115" }, { name = "fastapi", specifier = "==0.141.1" }, + { name = "opentelemetry-exporter-otlp-proto-http", specifier = "==1.36.0" }, + { name = "opentelemetry-sdk", specifier = "==1.36.0" }, { name = "pika", specifier = "==1.3.2" }, { name = "pillow", specifier = "==13.0.0" }, { name = "uvicorn", specifier = "==0.40.0" }, ] +[[package]] +name = "opentelemetry-api" +version = "1.36.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "importlib-metadata" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/27/d2/c782c88b8afbf961d6972428821c302bd1e9e7bc361352172f0ca31296e2/opentelemetry_api-1.36.0.tar.gz", hash = "sha256:9a72572b9c416d004d492cbc6e61962c0501eaf945ece9b5a0f56597d8348aa0", size = 64780, upload-time = "2025-07-29T15:12:06.02Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bb/ee/6b08dde0a022c463b88f55ae81149584b125a42183407dc1045c486cc870/opentelemetry_api-1.36.0-py3-none-any.whl", hash = "sha256:02f20bcacf666e1333b6b1f04e647dc1d5111f86b8e510238fcc56d7762cda8c", size = 65564, upload-time = "2025-07-29T15:11:47.998Z" }, +] + +[[package]] +name = "opentelemetry-exporter-otlp-proto-common" +version = "1.36.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-proto" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/34/da/7747e57eb341c59886052d733072bc878424bf20f1d8cf203d508bbece5b/opentelemetry_exporter_otlp_proto_common-1.36.0.tar.gz", hash = "sha256:6c496ccbcbe26b04653cecadd92f73659b814c6e3579af157d8716e5f9f25cbf", size = 20302, upload-time = "2025-07-29T15:12:07.71Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d0/ed/22290dca7db78eb32e0101738366b5bbda00d0407f00feffb9bf8c3fdf87/opentelemetry_exporter_otlp_proto_common-1.36.0-py3-none-any.whl", hash = "sha256:0fc002a6ed63eac235ada9aa7056e5492e9a71728214a61745f6ad04b923f840", size = 18349, upload-time = "2025-07-29T15:11:51.327Z" }, +] + +[[package]] +name = "opentelemetry-exporter-otlp-proto-http" +version = "1.36.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "googleapis-common-protos" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-exporter-otlp-proto-common" }, + { name = "opentelemetry-proto" }, + { name = "opentelemetry-sdk" }, + { name = "requests" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/25/85/6632e7e5700ba1ce5b8a065315f92c1e6d787ccc4fb2bdab15139eaefc82/opentelemetry_exporter_otlp_proto_http-1.36.0.tar.gz", hash = "sha256:dd3637f72f774b9fc9608ab1ac479f8b44d09b6fb5b2f3df68a24ad1da7d356e", size = 16213, upload-time = "2025-07-29T15:12:08.932Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7f/41/a680d38b34f8f5ddbd78ed9f0042e1cc712d58ec7531924d71cb1e6c629d/opentelemetry_exporter_otlp_proto_http-1.36.0-py3-none-any.whl", hash = "sha256:3d769f68e2267e7abe4527f70deb6f598f40be3ea34c6adc35789bea94a32902", size = 18752, upload-time = "2025-07-29T15:11:53.164Z" }, +] + +[[package]] +name = "opentelemetry-proto" +version = "1.36.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fd/02/f6556142301d136e3b7e95ab8ea6a5d9dc28d879a99f3dd673b5f97dca06/opentelemetry_proto-1.36.0.tar.gz", hash = "sha256:0f10b3c72f74c91e0764a5ec88fd8f1c368ea5d9c64639fb455e2854ef87dd2f", size = 46152, upload-time = "2025-07-29T15:12:15.717Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/57/3361e06136225be8180e879199caea520f38026f8071366241ac458beb8d/opentelemetry_proto-1.36.0-py3-none-any.whl", hash = "sha256:151b3bf73a09f94afc658497cf77d45a565606f62ce0c17acb08cd9937ca206e", size = 72537, upload-time = "2025-07-29T15:12:02.243Z" }, +] + +[[package]] +name = "opentelemetry-sdk" +version = "1.36.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "opentelemetry-semantic-conventions" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/4c/85/8567a966b85a2d3f971c4d42f781c305b2b91c043724fa08fd37d158e9dc/opentelemetry_sdk-1.36.0.tar.gz", hash = "sha256:19c8c81599f51b71670661ff7495c905d8fdf6976e41622d5245b791b06fa581", size = 162557, upload-time = "2025-07-29T15:12:16.76Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0b/59/7bed362ad1137ba5886dac8439e84cd2df6d087be7c09574ece47ae9b22c/opentelemetry_sdk-1.36.0-py3-none-any.whl", hash = "sha256:19fe048b42e98c5c1ffe85b569b7073576ad4ce0bcb6e9b4c6a39e890a6c45fb", size = 119995, upload-time = "2025-07-29T15:12:03.181Z" }, +] + +[[package]] +name = "opentelemetry-semantic-conventions" +version = "0.57b0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7e/31/67dfa252ee88476a29200b0255bda8dfc2cf07b56ad66dc9a6221f7dc787/opentelemetry_semantic_conventions-0.57b0.tar.gz", hash = "sha256:609a4a79c7891b4620d64c7aac6898f872d790d75f22019913a660756f27ff32", size = 124225, upload-time = "2025-07-29T15:12:17.873Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/05/75/7d591371c6c39c73de5ce5da5a2cc7b72d1d1cd3f8f4638f553c01c37b11/opentelemetry_semantic_conventions-0.57b0-py3-none-any.whl", hash = "sha256:757f7e76293294f124c827e514c2a3144f191ef175b069ce8d1211e1e38e9e78", size = 201627, upload-time = "2025-07-29T15:12:04.174Z" }, +] + [[package]] name = "pika" version = "1.3.2" @@ -407,6 +657,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c0/37/66b12dc476041fe523bca29c9d58fbf1ea92a4eaf73acfb9e993948f1191/pillow-13.0.0-pp312-pypy312_pp80-win_amd64.whl", hash = "sha256:567f20f1c75897d7663830da6ea8af719e03d712b1d9ee7f813319f9ece42e5a", size = 7366030, upload-time = "2026-10-16T22:18:06.638Z" }, ] +[[package]] +name = "protobuf" +version = "6.33.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/66/70/e908e9c5e52ef7c3a6c7902c9dfbb34c7e29c25d2f81ade3856445fd5c94/protobuf-6.33.6.tar.gz", hash = "sha256:a6768d25248312c297558af96a9f9c929e8c4cee0659cb07e780731095f38135", size = 444531, upload-time = "2026-03-18T19:05:00.988Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fc/9f/2f509339e89cfa6f6a4c4ff50438db9ca488dec341f7e454adad60150b00/protobuf-6.33.6-cp310-abi3-win32.whl", hash = "sha256:7d29d9b65f8afef196f8334e80d6bc1d5d4adedb449971fefd3723824e6e77d3", size = 425739, upload-time = "2026-03-18T19:04:48.373Z" }, + { url = "https://files.pythonhosted.org/packages/76/5d/683efcd4798e0030c1bab27374fd13a89f7c2515fb1f3123efdfaa5eab57/protobuf-6.33.6-cp310-abi3-win_amd64.whl", hash = "sha256:0cd27b587afca21b7cfa59a74dcbd48a50f0a6400cfb59391340ad729d91d326", size = 437089, upload-time = "2026-03-18T19:04:50.381Z" }, + { url = "https://files.pythonhosted.org/packages/5c/01/a3c3ed5cd186f39e7880f8303cc51385a198a81469d53d0fdecf1f64d929/protobuf-6.33.6-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:9720e6961b251bde64edfdab7d500725a2af5280f3f4c87e57c0208376aa8c3a", size = 427737, upload-time = "2026-03-18T19:04:51.866Z" }, + { url = "https://files.pythonhosted.org/packages/ee/90/b3c01fdec7d2f627b3a6884243ba328c1217ed2d978def5c12dc50d328a3/protobuf-6.33.6-cp39-abi3-manylinux2014_aarch64.whl", hash = "sha256:e2afbae9b8e1825e3529f88d514754e094278bb95eadc0e199751cdd9a2e82a2", size = 324610, upload-time = "2026-03-18T19:04:53.096Z" }, + { url = "https://files.pythonhosted.org/packages/9b/ca/25afc144934014700c52e05103c2421997482d561f3101ff352e1292fb81/protobuf-6.33.6-cp39-abi3-manylinux2014_s390x.whl", hash = "sha256:c96c37eec15086b79762ed265d59ab204dabc53056e3443e702d2681f4b39ce3", size = 339381, upload-time = "2026-03-18T19:04:54.616Z" }, + { url = "https://files.pythonhosted.org/packages/16/92/d1e32e3e0d894fe00b15ce28ad4944ab692713f2e7f0a99787405e43533a/protobuf-6.33.6-cp39-abi3-manylinux2014_x86_64.whl", hash = "sha256:e9db7e292e0ab79dd108d7f1a94fe31601ce1ee3f7b79e0692043423020b0593", size = 323436, upload-time = "2026-03-18T19:04:55.768Z" }, + { url = "https://files.pythonhosted.org/packages/c4/72/02445137af02769918a93807b2b7890047c32bfb9f90371cbc12688819eb/protobuf-6.33.6-py3-none-any.whl", hash = "sha256:77179e006c476e69bf8e8ce866640091ec42e1beb80b213c3900006ecfba6901", size = 170656, upload-time = "2026-03-18T19:04:59.826Z" }, +] + [[package]] name = "pycparser" version = "3.1" @@ -518,6 +783,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, ] +[[package]] +name = "requests" +version = "2.34.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "charset-normalizer" }, + { name = "idna" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ac/c3/e2a2b89f2d3e2179abd6d00ebd70bff6273f37fb3e0cc209f48b39d00cbf/requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed", size = 142856, upload-time = "2026-05-14T19:25:27.735Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", size = 73075, upload-time = "2026-05-14T19:25:26.443Z" }, +] + [[package]] name = "s3transfer" version = "0.19.3" @@ -594,3 +874,12 @@ sdist = { url = "https://files.pythonhosted.org/packages/c3/d1/8f3c683c9561a4e66 wheels = [ { url = "https://files.pythonhosted.org/packages/3d/d8/2083a1daa7439a66f3a48589a57d576aa117726762618f6bb09fe3798796/uvicorn-0.40.0-py3-none-any.whl", hash = "sha256:c6c8f55bc8bf13eb6fa9ff87ad62308bbbc33d0b67f84293151efe87e0d5f2ee", size = 68502, upload-time = "2025-12-21T14:16:21.041Z" }, ] + +[[package]] +name = "zipp" +version = "4.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/18/05/ba1458dee0ac485b2a01071c1b88f5199fe878a485df482e10e1ef2205cc/zipp-4.1.2.tar.gz", hash = "sha256:cf474b90b07ea932637e17c6b9479de2b2882900b0d311bdaf4f5c000a15a97f", size = 27864, upload-time = "2026-10-21T18:57:04.743Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b8/44/7bd2007260da11cccca29ba2d4c345f1287aa663db38ee7cefeabf6386a5/zipp-4.1.2-py3-none-any.whl", hash = "sha256:b30cb6cd253122169c47de245ca24a86bac2a5d3802ba15bf7ba6414eb9e144d", size = 10581, upload-time = "2026-10-21T18:57:03.79Z" }, +] -- 2.54.0 From c885462c044f560ed4ea12e3c0048be40766da13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9C=D0=B0=D1=82=D0=B2=D0=B5=D0=B9?= Date: Fri, 6 Nov 2026 09:39:53 +0300 Subject: [PATCH 3/4] add level 5 --- REPORT.md | 12 +- docs/observability.md | 2 +- docs/slo-runbook.md | 41 + evidence/slo-analysis.json | 53 + evidence/slo-incidents.json | 1170 +++++++++++++++++++++++ evidence/slo-ready-incident.json | 211 ++++ evidence/slo-track.md | 34 + infra/grafana/dashboards/messenger.json | 110 ++- infra/prometheus/alerts.yaml | 16 + src/outbox.py | 14 +- src/telemetry.py | 5 +- student-tests/burn_rules.test.yaml | 83 ++ student-tests/slo_analyze.py | 32 + student-tests/slo_incidents.py | 121 +++ student-tests/slo_ready_incident.py | 70 ++ 15 files changed, 1966 insertions(+), 8 deletions(-) create mode 100644 docs/slo-runbook.md create mode 100644 evidence/slo-analysis.json create mode 100644 evidence/slo-incidents.json create mode 100644 evidence/slo-ready-incident.json create mode 100644 evidence/slo-track.md create mode 100644 student-tests/burn_rules.test.yaml create mode 100644 student-tests/slo_analyze.py create mode 100644 student-tests/slo_incidents.py create mode 100644 student-tests/slo_ready_incident.py diff --git a/REPORT.md b/REPORT.md index e23516b..6a79e7e 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,7 +1,7 @@ # Отчёт — лабораторная 6 - Автор, группа: Kozyrev, BVT2301 -- Целевая оценка: 4 +- Целевая оценка: 5, трек 2 — SLO и обнаружение инцидентов - Commit/PR: TODO после публикации - Источники: [контракт телеметрии](contracts/observability.md), [OTel Collector](https://opentelemetry.io/docs/collector/configuration/), [OTel Python exporters](https://opentelemetry.io/docs/languages/python/exporters/), [Loki OTLP](https://grafana.com/docs/loki/latest/send-data/otel/), [Grafana provisioning](https://grafana.com/docs/grafana/latest/administration/provisioning/), исходники lab5 commit `6bc0712`. При реализации использован Codex. @@ -42,8 +42,16 @@ API, outbox и worker отправляют OTLP через Collector в Promethe Идентификаторы трасс, результаты, формулы и границы проверки приведены в [evidence/observability.md](evidence/observability.md). Raw summaries нагрузки: [инцидент БД](evidence/db-incident-load-repeat.json), [отказ Tempo](evidence/tempo-outage-load.json). Полный набор наблюдений живёт в локальных volumes, а не в Git. +## Трек 5: SLO и обнаружение инцидентов + +Заданы учебные SLO 99% за 24 часа для API и времени готовности принятых вложений. Для каждого настроены быстрые и медленные multiwindow burn-rate alerts; формулы, допущения и пошаговая диагностика — в [runbook](docs/slo-runbook.md). Последние 30 секунд заданий исключены из зрелой когорты; просроченные, failed и неготовые задания входят в ошибки SLO. Моменты начала и окончания приблизительны из-за транзакционных границ. При недоступном outbox счётчики устаревают. + +Повторяемые сценарии и наблюдения — в [отчёте трека](evidence/slo-track.md), исходные отсчёты — [API](evidence/slo-incidents.json), [расчёт](evidence/slo-analysis.json) и [вложения](evidence/slo-ready-incident.json). При короткой и длительной блокировке API fast alert обнаружен через 55,088 и 50,278 секунды; slow в первом случае впервые появился после конца инцидента (одна новая тревога в фазе восстановления), во втором — через 90,428 секунды. На коротком baseline из 13 проб не наблюдалось burn alerts; долгосрочная частота ложных срабатываний не измерена. При тесте восьми вложений обнаружение — 40,279 секунды после последнего принятия, после запуска worker все восемь стали ready. Опросный метод, ограничения и заметный скачок UTC-часов описаны в отчёте трека. Проведена собственная проверка runbook по Loki/Tempo; прохождение другим студентом не подтверждено. + +Проверка правил: `docker run --rm -v "$PWD:/workspace:ro" --entrypoint promtool prom/prometheus:v3.2.1 test rules /workspace/student-tests/burn_rules.test.yaml`. Синтетические проверки нормальной нагрузки, всплеска, устойчивой деградации и малого числа событий прошли. + ## Эксперименты и ограничения Docker Engine 29.8.0, `aarch64`; Docker VM сообщил 10 CPU и 10 416 644 096 байт RAM. Лимиты контейнеров указаны в Compose: например, Collector — 256 MiB, каждая API-реплика — 0,5 CPU и 512 MB. Ограниченная нагрузка была closed-loop; прогрев и статистические повторы для сравнения производительности не проводились. Числа не являются оценкой максимальной пропускной способности. В отчёте о первом опыте блокировки БД alert не успел перейти в firing; после изменения выдержки до 10 секунд повторный опыт показал firing и восстановление. -`messenger_job_ready_duration_seconds` измеряет время от создания DB-записи о вложении до фиксации ready; оно включает время до commit принятия. Измерение SLO за полное 24-часовое окно ещё не проведено. Данные queue age обновляет outbox: при его остановке gauge устаревает. RSS/CPU worker не включают отдельный render-процесс; pool waiting зависит от внутреннего API закреплённой версии asyncpg. Один Docker-host и единственные экземпляры DB, broker и LB не обеспечивают HA при их потере. Три замера RAM при отказе Tempo не доказывают поведение для произвольной длительности; переполнение очереди приводит к потере spans. При необходимости переносимой защиты следует отдельно сохранить наблюдения и подтвердить состояние alerts и трасс после нового clone. +`messenger_job_ready_duration_seconds` измеряет время от создания DB-записи о вложении до фиксации ready; оно включает время до commit принятия. Репрезентативность 24-часового окна и долгосрочная частота ложных срабатываний не проверены. Данные queue age обновляет outbox: при его остановке gauge устаревает. RSS/CPU worker не включают отдельный render-процесс; pool waiting зависит от внутреннего API закреплённой версии asyncpg. Один Docker-host и единственные экземпляры DB, broker и LB не обеспечивают HA при их потере. Три замера RAM при отказе Tempo не доказывают поведение для произвольной длительности; переполнение очереди приводит к потере spans. При необходимости переносимой защиты следует отдельно сохранить наблюдения и подтвердить состояние alerts и трасс после нового clone. diff --git a/docs/observability.md b/docs/observability.md index 9caf9a3..60a2da1 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -10,7 +10,7 @@ API, worker и outbox отправляют OTLP/HTTP в Collector. Prometheus х Dashboard использует счётчики и гистограммы с единицами секунд: p50/p95/p99 HTTP вычисляются через `histogram_quantile` по сумме buckets, не по среднему перцентилей. `messenger_job_ready_duration_seconds` измеряет от времени создания записи о вложении в DB до фиксации ready; эта оценка включает время до commit принятия и не претендует на точную границу durable acceptance. Queue depth и age — число и возраст самого старого задания в состоянии queued из PostgreSQL, включая ожидающие retry; это не длина очереди RabbitMQ. Данные обновляет outbox. DB pool отражает пул каждой API-реплики; число ожидающих читается из внутренней очереди asyncpg 0.31.0 и зависит от этой версии. Worker busy/capacity — число занятых executor threads и их лимит; повторная доставка уже завершённой задачи учитывается как успешная обработка. CPU и RSS — процессы API/worker/outbox, без дочерних процессов render. При остановленном outbox queue gauge устаревает. -SLI: доля ответов 5xx среди всех HTTP бизнес-запросов API, исключая health. Целевой SLO: не менее 99% запросов без 5xx за скользящие 24 часа; ошибки балансировщика, которые не дошли до API, SLI не учитывает. Целевое значение выбрано для учебного стенда, а не по измеренным требованиям пользователей. Alert `MessengerHighErrorRate` срабатывает при >5% за минуту в течение 10 секунд; это оперативный порог, не burn-rate alert. `MessengerQueueLag` срабатывает при возрасте queued >30 секунд в течение 15 секунд. Состояния доступны в Prometheus через Grafana Explore или локальный API Prometheus внутри контейнера. +SLI: доля ответов 5xx среди всех HTTP бизнес-запросов API, исключая health. Целевой SLO: не менее 99% запросов без 5xx за скользящие 24 часа; ошибки балансировщика, которые не дошли до API, SLI не учитывает. Целевое значение выбрано для учебного стенда, а не по измеренным требованиям пользователей. Второй SLO и расчёт burn-rate описаны в [runbook](slo-runbook.md); его DB-счётчики учитывают и невыполненные в срок задания. Alert `MessengerHighErrorRate` срабатывает при >5% за минуту в течение 10 секунд; это оперативный порог, не burn-rate alert. `MessengerQueueLag` срабатывает при возрасте queued >30 секунд в течение 15 секунд. Состояния доступны в Prometheus через Grafana Explore или локальный API Prometheus внутри контейнера. Collector использует memory limiter 192 MiB, batch до 1024 записей, очереди экспортёров по 128 batches и retry до 60 секунд; лимит контейнера 256 MiB. SDK использует ограниченные очереди spans/logs по 2048 и timeout экспорта 2 секунды. При долгом отказе хранилища возможна потеря телеметрии; это не должно блокировать бизнес-обработку. Наблюдайте внутренние метрики `otelcol_exporter_queue_size`, `otelcol_exporter_send_failed_spans_total`, `otelcol_exporter_enqueue_failed_spans_total`, `otelcol_exporter_send_failed_log_records_total` и `otelcol_receiver_refused_*`. Очереди не сохраняются после перезапуска Collector. diff --git a/docs/slo-runbook.md b/docs/slo-runbook.md new file mode 100644 index 0000000..d449ed1 --- /dev/null +++ b/docs/slo-runbook.md @@ -0,0 +1,41 @@ +# Runbook: расход бюджета SLO + +Используйте только учебный стенд с синтетическими данными. Grafana: `http://localhost:3000`, имя `admin`, пароль в `secrets/grafana_password`. На dashboard `Messenger — observability` находятся панели API/ready SLI и burn rates. В Explore выберите Prometheus; логи и traces доступны через Loki и Tempo. Правила — [alerts.yaml](../infra/prometheus/alerts.yaml). + +## Что означает сигнал + +Два учебных SLO на скользящие 24 часа: 99% бизнес-запросов API без 5xx и 99% принятых вложений становятся ready не позднее чем через 30 секунд после создания записи о задании. API исключает health и ошибки LB до попадания в приложение; 4xx остаются в знаменателе. Для готовности учтены только задания, достигшие 30-секундного возраста: неготовые, failed и завершённые позднее срока считаются плохими. Счётчики вычисляет outbox по PostgreSQL, поэтому при его недоступности данные не обновляются; события между последним scrape старого процесса и первым scrape нового могут не попасть в расчёт `increase`. Момент создания записи немного предшествует commit принятия, а время ready фиксируется внутри транзакции до её commit: это приближение к пользовательским границам. Порог 30 секунд — половина 60-секундного бюджета smoke-теста для одного файла при свободном worker; это учебная цель, не измеренное требование пользователей. Цель 99% также выбрана для демонстрации, а не выведена из пользовательского исследования. + +Error budget каждого SLO — `1 − 0,99 = 0,01`. Burn rate — доля плохих событий за окно / 0,01. Приближение расхода бюджета `burn × окно / 24 часа` предполагает стабильную интенсивность событий; при сильно меняющейся нагрузке бюджет, выраженный в доле событий за 24 часа, расходуется иначе. Значение 1 означает, что при сохранении текущей доли плохих событий все допустимые ошибки будут израсходованы за период SLO. + +- Fast: одновременно 1 минута и 20 секунд выше `28,8 × 0,01 = 28,8%`; в коротком окне не менее 5 событий. При постоянной интенсивности это `28,8 × 1 / 1440 = 0,02`, то есть 2% суточного бюджета за минуту. +- Slow: одновременно 5 минут и 1 минута выше `14,4 × 0,01 = 14,4%`; в коротком окне не менее 5 событий. При постоянной интенсивности это `14,4 × 5 / 1440 = 0,05`, то есть 5% бюджета за пять минут. + +В окне недостаточно событий — alert молчит; отсутствие ряда не является доказательством здоровья. Короткие окна и пороги выбраны для воспроизводимого лабораторного опыта, не являются рекомендацией для production. Источник метода multiwindow/multi-burn-rate и ограничений: [Google SRE Workbook](https://sre.google/workbook/alerting-on-slos/). Гистограмма: [Prometheus histograms](https://prometheus.io/docs/practices/histograms/). + +## API burn + +1. Убедитесь в наличии запросов: `sum(rate(messenger_http_requests_total{route!~"/health/.*"}[1m]))`. Отсутствие запросов требует отдельной проверки LB/readiness, а не вывода об отсутствии ошибок. +2. Сравните на dashboard 20s/1m/5m и долю 5xx по маршрутам: `sum by(route) (rate(messenger_http_requests_total{status_class="5xx"}[1m]))`. Если burn high, перейдите в Loki: `{service_name="messenger-api"} | status="503"`. Ограничьте диапазон временем alert. +3. Откройте trace из metadata лога в Tempo. Если `postgres.user_for_token` или другая DB-операция занимает большую часть HTTP span, проверьте подключение к PostgreSQL и блокировки (`pg_locks`, `pg_stat_activity`); не печатайте параметры SQL или credentials. Если вместо этого длителен S3 span — проверьте S3 и сеть. Проверьте также pool used/waiting. +4. Устраните причину в рамках полномочий: в учебном опыте дождитесь окончания контролируемой блокировки. Проверьте уменьшение error ratio, отсутствие активного burn alert и успешный `make test`. Сообщение об устранении возможно после короткого окна; долгая часть условия может ещё некоторое время оставаться выше порога. + +## Ready burn / очередь + +1. Проверьте `sum(increase(messenger_slo_ready_eligible_total{service_name="messenger-outbox"}[1m]))`, p95 ready и исходы jobs. Счётчик `messenger_slo_ready_bad_total` включает задания, нарушившие срок. Отсутствие событий не делает SLI хорошим: проверьте `messenger_queue_depth{service_name="messenger-outbox"}` и `messenger_queue_age_seconds{service_name="messenger-outbox"}`. +2. В Loki откройте `{service_name="messenger-worker"}`, найдите событие и trace ID. В Tempo проверьте цепочку upload → producer → consumer → S3/DB; посмотрите busy/capacity и состояние broker/worker через `docker compose ps`. +3. В учебном опыте запустите остановленный worker, наблюдайте уменьшение глубины/возраста и новые ready. Проверьте resolved и метаданные вложений. Не удаляйте volumes и не запускайте replay вслепую. + +## Повторение экспериментов + +`python3 student-tests/slo_incidents.py` создаёт синтетические данные, ведёт три клиента и выполняет краткую и длительную блокировку таблицы `users`. Скрипт пишет временные отметки и наблюдения в [evidence/slo-incidents.json](../evidence/slo-incidents.json), не сохраняет tokens; опрашивает alert примерно каждые 5 секунд. `python3 student-tests/slo_analyze.py` строит сводку, включая срабатывания уже после завершения воздействия. Тест намеренно ухудшает локальный API и занимает несколько минут. Первая отметка воздействия — момент, когда скрипт увидел granted `AccessExclusiveLock`; окончание — возврат команды транзакции, поэтому обе оценки имеют задержку наблюдения. Длительности считаются по монотонным часам: при выполненном опыте системное UTC скачкообразно изменилось. Аварийное прерывание клиентской команды может оставить server-side `pg_sleep` до окончания заданного ограниченного времени; проверьте состояние блокировки. + +`python3 student-tests/slo_ready_incident.py` проверяет отсутствие незавершённых заданий, останавливает worker, принимает восемь синтетических файлов и восстанавливает worker в `finally`. До запуска убедитесь, что API и worker свободны. Результат — [evidence/slo-ready-incident.json](../evidence/slo-ready-incident.json). Снятие alerts при малом количестве новых событий может означать недостаток событий в коротком окне, а не доказательство выполнения 24-часового SLO. + +Проверка правил без стенда и в ускоренном времени: + +```sh +docker run --rm -v "$PWD:/workspace:ro" --entrypoint promtool prom/prometheus:v3.2.1 test rules /workspace/student-tests/burn_rules.test.yaml +``` + +В runbook зафиксируйте фазу, UTC начала/окончания, первый firing и первое наблюдение после исчезновения alert. Первый и последний интервалы ограничены частотой опроса, а отсутствие ложного firing в одном baseline не оценивает долгосрочный false-positive rate. diff --git a/evidence/slo-analysis.json b/evidence/slo-analysis.json new file mode 100644 index 0000000..9c5d11c --- /dev/null +++ b/evidence/slo-analysis.json @@ -0,0 +1,53 @@ +{ + "sampling_interval_seconds": 5, + "monitor_errors": 0, + "phases": { + "burst": { + "observed_lock_seconds": 75.033, + "alerts": { + "MessengerApiBurnFast": { + "active_before_lock": false, + "first_firing": "2026-11-05T19:30:49.560423+00:00", + "first_firing_after_incident": false, + "detection_seconds": 55.088, + "first_clear": "2026-11-05T19:31:24.687321+00:00", + "recovery_seconds": 15.181 + }, + "MessengerApiBurnSlow": { + "active_before_lock": false, + "first_firing": "2026-11-05T19:31:14.651231+00:00", + "first_firing_after_incident": true, + "detection_seconds": 80.179, + "first_clear": "2026-11-05T19:31:24.687321+00:00", + "recovery_seconds": 15.181 + } + } + }, + "sustained": { + "observed_lock_seconds": 180.134, + "alerts": { + "MessengerApiBurnFast": { + "active_before_lock": false, + "first_firing": "2026-11-05T19:33:40.096081+00:00", + "first_firing_after_incident": false, + "detection_seconds": 50.278, + "first_clear": "2026-11-05T19:36:10.387240+00:00", + "recovery_seconds": 10.648 + }, + "MessengerApiBurnSlow": { + "active_before_lock": false, + "first_firing": "2026-11-05T19:34:20.245337+00:00", + "first_firing_after_incident": false, + "detection_seconds": 90.428, + "first_clear": "2026-11-05T19:37:53.534744+00:00", + "recovery_seconds": 30.724 + } + } + } + }, + "baseline": { + "samples": 13, + "samples_with_burn_alert": 0, + "scope": "one observed healthy baseline, not a long-term false-positive rate" + } +} diff --git a/evidence/slo-incidents.json b/evidence/slo-incidents.json new file mode 100644 index 0000000..742ed8f --- /dev/null +++ b/evidence/slo-incidents.json @@ -0,0 +1,1170 @@ +{ + "description": "controlled DB table lock; three synthetic clients; 5s alert sampling", + "periods": { + "burst": { + "lock_observed": { + "time": "2026-11-05T19:29:54.472744+00:00", + "monotonic": 43527.079238041, + "command_started": { + "time": "2026-11-05T19:29:54.163828+00:00", + "monotonic": 43526.770364958 + } + }, + "command_completed": { + "time": "2026-11-05T19:31:09.505932+00:00", + "monotonic": 43602.112587416 + } + }, + "sustained": { + "lock_observed": { + "time": "2026-11-05T19:32:49.817996+00:00", + "monotonic": 43702.424801208, + "command_started": { + "time": "2026-11-05T19:32:49.516057+00:00", + "monotonic": 43702.122908708 + } + }, + "command_completed": { + "time": "2026-11-05T19:35:59.744503+00:00", + "monotonic": 43882.558427291 + } + } + }, + "samples": [ + { + "time": "2026-11-05T19:28:54.176617+00:00", + "monotonic": 43466.783021166, + "phase": "baseline", + "counts": { + "ok": 1, + "failed": 0 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:28:59.184768+00:00", + "monotonic": 43471.791177208, + "phase": "baseline", + "counts": { + "ok": 15, + "failed": 0 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:29:04.205356+00:00", + "monotonic": 43476.8117975, + "phase": "baseline", + "counts": { + "ok": 30, + "failed": 0 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:29:09.226269+00:00", + "monotonic": 43481.832718708, + "phase": "baseline", + "counts": { + "ok": 45, + "failed": 0 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:29:14.243046+00:00", + "monotonic": 43486.849499708, + "phase": "baseline", + "counts": { + "ok": 60, + "failed": 0 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:29:19.249330+00:00", + "monotonic": 43491.855772208, + "phase": "baseline", + "counts": { + "ok": 75, + "failed": 0 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:29:24.265281+00:00", + "monotonic": 43496.871750375, + "phase": "baseline", + "counts": { + "ok": 90, + "failed": 0 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:29:29.284910+00:00", + "monotonic": 43501.891384583, + "phase": "baseline", + "counts": { + "ok": 105, + "failed": 0 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:29:34.295325+00:00", + "monotonic": 43506.901793458, + "phase": "baseline", + "counts": { + "ok": 120, + "failed": 0 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:29:39.306441+00:00", + "monotonic": 43511.91291825, + "phase": "baseline", + "counts": { + "ok": 135, + "failed": 0 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:29:44.313768+00:00", + "monotonic": 43516.920254333, + "phase": "baseline", + "counts": { + "ok": 150, + "failed": 0 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:29:49.331915+00:00", + "monotonic": 43521.938423708, + "phase": "baseline", + "counts": { + "ok": 165, + "failed": 0 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:29:54.351411+00:00", + "monotonic": 43526.957913708, + "phase": "baseline", + "counts": { + "ok": 180, + "failed": 0 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:29:59.372201+00:00", + "monotonic": 43531.978732458, + "phase": "burst", + "counts": { + "ok": 180, + "failed": 0 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:30:04.386019+00:00", + "monotonic": 43536.992552291, + "phase": "burst", + "counts": { + "ok": 180, + "failed": 3 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:30:09.401930+00:00", + "monotonic": 43542.0084625, + "phase": "burst", + "counts": { + "ok": 180, + "failed": 6 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:30:14.419029+00:00", + "monotonic": 43547.025585291, + "phase": "burst", + "counts": { + "ok": 180, + "failed": 9 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:30:19.433563+00:00", + "monotonic": 43552.040130916, + "phase": "burst", + "counts": { + "ok": 180, + "failed": 12 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:30:24.450146+00:00", + "monotonic": 43557.05672225, + "phase": "burst", + "counts": { + "ok": 180, + "failed": 15 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:30:29.473233+00:00", + "monotonic": 43562.079822583, + "phase": "burst", + "counts": { + "ok": 180, + "failed": 15 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:30:34.493955+00:00", + "monotonic": 43567.1005505, + "phase": "burst", + "counts": { + "ok": 180, + "failed": 18 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:30:39.519370+00:00", + "monotonic": 43572.125976166, + "phase": "burst", + "counts": { + "ok": 180, + "failed": 21 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:30:44.535535+00:00", + "monotonic": 43577.142128416, + "phase": "burst", + "counts": { + "ok": 180, + "failed": 24 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:30:49.560423+00:00", + "monotonic": 43582.167042875, + "phase": "burst", + "counts": { + "ok": 180, + "failed": 27 + }, + "firing": [ + "MessengerApiBurnFast" + ] + }, + { + "time": "2026-11-05T19:30:54.583572+00:00", + "monotonic": 43587.190215125, + "phase": "burst", + "counts": { + "ok": 180, + "failed": 30 + }, + "firing": [ + "MessengerApiBurnFast" + ] + }, + { + "time": "2026-11-05T19:30:59.600250+00:00", + "monotonic": 43592.206889708, + "phase": "burst", + "counts": { + "ok": 180, + "failed": 30 + }, + "firing": [ + "MessengerApiBurnFast" + ] + }, + { + "time": "2026-11-05T19:31:04.616531+00:00", + "monotonic": 43597.223167708, + "phase": "burst", + "counts": { + "ok": 180, + "failed": 33 + }, + "firing": [ + "MessengerApiBurnFast" + ] + }, + { + "time": "2026-11-05T19:31:09.632643+00:00", + "monotonic": 43602.239279708, + "phase": "burst_recovery", + "counts": { + "ok": 183, + "failed": 36 + }, + "firing": [ + "MessengerApiBurnFast" + ] + }, + { + "time": "2026-11-05T19:31:14.651231+00:00", + "monotonic": 43607.257891625, + "phase": "burst_recovery", + "counts": { + "ok": 198, + "failed": 36 + }, + "firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow" + ] + }, + { + "time": "2026-11-05T19:31:19.663273+00:00", + "monotonic": 43612.269926416, + "phase": "burst_recovery", + "counts": { + "ok": 211, + "failed": 36 + }, + "firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow" + ] + }, + { + "time": "2026-11-05T19:31:24.687321+00:00", + "monotonic": 43617.294002291, + "phase": "burst_recovery", + "counts": { + "ok": 225, + "failed": 36 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:31:29.708468+00:00", + "monotonic": 43622.315158041, + "phase": "burst_recovery", + "counts": { + "ok": 240, + "failed": 36 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:31:34.728166+00:00", + "monotonic": 43627.334856708, + "phase": "burst_recovery", + "counts": { + "ok": 255, + "failed": 36 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:31:39.733767+00:00", + "monotonic": 43632.340459958, + "phase": "burst_recovery", + "counts": { + "ok": 270, + "failed": 36 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:31:44.755480+00:00", + "monotonic": 43637.36218675, + "phase": "burst_recovery", + "counts": { + "ok": 285, + "failed": 36 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:31:49.769677+00:00", + "monotonic": 43642.376390333, + "phase": "burst_recovery", + "counts": { + "ok": 300, + "failed": 36 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:31:54.778133+00:00", + "monotonic": 43647.384860875, + "phase": "burst_recovery", + "counts": { + "ok": 315, + "failed": 36 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:31:59.788234+00:00", + "monotonic": 43652.394963833, + "phase": "burst_recovery", + "counts": { + "ok": 330, + "failed": 36 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:32:04.797259+00:00", + "monotonic": 43657.403998, + "phase": "burst_recovery", + "counts": { + "ok": 345, + "failed": 36 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:32:09.813581+00:00", + "monotonic": 43662.42033325, + "phase": "burst_recovery", + "counts": { + "ok": 360, + "failed": 36 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:32:14.823961+00:00", + "monotonic": 43667.430715458, + "phase": "burst_recovery", + "counts": { + "ok": 375, + "failed": 36 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:32:19.834831+00:00", + "monotonic": 43672.441596833, + "phase": "burst_recovery", + "counts": { + "ok": 390, + "failed": 36 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:32:24.847866+00:00", + "monotonic": 43677.454636958, + "phase": "burst_recovery", + "counts": { + "ok": 405, + "failed": 36 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:32:29.862747+00:00", + "monotonic": 43682.469528375, + "phase": "burst_recovery", + "counts": { + "ok": 417, + "failed": 36 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:32:34.870564+00:00", + "monotonic": 43687.47735275, + "phase": "burst_recovery", + "counts": { + "ok": 432, + "failed": 36 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:32:39.880308+00:00", + "monotonic": 43692.487106, + "phase": "burst_recovery", + "counts": { + "ok": 447, + "failed": 36 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:32:44.899942+00:00", + "monotonic": 43697.506751, + "phase": "burst_recovery", + "counts": { + "ok": 462, + "failed": 36 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:32:49.916853+00:00", + "monotonic": 43702.523669458, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 36 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:32:54.942437+00:00", + "monotonic": 43707.549288958, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 36 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:32:59.957394+00:00", + "monotonic": 43712.564228083, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 39 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:33:04.970687+00:00", + "monotonic": 43717.577532333, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 42 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:33:09.993536+00:00", + "monotonic": 43722.600420875, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 45 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:33:15.009372+00:00", + "monotonic": 43727.616250583, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 48 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:33:20.031828+00:00", + "monotonic": 43732.638718, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 51 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:33:25.051997+00:00", + "monotonic": 43737.658880125, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 51 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:33:30.067799+00:00", + "monotonic": 43742.674685791, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 54 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:33:35.075453+00:00", + "monotonic": 43747.682349166, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 57 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:33:40.096081+00:00", + "monotonic": 43752.703001416, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 60 + }, + "firing": [ + "MessengerApiBurnFast" + ] + }, + { + "time": "2026-11-05T19:33:45.119089+00:00", + "monotonic": 43757.7260235, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 63 + }, + "firing": [ + "MessengerApiBurnFast" + ] + }, + { + "time": "2026-11-05T19:33:50.133709+00:00", + "monotonic": 43762.740644916, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 66 + }, + "firing": [ + "MessengerApiBurnFast" + ] + }, + { + "time": "2026-11-05T19:33:55.161233+00:00", + "monotonic": 43767.768187291, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 66 + }, + "firing": [ + "MessengerApiBurnFast" + ] + }, + { + "time": "2026-11-05T19:34:00.176785+00:00", + "monotonic": 43772.783745208, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 69 + }, + "firing": [ + "MessengerApiBurnFast" + ] + }, + { + "time": "2026-11-05T19:34:05.196858+00:00", + "monotonic": 43777.803812291, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 72 + }, + "firing": [ + "MessengerApiBurnFast" + ] + }, + { + "time": "2026-11-05T19:34:10.212935+00:00", + "monotonic": 43782.819901041, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 75 + }, + "firing": [ + "MessengerApiBurnFast" + ] + }, + { + "time": "2026-11-05T19:34:15.231470+00:00", + "monotonic": 43787.838454541, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 78 + }, + "firing": [ + "MessengerApiBurnFast" + ] + }, + { + "time": "2026-11-05T19:34:20.245337+00:00", + "monotonic": 43792.852329875, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 81 + }, + "firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow" + ] + }, + { + "time": "2026-11-05T19:34:25.257984+00:00", + "monotonic": 43797.864980958, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 81 + }, + "firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow" + ] + }, + { + "time": "2026-11-05T19:34:30.282024+00:00", + "monotonic": 43802.889036875, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 84 + }, + "firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow" + ] + }, + { + "time": "2026-11-05T19:34:35.303328+00:00", + "monotonic": 43807.910348833, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 87 + }, + "firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow" + ] + }, + { + "time": "2026-11-05T19:34:40.328342+00:00", + "monotonic": 43812.935370625, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 90 + }, + "firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow" + ] + }, + { + "time": "2026-11-05T19:34:45.343604+00:00", + "monotonic": 43817.950633291, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 93 + }, + "firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow" + ] + }, + { + "time": "2026-11-05T19:34:50.359471+00:00", + "monotonic": 43822.966518166, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 96 + }, + "firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow" + ] + }, + { + "time": "2026-11-05T19:34:55.367578+00:00", + "monotonic": 43827.974616416, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 96 + }, + "firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow" + ] + }, + { + "time": "2026-11-05T19:35:00.386215+00:00", + "monotonic": 43832.993284208, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 99 + }, + "firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow" + ] + }, + { + "time": "2026-11-05T19:35:05.400579+00:00", + "monotonic": 43838.0076475, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 102 + }, + "firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow" + ] + }, + { + "time": "2026-11-05T19:35:10.422157+00:00", + "monotonic": 43843.0292365, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 105 + }, + "firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow" + ] + }, + { + "time": "2026-11-05T19:35:15.442075+00:00", + "monotonic": 43848.049173416, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 108 + }, + "firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow" + ] + }, + { + "time": "2026-11-05T19:35:20.452042+00:00", + "monotonic": 43853.05914025, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 111 + }, + "firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow" + ] + }, + { + "time": "2026-11-05T19:35:25.461182+00:00", + "monotonic": 43858.068280916, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 111 + }, + "firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow" + ] + }, + { + "time": "2026-11-05T19:35:30.480197+00:00", + "monotonic": 43863.087333625, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 114 + }, + "firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow" + ] + }, + { + "time": "2026-11-05T19:35:35.504075+00:00", + "monotonic": 43868.111188291, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 117 + }, + "firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow" + ] + }, + { + "time": "2026-11-05T19:35:40.521486+00:00", + "monotonic": 43873.129961791, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 120 + }, + "firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow" + ] + }, + { + "time": "2026-11-05T19:35:55.348285+00:00", + "monotonic": 43878.158763833, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 123 + }, + "firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow" + ] + }, + { + "time": "2026-11-05T19:36:00.359177+00:00", + "monotonic": 43883.173501041, + "phase": "sustained_recovery", + "counts": { + "ok": 480, + "failed": 123 + }, + "firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow" + ] + }, + { + "time": "2026-11-05T19:36:05.367876+00:00", + "monotonic": 43888.184996, + "phase": "sustained_recovery", + "counts": { + "ok": 495, + "failed": 123 + }, + "firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow" + ] + }, + { + "time": "2026-11-05T19:36:10.387240+00:00", + "monotonic": 43893.206441208, + "phase": "sustained_recovery", + "counts": { + "ok": 510, + "failed": 123 + }, + "firing": [ + "MessengerApiBurnSlow" + ] + }, + { + "time": "2026-11-05T19:36:15.397936+00:00", + "monotonic": 43898.218563625, + "phase": "sustained_recovery", + "counts": { + "ok": 525, + "failed": 123 + }, + "firing": [ + "MessengerApiBurnSlow" + ] + }, + { + "time": "2026-11-05T19:36:20.421727+00:00", + "monotonic": 43903.243449875, + "phase": "sustained_recovery", + "counts": { + "ok": 540, + "failed": 123 + }, + "firing": [ + "MessengerApiBurnSlow" + ] + }, + { + "time": "2026-11-05T19:37:48.510428+00:00", + "monotonic": 43908.257572375, + "phase": "sustained_recovery", + "counts": { + "ok": 555, + "failed": 123 + }, + "firing": [ + "MessengerApiBurnSlow" + ] + }, + { + "time": "2026-11-05T19:37:53.534744+00:00", + "monotonic": 43913.282484708, + "phase": "sustained_recovery", + "counts": { + "ok": 570, + "failed": 123 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:37:58.549296+00:00", + "monotonic": 43918.29743975, + "phase": "sustained_recovery", + "counts": { + "ok": 585, + "failed": 123 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:38:03.567430+00:00", + "monotonic": 43923.315891, + "phase": "sustained_recovery", + "counts": { + "ok": 600, + "failed": 123 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:38:08.593041+00:00", + "monotonic": 43928.341731291, + "phase": "sustained_recovery", + "counts": { + "ok": 615, + "failed": 123 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:38:13.607305+00:00", + "monotonic": 43933.356177, + "phase": "sustained_recovery", + "counts": { + "ok": 630, + "failed": 123 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:38:18.617266+00:00", + "monotonic": 43938.366246166, + "phase": "sustained_recovery", + "counts": { + "ok": 642, + "failed": 123 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:38:23.636630+00:00", + "monotonic": 43943.385724875, + "phase": "sustained_recovery", + "counts": { + "ok": 657, + "failed": 123 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:38:28.654169+00:00", + "monotonic": 43948.403359125, + "phase": "sustained_recovery", + "counts": { + "ok": 672, + "failed": 123 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:38:33.670193+00:00", + "monotonic": 43953.419418, + "phase": "sustained_recovery", + "counts": { + "ok": 687, + "failed": 123 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:38:38.676034+00:00", + "monotonic": 43958.425318541, + "phase": "sustained_recovery", + "counts": { + "ok": 702, + "failed": 123 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:38:43.699279+00:00", + "monotonic": 43963.448615416, + "phase": "sustained_recovery", + "counts": { + "ok": 717, + "failed": 123 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:38:48.718747+00:00", + "monotonic": 43968.468122083, + "phase": "sustained_recovery", + "counts": { + "ok": 732, + "failed": 123 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:38:53.742289+00:00", + "monotonic": 43973.491697375, + "phase": "sustained_recovery", + "counts": { + "ok": 747, + "failed": 123 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:38:58.757203+00:00", + "monotonic": 43978.50662725, + "phase": "sustained_recovery", + "counts": { + "ok": 762, + "failed": 123 + }, + "firing": [] + } + ], + "end_time": "2026-11-05T19:39:02.820853+00:00", + "final_counts": { + "ok": 774, + "failed": 123 + } +} diff --git a/evidence/slo-ready-incident.json b/evidence/slo-ready-incident.json new file mode 100644 index 0000000..5eff503 --- /dev/null +++ b/evidence/slo-ready-incident.json @@ -0,0 +1,211 @@ +{ + "samples": [ + { + "time": "2026-11-06T06:18:43.857552+00:00", + "monotonic": 49295.73000975, + "phase": "baseline", + "firing": [], + "eligible": 9.0, + "bad": 4.0 + }, + { + "time": "2026-11-06T06:18:49.195942+00:00", + "monotonic": 49301.068345333, + "phase": "worker_stopped", + "firing": [], + "eligible": 9.0, + "bad": 4.0 + }, + { + "time": "2026-11-06T06:18:54.211236+00:00", + "monotonic": 49306.083585125, + "phase": "worker_stopped", + "firing": [], + "eligible": 9.0, + "bad": 4.0 + }, + { + "time": "2026-11-06T06:18:59.238528+00:00", + "monotonic": 49311.110847333, + "phase": "worker_stopped", + "firing": [], + "eligible": 9.0, + "bad": 4.0 + }, + { + "time": "2026-11-06T06:19:04.276851+00:00", + "monotonic": 49316.149107833, + "phase": "worker_stopped", + "firing": [], + "eligible": 9.0, + "bad": 4.0 + }, + { + "time": "2026-11-06T06:19:09.316299+00:00", + "monotonic": 49321.18853025, + "phase": "worker_stopped", + "firing": [], + "eligible": 9.0, + "bad": 4.0 + }, + { + "time": "2026-11-06T06:19:14.361014+00:00", + "monotonic": 49326.233190833, + "phase": "worker_stopped", + "firing": [], + "eligible": 9.0, + "bad": 4.0 + }, + { + "time": "2026-11-06T06:19:19.399292+00:00", + "monotonic": 49331.271419, + "phase": "worker_stopped", + "firing": [], + "eligible": 9.0, + "bad": 4.0 + }, + { + "time": "2026-11-06T06:19:24.439389+00:00", + "monotonic": 49336.31146675, + "phase": "worker_stopped", + "firing": [], + "eligible": 9.0, + "bad": 4.0 + }, + { + "time": "2026-11-06T06:19:29.469853+00:00", + "monotonic": 49341.341864875, + "phase": "worker_stopped", + "firing": [ + "MessengerReadyBurnFast", + "MessengerReadyBurnSlow" + ], + "eligible": 17.0, + "bad": 12.0 + }, + { + "time": "2026-11-06T06:19:35.732760+00:00", + "monotonic": 49347.604713666, + "phase": "recovery", + "firing": [ + "MessengerReadyBurnFast", + "MessengerReadyBurnSlow" + ], + "eligible": 17.0, + "bad": 12.0 + }, + { + "time": "2026-11-06T06:19:40.766904+00:00", + "monotonic": 49352.638829583, + "phase": "recovery", + "firing": [ + "MessengerReadyBurnFast", + "MessengerReadyBurnSlow" + ], + "eligible": 17.0, + "bad": 12.0 + }, + { + "time": "2026-11-06T06:19:45.800827+00:00", + "monotonic": 49357.672704333, + "phase": "recovery", + "firing": [ + "MessengerReadyBurnSlow" + ], + "eligible": 17.0, + "bad": 12.0 + }, + { + "time": "2026-11-06T06:19:50.839444+00:00", + "monotonic": 49362.711278166, + "phase": "recovery", + "firing": [ + "MessengerReadyBurnSlow" + ], + "eligible": 17.0, + "bad": 12.0 + }, + { + "time": "2026-11-06T06:19:55.882108+00:00", + "monotonic": 49367.753864791, + "phase": "recovery", + "firing": [ + "MessengerReadyBurnSlow" + ], + "eligible": 17.0, + "bad": 12.0 + }, + { + "time": "2026-11-06T06:20:00.935125+00:00", + "monotonic": 49372.806871, + "phase": "recovery", + "firing": [ + "MessengerReadyBurnSlow" + ], + "eligible": 17.0, + "bad": 12.0 + }, + { + "time": "2026-11-06T06:20:05.976123+00:00", + "monotonic": 49377.847798625, + "phase": "recovery", + "firing": [ + "MessengerReadyBurnSlow" + ], + "eligible": 17.0, + "bad": 12.0 + }, + { + "time": "2026-11-06T06:20:11.006201+00:00", + "monotonic": 49382.877811166, + "phase": "recovery", + "firing": [ + "MessengerReadyBurnSlow" + ], + "eligible": 17.0, + "bad": 12.0 + }, + { + "time": "2026-11-06T06:20:16.034665+00:00", + "monotonic": 49387.906272875, + "phase": "recovery", + "firing": [ + "MessengerReadyBurnSlow" + ], + "eligible": 17.0, + "bad": 12.0 + }, + { + "time": "2026-11-06T06:20:21.073601+00:00", + "monotonic": 49392.945128916, + "phase": "recovery", + "firing": [ + "MessengerReadyBurnSlow" + ], + "eligible": 17.0, + "bad": 12.0 + }, + { + "time": "2026-11-06T06:20:26.112334+00:00", + "monotonic": 49397.983810916, + "phase": "recovery", + "firing": [], + "eligible": 17.0, + "bad": 12.0 + } + ], + "jobs": 8, + "accepted_at": { + "time": "2026-11-06T06:18:49.190891+00:00", + "monotonic": 49301.063291333 + }, + "worker_started": { + "time": "2026-11-06T06:19:31.951591+00:00", + "monotonic": 49343.823643916 + }, + "all_ready": { + "time": "2026-11-06T06:19:35.725614+00:00", + "monotonic": 49347.597564333 + }, + "result": "PASS" +} diff --git a/evidence/slo-track.md b/evidence/slo-track.md new file mode 100644 index 0000000..a24838f --- /dev/null +++ b/evidence/slo-track.md @@ -0,0 +1,34 @@ +# Трек 2: SLO и обнаружение инцидентов + +## Гипотеза и методика + +Гипотеза: при контролируемой деградации многооконные burn-rate alerts обнаружат ускоренный расход бюджета; короткое окно позволит снять сигнал после прекращения воздействия. Это гипотеза об учебной конфигурации, а не утверждение о любой нагрузке. + +Цели, формулы, ограничения SLI и команды диагностики: [runbook](../docs/slo-runbook.md). Оба SLO имеют целевое значение 99% на скользящем окне 24 часа. Пары окон и пороги намеренно сокращены для локального испытания: fast — 20 секунд и 1 минута при burn >28,8; slow — 1 и 5 минут при burn >14,4. В коротком окне нужно не менее 5 событий. Правила и панели версионируются в Git. + +Испытания на существующем локальном стенде Docker Compose. API: три клиентских потока создавали синтетические сообщения, каждый делал паузу 1 секунду между завершившимися запросами; до воздействия — 60 секунд baseline. На время воздействия таблица `users` блокировалась транзакцией на 75 и 180 секунд, между опытами было 100 секунд восстановления. Это воспроизводимая искусственная блокировка, а не реальный пользовательский трафик. Каждые примерно 5 секунд сценарий запрашивал firing alerts через Grafana/Prometheus и сохранял монотонное время. Начало — момент, когда скрипт увидел lock, конец — возврат транзакции. Эти точки и опрос имеют собственную задержку; полученные длительности — наблюдательные оценки, не точные времена переключения Prometheus. + +Исходные агрегаты и временные отметки: [slo-incidents.json](slo-incidents.json). Пересчёт: `python3 student-tests/slo_analyze.py` → [slo-analysis.json](slo-analysis.json). Ошибок опроса не зафиксировано. В ходе длительного опыта наблюдался скачок системных часов UTC: разность некоторых соседних UTC-отметок не равна разности монотонных отметок. Длительности ниже рассчитаны по `monotonic`; скачок также мог повлиять на временные окна Prometheus, поэтому эксперимент следует повторить на системе со стабильными часами. + +## Результаты API + +| Воздействие | Наблюдаемая длительность блокировки | Fast: первое firing / снятие после конца | Slow: первое firing / снятие после конца | +| --- | ---: | ---: | ---: | +| Короткое | 75,033 с | 55,088 с / 15,181 с | 80,179 с / 15,181 с | +| Длительное | 180,134 с | 50,278 с / 10,648 с | 90,428 с / 30,724 с | + +В коротком сценарии slow впервые появился уже через 5,145 с после окончания воздействия и исчез вместе с fast. По строгой классификации «блокировка уже завершилась — система восстановлена» зафиксирована одна новая тревога вне воздействия; по смыслу окна это запоздалый сигнал о предыдущем расходе бюджета. При длительном сценарии оба сигнала впервые появились во время блокировки. За один baseline получено 13 проб опроса и 0 проб с burn alert: наблюдаемых ложных срабатываний на этом коротком интервале нет, но данных недостаточно для оценки долгосрочной частоты ложных тревог. + +Разность клиентских накопленных счётчиков между первой и последней пробами фазы: baseline 179 успешных / 0 ошибок; короткая блокировка 0 / 33; длительная 0 / 87. Они не включают полные пограничные интервалы и могут включать timeout; это не числитель HTTP 5xx SLI. Срабатывания рассчитаны Prometheus по серверным метрикам. В Loki найден синтетический HTTP 503 с trace `47caf6293f35aabf09f71f307ed860a3`, где span `postgres.user_for_token` занял 5,004 с: это проверка пути диагностики в runbook. Проверка выполнена автором сценария; прохождение другим студентом пока не зафиксировано. + +## Проверка SLO вложений + +`python3 student-tests/slo_ready_incident.py` убедился, что незавершённых задач нет, остановил worker, принял восемь синтетических вложений и дождался alert. Затем worker запущен, все вложения достигли ready. Исходные измерения: [slo-ready-incident.json](slo-ready-incident.json). + +До воздействия счётчики matured/плохих заданий были 9/4. В первой пробе firing — 17/12: прирост обоих счётчиков равен 8, то есть все восемь заданий попали в знаменатель и нарушили 30-секундный срок. Fast и slow стали firing через 40,279 с после последнего принятия. Все задания были ready через 3,774 с после команды восстановления worker. Fast исчез через 13,849 с, slow — через 54,160 с после команды запуска. Значения опросные, включая задержку запуска Docker и опроса. При снятии slow новых заданий не поступало: условие минимального числа событий перестало выполняться. Это не доказывает, что 24-часовой SLO снова выполнен; факт просрочки остаётся в счётчиках. + +## Дополнительная проверка правил и ограничения + +`promtool test rules` на [burn_rules.test.yaml](../student-tests/burn_rules.test.yaml) проверил нормальную нагрузку без burn alerts, тяжёлый краткий всплеск с последующим снятием fast, устойчивую 20%-ную долю ошибок с firing slow без fast, а также поведение при малом числе событий и fast alert просроченных вложений. Это синтетические ряды, не замена эксперименту с реальными данными. + +Границы: API SLI не включает ошибки LB до приложения. Счётчики готовности вычисляются из состояний PostgreSQL раз в цикл outbox; при недоступном outbox данные устаревают, а события во время разрыва наблюдения могут не войти в `increase` после перезапуска процесса. Начало и конец срока фиксируются внутри DB-транзакций, поэтому приближённо описывают пользовательские границы durable acceptance и ready. Возможные manual replay/редактирование БД, смена или удаление заданий могут нарушить монотонность DB-счётчиков; учебный API не удаляет задания. Нет оценки ложных срабатываний за длительный период или проверки runbook другим студентом. Ошибки, произошедшие до восстановления, учитываются в бюджетном окне до истечения окна и могут вызвать задержанный alert. diff --git a/infra/grafana/dashboards/messenger.json b/infra/grafana/dashboards/messenger.json index 05cb19a..57e8548 100644 --- a/infra/grafana/dashboards/messenger.json +++ b/infra/grafana/dashboards/messenger.json @@ -60,7 +60,7 @@ "targets": [ { "refId": "A", - "expr": "(sum(rate(messenger_http_requests_total{status_class=\"5xx\",route!~\"/health/.*\"}[1m])) or vector(0)) / clamp_min(sum(rate(messenger_http_requests_total{route!~\"/health/.*\"}[1m])),0.001)", + "expr": "(sum(rate(messenger_http_requests_total{status_class=\"5xx\",route!~\"/health/.*\"}[1m])) or vector(0)) / clamp_min(sum(rate(messenger_http_requests_total{route!~\"/health/.*\"}[1m])),0.001) and sum(increase(messenger_http_requests_total{route!~\"/health/.*\"}[1m])) > 0", "legendFormat": "{{service_name}} {{outcome}}" } ] @@ -428,10 +428,116 @@ "targets": [ { "refId": "A", - "expr": "1 - (sum(increase(messenger_http_requests_total{status_class=\"5xx\",route!~\"/health/.*\"}[24h])) or vector(0)) / clamp_min(sum(increase(messenger_http_requests_total{route!~\"/health/.*\"}[24h])),1)", + "expr": "1 - (sum(increase(messenger_http_requests_total{status_class=\"5xx\",route!~\"/health/.*\"}[24h])) or vector(0)) / clamp_min(sum(increase(messenger_http_requests_total{route!~\"/health/.*\"}[24h])),1) and sum(increase(messenger_http_requests_total{route!~\"/health/.*\"}[24h])) > 0", "legendFormat": "SLI" } ] + }, + { + "id": 15, + "title": "API SLO burn rate (budget 1%)", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 56 + }, + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "expr": "(((sum(increase(messenger_http_requests_total{status_class=\"5xx\",route!~\"/health/.*\"}[20s])) or vector(0)) / clamp_min(sum(increase(messenger_http_requests_total{route!~\"/health/.*\"}[20s])), 1)) / 0.01) and sum(increase(messenger_http_requests_total{route!~\"/health/.*\"}[20s])) > 0", + "legendFormat": "burn 20s" + }, + { + "refId": "B", + "expr": "(((sum(increase(messenger_http_requests_total{status_class=\"5xx\",route!~\"/health/.*\"}[1m])) or vector(0)) / clamp_min(sum(increase(messenger_http_requests_total{route!~\"/health/.*\"}[1m])), 1)) / 0.01) and sum(increase(messenger_http_requests_total{route!~\"/health/.*\"}[1m])) > 0", + "legendFormat": "burn 1m" + }, + { + "refId": "C", + "expr": "(((sum(increase(messenger_http_requests_total{status_class=\"5xx\",route!~\"/health/.*\"}[5m])) or vector(0)) / clamp_min(sum(increase(messenger_http_requests_total{route!~\"/health/.*\"}[5m])), 1)) / 0.01) and sum(increase(messenger_http_requests_total{route!~\"/health/.*\"}[5m])) > 0", + "legendFormat": "burn 5m" + } + ] + }, + { + "id": 16, + "title": "READY SLO burn rate (budget 1%)", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 56 + }, + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "expr": "(((sum(increase(messenger_slo_ready_bad_total{service_name=\"messenger-outbox\"}[20s])) or vector(0)) / clamp_min(sum(increase(messenger_slo_ready_eligible_total{service_name=\"messenger-outbox\"}[20s])), 1)) / 0.01) and sum(increase(messenger_slo_ready_eligible_total{service_name=\"messenger-outbox\"}[20s])) > 0", + "legendFormat": "burn 20s" + }, + { + "refId": "B", + "expr": "(((sum(increase(messenger_slo_ready_bad_total{service_name=\"messenger-outbox\"}[1m])) or vector(0)) / clamp_min(sum(increase(messenger_slo_ready_eligible_total{service_name=\"messenger-outbox\"}[1m])), 1)) / 0.01) and sum(increase(messenger_slo_ready_eligible_total{service_name=\"messenger-outbox\"}[1m])) > 0", + "legendFormat": "burn 1m" + }, + { + "refId": "C", + "expr": "(((sum(increase(messenger_slo_ready_bad_total{service_name=\"messenger-outbox\"}[5m])) or vector(0)) / clamp_min(sum(increase(messenger_slo_ready_eligible_total{service_name=\"messenger-outbox\"}[5m])), 1)) / 0.01) and sum(increase(messenger_slo_ready_eligible_total{service_name=\"messenger-outbox\"}[5m])) > 0", + "legendFormat": "burn 5m" + } + ] + }, + { + "id": 17, + "title": "Accepted jobs ready within 30 s — 24h (matured cohort)", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 64 + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "min": 0, + "max": 1 + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "expr": "1 - ((sum(increase(messenger_slo_ready_bad_total{service_name=\"messenger-outbox\"}[24h])) or vector(0)) / clamp_min(sum(increase(messenger_slo_ready_eligible_total{service_name=\"messenger-outbox\"}[24h])), 1)) and sum(increase(messenger_slo_ready_eligible_total{service_name=\"messenger-outbox\"}[24h])) > 0", + "legendFormat": "ready SLI" + } + ] } ] } diff --git a/infra/prometheus/alerts.yaml b/infra/prometheus/alerts.yaml index 3bcc613..d7a700b 100644 --- a/infra/prometheus/alerts.yaml +++ b/infra/prometheus/alerts.yaml @@ -11,3 +11,19 @@ groups: for: 15s labels: {severity: warning} annotations: {summary: 'Oldest queued job exceeds 30 seconds', runbook: 'Check worker health, busy/capacity, broker and consumer traces.'} + - alert: MessengerApiBurnFast + expr: '((sum(increase(messenger_http_requests_total{status_class="5xx",route!~"/health/.*"}[1m])) or vector(0)) / clamp_min(sum(increase(messenger_http_requests_total{route!~"/health/.*"}[1m])), 1)) > 28.8 * 0.01 and ((sum(increase(messenger_http_requests_total{status_class="5xx",route!~"/health/.*"}[20s])) or vector(0)) / clamp_min(sum(increase(messenger_http_requests_total{route!~"/health/.*"}[20s])), 1)) > 28.8 * 0.01 and sum(increase(messenger_http_requests_total{route!~"/health/.*"}[20s])) >= 5' + labels: {severity: critical, slo: api, burn: fast} + annotations: {summary: 'api SLO fast burn', runbook: 'See docs/slo-runbook.md'} + - alert: MessengerApiBurnSlow + expr: '((sum(increase(messenger_http_requests_total{status_class="5xx",route!~"/health/.*"}[5m])) or vector(0)) / clamp_min(sum(increase(messenger_http_requests_total{route!~"/health/.*"}[5m])), 1)) > 14.4 * 0.01 and ((sum(increase(messenger_http_requests_total{status_class="5xx",route!~"/health/.*"}[1m])) or vector(0)) / clamp_min(sum(increase(messenger_http_requests_total{route!~"/health/.*"}[1m])), 1)) > 14.4 * 0.01 and sum(increase(messenger_http_requests_total{route!~"/health/.*"}[1m])) >= 5' + labels: {severity: warning, slo: api, burn: slow} + annotations: {summary: 'api SLO slow burn', runbook: 'See docs/slo-runbook.md'} + - alert: MessengerReadyBurnFast + expr: '((sum(increase(messenger_slo_ready_bad_total{service_name="messenger-outbox"}[1m])) or vector(0)) / clamp_min(sum(increase(messenger_slo_ready_eligible_total{service_name="messenger-outbox"}[1m])), 1)) > 28.8 * 0.01 and ((sum(increase(messenger_slo_ready_bad_total{service_name="messenger-outbox"}[20s])) or vector(0)) / clamp_min(sum(increase(messenger_slo_ready_eligible_total{service_name="messenger-outbox"}[20s])), 1)) > 28.8 * 0.01 and sum(increase(messenger_slo_ready_eligible_total{service_name="messenger-outbox"}[20s])) >= 5' + labels: {severity: critical, slo: ready, burn: fast} + annotations: {summary: 'ready SLO fast burn', runbook: 'See docs/slo-runbook.md'} + - alert: MessengerReadyBurnSlow + expr: '((sum(increase(messenger_slo_ready_bad_total{service_name="messenger-outbox"}[5m])) or vector(0)) / clamp_min(sum(increase(messenger_slo_ready_eligible_total{service_name="messenger-outbox"}[5m])), 1)) > 14.4 * 0.01 and ((sum(increase(messenger_slo_ready_bad_total{service_name="messenger-outbox"}[1m])) or vector(0)) / clamp_min(sum(increase(messenger_slo_ready_eligible_total{service_name="messenger-outbox"}[1m])), 1)) > 14.4 * 0.01 and sum(increase(messenger_slo_ready_eligible_total{service_name="messenger-outbox"}[1m])) >= 5' + labels: {severity: warning, slo: ready, burn: slow} + annotations: {summary: 'ready SLO slow burn', runbook: 'See docs/slo-runbook.md'} diff --git a/src/outbox.py b/src/outbox.py index 6e8e3d0..79522aa 100644 --- a/src/outbox.py +++ b/src/outbox.py @@ -22,12 +22,22 @@ async def run() -> None: while True: try: await store.schedule_due_jobs(int(os.environ.get('JOB_MAX_ATTEMPTS', '5'))) - queue = await store.pool.fetchrow("SELECT count(*) AS depth, COALESCE(EXTRACT(EPOCH FROM now()-min(created_at)), 0)::float8 AS age FROM attachment_jobs WHERE status='queued'") - tel.state.update(queue_depth=queue['depth'], queue_age=queue['age']) for event in await store.due_outbox_events(): context = event['trace_context'] await publish(event["payload"], json.loads(context) if isinstance(context, str) else context) await store.mark_outbox_published(event["id"]) + try: + queue = await store.pool.fetchrow("SELECT count(*) AS depth, COALESCE(EXTRACT(EPOCH FROM now()-min(created_at)), 0)::float8 AS age FROM attachment_jobs WHERE status='queued'") + slo = await store.pool.fetchrow(""" + SELECT count(*) FILTER (WHERE created_at <= now() - interval '30 seconds') AS eligible, + count(*) FILTER (WHERE created_at <= now() - interval '30 seconds' + AND (status <> 'ready' OR updated_at > created_at + interval '30 seconds')) AS bad + FROM attachment_jobs + """) + tel.state.update(queue_depth=queue['depth'], queue_age=queue['age'], + slo_ready_eligible=slo['eligible'], slo_ready_bad=slo['bad']) + except Exception: + logging.warning("queue telemetry unavailable") except Exception: logging.exception("outbox dispatch failed; durable event remains pending") await asyncio.sleep(interval) diff --git a/src/telemetry.py b/src/telemetry.py index 01ca0d4..c8c2969 100644 --- a/src/telemetry.py +++ b/src/telemetry.py @@ -21,9 +21,12 @@ job_duration = meter.create_histogram("messenger.job.duration", unit="s") ready_duration = meter.create_histogram("messenger.job.ready.duration", unit="s") busy = meter.create_up_down_counter("messenger.worker.busy", unit="{worker}") capacity = meter.create_up_down_counter("messenger.worker.capacity", unit="{worker}") -state = {"queue_depth": 0, "queue_age": 0, "pool_used": 0, "pool_capacity": 0, "pool_waiting": 0} +state = {"queue_depth": 0, "queue_age": 0, "pool_used": 0, "pool_capacity": 0, "pool_waiting": 0, + "slo_ready_eligible": 0, "slo_ready_bad": 0} for name, unit in (("queue_depth", "{job}"), ("queue_age", "s"), ("pool_used", "{connection}"), ("pool_capacity", "{connection}"), ("pool_waiting", "{request}")): meter.create_observable_gauge("messenger." + name, callbacks=[lambda options, key=name: [metrics.Observation(state[key])]], unit=unit) +for name in ("slo_ready_eligible", "slo_ready_bad"): + meter.create_observable_counter("messenger." + name, callbacks=[lambda options, key=name: [metrics.Observation(state[key])]], unit="{job}") def process_cpu(options): usage = resource.getrusage(resource.RUSAGE_SELF) diff --git a/student-tests/burn_rules.test.yaml b/student-tests/burn_rules.test.yaml new file mode 100644 index 0000000..7b25538 --- /dev/null +++ b/student-tests/burn_rules.test.yaml @@ -0,0 +1,83 @@ +rule_files: + - /workspace/infra/prometheus/alerts.yaml +evaluation_interval: 5s +tests: + - name: healthy + interval: 5s + input_series: + - series: 'messenger_http_requests_total{route="/api/v1/users",status_class="2xx"}' + values: '0+100x80' + - series: 'messenger_http_requests_total{route="/api/v1/users",status_class="5xx"}' + values: '0+0x80' + - series: 'messenger_slo_ready_eligible_total{service_name="messenger-outbox"}' + values: '0+100x80' + - series: 'messenger_slo_ready_bad_total{service_name="messenger-outbox"}' + values: '0+0x80' + alert_rule_test: + - {eval_time: 5m, alertname: MessengerApiBurnFast, exp_alerts: []} + - {eval_time: 5m, alertname: MessengerApiBurnSlow, exp_alerts: []} + - {eval_time: 5m, alertname: MessengerReadyBurnFast, exp_alerts: []} + - {eval_time: 5m, alertname: MessengerReadyBurnSlow, exp_alerts: []} + - name: brief severe spike and recovery + interval: 5s + input_series: + - series: 'messenger_http_requests_total{route="/api/v1/users",status_class="2xx"}' + values: '0+100x12 1200+0x8 1200+100x60' + - series: 'messenger_http_requests_total{route="/api/v1/users",status_class="5xx"}' + values: '0+0x12 0+100x8 800+0x60' + alert_rule_test: + - eval_time: 1m30s + alertname: MessengerApiBurnFast + exp_alerts: + - exp_labels: {severity: critical, slo: api, burn: fast} + exp_annotations: {summary: 'api SLO fast burn', runbook: 'See docs/slo-runbook.md'} + - {eval_time: 3m, alertname: MessengerApiBurnFast, exp_alerts: []} + - name: sustained moderate degradation + interval: 5s + input_series: + - series: 'messenger_http_requests_total{route="/api/v1/users",status_class="2xx"}' + values: '0+80x80' + - series: 'messenger_http_requests_total{route="/api/v1/users",status_class="5xx"}' + values: '0+20x80' + - series: 'messenger_slo_ready_eligible_total{service_name="messenger-outbox"}' + values: '0+100x80' + - series: 'messenger_slo_ready_bad_total{service_name="messenger-outbox"}' + values: '0+20x80' + alert_rule_test: + - {eval_time: 5m, alertname: MessengerApiBurnFast, exp_alerts: []} + - {eval_time: 5m, alertname: MessengerReadyBurnFast, exp_alerts: []} + - eval_time: 5m + alertname: MessengerApiBurnSlow + exp_alerts: + - exp_labels: {severity: warning, slo: api, burn: slow} + exp_annotations: {summary: 'api SLO slow burn', runbook: 'See docs/slo-runbook.md'} + - eval_time: 5m + alertname: MessengerReadyBurnSlow + exp_alerts: + - exp_labels: {severity: warning, slo: ready, burn: slow} + exp_annotations: {summary: 'ready SLO slow burn', runbook: 'See docs/slo-runbook.md'} + - name: low-volume guard + interval: 5s + input_series: + - series: 'messenger_http_requests_total{route="/api/v1/users",status_class="5xx"}' + values: '0+1x80' + - series: 'messenger_slo_ready_eligible_total{service_name="messenger-outbox"}' + values: '0+1x80' + - series: 'messenger_slo_ready_bad_total{service_name="messenger-outbox"}' + values: '0+1x80' + alert_rule_test: + - {eval_time: 5m, alertname: MessengerApiBurnFast, exp_alerts: []} + - {eval_time: 5m, alertname: MessengerReadyBurnFast, exp_alerts: []} + - name: late ready events fast burn + interval: 5s + input_series: + - series: 'messenger_slo_ready_eligible_total{service_name="messenger-outbox"}' + values: '0+100x30' + - series: 'messenger_slo_ready_bad_total{service_name="messenger-outbox"}' + values: '0+100x30' + alert_rule_test: + - eval_time: 1m + alertname: MessengerReadyBurnFast + exp_alerts: + - exp_labels: {severity: critical, slo: ready, burn: fast} + exp_annotations: {summary: 'ready SLO fast burn', runbook: 'See docs/slo-runbook.md'} diff --git a/student-tests/slo_analyze.py b/student-tests/slo_analyze.py new file mode 100644 index 0000000..e4dfb87 --- /dev/null +++ b/student-tests/slo_analyze.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 +"""Summarize observed alert transitions; sampling bounds detection precision.""" +import json +from pathlib import Path + +root = Path(__file__).resolve().parents[1] +data = json.loads((root / 'evidence/slo-incidents.json').read_text()) +samples = sorted((s for s in data['samples'] if 'firing' in s), key=lambda s: s['monotonic']) +alerts = ['MessengerApiBurnFast', 'MessengerApiBurnSlow'] +output = {'sampling_interval_seconds': 5, 'monitor_errors': sum('monitor_error' in s for s in data['samples']), 'phases': {}} +base = [s for s in samples if s['phase'] == 'baseline'] +output['baseline'] = {'samples': len(base), 'samples_with_burn_alert': sum(bool(s['firing']) for s in base), 'scope': 'one observed healthy baseline, not a long-term false-positive rate'} +for name, period in data['periods'].items(): + start, end = period['lock_observed']['monotonic'], period['command_completed']['monotonic'] + active = [s for s in samples if start <= s['monotonic'] <= end] + recovery = [s for s in samples if s['monotonic'] > end and s['phase'] == name + '_recovery'] + previous = [s for s in samples if s['monotonic'] < start] + row = {'observed_lock_seconds': round(end-start, 3), 'alerts': {}} + for alert in alerts: + before = bool(previous and alert in previous[-1]['firing']) + hits = [s for s in active + recovery if alert in s['firing']] + clear = [s for s in recovery if hits and s['monotonic'] > hits[0]['monotonic'] and alert not in s['firing']] + row['alerts'][alert] = {'active_before_lock': before, + 'first_firing': hits[0]['time'] if hits else None, + 'first_firing_after_incident': bool(hits and hits[0]['monotonic'] > end), + 'detection_seconds': round(hits[0]['monotonic']-start, 3) if hits and not before else None, + 'first_clear': clear[0]['time'] if hits and clear else None, + 'recovery_seconds': round(clear[0]['monotonic']-end, 3) if hits and clear else None} + output['phases'][name] = row +path = root / 'evidence/slo-analysis.json' +path.write_text(json.dumps(output, ensure_ascii=False, indent=2) + '\n') +print(json.dumps(output, ensure_ascii=False, indent=2)) diff --git a/student-tests/slo_incidents.py b/student-tests/slo_incidents.py new file mode 100644 index 0000000..241c504 --- /dev/null +++ b/student-tests/slo_incidents.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python3 +"""Run two bounded DB-lock incidents against the local Compose project. + +Creates synthetic data, records only aggregate counts/alert states, and releases +all locks on exit. Run on the isolated lab6 stack, never a shared database. +""" +import base64 +from concurrent.futures import ThreadPoolExecutor +from datetime import datetime, timezone +import json +from pathlib import Path +import subprocess +import sys +import threading +import time +import urllib.parse +import urllib.request + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client, Failure + +OUTPUT = ROOT / 'evidence' / 'slo-incidents.json' +STOP = threading.Event() +LOCK = threading.Lock() +COUNTS = {'ok': 0, 'failed': 0} +SCENARIO = {'phase': 'bootstrap'} +SAMPLES = [] +AUTH = 'Basic ' + base64.b64encode(('admin:' + (ROOT / 'secrets/grafana_password').read_text().strip()).encode()).decode() + +def utc(): return datetime.now(timezone.utc).isoformat() + +def get(path, params=None): + url = 'http://127.0.0.1:3000/api/datasources/proxy/uid/prometheus/api/v1/' + path + if params: url += '?' + urllib.parse.urlencode(params) + with urllib.request.urlopen(urllib.request.Request(url, headers={'Authorization': AUTH}), timeout=5) as response: + result = json.load(response) + if result.get('status') != 'success': raise RuntimeError('Prometheus query failed') + return result['data'] + +def monitor(): + while not STOP.is_set(): + try: + alerts = get('alerts')['alerts'] + firing = sorted(a['labels']['alertname'] for a in alerts if a['state'] == 'firing' and 'Burn' in a['labels']['alertname']) + with LOCK: sample = {'time': utc(), 'monotonic': time.monotonic(), 'phase': SCENARIO['phase'], 'counts': dict(COUNTS), 'firing': firing} + SAMPLES.append(sample) + except Exception as error: + SAMPLES.append({'time': utc(), 'monotonic': time.monotonic(), 'phase': SCENARIO['phase'], 'monitor_error': type(error).__name__}) + STOP.wait(5) + +def worker(actor, chat): + client = Client(timeout=9) + while not STOP.is_set(): + try: + client.message(actor, chat, 'synthetic SLO experiment') + outcome = 'ok' + except Failure: + outcome = 'failed' + with LOCK: COUNTS[outcome] += 1 + STOP.wait(1) + +def phase(name): + with LOCK: SCENARIO['phase'] = name + print(utc(), name, flush=True) + +def db_is_locked(): + result = subprocess.run(['docker','compose','exec','-T','postgres','psql','-U','messenger','-d','messenger','-Atc', + "SELECT EXISTS (SELECT 1 FROM pg_locks WHERE relation='users'::regclass AND mode='AccessExclusiveLock' AND granted)"], + cwd=ROOT, capture_output=True, text=True, timeout=10, check=True) + return result.stdout.strip() == 't' + +def incident(name, seconds): + cmd = ['docker','compose','exec','-T','postgres','psql','-U','messenger','-d','messenger','-c', + f'BEGIN; LOCK TABLE users IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep({seconds}); COMMIT;'] + start_command = {'time': utc(), 'monotonic': time.monotonic()} + proc = subprocess.Popen(cmd, cwd=ROOT, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE) + try: + deadline = time.monotonic() + 15 + while time.monotonic() < deadline: + if proc.poll() is not None: raise RuntimeError('DB lock command ended early') + if db_is_locked(): break + time.sleep(.5) + else: raise RuntimeError('DB lock was not observed') + phase(name) + start = {'time': utc(), 'monotonic': time.monotonic(), 'command_started': start_command} + code = proc.wait(timeout=seconds+20) + if code: raise RuntimeError('DB lock command failed') + end = {'time': utc(), 'monotonic': time.monotonic()} + phase(name + '_recovery') + return {'lock_observed': start, 'command_completed': end} + finally: + if proc.poll() is None: + proc.terminate() + try: proc.wait(timeout=5) + except subprocess.TimeoutExpired: proc.kill(); proc.wait() + +def main(): + client = Client(timeout=9) + actor, _, _, chat = client.bootstrap() + result = {'description': 'controlled DB table lock; three synthetic clients; 5s alert sampling', 'periods': {}, 'samples': SAMPLES} + monitor_thread = threading.Thread(target=monitor, daemon=True); monitor_thread.start() + try: + with ThreadPoolExecutor(max_workers=3) as pool: + futures = [pool.submit(worker, actor, chat) for _ in range(3)] + phase('baseline'); time.sleep(60) + result['periods']['burst'] = incident('burst', 75) + time.sleep(100) + result['periods']['sustained'] = incident('sustained', 180) + time.sleep(100) + phase('complete') + STOP.set() + for future in futures: future.result(timeout=15) + finally: + STOP.set(); monitor_thread.join(timeout=10) + result['end_time'] = utc() + result['final_counts'] = COUNTS + OUTPUT.write_text(json.dumps(result, ensure_ascii=False, indent=2) + '\n') + print('Aggregate evidence:', OUTPUT, flush=True) + +if __name__ == '__main__': main() diff --git a/student-tests/slo_ready_incident.py b/student-tests/slo_ready_incident.py new file mode 100644 index 0000000..c816668 --- /dev/null +++ b/student-tests/slo_ready_incident.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 +"""Stop only this project's idle worker, submit 8 jobs, verify ready SLO.""" +import base64 +from datetime import datetime, timezone +import json +from pathlib import Path +import subprocess +import sys +import time +import urllib.parse +import urllib.request + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client + +secret = (ROOT / 'secrets/grafana_password').read_text().strip() +auth = 'Basic ' + base64.b64encode(('admin:' + secret).encode()).decode() +def utc(): return datetime.now(timezone.utc).isoformat() +def prom(path, query=None): + url = 'http://127.0.0.1:3000/api/datasources/proxy/uid/prometheus/api/v1/' + path + if query is not None: url += '?' + urllib.parse.urlencode({'query': query}) + with urllib.request.urlopen(urllib.request.Request(url, headers={'Authorization': auth}), timeout=5) as r: return json.load(r)['data'] +def metric(name): + result = prom('query', name + '{service_name="messenger-outbox"}')['result'] + return float(result[0]['value'][1]) if result else None +def sample(phase): + alerts = prom('alerts')['alerts'] + return {'time': utc(), 'monotonic': time.monotonic(), 'phase': phase, + 'firing': sorted(a['labels']['alertname'] for a in alerts if a['state']=='firing' and a['labels']['alertname'].startswith('MessengerReadyBurn')), + 'eligible': metric('messenger_slo_ready_eligible_total'), 'bad': metric('messenger_slo_ready_bad_total')} +def docker(*args): subprocess.run(['docker','compose',*args], cwd=ROOT, check=True, stdout=subprocess.DEVNULL) +def run(): + status = subprocess.run(['docker','compose','exec','-T','postgres','psql','-U','messenger','-d','messenger','-Atc', + "SELECT count(*) FROM attachment_jobs WHERE status IN ('queued','processing')"], cwd=ROOT, check=True, capture_output=True, text=True) + if int(status.stdout.strip()): raise RuntimeError('Pending jobs exist; refusing to stop worker') + client=Client(); actor,_,_,chat=client.bootstrap(); samples=[]; out={'samples':samples,'jobs':8} + worker_stopped=False + try: + samples.append(sample('baseline')) + docker('stop','-t','5','worker'); worker_stopped=True + accepted=[] + for _ in range(8): accepted.append(client.attachment(actor,chat)[0]) + out['accepted_at']={'time':utc(),'monotonic':time.monotonic()} + print('8 synthetic jobs accepted; waiting for burn alert', flush=True) + deadline=time.monotonic()+100 + while time.monotonic() Date: Fri, 6 Nov 2026 11:12:38 +0300 Subject: [PATCH 4/4] refactor code --- REPORT.md | 71 +- compose.yaml | 1 + docs/observability.md | 2 +- docs/review.md | 36 + docs/slo-runbook.md | 10 +- .../pre-review/ready-scenario-failed.json | 300 +++++ evidence/pre-review/slo-analysis.json | 59 + evidence/pre-review/slo-incidents.json | 1170 +++++++++++++++++ evidence/pre-review/slo-ready-incident.json | 211 +++ evidence/review-checks.json | 109 ++ evidence/review-dashboard.json | 137 ++ evidence/review-db-trace.json | 909 +++++++++++++ evidence/review-observability.md | 29 + evidence/review-resilience.json | 27 + evidence/review-retry-job.json | 9 + evidence/review-retry-summary.json | 27 + evidence/review-retry-trace.json | 870 ++++++++++++ evidence/review-tempo-load.json | 17 + evidence/review-tempo-outage.json | 245 ++++ evidence/review-tempo-persistence.json | 93 ++ evidence/slo-analysis.json | 44 +- evidence/slo-incidents.json | 974 +++++++++----- evidence/slo-ready-incident.json | 283 ++-- evidence/slo-track.md | 20 +- infra/prometheus/alerts.yaml | 10 + migrations/009_slo_commits.sql | 11 + pyproject.toml | 6 +- src/broker.py | 6 +- src/main.py | 11 +- src/outbox.py | 21 +- src/storage/postgres.py | 37 + src/telemetry.py | 31 +- src/worker.py | 36 +- student-tests/capture_observability.py | 41 + student-tests/check_dashboard.py | 18 + student-tests/check_retry_trace.py | 31 + student-tests/pipeline_critical.py | 12 +- student-tests/retry_evidence.py | 30 + student-tests/review_checks.py | 22 + student-tests/review_queries.py | 13 + student-tests/review_resilience.py | 20 + student-tests/slo_analyze.py | 8 + student-tests/slo_commit_integration.py | 45 + student-tests/slo_incidents.py | 20 +- student-tests/slo_ready_incident.py | 16 +- student-tests/telemetry_alerts.test.yaml | 31 + student-tests/telemetry_unit.py | 38 + student-tests/tempo_outage.py | 60 + student-tests/trace_retry.py | 7 + uv.lock | 4 +- 50 files changed, 5674 insertions(+), 564 deletions(-) create mode 100644 docs/review.md create mode 100644 evidence/pre-review/ready-scenario-failed.json create mode 100644 evidence/pre-review/slo-analysis.json create mode 100644 evidence/pre-review/slo-incidents.json create mode 100644 evidence/pre-review/slo-ready-incident.json create mode 100644 evidence/review-checks.json create mode 100644 evidence/review-dashboard.json create mode 100644 evidence/review-db-trace.json create mode 100644 evidence/review-observability.md create mode 100644 evidence/review-resilience.json create mode 100644 evidence/review-retry-job.json create mode 100644 evidence/review-retry-summary.json create mode 100644 evidence/review-retry-trace.json create mode 100644 evidence/review-tempo-load.json create mode 100644 evidence/review-tempo-outage.json create mode 100644 evidence/review-tempo-persistence.json create mode 100644 migrations/009_slo_commits.sql create mode 100644 student-tests/capture_observability.py create mode 100644 student-tests/check_dashboard.py create mode 100644 student-tests/check_retry_trace.py create mode 100644 student-tests/retry_evidence.py create mode 100644 student-tests/review_checks.py create mode 100644 student-tests/review_queries.py create mode 100644 student-tests/review_resilience.py create mode 100644 student-tests/slo_commit_integration.py create mode 100644 student-tests/telemetry_alerts.test.yaml create mode 100644 student-tests/tempo_outage.py diff --git a/REPORT.md b/REPORT.md index 6a79e7e..6adc761 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,57 +1,66 @@ # Отчёт — лабораторная 6 -- Автор, группа: Kozyrev, BVT2301 -- Целевая оценка: 5, трек 2 — SLO и обнаружение инцидентов -- Commit/PR: TODO после публикации -- Источники: [контракт телеметрии](contracts/observability.md), [OTel Collector](https://opentelemetry.io/docs/collector/configuration/), [OTel Python exporters](https://opentelemetry.io/docs/languages/python/exporters/), [Loki OTLP](https://grafana.com/docs/loki/latest/send-data/otel/), [Grafana provisioning](https://grafana.com/docs/grafana/latest/administration/provisioning/), исходники lab5 commit `6bc0712`. При реализации использован Codex. +- Автор: Kozyrev, BVT2301. +- Цель: уровень 5, трек 2 — SLO и обнаружение инцидентов. +- Рабочая ветка: `solution/BVT2301-Kozyrev`. Результаты повторной проверки — [docs/review.md](docs/review.md). Независимое прохождение runbook другим студентом не подтверждено; полностью закрытым уровень 5 не объявляется. +- При реализации и проверке использован Codex. Исходная основа: lab5, commit `6bc0712`. -## Запуск +## Воспроизводимый запуск -Нужны Python 3, Docker Engine и Compose v2. API слушает `127.0.0.1:8080`, Grafana — `127.0.0.1:3000`. Версии образов закреплены в [Dockerfile](Dockerfile) и [compose.yaml](compose.yaml). +Нужны Docker Engine с Compose v2 и Python 3 для публичных тестов. Для локальных SDK-тестов нужен Python ≥3.12 согласно [pyproject.toml](pyproject.toml); их также можно запускать внутри контейнера. ```sh python3 scripts/init-secrets.py docker compose config --quiet docker compose up -d --build -docker compose ps -a -curl --fail http://localhost:8080/health/ready +# Дождаться успешного ответа; старт API может занять время: +python3 - <<'PY' +import time, urllib.request +for attempt in range(90): + try: + urllib.request.urlopen('http://localhost:8080/health/ready', timeout=2).close() + break + except OSError: + time.sleep(1) +else: + raise SystemExit('API readiness timeout; inspect docker compose ps -a') +PY make check make test ``` -Миграции и создание S3 bucket выполняют сервисы `migrate` и `s3-init`. Секреты создаются в игнорируемом `secrets/`; `.env` для значений Compose по умолчанию не обязателен. В Grafana логин `admin`, пароль — в локальном `secrets/grafana_password`. `docker compose down` сохраняет volumes; `docker compose down -v` намеренно удаляет все именованные volumes этого проекта, включая учебные данные и историю телеметрии. +Миграции выполняет `migrate`, bucket создаёт `s3-init`; [Compose](compose.yaml) закрепляет образы и зависимости. API: `127.0.0.1:8080`; Grafana: `127.0.0.1:3000`, логин `admin`, пароль в игнорируемом `secrets/grafana_password`. `.env` при значениях по умолчанию не обязателен. `docker compose down` сохраняет volumes; `down -v` удаляет данные и историю. -## Архитектура и решения +Для полной проверки OpenAPI установите `requirements-dev.txt` в `.venv`, как описано в [README](README.md), затем `make validate PYTHON=.venv/bin/python`. Короткие проверки повторяются командой `python3 student-tests/review_checks.py`; журнал сохраняется в [review-checks.json](evidence/review-checks.json). Инциденты запускаются отдельно и намеренно ухудшают учебный стенд. -Клиент обращается через HAProxy к двум API-репликам. PostgreSQL хранит бизнес-данные, задания и outbox; SeaweedFS — байты вложений. Dispatcher публикует события из outbox в RabbitMQ, worker формирует варианты; reconcile выполняет сверку. S3 и PostgreSQL не образуют общую транзакцию. Из lab5 перенесены исходники, миграции, инфраструктура и собственные тесты; контракты и публичные тесты lab6 сохранены. +## Уровень 3 -API, outbox и worker отправляют OTLP через Collector в Prometheus, Loki и Tempo; Grafana использует эти хранилища через provisioned datasources. [Dashboard](infra/grafana/dashboards/messenger.json), [правила alerts](infra/prometheus/alerts.yaml), конфигурации и миграция context находятся в репозитории. Схема сигналов, запросы, SLI/SLO, ограничения и сценарии — в [docs/observability.md](docs/observability.md). `.env.example` описывает логические настройки; Compose часть значений задаёт явно. OTLP включён в Compose для API, worker и outbox. +HAProxy распределяет запросы между двумя API. PostgreSQL хранит бизнес-данные, outbox и задания; SeaweedFS — файлы; RabbitMQ доставляет задания worker. Сигналы API/worker/outbox идут по OTLP в Collector. Prometheus хранит метрики, Loki — логи, Tempo — трассы; Grafana запрашивает эти хранилища. Источники и dashboard загружаются provisioning; нужные данные находятся в named volumes. Исходники: [compose.yaml](compose.yaml), [Collector](infra/otel/collector.yaml), [dashboard](infra/grafana/dashboards/messenger.json). -## Проверки и доказательства +[Telemetry](src/telemetry.py) ограничивает metric labels, очищает log record перед stdout и OTLP. Неизвестные HTTP methods сводятся к `_OTHER`. Jobs имеют `success`, `failed`, `skipped`: повторная доставка без нового результата не считается успешной обработкой. Проверки реального SDK exporter, W3C propagation и исходов job — [telemetry_unit.py](student-tests/telemetry_unit.py), [pipeline_critical.py](student-tests/pipeline_critical.py). -| Критерий | Команда/сценарий | Результат | -| --- | --- | --- | -| Целостность и smoke | `make check`, `make validate PYTHON=.venv/bin/python`, `make test` | PASS; smoke повторён после восстановления. | -| Конфигурации | `docker compose config --quiet`; штатные проверки Collector, Loki, Tempo и `promtool` | PASS; Grafana API вернул три datasource и dashboard, запросы панелей выполнены. | -| Контекст и конфиденциальность | `.venv/bin/python student-tests/telemetry_unit.py` | 3 теста PASS: parent/child через AMQP, ограниченные поля логов и шаблон маршрута метрики. | -| Обработка вложений | `.venv/bin/python student-tests/pipeline_critical.py` | 8 тестов PASS. | -| Данные и реплики | `make test-persistence ACTION_SCRIPT=scripts/restart.sh`; `make test-failover ACTION_SCRIPT=scripts/stop-one.sh` | Оба PASS; две реплики восстановлены. Фактический состав hook описан в скриптах; отдельного снимка состояний до/после для persistence нет. | -| Связь API → worker, включая retry | upload с тестовой инъекцией crash worker | ready на второй попытке; 2 producer и 2 consumer spans в одном trace. | -| Alert очереди и alert ошибок API | остановка worker; блокировка таблицы users при нагрузке; `promtool test rules student-tests/alerts.test.yaml` в контейнере | Оба наблюдались в firing и отсутствовали после восстановления; синтетический тест финальных правил PASS. | -| Отказ хранилища telemetry | остановить Tempo, выполнить smoke и отдельную нагрузку 75 с | smoke PASS; очередь ограничена, зафиксировано переполнение и потери traces. | +Повторные публичные проверки сохранности и отказа реплики: [review-resilience.json](evidence/review-resilience.json), сценарий [review_resilience.py](student-tests/review_resilience.py). -Идентификаторы трасс, результаты, формулы и границы проверки приведены в [evidence/observability.md](evidence/observability.md). Raw summaries нагрузки: [инцидент БД](evidence/db-incident-load-repeat.json), [отказ Tempo](evidence/tempo-outage-load.json). Полный набор наблюдений живёт в локальных volumes, а не в Git. +## Уровень 4 -## Трек 5: SLO и обнаружение инцидентов +Контекст сохраняется вместе с заданием и outbox, передаётся через AMQP headers и восстанавливается worker. Ошибочный producer span получает ERROR без текста исключения. Dashboard содержит histogram p50/p95/p99, ошибки, очередь, pool и worker saturation; все запросы проверяет [check_dashboard.py](student-tests/check_dashboard.py), результаты — [review-dashboard.json](evidence/review-dashboard.json). Описание метрик, единиц, запросов и ограничений — [observability.md](docs/observability.md). -Заданы учебные SLO 99% за 24 часа для API и времени готовности принятых вложений. Для каждого настроены быстрые и медленные multiwindow burn-rate alerts; формулы, допущения и пошаговая диагностика — в [runbook](docs/slo-runbook.md). Последние 30 секунд заданий исключены из зрелой когорты; просроченные, failed и неготовые задания входят в ошибки SLO. Моменты начала и окончания приблизительны из-за транзакционных границ. При недоступном outbox счётчики устаревают. +Время ready теперь вычисляется между commit принятия и commit первого ready в PostgreSQL. [Миграция 009](migrations/009_slo_commits.sql) сохраняет отдельную когорту; outbox материализует timestamps и однократно фиксирует результат 30-секундного срока. `track_commit_timestamp=on` включён в Compose. Старые задания не backfill-ятся приближёнными датами. [Интеграционный тест](student-tests/slo_commit_integration.py) проверяет commit-границы и неизменность good/bad без изменения бизнес-данных. -Повторяемые сценарии и наблюдения — в [отчёте трека](evidence/slo-track.md), исходные отсчёты — [API](evidence/slo-incidents.json), [расчёт](evidence/slo-analysis.json) и [вложения](evidence/slo-ready-incident.json). При короткой и длительной блокировке API fast alert обнаружен через 55,088 и 50,278 секунды; slow в первом случае впервые появился после конца инцидента (одна новая тревога в фазе восстановления), во втором — через 90,428 секунды. На коротком baseline из 13 проб не наблюдалось burn alerts; долгосрочная частота ложных срабатываний не измерена. При тесте восьми вложений обнаружение — 40,279 секунды после последнего принятия, после запуска worker все восемь стали ready. Опросный метод, ограничения и заметный скачок UTC-часов описаны в отчёте трека. Проведена собственная проверка runbook по Loki/Tempo; прохождение другим студентом не подтверждено. +Повторный crash/retry проверен [сценарием](student-tests/retry_evidence.py): [job/attempt/restart](evidence/review-retry-job.json), [логи и trace](evidence/review-retry-trace.json), [проверка дерева spans](evidence/review-retry-summary.json). Незавершённые spans аварийной попытки могут потеряться при `os._exit`; сохранённые consumer spans не доказывают полноту упавшей попытки. -Проверка правил: `docker run --rm -v "$PWD:/workspace:ro" --entrypoint promtool prom/prometheus:v3.2.1 test rules /workspace/student-tests/burn_rules.test.yaml`. Синтетические проверки нормальной нагрузки, всплеска, устойчивой деградации и малого числа событий прошли. +Сценарии очереди и БД сохранены в [slo-ready-incident.json](evidence/slo-ready-incident.json) и [slo-incidents.json](evidence/slo-incidents.json); коррелированный лог 503 и trace — [review-db-trace.json](evidence/review-db-trace.json). Ограниченный отказ Tempo воспроизводится [tempo_outage.py](student-tests/tempo_outage.py), включая baseline счётчиков, их дельты и память Collector: [сырые результаты](evidence/review-tempo-outage.json), [разбор](evidence/review-observability.md). Старые эксперименты и их ограничения сохранены отдельно в [observability.md](evidence/observability.md). -## Эксперименты и ограничения +## Уровень 5, трек 2 -Docker Engine 29.8.0, `aarch64`; Docker VM сообщил 10 CPU и 10 416 644 096 байт RAM. Лимиты контейнеров указаны в Compose: например, Collector — 256 MiB, каждая API-реплика — 0,5 CPU и 512 MB. Ограниченная нагрузка была closed-loop; прогрев и статистические повторы для сравнения производительности не проводились. Числа не являются оценкой максимальной пропускной способности. В отчёте о первом опыте блокировки БД alert не успел перейти в firing; после изменения выдержки до 10 секунд повторный опыт показал firing и восстановление. +Гипотеза: multiwindow burn-rate alerts обнаруживают контролируемый расход бюджета и снимаются после восстановления короткого окна. Учебные цели — 99% API без 5xx и 99% принятых файлов ready ≤30 с, окно SLO 24 часа. Порог 30 с выбран относительно 60-секундного ожидания smoke; 99% — учебная цель, не исследование пользователей. Формулы, числители, знаменатели, low-volume guard и диагностика приведены в [runbook](docs/slo-runbook.md). -`messenger_job_ready_duration_seconds` измеряет время от создания DB-записи о вложении до фиксации ready; оно включает время до commit принятия. Репрезентативность 24-часового окна и долгосрочная частота ложных срабатываний не проверены. Данные queue age обновляет outbox: при его остановке gauge устаревает. RSS/CPU worker не включают отдельный render-процесс; pool waiting зависит от внутреннего API закреплённой версии asyncpg. Один Docker-host и единственные экземпляры DB, broker и LB не обеспечивают HA при их потере. Три замера RAM при отказе Tempo не доказывают поведение для произвольной длительности; переполнение очереди приводит к потере spans. При необходимости переносимой защиты следует отдельно сохранить наблюдения и подтвердить состояние alerts и трасс после нового clone. +Методика, свежие результаты, оценка задержанных/ложных тревог и ограничения — [slo-track.md](evidence/slo-track.md). Расчёт выполняет [slo_analyze.py](student-tests/slo_analyze.py). Старый опыт имел расхождение приращений UTC/monotonic до 83,075 с: данные сохранены в [pre-review](evidence/pre-review/slo-analysis.json), но не принимаются как надёжный замер длительностей. Причину расхождения установить нельзя. + +Снятие alert из-за недостатка новых событий не означает восстановления суточного SLO. Короткий baseline не измеряет долгосрочную частоту ложных срабатываний. Подготовлен протокол независимой проверки runbook; фактического прохождения другим студентом пока нет. + +## Практические ограничения и источники + +Один Docker-host и единственные DB/broker/LB не обеспечивают HA при их потере. CPU/RSS worker не включают дочерний render; pool waiting использует внутреннюю очередь закреплённой версии asyncpg. Потеря отдельных histogram samples возможна при crash после commit; durable SLO-когорта сохраняется. При разрыве экспорта `increase` может пропускать события; stale/unknown alerts показывают неполноту наблюдения. Commit timestamps PostgreSQL хранятся ограниченное время, поэтому outbox сохраняет их в таблицу. Репрезентативные 24 часа и долговременная устойчивость по коротким экспериментам не доказаны. + +Источники: [контракт](contracts/observability.md), [PostgreSQL commit timestamps](https://www.postgresql.org/docs/17/functions-info.html#FUNCTIONS-COMMIT-TIMESTAMP), [OTel Collector](https://opentelemetry.io/docs/collector/configuration/), [OTel Python exporters](https://opentelemetry.io/docs/languages/python/exporters/), [W3C Trace Context](https://www.w3.org/TR/trace-context/), [Prometheus histograms](https://prometheus.io/docs/practices/histograms/), [SRE Workbook: SLO alerts](https://sre.google/workbook/alerting-on-slos/). diff --git a/compose.yaml b/compose.yaml index 551c0cd..4d58c8e 100644 --- a/compose.yaml +++ b/compose.yaml @@ -114,6 +114,7 @@ services: postgres: image: postgres:17.6-alpine + command: ["postgres", "-c", "track_commit_timestamp=on"] environment: POSTGRES_DB: messenger POSTGRES_USER: messenger diff --git a/docs/observability.md b/docs/observability.md index 60a2da1..9318302 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -8,7 +8,7 @@ API, worker и outbox отправляют OTLP/HTTP в Collector. Prometheus х При upload текущий W3C trace context сохраняется в `attachment_jobs` одной транзакцией с outbox. Dispatcher извлекает его и публикует сообщение с дочерним producer span и AMQP headers; worker извлекает контекст и создаёт consumer span. Повторная публикация и retry используют сохранённый контекст исходной загрузки. Каждая попытка имеет собственные spans; они являются потомками исходной транзакции загрузки. Baggage не переносится. Контекст — технические данные, не содержимое изображения. -Dashboard использует счётчики и гистограммы с единицами секунд: p50/p95/p99 HTTP вычисляются через `histogram_quantile` по сумме buckets, не по среднему перцентилей. `messenger_job_ready_duration_seconds` измеряет от времени создания записи о вложении в DB до фиксации ready; эта оценка включает время до commit принятия и не претендует на точную границу durable acceptance. Queue depth и age — число и возраст самого старого задания в состоянии queued из PostgreSQL, включая ожидающие retry; это не длина очереди RabbitMQ. Данные обновляет outbox. DB pool отражает пул каждой API-реплики; число ожидающих читается из внутренней очереди asyncpg 0.31.0 и зависит от этой версии. Worker busy/capacity — число занятых executor threads и их лимит; повторная доставка уже завершённой задачи учитывается как успешная обработка. CPU и RSS — процессы API/worker/outbox, без дочерних процессов render. При остановленном outbox queue gauge устаревает. +Dashboard использует счётчики и гистограммы с единицами секунд: p50/p95/p99 HTTP вычисляются через `histogram_quantile` по сумме buckets, не по среднему перцентилей. `messenger_job_ready_duration_seconds` измеряет разность commit публикации ready и commit принятия задания из `attachment_slo` (миграция 009). Оба момента берутся из PostgreSQL; часы worker не участвуют. Гистограмма записывается после успешного commit: аварийное завершение worker между commit и записью метрики может потерять этот histogram sample; DB-когорта SLO при этом сохраняется. Queue depth и age — число и возраст самого старого задания в состоянии queued из PostgreSQL, включая ожидающие retry; это не длина очереди RabbitMQ. Данные обновляет outbox. DB pool отражает пул каждой API-реплики; число ожидающих читается из внутренней очереди asyncpg 0.31.0 и зависит от этой версии. Worker busy/capacity — число занятых executor threads и их лимит; повторная доставка без нового результата имеет outcome `skipped`, успешный commit — `success`, ошибка — `failed`. CPU и RSS — процессы API/worker/outbox, без дочерних процессов render. Queue-метрики имеют label `queue="attachment.process.v1"`. `messenger_queue_sample_time_seconds` показывает время последнего успешного обновления; `MessengerQueueTelemetryStale` сигнализирует об устаревании/отсутствии ряда. SLI: доля ответов 5xx среди всех HTTP бизнес-запросов API, исключая health. Целевой SLO: не менее 99% запросов без 5xx за скользящие 24 часа; ошибки балансировщика, которые не дошли до API, SLI не учитывает. Целевое значение выбрано для учебного стенда, а не по измеренным требованиям пользователей. Второй SLO и расчёт burn-rate описаны в [runbook](slo-runbook.md); его DB-счётчики учитывают и невыполненные в срок задания. Alert `MessengerHighErrorRate` срабатывает при >5% за минуту в течение 10 секунд; это оперативный порог, не burn-rate alert. `MessengerQueueLag` срабатывает при возрасте queued >30 секунд в течение 15 секунд. Состояния доступны в Prometheus через Grafana Explore или локальный API Prometheus внутри контейнера. diff --git a/docs/review.md b/docs/review.md new file mode 100644 index 0000000..20489b9 --- /dev/null +++ b/docs/review.md @@ -0,0 +1,36 @@ +# Проверка по критериям README + +Проверялись [README](../README.md), [контракт наблюдаемости](../contracts/observability.md), исходный [REPORT](../REPORT.md), код и локальный Docker-стенд. Автоматический smoke не выставляет оценку за наблюдаемость. + +## Итог по уровням + +- **3: подтверждён после исправлений.** [Короткие проверки](../evidence/review-checks.json), [публичные persistence/failover](../evidence/review-resilience.json), [коррелированные логи API/worker и trace](../evidence/review-retry-trace.json). +- **4: подтверждён в проверенном локальном стенде.** [Разбор инцидентов и отказа Tempo](../evidence/review-observability.md), [QueueLag/ready](../evidence/slo-ready-incident.json), [проверка retry-дерева](../evidence/review-retry-summary.json), [сохранность trace после рестарта Tempo](../evidence/review-tempo-persistence.json). +- **5: техническая часть трека 2 проверена, полный зачёт не подтверждён.** [Повторные измерения](../evidence/slo-track.md) и runbook есть; независимый диагноз другого студента не задокументирован. Итоговая подтверждённая оценка этой проверки — **4**. + +## Исправления + +| Уровень | Найденная ошибка | Исправление и проверка | +| --- | --- | --- | +| 3 | Allowlist применялся только к stdout; OTLP LoggingHandler получал произвольные extra и exception | Фильтр до обоих handlers; тест реального SDK log exporter с секретом в extra и exception — [telemetry_unit.py](../student-tests/telemetry_unit.py) | +| 3 | Произвольные HTTP method могли создавать новые metric labels | Стандартные методы и `_OTHER`; шаблон маршрута сохранён — [main.py](../src/main.py), [telemetry.py](../src/telemetry.py) | +| 3 | Повторная доставка без нового результата считалась успешной job | Отдельный исход `skipped`; `success` только после commit ready — [worker.py](../src/worker.py), [pipeline_critical.py](../student-tests/pipeline_critical.py) | +| 4 | Исключение publish оставляло producer span без ERROR | ERROR без экспорта текста исключения; regression test — [broker.py](../src/broker.py), [telemetry_unit.py](../student-tests/telemetry_unit.py) | +| 4 | Histogram ready начинался до durable acceptance и использовал часы worker | Оба конца — commit timestamps PostgreSQL; [миграция 009](../migrations/009_slo_commits.sql), [изолированный интеграционный тест](../student-tests/slo_commit_integration.py) | +| 4 | Queue gauge не имел queue name и мог незаметно устареть | Label queue, timestamp последнего обновления и stale/absent alert; [telemetry.py](../src/telemetry.py), [правила](../infra/prometheus/alerts.yaml) | +| 4 | Потери Tempo приписывались накопленным счётчикам без baseline | Повторяемый опыт с baseline, дельтами, ID Collector и замерами памяти — [tempo_outage.py](../student-tests/tempo_outage.py) | +| 5 | Ready SLO зависел от изменяемых status/updated_at | Отдельная зрелая когорта с однократно зафиксированным bad; retry/replay не меняют результат — [postgres.py](../src/storage/postgres.py) | +| 5 | Старые временные результаты использовались при расхождении UTC/monotonic | Старые отсчёты сохранены в [pre-review](../evidence/pre-review/slo-analysis.json); анализатор выявил расхождение 83,075 с и пометил их непригодными для elapsed-time выводов; причина не установлена | +| 5 | При исключении сценарий мог зависнуть, ожидая бесконечные клиентские потоки | STOP устанавливается до выхода из ThreadPoolExecutor, включая исключения — [slo_incidents.py](../student-tests/slo_incidents.py) | + +Дополнительный прогон выявил дефект теста очереди: требовалось одновременное firing Fast и QueueLag, хотя их окна различаются. Кроме того, выбирался первый metric series старого outbox после пересоздания. [Неудачный прогон](../evidence/pre-review/ready-scenario-failed.json) сохранён; сценарий теперь проверяет наблюдение обоих сигналов и снятие каждого, а абсолютные DB-счётчики читает через max по экземплярам (они представляют одну БД). + +## Ограничения, которые нельзя скрыть оценкой + +- Прохождение runbook другим студентом не подтверждено. Подготовлен [протокол независимой проверки](slo-runbook.md#проверка-другим-студентом); собственная проверка не заменяет внешнюю. +- 99%/24 часа — учебная цель; короткий опыт не доказывает выполнение суточного SLO. Baseline оценивает наблюдавшиеся ложные тревоги только за этот интервал. +- Старые задания до миграции 009 не имеют восстановимых commit timestamps и исключены из новой когорты. Исторические Prometheus-ряды до миграции нельзя смешивать с новой методикой для доказательства SLO. +- PostgreSQL не хранит commit timestamps бессрочно. Outbox сохраняет их в таблицу; если потерял источник до материализации, unknown alert показывает неполноту данных. При перезапуске экспортёра `increase` может не учесть события в разрыве наблюдения. +- Histogram sample может потеряться при crash после commit ready; durable DB-когорта сохраняется. Collector с ограниченной очередью также может терять spans. Это не «нулевые потери». + +Источник границ commit timestamps: [PostgreSQL 17](https://www.postgresql.org/docs/17/functions-info.html#FUNCTIONS-COMMIT-TIMESTAMP). Источник критериев оценки — локальные README и contracts, ссылки выше. diff --git a/docs/slo-runbook.md b/docs/slo-runbook.md index d449ed1..f6e2c07 100644 --- a/docs/slo-runbook.md +++ b/docs/slo-runbook.md @@ -4,7 +4,7 @@ ## Что означает сигнал -Два учебных SLO на скользящие 24 часа: 99% бизнес-запросов API без 5xx и 99% принятых вложений становятся ready не позднее чем через 30 секунд после создания записи о задании. API исключает health и ошибки LB до попадания в приложение; 4xx остаются в знаменателе. Для готовности учтены только задания, достигшие 30-секундного возраста: неготовые, failed и завершённые позднее срока считаются плохими. Счётчики вычисляет outbox по PostgreSQL, поэтому при его недоступности данные не обновляются; события между последним scrape старого процесса и первым scrape нового могут не попасть в расчёт `increase`. Момент создания записи немного предшествует commit принятия, а время ready фиксируется внутри транзакции до её commit: это приближение к пользовательским границам. Порог 30 секунд — половина 60-секундного бюджета smoke-теста для одного файла при свободном worker; это учебная цель, не измеренное требование пользователей. Цель 99% также выбрана для демонстрации, а не выведена из пользовательского исследования. +Два учебных SLO на скользящие 24 часа: 99% бизнес-запросов API без 5xx и 99% принятых вложений становятся ready не позднее чем через 30 секунд после commit принятия задания. API исключает health и ошибки LB до попадания в приложение; 4xx остаются в знаменателе. Для готовности учтены только задания, достигшие 30-секундного возраста: неготовые, failed и завершённые позднее срока считаются плохими. Счётчики вычисляет outbox по PostgreSQL, поэтому при его недоступности данные не обновляются; события между последним scrape старого процесса и первым scrape нового могут не попасть в расчёт `increase`. Миграция 009 добавляет `attachment_slo`: моменты acceptance и первого ready берутся из commit timestamps PostgreSQL и сохраняются outbox. Результат зрелой когорты фиксируется один раз; retry/replay не меняют историю. Старые задания до миграции исключены, поскольку восстановить точные времена их приёма нельзя. `track_commit_timestamp=on` задан в Compose. PostgreSQL удаляет старые commit timestamps при vacuum; outbox материализует их раз в цикл. Если не успел, растёт `messenger_slo_unknown`, срабатывает `MessengerReadyTimestampUnknown`: неизвестные события нельзя считать доказательством выполнения SLO. Источник: [PostgreSQL 17, commit timestamps](https://www.postgresql.org/docs/17/functions-info.html#FUNCTIONS-COMMIT-TIMESTAMP). Порог 30 секунд — половина 60-секундного бюджета smoke-теста для одного файла при свободном worker; это учебная цель, не измеренное требование пользователей. Цель 99% также выбрана для демонстрации, а не выведена из пользовательского исследования. Error budget каждого SLO — `1 − 0,99 = 0,01`. Burn rate — доля плохих событий за окно / 0,01. Приближение расхода бюджета `burn × окно / 24 часа` предполагает стабильную интенсивность событий; при сильно меняющейся нагрузке бюджет, выраженный в доле событий за 24 часа, расходуется иначе. Значение 1 означает, что при сохранении текущей доли плохих событий все допустимые ошибки будут израсходованы за период SLO. @@ -22,13 +22,13 @@ Error budget каждого SLO — `1 − 0,99 = 0,01`. Burn rate — доля ## Ready burn / очередь -1. Проверьте `sum(increase(messenger_slo_ready_eligible_total{service_name="messenger-outbox"}[1m]))`, p95 ready и исходы jobs. Счётчик `messenger_slo_ready_bad_total` включает задания, нарушившие срок. Отсутствие событий не делает SLI хорошим: проверьте `messenger_queue_depth{service_name="messenger-outbox"}` и `messenger_queue_age_seconds{service_name="messenger-outbox"}`. +1. Сначала убедитесь, что `MessengerQueueTelemetryStale` и `MessengerReadyTimestampUnknown` не firing. Проверьте `sum(increase(messenger_slo_ready_eligible_total{service_name="messenger-outbox"}[1m]))`, p95 ready и исходы jobs. Счётчик `messenger_slo_ready_bad_total` включает задания, нарушившие срок. Отсутствие событий не делает SLI хорошим: проверьте `messenger_queue_depth{service_name="messenger-outbox"}` и `messenger_queue_age_seconds{service_name="messenger-outbox"}`. 2. В Loki откройте `{service_name="messenger-worker"}`, найдите событие и trace ID. В Tempo проверьте цепочку upload → producer → consumer → S3/DB; посмотрите busy/capacity и состояние broker/worker через `docker compose ps`. 3. В учебном опыте запустите остановленный worker, наблюдайте уменьшение глубины/возраста и новые ready. Проверьте resolved и метаданные вложений. Не удаляйте volumes и не запускайте replay вслепую. ## Повторение экспериментов -`python3 student-tests/slo_incidents.py` создаёт синтетические данные, ведёт три клиента и выполняет краткую и длительную блокировку таблицы `users`. Скрипт пишет временные отметки и наблюдения в [evidence/slo-incidents.json](../evidence/slo-incidents.json), не сохраняет tokens; опрашивает alert примерно каждые 5 секунд. `python3 student-tests/slo_analyze.py` строит сводку, включая срабатывания уже после завершения воздействия. Тест намеренно ухудшает локальный API и занимает несколько минут. Первая отметка воздействия — момент, когда скрипт увидел granted `AccessExclusiveLock`; окончание — возврат команды транзакции, поэтому обе оценки имеют задержку наблюдения. Длительности считаются по монотонным часам: при выполненном опыте системное UTC скачкообразно изменилось. Аварийное прерывание клиентской команды может оставить server-side `pg_sleep` до окончания заданного ограниченного времени; проверьте состояние блокировки. +`python3 student-tests/slo_incidents.py` создаёт синтетические данные, ведёт три клиента и выполняет краткую и длительную блокировку таблицы `users`. Скрипт пишет временные отметки и наблюдения в [evidence/slo-incidents.json](../evidence/slo-incidents.json), не сохраняет tokens; опрашивает alert примерно каждые 5 секунд. `python3 student-tests/slo_analyze.py` строит сводку, включая срабатывания уже после завершения воздействия. Тест намеренно ухудшает локальный API и занимает несколько минут. Первая отметка воздействия — момент, когда скрипт увидел granted `AccessExclusiveLock`; окончание — возврат команды транзакции, поэтому обе оценки имеют задержку наблюдения. Анализатор сравнивает приращения UTC и монотонных часов и сохраняет максимальный фактический интервал опроса. Если `timing_quality.valid_for_elapsed_time_claims=false`, опыт повторяют; причину расхождения часов по этим данным установить нельзя. Аварийное прерывание клиентской команды может оставить server-side `pg_sleep` до окончания заданного ограниченного времени; проверьте состояние блокировки. `python3 student-tests/slo_ready_incident.py` проверяет отсутствие незавершённых заданий, останавливает worker, принимает восемь синтетических файлов и восстанавливает worker в `finally`. До запуска убедитесь, что API и worker свободны. Результат — [evidence/slo-ready-incident.json](../evidence/slo-ready-incident.json). Снятие alerts при малом количестве новых событий может означать недостаток событий в коротком окне, а не доказательство выполнения 24-часового SLO. @@ -39,3 +39,7 @@ docker run --rm -v "$PWD:/workspace:ro" --entrypoint promtool prom/prometheus:v3 ``` В runbook зафиксируйте фазу, UTC начала/окончания, первый firing и первое наблюдение после исчезновения alert. Первый и последний интервалы ограничены частотой опроса, а отсутствие ложного firing в одном baseline не оценивает долгосрочный false-positive rate. + +## Проверка другим студентом + +Проверяющий не получает команду инъекции заранее. Автор запускает один из двух ограниченных сценариев; проверяющий получает время и имя alert, затем проходит шаги выше. В отдельный протокол записывают: имя проверяющего, UTC, запрос PromQL/LogQL, trace ID, медленный span или отсутствие consumer, диагноз, действие восстановления и свидетельство resolved. Совпадение диагноза с инъекцией проверяют после записи. Пока такого заполненного протокола нет, независимое прохождение runbook не подтверждено. diff --git a/evidence/pre-review/ready-scenario-failed.json b/evidence/pre-review/ready-scenario-failed.json new file mode 100644 index 0000000..feaff5a --- /dev/null +++ b/evidence/pre-review/ready-scenario-failed.json @@ -0,0 +1,300 @@ +{ + "samples": [ + { + "time": "2026-11-06T07:03:03.661461+00:00", + "monotonic": 51955.506616, + "phase": "baseline", + "firing": [], + "all_firing": [], + "queue_depth": 0.0, + "queue_age": 0.0, + "eligible": 9.0, + "bad": 8.0 + }, + { + "time": "2026-11-06T07:03:09.013918+00:00", + "monotonic": 51960.859020333, + "phase": "worker_stopped", + "firing": [], + "all_firing": [], + "queue_depth": 0.0, + "queue_age": 0.0, + "eligible": 9.0, + "bad": 8.0 + }, + { + "time": "2026-11-06T07:03:14.036654+00:00", + "monotonic": 51965.881710833, + "phase": "worker_stopped", + "firing": [], + "all_firing": [], + "queue_depth": 0.0, + "queue_age": 0.0, + "eligible": 9.0, + "bad": 8.0 + }, + { + "time": "2026-11-06T07:03:19.064110+00:00", + "monotonic": 51970.909114958, + "phase": "worker_stopped", + "firing": [], + "all_firing": [], + "queue_depth": 0.0, + "queue_age": 0.0, + "eligible": 9.0, + "bad": 8.0 + }, + { + "time": "2026-11-06T07:03:24.098276+00:00", + "monotonic": 51975.943249958, + "phase": "worker_stopped", + "firing": [], + "all_firing": [], + "queue_depth": 0.0, + "queue_age": 0.0, + "eligible": 9.0, + "bad": 8.0 + }, + { + "time": "2026-11-06T07:03:29.150984+00:00", + "monotonic": 51980.9959115, + "phase": "worker_stopped", + "firing": [], + "all_firing": [], + "queue_depth": 0.0, + "queue_age": 0.0, + "eligible": 9.0, + "bad": 8.0 + }, + { + "time": "2026-11-06T07:03:34.199153+00:00", + "monotonic": 51986.044040208, + "phase": "worker_stopped", + "firing": [], + "all_firing": [], + "queue_depth": 0.0, + "queue_age": 0.0, + "eligible": 9.0, + "bad": 8.0 + }, + { + "time": "2026-11-06T07:03:39.245809+00:00", + "monotonic": 51991.09063275, + "phase": "worker_stopped", + "firing": [], + "all_firing": [], + "queue_depth": 0.0, + "queue_age": 0.0, + "eligible": 9.0, + "bad": 8.0 + }, + { + "time": "2026-11-06T07:03:44.282804+00:00", + "monotonic": 51996.127578458, + "phase": "worker_stopped", + "firing": [], + "all_firing": [], + "queue_depth": 0.0, + "queue_age": 0.0, + "eligible": 9.0, + "bad": 8.0 + }, + { + "time": "2026-11-06T07:03:49.317995+00:00", + "monotonic": 52001.162700583, + "phase": "worker_stopped", + "firing": [ + "MessengerReadyBurnFast", + "MessengerReadyBurnSlow" + ], + "all_firing": [ + "MessengerReadyBurnFast", + "MessengerReadyBurnSlow" + ], + "queue_depth": 0.0, + "queue_age": 0.0, + "eligible": 9.0, + "bad": 8.0 + }, + { + "time": "2026-11-06T07:03:54.345636+00:00", + "monotonic": 52006.190306166, + "phase": "worker_stopped", + "firing": [ + "MessengerReadyBurnFast", + "MessengerReadyBurnSlow" + ], + "all_firing": [ + "MessengerReadyBurnFast", + "MessengerReadyBurnSlow" + ], + "queue_depth": 0.0, + "queue_age": 0.0, + "eligible": 9.0, + "bad": 8.0 + }, + { + "time": "2026-11-06T07:03:59.376176+00:00", + "monotonic": 52011.220777958, + "phase": "worker_stopped", + "firing": [ + "MessengerReadyBurnFast", + "MessengerReadyBurnSlow" + ], + "all_firing": [ + "MessengerReadyBurnFast", + "MessengerReadyBurnSlow" + ], + "queue_depth": 0.0, + "queue_age": 0.0, + "eligible": 9.0, + "bad": 8.0 + }, + { + "time": "2026-11-06T07:04:04.416058+00:00", + "monotonic": 52016.260635708, + "phase": "worker_stopped", + "firing": [ + "MessengerReadyBurnSlow" + ], + "all_firing": [ + "MessengerQueueLag", + "MessengerReadyBurnSlow" + ], + "queue_depth": 0.0, + "queue_age": 0.0, + "eligible": 9.0, + "bad": 8.0 + }, + { + "time": "2026-11-06T07:04:09.450875+00:00", + "monotonic": 52021.295390541, + "phase": "worker_stopped", + "firing": [ + "MessengerReadyBurnSlow" + ], + "all_firing": [ + "MessengerQueueLag", + "MessengerReadyBurnSlow" + ], + "queue_depth": 0.0, + "queue_age": 0.0, + "eligible": 9.0, + "bad": 8.0 + }, + { + "time": "2026-11-06T07:04:14.481279+00:00", + "monotonic": 52026.325748, + "phase": "worker_stopped", + "firing": [ + "MessengerReadyBurnSlow" + ], + "all_firing": [ + "MessengerQueueLag", + "MessengerReadyBurnSlow" + ], + "queue_depth": 0.0, + "queue_age": 0.0, + "eligible": 9.0, + "bad": 8.0 + }, + { + "time": "2026-11-06T07:04:19.519443+00:00", + "monotonic": 52031.36385225, + "phase": "worker_stopped", + "firing": [ + "MessengerReadyBurnSlow" + ], + "all_firing": [ + "MessengerQueueLag", + "MessengerReadyBurnSlow" + ], + "queue_depth": 0.0, + "queue_age": 0.0, + "eligible": 9.0, + "bad": 8.0 + }, + { + "time": "2026-11-06T07:04:24.550837+00:00", + "monotonic": 52036.395194208, + "phase": "worker_stopped", + "firing": [ + "MessengerReadyBurnSlow" + ], + "all_firing": [ + "MessengerQueueLag", + "MessengerReadyBurnSlow" + ], + "queue_depth": 0.0, + "queue_age": 0.0, + "eligible": 9.0, + "bad": 8.0 + }, + { + "time": "2026-11-06T07:04:29.576411+00:00", + "monotonic": 52041.420720208, + "phase": "worker_stopped", + "firing": [ + "MessengerReadyBurnSlow" + ], + "all_firing": [ + "MessengerQueueLag", + "MessengerReadyBurnSlow" + ], + "queue_depth": 0.0, + "queue_age": 0.0, + "eligible": 9.0, + "bad": 8.0 + }, + { + "time": "2026-11-06T07:04:34.607663+00:00", + "monotonic": 52046.451919791, + "phase": "worker_stopped", + "firing": [ + "MessengerReadyBurnSlow" + ], + "all_firing": [ + "MessengerQueueLag", + "MessengerReadyBurnSlow" + ], + "queue_depth": 0.0, + "queue_age": 0.0, + "eligible": 9.0, + "bad": 8.0 + }, + { + "time": "2026-11-06T07:04:39.633257+00:00", + "monotonic": 52051.477466458, + "phase": "worker_stopped", + "firing": [ + "MessengerReadyBurnSlow" + ], + "all_firing": [ + "MessengerQueueLag", + "MessengerReadyBurnSlow" + ], + "queue_depth": 0.0, + "queue_age": 0.0, + "eligible": 9.0, + "bad": 8.0 + }, + { + "time": "2026-11-06T07:04:44.663734+00:00", + "monotonic": 52056.50792775, + "phase": "worker_stopped", + "firing": [], + "all_firing": [ + "MessengerQueueLag" + ], + "queue_depth": 0.0, + "queue_age": 0.0, + "eligible": 9.0, + "bad": 8.0 + } + ], + "jobs": 8, + "accepted_at": { + "time": "2026-11-06T07:03:09.009364+00:00", + "monotonic": 51960.854465416 + } +} diff --git a/evidence/pre-review/slo-analysis.json b/evidence/pre-review/slo-analysis.json new file mode 100644 index 0000000..180188b --- /dev/null +++ b/evidence/pre-review/slo-analysis.json @@ -0,0 +1,59 @@ +{ + "sampling_interval_seconds": 5, + "monitor_errors": 0, + "phases": { + "burst": { + "observed_lock_seconds": 75.033, + "alerts": { + "MessengerApiBurnFast": { + "active_before_lock": false, + "first_firing": "2026-11-05T19:30:49.560423+00:00", + "first_firing_after_incident": false, + "detection_seconds": 55.088, + "first_clear": "2026-11-05T19:31:24.687321+00:00", + "recovery_seconds": 15.181 + }, + "MessengerApiBurnSlow": { + "active_before_lock": false, + "first_firing": "2026-11-05T19:31:14.651231+00:00", + "first_firing_after_incident": true, + "detection_seconds": 80.179, + "first_clear": "2026-11-05T19:31:24.687321+00:00", + "recovery_seconds": 15.181 + } + } + }, + "sustained": { + "observed_lock_seconds": 180.134, + "alerts": { + "MessengerApiBurnFast": { + "active_before_lock": false, + "first_firing": "2026-11-05T19:33:40.096081+00:00", + "first_firing_after_incident": false, + "detection_seconds": 50.278, + "first_clear": "2026-11-05T19:36:10.387240+00:00", + "recovery_seconds": 10.648 + }, + "MessengerApiBurnSlow": { + "active_before_lock": false, + "first_firing": "2026-11-05T19:34:20.245337+00:00", + "first_firing_after_incident": false, + "detection_seconds": 90.428, + "first_clear": "2026-11-05T19:37:53.534744+00:00", + "recovery_seconds": 30.724 + } + } + } + }, + "timing_quality": { + "max_wall_monotonic_disagreement_seconds": 83.075, + "max_observed_sampling_gap_seconds": 5.029, + "valid_for_elapsed_time_claims": false, + "note": "Disagreement does not identify its cause; rerun if invalid. Precision is bounded by actual gaps, not the configured sleep." + }, + "baseline": { + "samples": 13, + "samples_with_burn_alert": 0, + "scope": "one observed healthy baseline, not a long-term false-positive rate" + } +} diff --git a/evidence/pre-review/slo-incidents.json b/evidence/pre-review/slo-incidents.json new file mode 100644 index 0000000..742ed8f --- /dev/null +++ b/evidence/pre-review/slo-incidents.json @@ -0,0 +1,1170 @@ +{ + "description": "controlled DB table lock; three synthetic clients; 5s alert sampling", + "periods": { + "burst": { + "lock_observed": { + "time": "2026-11-05T19:29:54.472744+00:00", + "monotonic": 43527.079238041, + "command_started": { + "time": "2026-11-05T19:29:54.163828+00:00", + "monotonic": 43526.770364958 + } + }, + "command_completed": { + "time": "2026-11-05T19:31:09.505932+00:00", + "monotonic": 43602.112587416 + } + }, + "sustained": { + "lock_observed": { + "time": "2026-11-05T19:32:49.817996+00:00", + "monotonic": 43702.424801208, + "command_started": { + "time": "2026-11-05T19:32:49.516057+00:00", + "monotonic": 43702.122908708 + } + }, + "command_completed": { + "time": "2026-11-05T19:35:59.744503+00:00", + "monotonic": 43882.558427291 + } + } + }, + "samples": [ + { + "time": "2026-11-05T19:28:54.176617+00:00", + "monotonic": 43466.783021166, + "phase": "baseline", + "counts": { + "ok": 1, + "failed": 0 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:28:59.184768+00:00", + "monotonic": 43471.791177208, + "phase": "baseline", + "counts": { + "ok": 15, + "failed": 0 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:29:04.205356+00:00", + "monotonic": 43476.8117975, + "phase": "baseline", + "counts": { + "ok": 30, + "failed": 0 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:29:09.226269+00:00", + "monotonic": 43481.832718708, + "phase": "baseline", + "counts": { + "ok": 45, + "failed": 0 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:29:14.243046+00:00", + "monotonic": 43486.849499708, + "phase": "baseline", + "counts": { + "ok": 60, + "failed": 0 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:29:19.249330+00:00", + "monotonic": 43491.855772208, + "phase": "baseline", + "counts": { + "ok": 75, + "failed": 0 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:29:24.265281+00:00", + "monotonic": 43496.871750375, + "phase": "baseline", + "counts": { + "ok": 90, + "failed": 0 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:29:29.284910+00:00", + "monotonic": 43501.891384583, + "phase": "baseline", + "counts": { + "ok": 105, + "failed": 0 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:29:34.295325+00:00", + "monotonic": 43506.901793458, + "phase": "baseline", + "counts": { + "ok": 120, + "failed": 0 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:29:39.306441+00:00", + "monotonic": 43511.91291825, + "phase": "baseline", + "counts": { + "ok": 135, + "failed": 0 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:29:44.313768+00:00", + "monotonic": 43516.920254333, + "phase": "baseline", + "counts": { + "ok": 150, + "failed": 0 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:29:49.331915+00:00", + "monotonic": 43521.938423708, + "phase": "baseline", + "counts": { + "ok": 165, + "failed": 0 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:29:54.351411+00:00", + "monotonic": 43526.957913708, + "phase": "baseline", + "counts": { + "ok": 180, + "failed": 0 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:29:59.372201+00:00", + "monotonic": 43531.978732458, + "phase": "burst", + "counts": { + "ok": 180, + "failed": 0 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:30:04.386019+00:00", + "monotonic": 43536.992552291, + "phase": "burst", + "counts": { + "ok": 180, + "failed": 3 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:30:09.401930+00:00", + "monotonic": 43542.0084625, + "phase": "burst", + "counts": { + "ok": 180, + "failed": 6 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:30:14.419029+00:00", + "monotonic": 43547.025585291, + "phase": "burst", + "counts": { + "ok": 180, + "failed": 9 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:30:19.433563+00:00", + "monotonic": 43552.040130916, + "phase": "burst", + "counts": { + "ok": 180, + "failed": 12 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:30:24.450146+00:00", + "monotonic": 43557.05672225, + "phase": "burst", + "counts": { + "ok": 180, + "failed": 15 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:30:29.473233+00:00", + "monotonic": 43562.079822583, + "phase": "burst", + "counts": { + "ok": 180, + "failed": 15 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:30:34.493955+00:00", + "monotonic": 43567.1005505, + "phase": "burst", + "counts": { + "ok": 180, + "failed": 18 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:30:39.519370+00:00", + "monotonic": 43572.125976166, + "phase": "burst", + "counts": { + "ok": 180, + "failed": 21 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:30:44.535535+00:00", + "monotonic": 43577.142128416, + "phase": "burst", + "counts": { + "ok": 180, + "failed": 24 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:30:49.560423+00:00", + "monotonic": 43582.167042875, + "phase": "burst", + "counts": { + "ok": 180, + "failed": 27 + }, + "firing": [ + "MessengerApiBurnFast" + ] + }, + { + "time": "2026-11-05T19:30:54.583572+00:00", + "monotonic": 43587.190215125, + "phase": "burst", + "counts": { + "ok": 180, + "failed": 30 + }, + "firing": [ + "MessengerApiBurnFast" + ] + }, + { + "time": "2026-11-05T19:30:59.600250+00:00", + "monotonic": 43592.206889708, + "phase": "burst", + "counts": { + "ok": 180, + "failed": 30 + }, + "firing": [ + "MessengerApiBurnFast" + ] + }, + { + "time": "2026-11-05T19:31:04.616531+00:00", + "monotonic": 43597.223167708, + "phase": "burst", + "counts": { + "ok": 180, + "failed": 33 + }, + "firing": [ + "MessengerApiBurnFast" + ] + }, + { + "time": "2026-11-05T19:31:09.632643+00:00", + "monotonic": 43602.239279708, + "phase": "burst_recovery", + "counts": { + "ok": 183, + "failed": 36 + }, + "firing": [ + "MessengerApiBurnFast" + ] + }, + { + "time": "2026-11-05T19:31:14.651231+00:00", + "monotonic": 43607.257891625, + "phase": "burst_recovery", + "counts": { + "ok": 198, + "failed": 36 + }, + "firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow" + ] + }, + { + "time": "2026-11-05T19:31:19.663273+00:00", + "monotonic": 43612.269926416, + "phase": "burst_recovery", + "counts": { + "ok": 211, + "failed": 36 + }, + "firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow" + ] + }, + { + "time": "2026-11-05T19:31:24.687321+00:00", + "monotonic": 43617.294002291, + "phase": "burst_recovery", + "counts": { + "ok": 225, + "failed": 36 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:31:29.708468+00:00", + "monotonic": 43622.315158041, + "phase": "burst_recovery", + "counts": { + "ok": 240, + "failed": 36 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:31:34.728166+00:00", + "monotonic": 43627.334856708, + "phase": "burst_recovery", + "counts": { + "ok": 255, + "failed": 36 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:31:39.733767+00:00", + "monotonic": 43632.340459958, + "phase": "burst_recovery", + "counts": { + "ok": 270, + "failed": 36 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:31:44.755480+00:00", + "monotonic": 43637.36218675, + "phase": "burst_recovery", + "counts": { + "ok": 285, + "failed": 36 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:31:49.769677+00:00", + "monotonic": 43642.376390333, + "phase": "burst_recovery", + "counts": { + "ok": 300, + "failed": 36 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:31:54.778133+00:00", + "monotonic": 43647.384860875, + "phase": "burst_recovery", + "counts": { + "ok": 315, + "failed": 36 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:31:59.788234+00:00", + "monotonic": 43652.394963833, + "phase": "burst_recovery", + "counts": { + "ok": 330, + "failed": 36 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:32:04.797259+00:00", + "monotonic": 43657.403998, + "phase": "burst_recovery", + "counts": { + "ok": 345, + "failed": 36 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:32:09.813581+00:00", + "monotonic": 43662.42033325, + "phase": "burst_recovery", + "counts": { + "ok": 360, + "failed": 36 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:32:14.823961+00:00", + "monotonic": 43667.430715458, + "phase": "burst_recovery", + "counts": { + "ok": 375, + "failed": 36 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:32:19.834831+00:00", + "monotonic": 43672.441596833, + "phase": "burst_recovery", + "counts": { + "ok": 390, + "failed": 36 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:32:24.847866+00:00", + "monotonic": 43677.454636958, + "phase": "burst_recovery", + "counts": { + "ok": 405, + "failed": 36 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:32:29.862747+00:00", + "monotonic": 43682.469528375, + "phase": "burst_recovery", + "counts": { + "ok": 417, + "failed": 36 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:32:34.870564+00:00", + "monotonic": 43687.47735275, + "phase": "burst_recovery", + "counts": { + "ok": 432, + "failed": 36 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:32:39.880308+00:00", + "monotonic": 43692.487106, + "phase": "burst_recovery", + "counts": { + "ok": 447, + "failed": 36 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:32:44.899942+00:00", + "monotonic": 43697.506751, + "phase": "burst_recovery", + "counts": { + "ok": 462, + "failed": 36 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:32:49.916853+00:00", + "monotonic": 43702.523669458, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 36 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:32:54.942437+00:00", + "monotonic": 43707.549288958, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 36 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:32:59.957394+00:00", + "monotonic": 43712.564228083, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 39 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:33:04.970687+00:00", + "monotonic": 43717.577532333, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 42 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:33:09.993536+00:00", + "monotonic": 43722.600420875, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 45 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:33:15.009372+00:00", + "monotonic": 43727.616250583, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 48 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:33:20.031828+00:00", + "monotonic": 43732.638718, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 51 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:33:25.051997+00:00", + "monotonic": 43737.658880125, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 51 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:33:30.067799+00:00", + "monotonic": 43742.674685791, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 54 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:33:35.075453+00:00", + "monotonic": 43747.682349166, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 57 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:33:40.096081+00:00", + "monotonic": 43752.703001416, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 60 + }, + "firing": [ + "MessengerApiBurnFast" + ] + }, + { + "time": "2026-11-05T19:33:45.119089+00:00", + "monotonic": 43757.7260235, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 63 + }, + "firing": [ + "MessengerApiBurnFast" + ] + }, + { + "time": "2026-11-05T19:33:50.133709+00:00", + "monotonic": 43762.740644916, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 66 + }, + "firing": [ + "MessengerApiBurnFast" + ] + }, + { + "time": "2026-11-05T19:33:55.161233+00:00", + "monotonic": 43767.768187291, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 66 + }, + "firing": [ + "MessengerApiBurnFast" + ] + }, + { + "time": "2026-11-05T19:34:00.176785+00:00", + "monotonic": 43772.783745208, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 69 + }, + "firing": [ + "MessengerApiBurnFast" + ] + }, + { + "time": "2026-11-05T19:34:05.196858+00:00", + "monotonic": 43777.803812291, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 72 + }, + "firing": [ + "MessengerApiBurnFast" + ] + }, + { + "time": "2026-11-05T19:34:10.212935+00:00", + "monotonic": 43782.819901041, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 75 + }, + "firing": [ + "MessengerApiBurnFast" + ] + }, + { + "time": "2026-11-05T19:34:15.231470+00:00", + "monotonic": 43787.838454541, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 78 + }, + "firing": [ + "MessengerApiBurnFast" + ] + }, + { + "time": "2026-11-05T19:34:20.245337+00:00", + "monotonic": 43792.852329875, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 81 + }, + "firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow" + ] + }, + { + "time": "2026-11-05T19:34:25.257984+00:00", + "monotonic": 43797.864980958, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 81 + }, + "firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow" + ] + }, + { + "time": "2026-11-05T19:34:30.282024+00:00", + "monotonic": 43802.889036875, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 84 + }, + "firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow" + ] + }, + { + "time": "2026-11-05T19:34:35.303328+00:00", + "monotonic": 43807.910348833, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 87 + }, + "firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow" + ] + }, + { + "time": "2026-11-05T19:34:40.328342+00:00", + "monotonic": 43812.935370625, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 90 + }, + "firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow" + ] + }, + { + "time": "2026-11-05T19:34:45.343604+00:00", + "monotonic": 43817.950633291, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 93 + }, + "firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow" + ] + }, + { + "time": "2026-11-05T19:34:50.359471+00:00", + "monotonic": 43822.966518166, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 96 + }, + "firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow" + ] + }, + { + "time": "2026-11-05T19:34:55.367578+00:00", + "monotonic": 43827.974616416, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 96 + }, + "firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow" + ] + }, + { + "time": "2026-11-05T19:35:00.386215+00:00", + "monotonic": 43832.993284208, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 99 + }, + "firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow" + ] + }, + { + "time": "2026-11-05T19:35:05.400579+00:00", + "monotonic": 43838.0076475, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 102 + }, + "firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow" + ] + }, + { + "time": "2026-11-05T19:35:10.422157+00:00", + "monotonic": 43843.0292365, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 105 + }, + "firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow" + ] + }, + { + "time": "2026-11-05T19:35:15.442075+00:00", + "monotonic": 43848.049173416, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 108 + }, + "firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow" + ] + }, + { + "time": "2026-11-05T19:35:20.452042+00:00", + "monotonic": 43853.05914025, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 111 + }, + "firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow" + ] + }, + { + "time": "2026-11-05T19:35:25.461182+00:00", + "monotonic": 43858.068280916, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 111 + }, + "firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow" + ] + }, + { + "time": "2026-11-05T19:35:30.480197+00:00", + "monotonic": 43863.087333625, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 114 + }, + "firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow" + ] + }, + { + "time": "2026-11-05T19:35:35.504075+00:00", + "monotonic": 43868.111188291, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 117 + }, + "firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow" + ] + }, + { + "time": "2026-11-05T19:35:40.521486+00:00", + "monotonic": 43873.129961791, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 120 + }, + "firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow" + ] + }, + { + "time": "2026-11-05T19:35:55.348285+00:00", + "monotonic": 43878.158763833, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 123 + }, + "firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow" + ] + }, + { + "time": "2026-11-05T19:36:00.359177+00:00", + "monotonic": 43883.173501041, + "phase": "sustained_recovery", + "counts": { + "ok": 480, + "failed": 123 + }, + "firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow" + ] + }, + { + "time": "2026-11-05T19:36:05.367876+00:00", + "monotonic": 43888.184996, + "phase": "sustained_recovery", + "counts": { + "ok": 495, + "failed": 123 + }, + "firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow" + ] + }, + { + "time": "2026-11-05T19:36:10.387240+00:00", + "monotonic": 43893.206441208, + "phase": "sustained_recovery", + "counts": { + "ok": 510, + "failed": 123 + }, + "firing": [ + "MessengerApiBurnSlow" + ] + }, + { + "time": "2026-11-05T19:36:15.397936+00:00", + "monotonic": 43898.218563625, + "phase": "sustained_recovery", + "counts": { + "ok": 525, + "failed": 123 + }, + "firing": [ + "MessengerApiBurnSlow" + ] + }, + { + "time": "2026-11-05T19:36:20.421727+00:00", + "monotonic": 43903.243449875, + "phase": "sustained_recovery", + "counts": { + "ok": 540, + "failed": 123 + }, + "firing": [ + "MessengerApiBurnSlow" + ] + }, + { + "time": "2026-11-05T19:37:48.510428+00:00", + "monotonic": 43908.257572375, + "phase": "sustained_recovery", + "counts": { + "ok": 555, + "failed": 123 + }, + "firing": [ + "MessengerApiBurnSlow" + ] + }, + { + "time": "2026-11-05T19:37:53.534744+00:00", + "monotonic": 43913.282484708, + "phase": "sustained_recovery", + "counts": { + "ok": 570, + "failed": 123 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:37:58.549296+00:00", + "monotonic": 43918.29743975, + "phase": "sustained_recovery", + "counts": { + "ok": 585, + "failed": 123 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:38:03.567430+00:00", + "monotonic": 43923.315891, + "phase": "sustained_recovery", + "counts": { + "ok": 600, + "failed": 123 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:38:08.593041+00:00", + "monotonic": 43928.341731291, + "phase": "sustained_recovery", + "counts": { + "ok": 615, + "failed": 123 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:38:13.607305+00:00", + "monotonic": 43933.356177, + "phase": "sustained_recovery", + "counts": { + "ok": 630, + "failed": 123 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:38:18.617266+00:00", + "monotonic": 43938.366246166, + "phase": "sustained_recovery", + "counts": { + "ok": 642, + "failed": 123 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:38:23.636630+00:00", + "monotonic": 43943.385724875, + "phase": "sustained_recovery", + "counts": { + "ok": 657, + "failed": 123 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:38:28.654169+00:00", + "monotonic": 43948.403359125, + "phase": "sustained_recovery", + "counts": { + "ok": 672, + "failed": 123 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:38:33.670193+00:00", + "monotonic": 43953.419418, + "phase": "sustained_recovery", + "counts": { + "ok": 687, + "failed": 123 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:38:38.676034+00:00", + "monotonic": 43958.425318541, + "phase": "sustained_recovery", + "counts": { + "ok": 702, + "failed": 123 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:38:43.699279+00:00", + "monotonic": 43963.448615416, + "phase": "sustained_recovery", + "counts": { + "ok": 717, + "failed": 123 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:38:48.718747+00:00", + "monotonic": 43968.468122083, + "phase": "sustained_recovery", + "counts": { + "ok": 732, + "failed": 123 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:38:53.742289+00:00", + "monotonic": 43973.491697375, + "phase": "sustained_recovery", + "counts": { + "ok": 747, + "failed": 123 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:38:58.757203+00:00", + "monotonic": 43978.50662725, + "phase": "sustained_recovery", + "counts": { + "ok": 762, + "failed": 123 + }, + "firing": [] + } + ], + "end_time": "2026-11-05T19:39:02.820853+00:00", + "final_counts": { + "ok": 774, + "failed": 123 + } +} diff --git a/evidence/pre-review/slo-ready-incident.json b/evidence/pre-review/slo-ready-incident.json new file mode 100644 index 0000000..5eff503 --- /dev/null +++ b/evidence/pre-review/slo-ready-incident.json @@ -0,0 +1,211 @@ +{ + "samples": [ + { + "time": "2026-11-06T06:18:43.857552+00:00", + "monotonic": 49295.73000975, + "phase": "baseline", + "firing": [], + "eligible": 9.0, + "bad": 4.0 + }, + { + "time": "2026-11-06T06:18:49.195942+00:00", + "monotonic": 49301.068345333, + "phase": "worker_stopped", + "firing": [], + "eligible": 9.0, + "bad": 4.0 + }, + { + "time": "2026-11-06T06:18:54.211236+00:00", + "monotonic": 49306.083585125, + "phase": "worker_stopped", + "firing": [], + "eligible": 9.0, + "bad": 4.0 + }, + { + "time": "2026-11-06T06:18:59.238528+00:00", + "monotonic": 49311.110847333, + "phase": "worker_stopped", + "firing": [], + "eligible": 9.0, + "bad": 4.0 + }, + { + "time": "2026-11-06T06:19:04.276851+00:00", + "monotonic": 49316.149107833, + "phase": "worker_stopped", + "firing": [], + "eligible": 9.0, + "bad": 4.0 + }, + { + "time": "2026-11-06T06:19:09.316299+00:00", + "monotonic": 49321.18853025, + "phase": "worker_stopped", + "firing": [], + "eligible": 9.0, + "bad": 4.0 + }, + { + "time": "2026-11-06T06:19:14.361014+00:00", + "monotonic": 49326.233190833, + "phase": "worker_stopped", + "firing": [], + "eligible": 9.0, + "bad": 4.0 + }, + { + "time": "2026-11-06T06:19:19.399292+00:00", + "monotonic": 49331.271419, + "phase": "worker_stopped", + "firing": [], + "eligible": 9.0, + "bad": 4.0 + }, + { + "time": "2026-11-06T06:19:24.439389+00:00", + "monotonic": 49336.31146675, + "phase": "worker_stopped", + "firing": [], + "eligible": 9.0, + "bad": 4.0 + }, + { + "time": "2026-11-06T06:19:29.469853+00:00", + "monotonic": 49341.341864875, + "phase": "worker_stopped", + "firing": [ + "MessengerReadyBurnFast", + "MessengerReadyBurnSlow" + ], + "eligible": 17.0, + "bad": 12.0 + }, + { + "time": "2026-11-06T06:19:35.732760+00:00", + "monotonic": 49347.604713666, + "phase": "recovery", + "firing": [ + "MessengerReadyBurnFast", + "MessengerReadyBurnSlow" + ], + "eligible": 17.0, + "bad": 12.0 + }, + { + "time": "2026-11-06T06:19:40.766904+00:00", + "monotonic": 49352.638829583, + "phase": "recovery", + "firing": [ + "MessengerReadyBurnFast", + "MessengerReadyBurnSlow" + ], + "eligible": 17.0, + "bad": 12.0 + }, + { + "time": "2026-11-06T06:19:45.800827+00:00", + "monotonic": 49357.672704333, + "phase": "recovery", + "firing": [ + "MessengerReadyBurnSlow" + ], + "eligible": 17.0, + "bad": 12.0 + }, + { + "time": "2026-11-06T06:19:50.839444+00:00", + "monotonic": 49362.711278166, + "phase": "recovery", + "firing": [ + "MessengerReadyBurnSlow" + ], + "eligible": 17.0, + "bad": 12.0 + }, + { + "time": "2026-11-06T06:19:55.882108+00:00", + "monotonic": 49367.753864791, + "phase": "recovery", + "firing": [ + "MessengerReadyBurnSlow" + ], + "eligible": 17.0, + "bad": 12.0 + }, + { + "time": "2026-11-06T06:20:00.935125+00:00", + "monotonic": 49372.806871, + "phase": "recovery", + "firing": [ + "MessengerReadyBurnSlow" + ], + "eligible": 17.0, + "bad": 12.0 + }, + { + "time": "2026-11-06T06:20:05.976123+00:00", + "monotonic": 49377.847798625, + "phase": "recovery", + "firing": [ + "MessengerReadyBurnSlow" + ], + "eligible": 17.0, + "bad": 12.0 + }, + { + "time": "2026-11-06T06:20:11.006201+00:00", + "monotonic": 49382.877811166, + "phase": "recovery", + "firing": [ + "MessengerReadyBurnSlow" + ], + "eligible": 17.0, + "bad": 12.0 + }, + { + "time": "2026-11-06T06:20:16.034665+00:00", + "monotonic": 49387.906272875, + "phase": "recovery", + "firing": [ + "MessengerReadyBurnSlow" + ], + "eligible": 17.0, + "bad": 12.0 + }, + { + "time": "2026-11-06T06:20:21.073601+00:00", + "monotonic": 49392.945128916, + "phase": "recovery", + "firing": [ + "MessengerReadyBurnSlow" + ], + "eligible": 17.0, + "bad": 12.0 + }, + { + "time": "2026-11-06T06:20:26.112334+00:00", + "monotonic": 49397.983810916, + "phase": "recovery", + "firing": [], + "eligible": 17.0, + "bad": 12.0 + } + ], + "jobs": 8, + "accepted_at": { + "time": "2026-11-06T06:18:49.190891+00:00", + "monotonic": 49301.063291333 + }, + "worker_started": { + "time": "2026-11-06T06:19:31.951591+00:00", + "monotonic": 49343.823643916 + }, + "all_ready": { + "time": "2026-11-06T06:19:35.725614+00:00", + "monotonic": 49347.597564333 + }, + "result": "PASS" +} diff --git a/evidence/review-checks.json b/evidence/review-checks.json new file mode 100644 index 0000000..cd47500 --- /dev/null +++ b/evidence/review-checks.json @@ -0,0 +1,109 @@ +{ + "utc": "2026-11-06T07:13:15.774667+00:00", + "checks": [ + { + "command": [ + "make", + "check" + ], + "exit_code": 0, + "output": "python3 scripts/check_contract.py\nPASS: offline skeleton structure, local refs and Python syntax; application NOT tested.\n" + }, + { + "command": [ + "make", + "validate", + "PYTHON=.venv/bin/python" + ], + "exit_code": 0, + "output": ".venv/bin/python scripts/validate_openapi.py\nPASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.\n" + }, + { + "command": [ + "docker", + "compose", + "config", + "--quiet" + ], + "exit_code": 0, + "output": "" + }, + { + "command": [ + "docker", + "run", + "--rm", + "-v", + "/Users/garfieldmk/Desktop/study/highLoad/mtusi2026lab6:/workspace:ro", + "--entrypoint", + "promtool", + "prom/prometheus:v3.2.1", + "test", + "rules", + "/workspace/student-tests/alerts.test.yaml", + "/workspace/student-tests/burn_rules.test.yaml", + "/workspace/student-tests/telemetry_alerts.test.yaml" + ], + "exit_code": 0, + "output": " SUCCESS\n\n SUCCESS\n\n SUCCESS\n\n" + }, + { + "command": [ + "make", + "test" + ], + "exit_code": 0, + "output": "python3 tests/smoke.py --base-url \"http://localhost:8080\" \nPASS: lab6 public API scenario; infrastructure and grading evidence remain separate.\n" + }, + { + "command": [ + "python3", + "student-tests/check_dashboard.py" + ], + "exit_code": 0, + "output": "PASS: 27 dashboard queries; empty targets: 5\n" + }, + { + "command": [ + "docker", + "compose", + "exec", + "-T", + "api-a", + "/app/.venv/bin/python", + "-" + ], + "stdin": "student-tests/telemetry_unit.py", + "exit_code": 0, + "output": "......\n----------------------------------------------------------------------\nRan 6 tests in 0.097s\n\nOK\n" + }, + { + "command": [ + "docker", + "compose", + "exec", + "-T", + "api-a", + "/app/.venv/bin/python", + "-" + ], + "stdin": "student-tests/pipeline_critical.py", + "exit_code": 0, + "output": "2026-11-06 07:13:14,688 WARNING worker staging cleanup deferred attachment_id=e9a74014-aaa3-4d97-9012-baf9611cc58b\n.........\n----------------------------------------------------------------------\nRan 9 tests in 0.010s\n\nOK\n" + }, + { + "command": [ + "docker", + "compose", + "exec", + "-T", + "api-a", + "/app/.venv/bin/python", + "-" + ], + "stdin": "student-tests/slo_commit_integration.py", + "exit_code": 0, + "output": "PASS: commit boundaries; immutable good/bad cohorts; late completion remains bad\n" + } + ] +} diff --git a/evidence/review-dashboard.json b/evidence/review-dashboard.json new file mode 100644 index 0000000..e02837c --- /dev/null +++ b/evidence/review-dashboard.json @@ -0,0 +1,137 @@ +[ + { + "panel": "API RPS", + "query": "sum(rate(messenger_http_requests_total{route!~\"/health/.*\"}[1m]))", + "series": 1 + }, + { + "panel": "API error ratio", + "query": "(sum(rate(messenger_http_requests_total{status_class=\"5xx\",route!~\"/health/.*\"}[1m])) or vector(0)) / clamp_min(sum(rate(messenger_http_requests_total{route!~\"/health/.*\"}[1m])),0.001) and sum(increase(messenger_http_requests_total{route!~\"/health/.*\"}[1m])) > 0", + "series": 0 + }, + { + "panel": "API latency p50 / p95 / p99", + "query": "histogram_quantile(0.5, sum by (le) (rate(messenger_http_duration_seconds_bucket{route!~\"/health/.*\"}[5m])))", + "series": 1 + }, + { + "panel": "API latency p50 / p95 / p99", + "query": "histogram_quantile(0.95, sum by (le) (rate(messenger_http_duration_seconds_bucket{route!~\"/health/.*\"}[5m])))", + "series": 1 + }, + { + "panel": "API latency p50 / p95 / p99", + "query": "histogram_quantile(0.99, sum by (le) (rate(messenger_http_duration_seconds_bucket{route!~\"/health/.*\"}[5m])))", + "series": 1 + }, + { + "panel": "Jobs by outcome", + "query": "sum by (outcome) (rate(messenger_jobs_total[1m]))", + "series": 2 + }, + { + "panel": "Job duration p95", + "query": "histogram_quantile(0.95, sum by (le) (rate(messenger_job_duration_seconds_bucket[5m])))", + "series": 1 + }, + { + "panel": "Acceptance to ready p95", + "query": "histogram_quantile(0.95, sum by (le) (rate(messenger_job_ready_duration_seconds_bucket[5m])))", + "series": 1 + }, + { + "panel": "Queued jobs", + "query": "max(messenger_queue_depth{service_name=\"messenger-outbox\"})", + "series": 1 + }, + { + "panel": "Oldest queued age", + "query": "max(messenger_queue_age_seconds{service_name=\"messenger-outbox\"})", + "series": 1 + }, + { + "panel": "DB pool used / capacity / waiting", + "query": "messenger_pool_used{service_name=\"messenger-api\"}", + "series": 2 + }, + { + "panel": "DB pool used / capacity / waiting", + "query": "messenger_pool_capacity{service_name=\"messenger-api\"}", + "series": 2 + }, + { + "panel": "DB pool used / capacity / waiting", + "query": "messenger_pool_waiting{service_name=\"messenger-api\"}", + "series": 2 + }, + { + "panel": "Worker busy / capacity", + "query": "messenger_worker_busy{service_name=\"messenger-worker\"}", + "series": 2 + }, + { + "panel": "Worker busy / capacity", + "query": "messenger_worker_capacity{service_name=\"messenger-worker\"}", + "series": 2 + }, + { + "panel": "Process CPU", + "query": "sum by (service_name, service_instance_id) (rate(messenger_process_cpu_seconds_total[1m]))", + "series": 6 + }, + { + "panel": "Process resident memory", + "query": "messenger_process_memory_bytes", + "series": 5 + }, + { + "panel": "Collector failed / dropped spans per second", + "query": "sum by (exporter) (rate(otelcol_exporter_send_failed_spans_total[1m]))", + "series": 1 + }, + { + "panel": "Collector failed / dropped spans per second", + "query": "sum by (exporter) (rate(otelcol_exporter_enqueue_failed_spans_total[1m]))", + "series": 1 + }, + { + "panel": "API SLI: successful ratio over 24h", + "query": "1 - (sum(increase(messenger_http_requests_total{status_class=\"5xx\",route!~\"/health/.*\"}[24h])) or vector(0)) / clamp_min(sum(increase(messenger_http_requests_total{route!~\"/health/.*\"}[24h])),1) and sum(increase(messenger_http_requests_total{route!~\"/health/.*\"}[24h])) > 0", + "series": 1 + }, + { + "panel": "API SLO burn rate (budget 1%)", + "query": "(((sum(increase(messenger_http_requests_total{status_class=\"5xx\",route!~\"/health/.*\"}[20s])) or vector(0)) / clamp_min(sum(increase(messenger_http_requests_total{route!~\"/health/.*\"}[20s])), 1)) / 0.01) and sum(increase(messenger_http_requests_total{route!~\"/health/.*\"}[20s])) > 0", + "series": 0 + }, + { + "panel": "API SLO burn rate (budget 1%)", + "query": "(((sum(increase(messenger_http_requests_total{status_class=\"5xx\",route!~\"/health/.*\"}[1m])) or vector(0)) / clamp_min(sum(increase(messenger_http_requests_total{route!~\"/health/.*\"}[1m])), 1)) / 0.01) and sum(increase(messenger_http_requests_total{route!~\"/health/.*\"}[1m])) > 0", + "series": 0 + }, + { + "panel": "API SLO burn rate (budget 1%)", + "query": "(((sum(increase(messenger_http_requests_total{status_class=\"5xx\",route!~\"/health/.*\"}[5m])) or vector(0)) / clamp_min(sum(increase(messenger_http_requests_total{route!~\"/health/.*\"}[5m])), 1)) / 0.01) and sum(increase(messenger_http_requests_total{route!~\"/health/.*\"}[5m])) > 0", + "series": 1 + }, + { + "panel": "READY SLO burn rate (budget 1%)", + "query": "(((sum(increase(messenger_slo_ready_bad_total{service_name=\"messenger-outbox\"}[20s])) or vector(0)) / clamp_min(sum(increase(messenger_slo_ready_eligible_total{service_name=\"messenger-outbox\"}[20s])), 1)) / 0.01) and sum(increase(messenger_slo_ready_eligible_total{service_name=\"messenger-outbox\"}[20s])) > 0", + "series": 0 + }, + { + "panel": "READY SLO burn rate (budget 1%)", + "query": "(((sum(increase(messenger_slo_ready_bad_total{service_name=\"messenger-outbox\"}[1m])) or vector(0)) / clamp_min(sum(increase(messenger_slo_ready_eligible_total{service_name=\"messenger-outbox\"}[1m])), 1)) / 0.01) and sum(increase(messenger_slo_ready_eligible_total{service_name=\"messenger-outbox\"}[1m])) > 0", + "series": 0 + }, + { + "panel": "READY SLO burn rate (budget 1%)", + "query": "(((sum(increase(messenger_slo_ready_bad_total{service_name=\"messenger-outbox\"}[5m])) or vector(0)) / clamp_min(sum(increase(messenger_slo_ready_eligible_total{service_name=\"messenger-outbox\"}[5m])), 1)) / 0.01) and sum(increase(messenger_slo_ready_eligible_total{service_name=\"messenger-outbox\"}[5m])) > 0", + "series": 1 + }, + { + "panel": "Accepted jobs ready within 30 s — 24h (matured cohort)", + "query": "1 - ((sum(increase(messenger_slo_ready_bad_total{service_name=\"messenger-outbox\"}[24h])) or vector(0)) / clamp_min(sum(increase(messenger_slo_ready_eligible_total{service_name=\"messenger-outbox\"}[24h])), 1)) and sum(increase(messenger_slo_ready_eligible_total{service_name=\"messenger-outbox\"}[24h])) > 0", + "series": 1 + } +] diff --git a/evidence/review-db-trace.json b/evidence/review-db-trace.json new file mode 100644 index 0000000..16cffb9 --- /dev/null +++ b/evidence/review-db-trace.json @@ -0,0 +1,909 @@ +{ + "captured_at": "2026-11-06T06:54:49.009234+00:00", + "query": "{service_name=\"messenger-api\"} | status=\"503\"", + "trace_id": "b9b17078824f1cc4e21a4434e129efc9", + "logs": { + "status": "success", + "data": { + "resultType": "streams", + "result": [ + { + "stream": { + "code_file_path": "/app/src/main.py", + "code_function_name": "add_request_id", + "code_line_number": "108", + "deployment_environment_name": "lab", + "detected_level": "info", + "flags": "1", + "method": "POST", + "observed_timestamp": "1793948029625964635", + "request_id": "51734099-b47b-433a-abb2-ce06a64c1478", + "route": "/api/v1/chats/{chat_id}/messages", + "scope_name": "messenger.telemetry", + "service_instance_id": "f1c146213339", + "service_name": "messenger-api", + "service_version": "1.6.0", + "severity_number": "9", + "severity_text": "INFO", + "span_id": "151f102afc35d89a", + "status": "503", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.36.0", + "trace_id": "b9b17078824f1cc4e21a4434e129efc9" + }, + "values": [ + [ + "1793948029625911040", + "http.request" + ] + ] + }, + { + "stream": { + "code_file_path": "/app/src/main.py", + "code_function_name": "add_request_id", + "code_line_number": "108", + "deployment_environment_name": "lab", + "detected_level": "info", + "flags": "1", + "method": "POST", + "observed_timestamp": "1793948029626627968", + "request_id": "55344573-05c5-474e-abd5-b260b03a332f", + "route": "/api/v1/chats/{chat_id}/messages", + "scope_name": "messenger.telemetry", + "service_instance_id": "f1c146213339", + "service_name": "messenger-api", + "service_version": "1.6.0", + "severity_number": "9", + "severity_text": "INFO", + "span_id": "60f1eaa06a9e9785", + "status": "503", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.36.0", + "trace_id": "fc93bdd340de079da209246d67f8a561" + }, + "values": [ + [ + "1793948029626592512", + "http.request" + ] + ] + }, + { + "stream": { + "code_file_path": "/app/src/main.py", + "code_function_name": "add_request_id", + "code_line_number": "108", + "deployment_environment_name": "lab", + "detected_level": "info", + "flags": "1", + "method": "POST", + "observed_timestamp": "1793948035645779263", + "request_id": "5b6032db-2e31-4f04-a385-456ebffebfdc", + "route": "/api/v1/chats/{chat_id}/messages", + "scope_name": "messenger.telemetry", + "service_instance_id": "d0867cbe3aca", + "service_name": "messenger-api", + "service_version": "1.6.0", + "severity_number": "9", + "severity_text": "INFO", + "span_id": "9a650ee7231a02bf", + "status": "503", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.36.0", + "trace_id": "1c58397c3ec8085b1e27609b27ae520c" + }, + "values": [ + [ + "1793948035645726464", + "http.request" + ] + ] + }, + { + "stream": { + "code_file_path": "/app/src/main.py", + "code_function_name": "add_request_id", + "code_line_number": "108", + "deployment_environment_name": "lab", + "detected_level": "info", + "flags": "1", + "method": "POST", + "observed_timestamp": "1793948035645787846", + "request_id": "8acd3330-9d9a-4c2c-8d84-f0e4ff441d4e", + "route": "/api/v1/chats/{chat_id}/messages", + "scope_name": "messenger.telemetry", + "service_instance_id": "f1c146213339", + "service_name": "messenger-api", + "service_version": "1.6.0", + "severity_number": "9", + "severity_text": "INFO", + "span_id": "b0df0e4132d5949b", + "status": "503", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.36.0", + "trace_id": "2498be54dd8888579ad47c9fb167cf18" + }, + "values": [ + [ + "1793948035645739520", + "http.request" + ] + ] + }, + { + "stream": { + "code_file_path": "/app/src/main.py", + "code_function_name": "add_request_id", + "code_line_number": "108", + "deployment_environment_name": "lab", + "detected_level": "info", + "flags": "1", + "method": "POST", + "observed_timestamp": "1793948035646862221", + "request_id": "5b0a1b31-a671-47a9-86e2-758ccb5dc675", + "route": "/api/v1/chats/{chat_id}/messages", + "scope_name": "messenger.telemetry", + "service_instance_id": "f1c146213339", + "service_name": "messenger-api", + "service_version": "1.6.0", + "severity_number": "9", + "severity_text": "INFO", + "span_id": "561098ed9ea394b0", + "status": "503", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.36.0", + "trace_id": "c94bf7e1d16e7c55357478fc8a25648e" + }, + "values": [ + [ + "1793948035646823168", + "http.request" + ] + ] + }, + { + "stream": { + "code_file_path": "/app/src/main.py", + "code_function_name": "add_request_id", + "code_line_number": "108", + "deployment_environment_name": "lab", + "detected_level": "info", + "flags": "1", + "method": "POST", + "observed_timestamp": "1793948041671332960", + "request_id": "7aff0420-ff29-4c90-96e3-cbc5a2532e30", + "route": "/api/v1/chats/{chat_id}/messages", + "scope_name": "messenger.telemetry", + "service_instance_id": "d0867cbe3aca", + "service_name": "messenger-api", + "service_version": "1.6.0", + "severity_number": "9", + "severity_text": "INFO", + "span_id": "6f8d3db9589f262e", + "status": "503", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.36.0", + "trace_id": "2c73801c22050e58905cf9d73060ff43" + }, + "values": [ + [ + "1793948041671150080", + "http.request" + ] + ] + }, + { + "stream": { + "code_file_path": "/app/src/main.py", + "code_function_name": "add_request_id", + "code_line_number": "108", + "deployment_environment_name": "lab", + "detected_level": "info", + "flags": "1", + "method": "POST", + "observed_timestamp": "1793948041671636210", + "request_id": "7f3e51d9-3057-47a6-9896-d53c1b52d498", + "route": "/api/v1/chats/{chat_id}/messages", + "scope_name": "messenger.telemetry", + "service_instance_id": "f1c146213339", + "service_name": "messenger-api", + "service_version": "1.6.0", + "severity_number": "9", + "severity_text": "INFO", + "span_id": "05853b814f44dd57", + "status": "503", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.36.0", + "trace_id": "134beebdc0479e59dacd0d3a91c6824c" + }, + "values": [ + [ + "1793948041671464960", + "http.request" + ] + ] + }, + { + "stream": { + "code_file_path": "/app/src/main.py", + "code_function_name": "add_request_id", + "code_line_number": "108", + "deployment_environment_name": "lab", + "detected_level": "info", + "flags": "1", + "method": "POST", + "observed_timestamp": "1793948041672626877", + "request_id": "fd9f4531-0d77-4abf-9232-6e621c2dc69a", + "route": "/api/v1/chats/{chat_id}/messages", + "scope_name": "messenger.telemetry", + "service_instance_id": "f1c146213339", + "service_name": "messenger-api", + "service_version": "1.6.0", + "severity_number": "9", + "severity_text": "INFO", + "span_id": "28115ec0c6bf870b", + "status": "503", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.36.0", + "trace_id": "d8f558f27d33ee11545f0c6c782092ab" + }, + "values": [ + [ + "1793948041672583424", + "http.request" + ] + ] + }, + { + "stream": { + "code_file_path": "/app/src/main.py", + "code_function_name": "add_request_id", + "code_line_number": "108", + "deployment_environment_name": "lab", + "detected_level": "info", + "flags": "1", + "method": "POST", + "observed_timestamp": "1793948047681994963", + "request_id": "c2dd6234-4f16-40a6-9ac8-adecbe4a2959", + "route": "/api/v1/chats/{chat_id}/messages", + "scope_name": "messenger.telemetry", + "service_instance_id": "d0867cbe3aca", + "service_name": "messenger-api", + "service_version": "1.6.0", + "severity_number": "9", + "severity_text": "INFO", + "span_id": "9c95cf9971a250f6", + "status": "503", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.36.0", + "trace_id": "a4d252574094985c29f71f4c76ad265e" + }, + "values": [ + [ + "1793948047681970944", + "http.request" + ] + ] + }, + { + "stream": { + "code_file_path": "/app/src/main.py", + "code_function_name": "add_request_id", + "code_line_number": "108", + "deployment_environment_name": "lab", + "detected_level": "info", + "flags": "1", + "method": "POST", + "observed_timestamp": "1793948047684209629", + "request_id": "d179ce27-8330-4b45-94bb-ade25970ac20", + "route": "/api/v1/chats/{chat_id}/messages", + "scope_name": "messenger.telemetry", + "service_instance_id": "f1c146213339", + "service_name": "messenger-api", + "service_version": "1.6.0", + "severity_number": "9", + "severity_text": "INFO", + "span_id": "d379e86b1b590491", + "status": "503", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.36.0", + "trace_id": "a161022e47d517d700bc98c8f1a61dff" + }, + "values": [ + [ + "1793948047684182528", + "http.request" + ] + ] + }, + { + "stream": { + "code_file_path": "/app/src/main.py", + "code_function_name": "add_request_id", + "code_line_number": "108", + "deployment_environment_name": "lab", + "detected_level": "info", + "flags": "1", + "method": "POST", + "observed_timestamp": "1793948047684229754", + "request_id": "f1bba5fc-b784-4326-b311-89e135e9cbf2", + "route": "/api/v1/chats/{chat_id}/messages", + "scope_name": "messenger.telemetry", + "service_instance_id": "d0867cbe3aca", + "service_name": "messenger-api", + "service_version": "1.6.0", + "severity_number": "9", + "severity_text": "INFO", + "span_id": "85dc24e82075299c", + "status": "503", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.36.0", + "trace_id": "300e5c652019f265efd75db929f5b52b" + }, + "values": [ + [ + "1793948047684212736", + "http.request" + ] + ] + }, + { + "stream": { + "code_file_path": "/app/src/main.py", + "code_function_name": "add_request_id", + "code_line_number": "108", + "deployment_environment_name": "lab", + "detected_level": "info", + "flags": "1", + "method": "POST", + "observed_timestamp": "1793948053706657382", + "request_id": "d9a079b7-3177-43f1-aadd-29fd78496ea9", + "route": "/api/v1/chats/{chat_id}/messages", + "scope_name": "messenger.telemetry", + "service_instance_id": "d0867cbe3aca", + "service_name": "messenger-api", + "service_version": "1.6.0", + "severity_number": "9", + "severity_text": "INFO", + "span_id": "76b413ac9de41425", + "status": "503", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.36.0", + "trace_id": "29a9abc1e5f3b9092d58b881f0736412" + }, + "values": [ + [ + "1793948053706575104", + "http.request" + ] + ] + }, + { + "stream": { + "code_file_path": "/app/src/main.py", + "code_function_name": "add_request_id", + "code_line_number": "108", + "deployment_environment_name": "lab", + "detected_level": "info", + "flags": "1", + "method": "POST", + "observed_timestamp": "1793948053706669174", + "request_id": "014723e9-26db-496d-883b-9b2db8bcd340", + "route": "/api/v1/chats/{chat_id}/messages", + "scope_name": "messenger.telemetry", + "service_instance_id": "f1c146213339", + "service_name": "messenger-api", + "service_version": "1.6.0", + "severity_number": "9", + "severity_text": "INFO", + "span_id": "d93bf6199b8546fc", + "status": "503", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.36.0", + "trace_id": "9088b24f0cd2fd1a018e0db423799194" + }, + "values": [ + [ + "1793948053706575360", + "http.request" + ] + ] + }, + { + "stream": { + "code_file_path": "/app/src/main.py", + "code_function_name": "add_request_id", + "code_line_number": "108", + "deployment_environment_name": "lab", + "detected_level": "info", + "flags": "1", + "method": "POST", + "observed_timestamp": "1793948053707987632", + "request_id": "353b4848-174a-4591-97bf-6d9f617fd313", + "route": "/api/v1/chats/{chat_id}/messages", + "scope_name": "messenger.telemetry", + "service_instance_id": "d0867cbe3aca", + "service_name": "messenger-api", + "service_version": "1.6.0", + "severity_number": "9", + "severity_text": "INFO", + "span_id": "f41d3c6bcba07369", + "status": "503", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.36.0", + "trace_id": "f7c3d5d493bbd9df21991623089fbdf0" + }, + "values": [ + [ + "1793948053707940096", + "http.request" + ] + ] + }, + { + "stream": { + "code_file_path": "/app/src/main.py", + "code_function_name": "add_request_id", + "code_line_number": "108", + "deployment_environment_name": "lab", + "detected_level": "info", + "flags": "1", + "method": "POST", + "observed_timestamp": "1793948059729947552", + "request_id": "6c9a13a3-4311-4f0d-8076-72c3d1e2f9f4", + "route": "/api/v1/chats/{chat_id}/messages", + "scope_name": "messenger.telemetry", + "service_instance_id": "f1c146213339", + "service_name": "messenger-api", + "service_version": "1.6.0", + "severity_number": "9", + "severity_text": "INFO", + "span_id": "f4ddfd85dcde62ee", + "status": "503", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.36.0", + "trace_id": "180f3189cf8a2830a6180baf8e0fcd08" + }, + "values": [ + [ + "1793948059729844992", + "http.request" + ] + ] + }, + { + "stream": { + "code_file_path": "/app/src/main.py", + "code_function_name": "add_request_id", + "code_line_number": "108", + "deployment_environment_name": "lab", + "detected_level": "info", + "flags": "1", + "method": "POST", + "observed_timestamp": "1793948059730063635", + "request_id": "fd704a6f-49b8-4fcd-93bd-12946f9a4244", + "route": "/api/v1/chats/{chat_id}/messages", + "scope_name": "messenger.telemetry", + "service_instance_id": "d0867cbe3aca", + "service_name": "messenger-api", + "service_version": "1.6.0", + "severity_number": "9", + "severity_text": "INFO", + "span_id": "2a4cfc787931f6ed", + "status": "503", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.36.0", + "trace_id": "b5d08507f638c052061e9b2b03323048" + }, + "values": [ + [ + "1793948059729981184", + "http.request" + ] + ] + }, + { + "stream": { + "code_file_path": "/app/src/main.py", + "code_function_name": "add_request_id", + "code_line_number": "108", + "deployment_environment_name": "lab", + "detected_level": "info", + "flags": "1", + "method": "POST", + "observed_timestamp": "1793948059730736968", + "request_id": "47ea9ee4-dbd2-4fe1-a369-90a43ebf49b6", + "route": "/api/v1/chats/{chat_id}/messages", + "scope_name": "messenger.telemetry", + "service_instance_id": "d0867cbe3aca", + "service_name": "messenger-api", + "service_version": "1.6.0", + "severity_number": "9", + "severity_text": "INFO", + "span_id": "9c11c85b2900685e", + "status": "503", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.36.0", + "trace_id": "8637ac13af6d5a61d8e3a3ea26d3968a" + }, + "values": [ + [ + "1793948059730694912", + "http.request" + ] + ] + }, + { + "stream": { + "code_file_path": "/app/src/main.py", + "code_function_name": "add_request_id", + "code_line_number": "108", + "deployment_environment_name": "lab", + "detected_level": "info", + "flags": "1", + "method": "POST", + "observed_timestamp": "1793948065753870555", + "request_id": "52575c60-c97b-4ebc-a214-f762c399e9ac", + "route": "/api/v1/chats/{chat_id}/messages", + "scope_name": "messenger.telemetry", + "service_instance_id": "d0867cbe3aca", + "service_name": "messenger-api", + "service_version": "1.6.0", + "severity_number": "9", + "severity_text": "INFO", + "span_id": "f13c6a8f6a55fe64", + "status": "503", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.36.0", + "trace_id": "ffe846cc93dfae2f682b097e9a304a57" + }, + "values": [ + [ + "1793948065753797120", + "http.request" + ] + ] + }, + { + "stream": { + "code_file_path": "/app/src/main.py", + "code_function_name": "add_request_id", + "code_line_number": "108", + "deployment_environment_name": "lab", + "detected_level": "info", + "flags": "1", + "method": "POST", + "observed_timestamp": "1793948065753873221", + "request_id": "2d6a57dc-1934-4403-bb8a-69f40d20049d", + "route": "/api/v1/chats/{chat_id}/messages", + "scope_name": "messenger.telemetry", + "service_instance_id": "f1c146213339", + "service_name": "messenger-api", + "service_version": "1.6.0", + "severity_number": "9", + "severity_text": "INFO", + "span_id": "ff28ad796a5c16fc", + "status": "503", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.36.0", + "trace_id": "3631b8f5fb534b466f0200d125ba50c2" + }, + "values": [ + [ + "1793948065753789696", + "http.request" + ] + ] + }, + { + "stream": { + "code_file_path": "/app/src/main.py", + "code_function_name": "add_request_id", + "code_line_number": "108", + "deployment_environment_name": "lab", + "detected_level": "info", + "flags": "1", + "method": "POST", + "observed_timestamp": "1793948065755343888", + "request_id": "cc3aecc7-1fe0-43b9-9f4d-46cd5f9dd63c", + "route": "/api/v1/chats/{chat_id}/messages", + "scope_name": "messenger.telemetry", + "service_instance_id": "d0867cbe3aca", + "service_name": "messenger-api", + "service_version": "1.6.0", + "severity_number": "9", + "severity_text": "INFO", + "span_id": "93d157788c8db59a", + "status": "503", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.36.0", + "trace_id": "78ff7789eadd30821784a80d298a9371" + }, + "values": [ + [ + "1793948065755291136", + "http.request" + ] + ] + } + ], + "stats": { + "summary": { + "bytesProcessedPerSecond": 55251140, + "linesProcessedPerSecond": 336897, + "totalBytesProcessed": 513976, + "totalLinesProcessed": 3134, + "execTime": 0.009303, + "queueTime": 8.4e-05, + "subqueries": 0, + "totalEntriesReturned": 20, + "splits": 0, + "shards": 1, + "totalPostFilterLines": 36, + "totalStructuredMetadataBytesProcessed": 476368 + }, + "querier": { + "store": { + "totalChunksRef": 0, + "totalChunksDownloaded": 0, + "chunksDownloadTime": 0, + "queryReferencedStructuredMetadata": false, + "chunk": { + "headChunkBytes": 0, + "headChunkLines": 0, + "decompressedBytes": 0, + "decompressedLines": 0, + "compressedBytes": 0, + "totalDuplicates": 0, + "postFilterLines": 0, + "headChunkStructuredMetadataBytes": 0, + "decompressedStructuredMetadataBytes": 0 + }, + "chunkRefsFetchTime": 0, + "congestionControlLatency": 0, + "pipelineWrapperFilteredLines": 0 + } + }, + "ingester": { + "totalReached": 1, + "totalChunksMatched": 4, + "totalBatches": 1, + "totalLinesSent": 20, + "store": { + "totalChunksRef": 0, + "totalChunksDownloaded": 0, + "chunksDownloadTime": 0, + "queryReferencedStructuredMetadata": true, + "chunk": { + "headChunkBytes": 513976, + "headChunkLines": 3134, + "decompressedBytes": 0, + "decompressedLines": 0, + "compressedBytes": 0, + "totalDuplicates": 0, + "postFilterLines": 36, + "headChunkStructuredMetadataBytes": 476368, + "decompressedStructuredMetadataBytes": 0 + }, + "chunkRefsFetchTime": 122375, + "congestionControlLatency": 0, + "pipelineWrapperFilteredLines": 0 + } + }, + "cache": { + "chunk": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "index": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "result": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "statsResult": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "volumeResult": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "seriesResult": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "labelResult": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "instantMetricResult": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + } + }, + "index": { + "totalChunks": 0, + "postFilterChunks": 0, + "shardsDuration": 0, + "usedBloomFilters": false + } + } + } + }, + "trace": { + "batches": [ + { + "resource": { + "attributes": [ + { + "key": "telemetry.sdk.language", + "value": { + "stringValue": "python" + } + }, + { + "key": "telemetry.sdk.name", + "value": { + "stringValue": "opentelemetry" + } + }, + { + "key": "telemetry.sdk.version", + "value": { + "stringValue": "1.36.0" + } + }, + { + "key": "service.version", + "value": { + "stringValue": "1.6.0" + } + }, + { + "key": "service.instance.id", + "value": { + "stringValue": "f1c146213339" + } + }, + { + "key": "deployment.environment.name", + "value": { + "stringValue": "lab" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "messenger-api" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "messenger" + }, + "spans": [ + { + "traceId": "ubFweIJPHMTiGkQ04SnvyQ==", + "spanId": "6fTEyNCjCc8=", + "parentSpanId": "FR8QKvw12Jo=", + "name": "postgres.user_for_token", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1793948024619601966", + "endTimeUnixNano": "1793948029625443968", + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "ubFweIJPHMTiGkQ04SnvyQ==", + "spanId": "FR8QKvw12Jo=", + "name": "POST /api/v1/chats/{chat_id}/messages", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1793948024616785508", + "endTimeUnixNano": "1793948029626105093", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "POST" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/chats/{chat_id}/messages" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + } + ] + } + ] + } + ] + } +} diff --git a/evidence/review-observability.md b/evidence/review-observability.md new file mode 100644 index 0000000..0ac8c85 --- /dev/null +++ b/evidence/review-observability.md @@ -0,0 +1,29 @@ +# Повторная проверка 2026-11-06 + +Команды и stdout/exit codes коротких проверок: [review-checks.json](review-checks.json). В Docker Python 3.12 прошли 6 SDK-тестов, 9 критических проверок pipeline и интеграционная проверка commit-границ во временной таблице. Публичный smoke выполнялся при включённом OTLP. Все 27 запросов dashboard корректны; пустые ready burn ряды при отсутствии новых событий не заменены искусственным значением «здоров» — [review-dashboard.json](review-dashboard.json). + +## Связь API и worker с retry + +[Сценарий](../student-tests/retry_evidence.py) включает fault после S3 PUT, затем возвращает обычную конфигурацию worker в finally. [Результат](review-retry-job.json): ready через 61,421 с, `attempt=2`, один рестарт контейнера. [Trace и логи](review-retry-trace.json) содержат 21 span, 2 producer и 2 consumer, операции S3 и завершения DB. [Проверка дерева](review-retry-summary.json) подтверждает одного API-предка для producer и producer-родителей consumer, общий trace ID и логи обоих сервисов. Сохранённый контекст находится внутри `postgres.create_attachment`, поэтому producer связан с API через DB span, а не обязан быть прямым ребёнком SERVER. Незавершённый span аварийной попытки может не экспортироваться при `os._exit`; полнота упавшей попытки не заявляется. + +## Два инцидента приложения + +- БД: [сырые пробы](slo-incidents.json) включают `MessengerHighErrorRate` firing и его отсутствие в конце. [Коррелированный лог и trace](review-db-trace.json) показывают 503, DB span 5,006 с и HTTP span 5,009 с; времена — разность timestamp полей /10⁹. Причина диагностируется по DB span. [Анализ](slo-analysis.json) сохраняет detection/recovery и качество времени. +- Worker: [пробы](slo-ready-incident.json) показывают очередь, ReadyBurnFast/Slow и QueueLag, затем depth/age = 0 и отсутствие alerts. Все 8 новых заданий нарушили срок: приращение eligible и bad равно 8. Fast исчез до восстановления worker вследствие короткого окна/minimum volume; это ограничение правил, а не доказательство здоровья. Числа и объяснение — [slo-track.md](slo-track.md). + +## Недоступность Tempo + +[Сценарий](../student-tests/tempo_outage.py), [метрики и память](review-tempo-outage.json), [нагрузка](review-tempo-load.json). Tempo остановлен, smoke прошёл. Затем concurrency=4 за 75,006 с дал 40 628 успешных запросов и 0 ошибок; успешные RPS = `40628 / 75,006 ≈ 541,663` (генератор сообщает 541,661 по неокруглённому времени). Это closed-loop опыт, а не максимальная производительность. + +Collector не пересоздавался: ID в начале и конце одинаков. По сохранённым Prometheus-пробам: + +- Очередь выросла от 0 до лимита 128, после восстановления опустела до 0. +- Память по `docker stats` — 114,6 MiB до опыта; максимум сохранённых проб — 128,8 MiB при лимите 256 MiB. Это выборочные наблюдения ограниченного опыта, не доказательство ограниченности на произвольном времени. +- Дельта `enqueue_failed_spans_total` — 57 268; дельта `send_failed_spans_total` — 772; дельта `sent_spans_total` — 64 860. Каждая дельта = последняя проба − baseline. Первые две величины характеризуют разные этапы ошибок экспорта и не складываются в число уникальных потерянных spans. +- Через первые 25 с после запуска Tempo очередь ещё была полной. Продолжение того же замера подтвердило опустошение; исходные пробы сохранены. Сценарий исправлен: теперь проверяет фактическое опустошение с пределом ожидания 90 с, а не только успешный запуск контейнера. + +Ограниченная очередь допускает потерю traces при переполнении; доступность API в данном опыте сохранена. Повторный запуск не обязан дать те же абсолютные числа: исходная нагрузка, моменты scrape и размер batches влияют на результат. + +## Сохранность и отказ реплики + +[Публичные тесты](review-resilience.json) persistence и failover завершились PASS; runner восстановил обе API-реплики. [Повторный запрос Tempo](review-tempo-persistence.json) после stop/start вернул те же 21 span трассы, записанной до отказа. Конфигурация persistent volumes находится в Compose; удаление volumes в этих сценариях не выполнялось. diff --git a/evidence/review-resilience.json b/evidence/review-resilience.json new file mode 100644 index 0000000..1bdf90e --- /dev/null +++ b/evidence/review-resilience.json @@ -0,0 +1,27 @@ +{ + "utc": "2026-11-06T07:15:28.687307+00:00", + "checks": [ + { + "command": [ + "make", + "test-persistence", + "ACTION_SCRIPT=scripts/restart.sh" + ], + "exit_code": 0, + "output": "python3 tests/resilience.py --mode persistence --action-script \"scripts/restart.sh\" --base-url \"http://localhost:8080\" \nExecuting explicit persistence hook; hook output suppressed to avoid leaking secrets.\nPASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.\n" + }, + { + "command": [ + "make", + "test-failover", + "ACTION_SCRIPT=scripts/stop-one.sh" + ], + "exit_code": 0, + "output": "python3 tests/resilience.py --mode failover --action-script \"scripts/stop-one.sh\" --base-url \"http://localhost:8080\" \nExecuting explicit failover hook; hook output suppressed to avoid leaking secrets.\n{\"observed_instances_before\": 2, \"successful_writes\": 112, \"failed_attempts\": 1, \"first_recovery_seconds\": 2.561}\nPASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.\n" + }, + { + "restore_exit_code": 0, + "output": " Container mtusi2026lab6-api-a-1 Running \n Container mtusi2026lab6-outbox-1 Running \n Container mtusi2026lab6-worker-1 Running \n Container mtusi2026lab6-api-b-1 Starting \n Container mtusi2026lab6-api-b-1 Started \n Container mtusi2026lab6-outbox-1 Waiting \n Container mtusi2026lab6-api-b-1 Waiting \n Container mtusi2026lab6-worker-1 Waiting \n Container mtusi2026lab6-api-a-1 Waiting \n Container mtusi2026lab6-outbox-1 Healthy \n Container mtusi2026lab6-api-a-1 Healthy \n Container mtusi2026lab6-worker-1 Healthy \n Container mtusi2026lab6-api-b-1 Healthy \n" + } + ] +} diff --git a/evidence/review-retry-job.json b/evidence/review-retry-job.json new file mode 100644 index 0000000..4f95172 --- /dev/null +++ b/evidence/review-retry-job.json @@ -0,0 +1,9 @@ +{ + "utc": "2026-11-06T07:09:13.514447+00:00", + "trace_id": "ae34626f1564506e278461f8e8e32fa6", + "attachment_id": "f000feb7-d0a9-4338-b7d5-0371343cf6d1", + "status": "ready", + "elapsed_seconds": 61.422, + "db_attempt": 2, + "restart_count": 1 +} diff --git a/evidence/review-retry-summary.json b/evidence/review-retry-summary.json new file mode 100644 index 0000000..1a69600 --- /dev/null +++ b/evidence/review-retry-summary.json @@ -0,0 +1,27 @@ +{ + "trace_id": "ae34626f1564506e278461f8e8e32fa6", + "span_count": 21, + "span_names": { + "POST /api/v1/chats/{chat_id}/attachments": 1, + "postgres.user_for_token": 1, + "postgres.get_chat_for_member": 1, + "postgres.attachment_transaction": 3, + "s3.put_object": 4, + "postgres.create_attachment": 1, + "attachment.publish": 2, + "attachment.consume": 2, + "postgres.claim_attachment_job": 2, + "postgres.complete_attachment_job": 1, + "s3.delete_object": 1, + "s3.copy_object": 1, + "s3.get_object": 1 + }, + "producer_count": 2, + "consumer_count": 2, + "log_services": [ + "messenger-api", + "messenger-worker" + ], + "parent_relationships": "PASS", + "note": "Abrupt os._exit may lose unfinished spans of the crashed attempt; redelivery and committed retry remain visible." +} diff --git a/evidence/review-retry-trace.json b/evidence/review-retry-trace.json new file mode 100644 index 0000000..7df38cc --- /dev/null +++ b/evidence/review-retry-trace.json @@ -0,0 +1,870 @@ +{ + "captured_at": "2026-11-06T07:09:24.120944+00:00", + "query": "{service_name=~\"messenger-api|messenger-worker\"} | trace_id=\"ae34626f1564506e278461f8e8e32fa6\"", + "trace_id": "ae34626f1564506e278461f8e8e32fa6", + "logs": { + "status": "success", + "data": { + "resultType": "streams", + "result": [ + { + "stream": { + "code_file_path": "/app/src/worker.py", + "code_function_name": "handle", + "code_line_number": "242", + "deployment_environment_name": "lab", + "detected_level": "info", + "flags": "1", + "observed_timestamp": "1793948953512429924", + "outcome": "success", + "scope_name": "messenger.telemetry", + "service_instance_id": "8ab056f473cf", + "service_name": "messenger-worker", + "service_version": "1.6.0", + "severity_number": "9", + "severity_text": "INFO", + "span_id": "5b57d22d2d3eabde", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.36.0", + "trace_id": "ae34626f1564506e278461f8e8e32fa6" + }, + "values": [ + [ + "1793948953512353792", + "job.processed" + ] + ] + }, + { + "stream": { + "code_file_path": "/app/src/worker.py", + "code_function_name": "handle", + "code_line_number": "242", + "deployment_environment_name": "lab", + "detected_level": "info", + "flags": "1", + "observed_timestamp": "1793948893930110257", + "outcome": "skipped", + "scope_name": "messenger.telemetry", + "service_instance_id": "8ab056f473cf", + "service_name": "messenger-worker", + "service_version": "1.6.0", + "severity_number": "9", + "severity_text": "INFO", + "span_id": "632c5bb6629fa77c", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.36.0", + "trace_id": "ae34626f1564506e278461f8e8e32fa6" + }, + "values": [ + [ + "1793948893930080512", + "job.processed" + ] + ] + }, + { + "stream": { + "code_file_path": "/app/src/main.py", + "code_function_name": "add_request_id", + "code_line_number": "108", + "deployment_environment_name": "lab", + "detected_level": "info", + "flags": "1", + "method": "POST", + "observed_timestamp": "1793948892089943715", + "request_id": "fd45bf79-92fd-40d9-b1b0-2faeaa9abd35", + "route": "/api/v1/chats/{chat_id}/attachments", + "scope_name": "messenger.telemetry", + "service_instance_id": "d81222953eb2", + "service_name": "messenger-api", + "service_version": "1.6.0", + "severity_number": "9", + "severity_text": "INFO", + "span_id": "8b7074677c4447e0", + "status": "202", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.36.0", + "trace_id": "ae34626f1564506e278461f8e8e32fa6" + }, + "values": [ + [ + "1793948892089921536", + "http.request" + ] + ] + } + ], + "stats": { + "summary": { + "bytesProcessedPerSecond": 50653466, + "linesProcessedPerSecond": 309238, + "totalBytesProcessed": 662083, + "totalLinesProcessed": 4042, + "execTime": 0.013071, + "queueTime": 0.000471, + "subqueries": 0, + "totalEntriesReturned": 3, + "splits": 2, + "shards": 2, + "totalPostFilterLines": 3, + "totalStructuredMetadataBytesProcessed": 613544 + }, + "querier": { + "store": { + "totalChunksRef": 0, + "totalChunksDownloaded": 0, + "chunksDownloadTime": 0, + "queryReferencedStructuredMetadata": false, + "chunk": { + "headChunkBytes": 0, + "headChunkLines": 0, + "decompressedBytes": 0, + "decompressedLines": 0, + "compressedBytes": 0, + "totalDuplicates": 0, + "postFilterLines": 0, + "headChunkStructuredMetadataBytes": 0, + "decompressedStructuredMetadataBytes": 0 + }, + "chunkRefsFetchTime": 0, + "congestionControlLatency": 0, + "pipelineWrapperFilteredLines": 0 + } + }, + "ingester": { + "totalReached": 2, + "totalChunksMatched": 11, + "totalBatches": 3, + "totalLinesSent": 3, + "store": { + "totalChunksRef": 0, + "totalChunksDownloaded": 0, + "chunksDownloadTime": 0, + "queryReferencedStructuredMetadata": true, + "chunk": { + "headChunkBytes": 662083, + "headChunkLines": 4042, + "decompressedBytes": 0, + "decompressedLines": 0, + "compressedBytes": 0, + "totalDuplicates": 0, + "postFilterLines": 3, + "headChunkStructuredMetadataBytes": 613544, + "decompressedStructuredMetadataBytes": 0 + }, + "chunkRefsFetchTime": 233000, + "congestionControlLatency": 0, + "pipelineWrapperFilteredLines": 0 + } + }, + "cache": { + "chunk": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "index": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "result": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "statsResult": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "volumeResult": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "seriesResult": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "labelResult": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "instantMetricResult": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + } + }, + "index": { + "totalChunks": 0, + "postFilterChunks": 0, + "shardsDuration": 0, + "usedBloomFilters": false + } + } + } + }, + "trace": { + "batches": [ + { + "resource": { + "attributes": [ + { + "key": "telemetry.sdk.language", + "value": { + "stringValue": "python" + } + }, + { + "key": "telemetry.sdk.name", + "value": { + "stringValue": "opentelemetry" + } + }, + { + "key": "telemetry.sdk.version", + "value": { + "stringValue": "1.36.0" + } + }, + { + "key": "service.version", + "value": { + "stringValue": "1.6.0" + } + }, + { + "key": "service.instance.id", + "value": { + "stringValue": "d81222953eb2" + } + }, + { + "key": "deployment.environment.name", + "value": { + "stringValue": "lab" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "messenger-api" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "messenger" + }, + "spans": [ + { + "traceId": "rjRibxVkUG4nhGH46OMvpg==", + "spanId": "i3B0Z3xER+A=", + "parentSpanId": "cPPbxzsVklc=", + "name": "POST /api/v1/chats/{chat_id}/attachments", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1793948892082137923", + "endTimeUnixNano": "1793948892089988006", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "POST" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/chats/{chat_id}/attachments" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "202" + } + } + ], + "status": {} + }, + { + "traceId": "rjRibxVkUG4nhGH46OMvpg==", + "spanId": "5ln8XViN22I=", + "parentSpanId": "i3B0Z3xER+A=", + "name": "postgres.user_for_token", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1793948892082560215", + "endTimeUnixNano": "1793948892083065215", + "status": {} + }, + { + "traceId": "rjRibxVkUG4nhGH46OMvpg==", + "spanId": "ENe80xA2A2k=", + "parentSpanId": "i3B0Z3xER+A=", + "name": "postgres.get_chat_for_member", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1793948892083141590", + "endTimeUnixNano": "1793948892083547006", + "status": {} + }, + { + "traceId": "rjRibxVkUG4nhGH46OMvpg==", + "spanId": "Z4dP2xX/VNg=", + "parentSpanId": "i3B0Z3xER+A=", + "name": "postgres.attachment_transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1793948892084939631", + "endTimeUnixNano": "1793948892089742131", + "status": {} + }, + { + "traceId": "rjRibxVkUG4nhGH46OMvpg==", + "spanId": "zohqiMpi+QU=", + "parentSpanId": "Z4dP2xX/VNg=", + "name": "s3.put_object", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1793948892085314173", + "endTimeUnixNano": "1793948892088217590", + "status": {} + }, + { + "traceId": "rjRibxVkUG4nhGH46OMvpg==", + "spanId": "TjCZy2qExRI=", + "parentSpanId": "Z4dP2xX/VNg=", + "name": "postgres.create_attachment", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1793948892088244965", + "endTimeUnixNano": "1793948892089214631", + "status": {} + } + ] + } + ] + }, + { + "resource": { + "attributes": [ + { + "key": "telemetry.sdk.language", + "value": { + "stringValue": "python" + } + }, + { + "key": "telemetry.sdk.name", + "value": { + "stringValue": "opentelemetry" + } + }, + { + "key": "telemetry.sdk.version", + "value": { + "stringValue": "1.36.0" + } + }, + { + "key": "service.version", + "value": { + "stringValue": "1.6.0" + } + }, + { + "key": "service.instance.id", + "value": { + "stringValue": "b106c9e70b85" + } + }, + { + "key": "deployment.environment.name", + "value": { + "stringValue": "lab" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "messenger-outbox" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "messenger" + }, + "spans": [ + { + "traceId": "rjRibxVkUG4nhGH46OMvpg==", + "spanId": "hHhcYiMN8r0=", + "parentSpanId": "TjCZy2qExRI=", + "name": "attachment.publish", + "kind": "SPAN_KIND_PRODUCER", + "startTimeUnixNano": "1793948892280212882", + "endTimeUnixNano": "1793948892293371132", + "attributes": [ + { + "key": "messaging.retry.attempt", + "value": { + "intValue": "1" + } + } + ], + "status": {} + } + ] + } + ] + }, + { + "resource": { + "attributes": [ + { + "key": "telemetry.sdk.language", + "value": { + "stringValue": "python" + } + }, + { + "key": "telemetry.sdk.name", + "value": { + "stringValue": "opentelemetry" + } + }, + { + "key": "telemetry.sdk.version", + "value": { + "stringValue": "1.36.0" + } + }, + { + "key": "service.version", + "value": { + "stringValue": "1.6.0" + } + }, + { + "key": "service.instance.id", + "value": { + "stringValue": "8ab056f473cf" + } + }, + { + "key": "deployment.environment.name", + "value": { + "stringValue": "lab" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "messenger" + }, + "spans": [ + { + "traceId": "rjRibxVkUG4nhGH46OMvpg==", + "spanId": "YyxbtmKfp3w=", + "parentSpanId": "hHhcYiMN8r0=", + "name": "attachment.consume", + "kind": "SPAN_KIND_CONSUMER", + "startTimeUnixNano": "1793948893852472632", + "endTimeUnixNano": "1793948893930183507", + "attributes": [ + { + "key": "messaging.retry.attempt", + "value": { + "intValue": "1" + } + } + ], + "status": {} + }, + { + "traceId": "rjRibxVkUG4nhGH46OMvpg==", + "spanId": "AoDGeW2ouFE=", + "parentSpanId": "YyxbtmKfp3w=", + "name": "postgres.attachment_transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1793948893915696007", + "endTimeUnixNano": "1793948893928589549", + "status": {} + }, + { + "traceId": "rjRibxVkUG4nhGH46OMvpg==", + "spanId": "/+BHiGdE7Mg=", + "parentSpanId": "AoDGeW2ouFE=", + "name": "postgres.claim_attachment_job", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1793948893916050591", + "endTimeUnixNano": "1793948893928426049", + "status": {} + } + ] + } + ] + }, + { + "resource": { + "attributes": [ + { + "key": "telemetry.sdk.language", + "value": { + "stringValue": "python" + } + }, + { + "key": "telemetry.sdk.name", + "value": { + "stringValue": "opentelemetry" + } + }, + { + "key": "telemetry.sdk.version", + "value": { + "stringValue": "1.36.0" + } + }, + { + "key": "service.version", + "value": { + "stringValue": "1.6.0" + } + }, + { + "key": "service.instance.id", + "value": { + "stringValue": "b106c9e70b85" + } + }, + { + "key": "deployment.environment.name", + "value": { + "stringValue": "lab" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "messenger-outbox" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "messenger" + }, + "spans": [ + { + "traceId": "rjRibxVkUG4nhGH46OMvpg==", + "spanId": "50KhQcpKHPM=", + "parentSpanId": "TjCZy2qExRI=", + "name": "attachment.publish", + "kind": "SPAN_KIND_PRODUCER", + "startTimeUnixNano": "1793948952742448132", + "endTimeUnixNano": "1793948952790699923", + "attributes": [ + { + "key": "messaging.retry.attempt", + "value": { + "intValue": "2" + } + } + ], + "status": {} + } + ] + } + ] + }, + { + "resource": { + "attributes": [ + { + "key": "telemetry.sdk.language", + "value": { + "stringValue": "python" + } + }, + { + "key": "telemetry.sdk.name", + "value": { + "stringValue": "opentelemetry" + } + }, + { + "key": "telemetry.sdk.version", + "value": { + "stringValue": "1.36.0" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + }, + { + "key": "service.version", + "value": { + "stringValue": "1.6.0" + } + }, + { + "key": "service.instance.id", + "value": { + "stringValue": "8ab056f473cf" + } + }, + { + "key": "deployment.environment.name", + "value": { + "stringValue": "lab" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "messenger" + }, + "spans": [ + { + "traceId": "rjRibxVkUG4nhGH46OMvpg==", + "spanId": "W1fSLS0+q94=", + "parentSpanId": "50KhQcpKHPM=", + "flags": 768, + "name": "attachment.consume", + "kind": "SPAN_KIND_CONSUMER", + "startTimeUnixNano": "1793948952789545048", + "endTimeUnixNano": "1793948953512550965", + "attributes": [ + { + "key": "messaging.retry.attempt", + "value": { + "intValue": "2" + } + } + ], + "status": {} + }, + { + "traceId": "rjRibxVkUG4nhGH46OMvpg==", + "spanId": "FNIzX117Ifc=", + "parentSpanId": "W1fSLS0+q94=", + "flags": 256, + "name": "postgres.attachment_transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1793948952811862298", + "endTimeUnixNano": "1793948953510339924", + "status": {} + }, + { + "traceId": "rjRibxVkUG4nhGH46OMvpg==", + "spanId": "/m/rSX5OYqA=", + "parentSpanId": "FNIzX117Ifc=", + "flags": 256, + "name": "s3.put_object", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1793948953499890965", + "endTimeUnixNano": "1793948953502638090", + "status": {} + }, + { + "traceId": "rjRibxVkUG4nhGH46OMvpg==", + "spanId": "N9dudhPIeg0=", + "parentSpanId": "FNIzX117Ifc=", + "flags": 256, + "name": "s3.put_object", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1793948953502674424", + "endTimeUnixNano": "1793948953504062215", + "status": {} + }, + { + "traceId": "rjRibxVkUG4nhGH46OMvpg==", + "spanId": "5WwABMSujUI=", + "parentSpanId": "FNIzX117Ifc=", + "flags": 256, + "name": "s3.put_object", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1793948953504093882", + "endTimeUnixNano": "1793948953505309674", + "status": {} + }, + { + "traceId": "rjRibxVkUG4nhGH46OMvpg==", + "spanId": "1EOfdNCSwms=", + "parentSpanId": "FNIzX117Ifc=", + "flags": 256, + "name": "postgres.complete_attachment_job", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1793948953505410007", + "endTimeUnixNano": "1793948953507989299", + "status": {} + }, + { + "traceId": "rjRibxVkUG4nhGH46OMvpg==", + "spanId": "mjqJgDBztVQ=", + "parentSpanId": "FNIzX117Ifc=", + "flags": 256, + "name": "s3.delete_object", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1793948953508583424", + "endTimeUnixNano": "1793948953510069090", + "status": {} + } + ] + } + ] + }, + { + "resource": { + "attributes": [ + { + "key": "telemetry.sdk.language", + "value": { + "stringValue": "python" + } + }, + { + "key": "telemetry.sdk.name", + "value": { + "stringValue": "opentelemetry" + } + }, + { + "key": "telemetry.sdk.version", + "value": { + "stringValue": "1.36.0" + } + }, + { + "key": "service.version", + "value": { + "stringValue": "1.6.0" + } + }, + { + "key": "service.instance.id", + "value": { + "stringValue": "8ab056f473cf" + } + }, + { + "key": "deployment.environment.name", + "value": { + "stringValue": "lab" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "messenger" + }, + "spans": [ + { + "traceId": "rjRibxVkUG4nhGH46OMvpg==", + "spanId": "lsvA6t1vfzY=", + "parentSpanId": "FNIzX117Ifc=", + "name": "postgres.claim_attachment_job", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1793948952812370090", + "endTimeUnixNano": "1793948952826915048", + "status": {} + }, + { + "traceId": "rjRibxVkUG4nhGH46OMvpg==", + "spanId": "otmRM1OBdj8=", + "parentSpanId": "FNIzX117Ifc=", + "name": "s3.copy_object", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1793948952827010590", + "endTimeUnixNano": "1793948952861476507", + "status": {} + }, + { + "traceId": "rjRibxVkUG4nhGH46OMvpg==", + "spanId": "DvSIq363rzM=", + "parentSpanId": "FNIzX117Ifc=", + "name": "s3.get_object", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1793948952861520423", + "endTimeUnixNano": "1793948952863334340", + "status": {} + } + ] + } + ] + } + ] + } +} diff --git a/evidence/review-tempo-load.json b/evidence/review-tempo-load.json new file mode 100644 index 0000000..231ce6c --- /dev/null +++ b/evidence/review-tempo-load.json @@ -0,0 +1,17 @@ +{ + "lab": 6, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 75.0, + "elapsed_seconds": 75.006, + "concurrency": 4, + "successful_requests": 40628, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 541.661, + "successful_latency_ms": { + "p50": 4.071, + "p95": 31.805, + "p99": 54.606 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/review-tempo-outage.json b/evidence/review-tempo-outage.json new file mode 100644 index 0000000..ec56245 --- /dev/null +++ b/evidence/review-tempo-outage.json @@ -0,0 +1,245 @@ +{ + "queries": { + "otelcol_exporter_enqueue_failed_spans_total": "sum(otelcol_exporter_enqueue_failed_spans_total{exporter=\"otlp/tempo\"}) or vector(0)", + "otelcol_exporter_send_failed_spans_total": "sum(otelcol_exporter_send_failed_spans_total{exporter=\"otlp/tempo\"}) or vector(0)", + "otelcol_exporter_sent_spans_total": "sum(otelcol_exporter_sent_spans_total{exporter=\"otlp/tempo\"}) or vector(0)", + "otelcol_exporter_queue_size": "sum(otelcol_exporter_queue_size{exporter=\"otlp/tempo\"}) or vector(0)", + "otelcol_exporter_queue_capacity": "sum(otelcol_exporter_queue_capacity{exporter=\"otlp/tempo\"}) or vector(0)" + }, + "samples": [ + { + "phase": "before", + "utc": "2026-11-06T07:09:41.743885+00:00", + "monotonic": 52353.585200125, + "metrics": { + "otelcol_exporter_enqueue_failed_spans_total": 56361.0, + "otelcol_exporter_send_failed_spans_total": 639.0, + "otelcol_exporter_sent_spans_total": 352596.0, + "otelcol_exporter_queue_size": 0.0, + "otelcol_exporter_queue_capacity": 128.0 + }, + "container_id": "022ca1971dd88b3dcfdd23ec5c4951830655070549631c29a2bddcedf354f45c", + "memory": "114.6MiB / 256MiB" + }, + { + "phase": "outage", + "utc": "2026-11-06T07:09:46.970039+00:00", + "monotonic": 52358.811228875, + "metrics": { + "otelcol_exporter_enqueue_failed_spans_total": 56361.0, + "otelcol_exporter_send_failed_spans_total": 639.0, + "otelcol_exporter_sent_spans_total": 352617.0, + "otelcol_exporter_queue_size": 0.0, + "otelcol_exporter_queue_capacity": 128.0 + }, + "container_id": "022ca1971dd88b3dcfdd23ec5c4951830655070549631c29a2bddcedf354f45c", + "memory": "116.5MiB / 256MiB" + }, + { + "phase": "outage", + "utc": "2026-11-06T07:09:58.085092+00:00", + "monotonic": 52369.9262515, + "metrics": { + "otelcol_exporter_enqueue_failed_spans_total": 56361.0, + "otelcol_exporter_send_failed_spans_total": 639.0, + "otelcol_exporter_sent_spans_total": 352617.0, + "otelcol_exporter_queue_size": 18.0, + "otelcol_exporter_queue_capacity": 128.0 + }, + "container_id": "022ca1971dd88b3dcfdd23ec5c4951830655070549631c29a2bddcedf354f45c", + "memory": "115.3MiB / 256MiB" + }, + { + "phase": "outage", + "utc": "2026-11-06T07:10:09.529528+00:00", + "monotonic": 52381.370598916, + "metrics": { + "otelcol_exporter_enqueue_failed_spans_total": 56361.0, + "otelcol_exporter_send_failed_spans_total": 639.0, + "otelcol_exporter_sent_spans_total": 352617.0, + "otelcol_exporter_queue_size": 60.0, + "otelcol_exporter_queue_capacity": 128.0 + }, + "container_id": "022ca1971dd88b3dcfdd23ec5c4951830655070549631c29a2bddcedf354f45c", + "memory": "112.3MiB / 256MiB" + }, + { + "phase": "outage", + "utc": "2026-11-06T07:10:20.835788+00:00", + "monotonic": 52392.676654541, + "metrics": { + "otelcol_exporter_enqueue_failed_spans_total": 56361.0, + "otelcol_exporter_send_failed_spans_total": 639.0, + "otelcol_exporter_sent_spans_total": 352617.0, + "otelcol_exporter_queue_size": 93.0, + "otelcol_exporter_queue_capacity": 128.0 + }, + "container_id": "022ca1971dd88b3dcfdd23ec5c4951830655070549631c29a2bddcedf354f45c", + "memory": "118.4MiB / 256MiB" + }, + { + "phase": "outage", + "utc": "2026-11-06T07:10:32.040068+00:00", + "monotonic": 52403.880798541, + "metrics": { + "otelcol_exporter_enqueue_failed_spans_total": 56361.0, + "otelcol_exporter_send_failed_spans_total": 639.0, + "otelcol_exporter_sent_spans_total": 352617.0, + "otelcol_exporter_queue_size": 125.0, + "otelcol_exporter_queue_capacity": 128.0 + }, + "container_id": "022ca1971dd88b3dcfdd23ec5c4951830655070549631c29a2bddcedf354f45c", + "memory": "120.5MiB / 256MiB" + }, + { + "phase": "outage", + "utc": "2026-11-06T07:10:43.165910+00:00", + "monotonic": 52415.006615083, + "metrics": { + "otelcol_exporter_enqueue_failed_spans_total": 71721.0, + "otelcol_exporter_send_failed_spans_total": 669.0, + "otelcol_exporter_sent_spans_total": 352617.0, + "otelcol_exporter_queue_size": 128.0, + "otelcol_exporter_queue_capacity": 128.0 + }, + "container_id": "022ca1971dd88b3dcfdd23ec5c4951830655070549631c29a2bddcedf354f45c", + "memory": "123MiB / 256MiB" + }, + { + "phase": "outage", + "utc": "2026-11-06T07:10:54.463041+00:00", + "monotonic": 52426.30365825, + "metrics": { + "otelcol_exporter_enqueue_failed_spans_total": 96297.0, + "otelcol_exporter_send_failed_spans_total": 669.0, + "otelcol_exporter_sent_spans_total": 352617.0, + "otelcol_exporter_queue_size": 128.0, + "otelcol_exporter_queue_capacity": 128.0 + }, + "container_id": "022ca1971dd88b3dcfdd23ec5c4951830655070549631c29a2bddcedf354f45c", + "memory": "125.9MiB / 256MiB" + }, + { + "phase": "before_restore", + "utc": "2026-11-06T07:11:05.578788+00:00", + "monotonic": 52437.419219458, + "metrics": { + "otelcol_exporter_enqueue_failed_spans_total": 113193.0, + "otelcol_exporter_send_failed_spans_total": 669.0, + "otelcol_exporter_sent_spans_total": 352617.0, + "otelcol_exporter_queue_size": 128.0, + "otelcol_exporter_queue_capacity": 128.0 + }, + "container_id": "022ca1971dd88b3dcfdd23ec5c4951830655070549631c29a2bddcedf354f45c", + "memory": "126.9MiB / 256MiB" + }, + { + "phase": "recovery", + "utc": "2026-11-06T07:11:12.171271+00:00", + "monotonic": 52444.011731041, + "metrics": { + "otelcol_exporter_enqueue_failed_spans_total": 113509.0, + "otelcol_exporter_send_failed_spans_total": 669.0, + "otelcol_exporter_sent_spans_total": 352617.0, + "otelcol_exporter_queue_size": 128.0, + "otelcol_exporter_queue_capacity": 128.0 + }, + "container_id": "022ca1971dd88b3dcfdd23ec5c4951830655070549631c29a2bddcedf354f45c", + "memory": "127MiB / 256MiB" + }, + { + "phase": "recovery", + "utc": "2026-11-06T07:11:18.348063+00:00", + "monotonic": 52450.188453375, + "metrics": { + "otelcol_exporter_enqueue_failed_spans_total": 113557.0, + "otelcol_exporter_send_failed_spans_total": 669.0, + "otelcol_exporter_sent_spans_total": 352617.0, + "otelcol_exporter_queue_size": 128.0, + "otelcol_exporter_queue_capacity": 128.0 + }, + "container_id": "022ca1971dd88b3dcfdd23ec5c4951830655070549631c29a2bddcedf354f45c", + "memory": "126.8MiB / 256MiB" + }, + { + "phase": "recovery", + "utc": "2026-11-06T07:11:24.531975+00:00", + "monotonic": 52456.372292375, + "metrics": { + "otelcol_exporter_enqueue_failed_spans_total": 113584.0, + "otelcol_exporter_send_failed_spans_total": 669.0, + "otelcol_exporter_sent_spans_total": 352617.0, + "otelcol_exporter_queue_size": 128.0, + "otelcol_exporter_queue_capacity": 128.0 + }, + "container_id": "022ca1971dd88b3dcfdd23ec5c4951830655070549631c29a2bddcedf354f45c", + "memory": "127.3MiB / 256MiB" + }, + { + "phase": "recovery", + "utc": "2026-11-06T07:11:30.708673+00:00", + "monotonic": 52462.548979291, + "metrics": { + "otelcol_exporter_enqueue_failed_spans_total": 113593.0, + "otelcol_exporter_send_failed_spans_total": 762.0, + "otelcol_exporter_sent_spans_total": 352617.0, + "otelcol_exporter_queue_size": 128.0, + "otelcol_exporter_queue_capacity": 128.0 + }, + "container_id": "022ca1971dd88b3dcfdd23ec5c4951830655070549631c29a2bddcedf354f45c", + "memory": "127.1MiB / 256MiB" + }, + { + "phase": "recovery", + "utc": "2026-11-06T07:11:36.891275+00:00", + "monotonic": 52468.731487041, + "metrics": { + "otelcol_exporter_enqueue_failed_spans_total": 113620.0, + "otelcol_exporter_send_failed_spans_total": 762.0, + "otelcol_exporter_sent_spans_total": 352617.0, + "otelcol_exporter_queue_size": 128.0, + "otelcol_exporter_queue_capacity": 128.0 + }, + "container_id": "022ca1971dd88b3dcfdd23ec5c4951830655070549631c29a2bddcedf354f45c", + "memory": "128MiB / 256MiB" + }, + { + "phase": "recovery_followup", + "utc": "2026-11-06T07:12:45.670764+00:00", + "monotonic": 52537.510424666, + "metrics": { + "otelcol_exporter_enqueue_failed_spans_total": 113629.0, + "otelcol_exporter_send_failed_spans_total": 1411.0, + "otelcol_exporter_sent_spans_total": 417456.0, + "otelcol_exporter_queue_size": 0.0, + "otelcol_exporter_queue_capacity": 128.0 + }, + "container_id": "022ca1971dd88b3dcfdd23ec5c4951830655070549631c29a2bddcedf354f45c", + "memory": "128.8MiB / 256MiB" + } + ], + "counter_deltas": { + "otelcol_exporter_enqueue_failed_spans_total": 57268.0, + "otelcol_exporter_send_failed_spans_total": 772.0, + "otelcol_exporter_sent_spans_total": 64860.0 + }, + "load": { + "lab": 6, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 75.0, + "elapsed_seconds": 75.006, + "concurrency": 4, + "successful_requests": 40628, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 541.661, + "successful_latency_ms": { + "p50": 4.071, + "p95": 31.805, + "p99": 54.606 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." + }, + "result": "PASS", + "recovery_verified": true +} diff --git a/evidence/review-tempo-persistence.json b/evidence/review-tempo-persistence.json new file mode 100644 index 0000000..7a65c4d --- /dev/null +++ b/evidence/review-tempo-persistence.json @@ -0,0 +1,93 @@ +{ + "utc": "2026-11-06T07:15:48.364407+00:00", + "trace_id": "ae34626f1564506e278461f8e8e32fa6", + "before_artifact": "review-retry-trace.json", + "query": "Tempo /api/traces/ae34626f1564506e278461f8e8e32fa6", + "after_spans": [ + [ + "/+BHiGdE7Mg=", + "postgres.claim_attachment_job" + ], + [ + "/m/rSX5OYqA=", + "s3.put_object" + ], + [ + "1EOfdNCSwms=", + "postgres.complete_attachment_job" + ], + [ + "50KhQcpKHPM=", + "attachment.publish" + ], + [ + "5WwABMSujUI=", + "s3.put_object" + ], + [ + "5ln8XViN22I=", + "postgres.user_for_token" + ], + [ + "AoDGeW2ouFE=", + "postgres.attachment_transaction" + ], + [ + "DvSIq363rzM=", + "s3.get_object" + ], + [ + "ENe80xA2A2k=", + "postgres.get_chat_for_member" + ], + [ + "FNIzX117Ifc=", + "postgres.attachment_transaction" + ], + [ + "N9dudhPIeg0=", + "s3.put_object" + ], + [ + "TjCZy2qExRI=", + "postgres.create_attachment" + ], + [ + "W1fSLS0+q94=", + "attachment.consume" + ], + [ + "YyxbtmKfp3w=", + "attachment.consume" + ], + [ + "Z4dP2xX/VNg=", + "postgres.attachment_transaction" + ], + [ + "hHhcYiMN8r0=", + "attachment.publish" + ], + [ + "i3B0Z3xER+A=", + "POST /api/v1/chats/{chat_id}/attachments" + ], + [ + "lsvA6t1vfzY=", + "postgres.claim_attachment_job" + ], + [ + "mjqJgDBztVQ=", + "s3.delete_object" + ], + [ + "otmRM1OBdj8=", + "s3.copy_object" + ], + [ + "zohqiMpi+QU=", + "s3.put_object" + ] + ], + "same_spans_after_restart": true +} diff --git a/evidence/slo-analysis.json b/evidence/slo-analysis.json index 9c5d11c..394d077 100644 --- a/evidence/slo-analysis.json +++ b/evidence/slo-analysis.json @@ -3,48 +3,54 @@ "monitor_errors": 0, "phases": { "burst": { - "observed_lock_seconds": 75.033, + "observed_lock_seconds": 75.001, "alerts": { "MessengerApiBurnFast": { "active_before_lock": false, - "first_firing": "2026-11-05T19:30:49.560423+00:00", + "first_firing": "2026-11-06T06:54:08.779613+00:00", "first_firing_after_incident": false, - "detection_seconds": 55.088, - "first_clear": "2026-11-05T19:31:24.687321+00:00", - "recovery_seconds": 15.181 + "detection_seconds": 54.676, + "first_clear": "2026-11-06T06:54:43.864649+00:00", + "recovery_seconds": 14.76 }, "MessengerApiBurnSlow": { "active_before_lock": false, - "first_firing": "2026-11-05T19:31:14.651231+00:00", - "first_firing_after_incident": true, - "detection_seconds": 80.179, - "first_clear": "2026-11-05T19:31:24.687321+00:00", - "recovery_seconds": 15.181 + "first_firing": null, + "first_firing_after_incident": false, + "detection_seconds": null, + "first_clear": null, + "recovery_seconds": null } } }, "sustained": { - "observed_lock_seconds": 180.134, + "observed_lock_seconds": 180.097, "alerts": { "MessengerApiBurnFast": { "active_before_lock": false, - "first_firing": "2026-11-05T19:33:40.096081+00:00", + "first_firing": "2026-11-06T06:56:59.279143+00:00", "first_firing_after_incident": false, - "detection_seconds": 50.278, - "first_clear": "2026-11-05T19:36:10.387240+00:00", - "recovery_seconds": 10.648 + "detection_seconds": 49.86, + "first_clear": "2026-11-06T06:59:24.818695+00:00", + "recovery_seconds": 15.301 }, "MessengerApiBurnSlow": { "active_before_lock": false, - "first_firing": "2026-11-05T19:34:20.245337+00:00", + "first_firing": "2026-11-06T06:57:39.433399+00:00", "first_firing_after_incident": false, - "detection_seconds": 90.428, - "first_clear": "2026-11-05T19:37:53.534744+00:00", - "recovery_seconds": 30.724 + "detection_seconds": 90.013, + "first_clear": "2026-11-06T06:59:49.911576+00:00", + "recovery_seconds": 40.394 } } } }, + "timing_quality": { + "max_wall_monotonic_disagreement_seconds": 0.0, + "max_observed_sampling_gap_seconds": 5.037, + "valid_for_elapsed_time_claims": true, + "note": "Disagreement does not identify its cause; rerun if invalid. Precision is bounded by actual gaps, not the configured sleep." + }, "baseline": { "samples": 13, "samples_with_burn_alert": 0, diff --git a/evidence/slo-incidents.json b/evidence/slo-incidents.json index 742ed8f..890db31 100644 --- a/evidence/slo-incidents.json +++ b/evidence/slo-incidents.json @@ -3,267 +3,296 @@ "periods": { "burst": { "lock_observed": { - "time": "2026-11-05T19:29:54.472744+00:00", - "monotonic": 43527.079238041, + "time": "2026-11-06T06:53:14.103079+00:00", + "monotonic": 51365.954057666, "command_started": { - "time": "2026-11-05T19:29:54.163828+00:00", - "monotonic": 43526.770364958 + "time": "2026-11-06T06:53:13.417102+00:00", + "monotonic": 51365.268133416 } }, "command_completed": { - "time": "2026-11-05T19:31:09.505932+00:00", - "monotonic": 43602.112587416 + "time": "2026-11-06T06:54:29.104850+00:00", + "monotonic": 51440.955130083 } }, "sustained": { "lock_observed": { - "time": "2026-11-05T19:32:49.817996+00:00", - "monotonic": 43702.424801208, + "time": "2026-11-06T06:56:09.419160+00:00", + "monotonic": 51541.268408583, "command_started": { - "time": "2026-11-05T19:32:49.516057+00:00", - "monotonic": 43702.122908708 + "time": "2026-11-06T06:56:09.111306+00:00", + "monotonic": 51540.960600125 } }, "command_completed": { - "time": "2026-11-05T19:35:59.744503+00:00", - "monotonic": 43882.558427291 + "time": "2026-11-06T06:59:09.517513+00:00", + "monotonic": 51721.365017541 } } }, "samples": [ { - "time": "2026-11-05T19:28:54.176617+00:00", - "monotonic": 43466.783021166, + "time": "2026-11-06T06:52:13.424045+00:00", + "monotonic": 51305.275633833, "phase": "baseline", "counts": { "ok": 1, "failed": 0 }, - "firing": [] + "firing": [], + "all_firing": [] }, { - "time": "2026-11-05T19:28:59.184768+00:00", - "monotonic": 43471.791177208, + "time": "2026-11-06T06:52:18.436944+00:00", + "monotonic": 51310.2884845, "phase": "baseline", "counts": { "ok": 15, "failed": 0 }, - "firing": [] + "firing": [], + "all_firing": [] }, { - "time": "2026-11-05T19:29:04.205356+00:00", - "monotonic": 43476.8117975, + "time": "2026-11-06T06:52:23.451635+00:00", + "monotonic": 51315.303133083, "phase": "baseline", "counts": { "ok": 30, "failed": 0 }, - "firing": [] + "firing": [], + "all_firing": [] }, { - "time": "2026-11-05T19:29:09.226269+00:00", - "monotonic": 43481.832718708, + "time": "2026-11-06T06:52:28.466169+00:00", + "monotonic": 51320.317629083, "phase": "baseline", "counts": { "ok": 45, "failed": 0 }, - "firing": [] + "firing": [], + "all_firing": [] }, { - "time": "2026-11-05T19:29:14.243046+00:00", - "monotonic": 43486.849499708, + "time": "2026-11-06T06:52:33.473068+00:00", + "monotonic": 51325.324459541, "phase": "baseline", "counts": { "ok": 60, "failed": 0 }, - "firing": [] + "firing": [], + "all_firing": [] }, { - "time": "2026-11-05T19:29:19.249330+00:00", - "monotonic": 43491.855772208, + "time": "2026-11-06T06:52:38.485045+00:00", + "monotonic": 51330.336387, "phase": "baseline", "counts": { "ok": 75, "failed": 0 }, - "firing": [] + "firing": [], + "all_firing": [] }, { - "time": "2026-11-05T19:29:24.265281+00:00", - "monotonic": 43496.871750375, + "time": "2026-11-06T06:52:43.495166+00:00", + "monotonic": 51335.346459208, "phase": "baseline", "counts": { "ok": 90, "failed": 0 }, - "firing": [] + "firing": [], + "all_firing": [] }, { - "time": "2026-11-05T19:29:29.284910+00:00", - "monotonic": 43501.891384583, + "time": "2026-11-06T06:52:48.528605+00:00", + "monotonic": 51340.379879458, "phase": "baseline", "counts": { "ok": 105, "failed": 0 }, - "firing": [] + "firing": [], + "all_firing": [] }, { - "time": "2026-11-05T19:29:34.295325+00:00", - "monotonic": 43506.901793458, + "time": "2026-11-06T06:52:53.543457+00:00", + "monotonic": 51345.394690083, "phase": "baseline", "counts": { "ok": 120, "failed": 0 }, - "firing": [] + "firing": [], + "all_firing": [] }, { - "time": "2026-11-05T19:29:39.306441+00:00", - "monotonic": 43511.91291825, + "time": "2026-11-06T06:52:58.563039+00:00", + "monotonic": 51350.414196125, "phase": "baseline", "counts": { "ok": 135, "failed": 0 }, - "firing": [] + "firing": [], + "all_firing": [] }, { - "time": "2026-11-05T19:29:44.313768+00:00", - "monotonic": 43516.920254333, + "time": "2026-11-06T06:53:03.585925+00:00", + "monotonic": 51355.437051291, "phase": "baseline", "counts": { "ok": 150, "failed": 0 }, - "firing": [] + "firing": [], + "all_firing": [] }, { - "time": "2026-11-05T19:29:49.331915+00:00", - "monotonic": 43521.938423708, + "time": "2026-11-06T06:53:08.603589+00:00", + "monotonic": 51360.454639416, "phase": "baseline", "counts": { "ok": 165, "failed": 0 }, - "firing": [] + "firing": [], + "all_firing": [] }, { - "time": "2026-11-05T19:29:54.351411+00:00", - "monotonic": 43526.957913708, + "time": "2026-11-06T06:53:13.615700+00:00", + "monotonic": 51365.466690666, "phase": "baseline", "counts": { "ok": 180, "failed": 0 }, - "firing": [] + "firing": [], + "all_firing": [] }, { - "time": "2026-11-05T19:29:59.372201+00:00", - "monotonic": 43531.978732458, + "time": "2026-11-06T06:53:18.628657+00:00", + "monotonic": 51370.479612125, "phase": "burst", "counts": { "ok": 180, "failed": 0 }, - "firing": [] + "firing": [], + "all_firing": [] }, { - "time": "2026-11-05T19:30:04.386019+00:00", - "monotonic": 43536.992552291, + "time": "2026-11-06T06:53:23.650959+00:00", + "monotonic": 51375.501877458, "phase": "burst", "counts": { "ok": 180, "failed": 3 }, - "firing": [] + "firing": [], + "all_firing": [] }, { - "time": "2026-11-05T19:30:09.401930+00:00", - "monotonic": 43542.0084625, + "time": "2026-11-06T06:53:28.671969+00:00", + "monotonic": 51380.522831125, "phase": "burst", "counts": { "ok": 180, "failed": 6 }, - "firing": [] + "firing": [], + "all_firing": [] }, { - "time": "2026-11-05T19:30:14.419029+00:00", - "monotonic": 43547.025585291, + "time": "2026-11-06T06:53:33.683094+00:00", + "monotonic": 51385.533890375, "phase": "burst", "counts": { "ok": 180, "failed": 9 }, - "firing": [] + "firing": [], + "all_firing": [] }, { - "time": "2026-11-05T19:30:19.433563+00:00", - "monotonic": 43552.040130916, + "time": "2026-11-06T06:53:38.695580+00:00", + "monotonic": 51390.546326541, "phase": "burst", "counts": { "ok": 180, "failed": 12 }, - "firing": [] + "firing": [], + "all_firing": [] }, { - "time": "2026-11-05T19:30:24.450146+00:00", - "monotonic": 43557.05672225, + "time": "2026-11-06T06:53:43.708619+00:00", + "monotonic": 51395.559316583, "phase": "burst", "counts": { "ok": 180, "failed": 15 }, - "firing": [] + "firing": [], + "all_firing": [] }, { - "time": "2026-11-05T19:30:29.473233+00:00", - "monotonic": 43562.079822583, + "time": "2026-11-06T06:53:48.719046+00:00", + "monotonic": 51400.569692583, "phase": "burst", "counts": { "ok": 180, "failed": 15 }, - "firing": [] + "firing": [], + "all_firing": [] }, { - "time": "2026-11-05T19:30:34.493955+00:00", - "monotonic": 43567.1005505, + "time": "2026-11-06T06:53:53.733123+00:00", + "monotonic": 51405.583727375, "phase": "burst", "counts": { "ok": 180, "failed": 18 }, - "firing": [] + "firing": [], + "all_firing": [ + "MessengerHighErrorRate" + ] }, { - "time": "2026-11-05T19:30:39.519370+00:00", - "monotonic": 43572.125976166, + "time": "2026-11-06T06:53:58.750268+00:00", + "monotonic": 51410.600817208, "phase": "burst", "counts": { "ok": 180, "failed": 21 }, - "firing": [] + "firing": [], + "all_firing": [ + "MessengerHighErrorRate" + ] }, { - "time": "2026-11-05T19:30:44.535535+00:00", - "monotonic": 43577.142128416, + "time": "2026-11-06T06:54:03.765059+00:00", + "monotonic": 51415.615563416, "phase": "burst", "counts": { "ok": 180, "failed": 24 }, - "firing": [] + "firing": [], + "all_firing": [ + "MessengerHighErrorRate" + ] }, { - "time": "2026-11-05T19:30:49.560423+00:00", - "monotonic": 43582.167042875, + "time": "2026-11-06T06:54:08.779613+00:00", + "monotonic": 51420.630081875, "phase": "burst", "counts": { "ok": 180, @@ -271,11 +300,15 @@ }, "firing": [ "MessengerApiBurnFast" + ], + "all_firing": [ + "MessengerApiBurnFast", + "MessengerHighErrorRate" ] }, { - "time": "2026-11-05T19:30:54.583572+00:00", - "monotonic": 43587.190215125, + "time": "2026-11-06T06:54:13.798324+00:00", + "monotonic": 51425.648740833, "phase": "burst", "counts": { "ok": 180, @@ -283,11 +316,15 @@ }, "firing": [ "MessengerApiBurnFast" + ], + "all_firing": [ + "MessengerApiBurnFast", + "MessengerHighErrorRate" ] }, { - "time": "2026-11-05T19:30:59.600250+00:00", - "monotonic": 43592.206889708, + "time": "2026-11-06T06:54:18.814338+00:00", + "monotonic": 51430.664704291, "phase": "burst", "counts": { "ok": 180, @@ -295,11 +332,15 @@ }, "firing": [ "MessengerApiBurnFast" + ], + "all_firing": [ + "MessengerApiBurnFast", + "MessengerHighErrorRate" ] }, { - "time": "2026-11-05T19:31:04.616531+00:00", - "monotonic": 43597.223167708, + "time": "2026-11-06T06:54:23.820189+00:00", + "monotonic": 51435.670496541, "phase": "burst", "counts": { "ok": 180, @@ -307,319 +348,378 @@ }, "firing": [ "MessengerApiBurnFast" + ], + "all_firing": [ + "MessengerApiBurnFast", + "MessengerHighErrorRate" ] }, { - "time": "2026-11-05T19:31:09.632643+00:00", - "monotonic": 43602.239279708, - "phase": "burst_recovery", + "time": "2026-11-06T06:54:28.827381+00:00", + "monotonic": 51440.677629875, + "phase": "burst", "counts": { - "ok": 183, + "ok": 180, "failed": 36 }, "firing": [ "MessengerApiBurnFast" + ], + "all_firing": [ + "MessengerApiBurnFast", + "MessengerHighErrorRate" ] }, { - "time": "2026-11-05T19:31:14.651231+00:00", - "monotonic": 43607.257891625, + "time": "2026-11-06T06:54:33.839856+00:00", + "monotonic": 51445.690054291, "phase": "burst_recovery", "counts": { - "ok": 198, + "ok": 195, "failed": 36 }, "firing": [ + "MessengerApiBurnFast" + ], + "all_firing": [ "MessengerApiBurnFast", - "MessengerApiBurnSlow" + "MessengerHighErrorRate" ] }, { - "time": "2026-11-05T19:31:19.663273+00:00", - "monotonic": 43612.269926416, + "time": "2026-11-06T06:54:38.847760+00:00", + "monotonic": 51450.697911083, "phase": "burst_recovery", "counts": { - "ok": 211, + "ok": 210, "failed": 36 }, "firing": [ + "MessengerApiBurnFast" + ], + "all_firing": [ "MessengerApiBurnFast", - "MessengerApiBurnSlow" + "MessengerHighErrorRate" ] }, { - "time": "2026-11-05T19:31:24.687321+00:00", - "monotonic": 43617.294002291, + "time": "2026-11-06T06:54:43.864649+00:00", + "monotonic": 51455.714782791, "phase": "burst_recovery", "counts": { "ok": 225, "failed": 36 }, - "firing": [] + "firing": [], + "all_firing": [ + "MessengerHighErrorRate" + ] }, { - "time": "2026-11-05T19:31:29.708468+00:00", - "monotonic": 43622.315158041, + "time": "2026-11-06T06:54:48.888857+00:00", + "monotonic": 51460.738907041, "phase": "burst_recovery", "counts": { "ok": 240, "failed": 36 }, - "firing": [] + "firing": [], + "all_firing": [ + "MessengerHighErrorRate" + ] }, { - "time": "2026-11-05T19:31:34.728166+00:00", - "monotonic": 43627.334856708, + "time": "2026-11-06T06:54:53.901671+00:00", + "monotonic": 51465.751684583, "phase": "burst_recovery", "counts": { "ok": 255, "failed": 36 }, - "firing": [] + "firing": [], + "all_firing": [ + "MessengerHighErrorRate" + ] }, { - "time": "2026-11-05T19:31:39.733767+00:00", - "monotonic": 43632.340459958, + "time": "2026-11-06T06:54:58.907581+00:00", + "monotonic": 51470.75753375, "phase": "burst_recovery", "counts": { "ok": 270, "failed": 36 }, - "firing": [] + "firing": [], + "all_firing": [ + "MessengerHighErrorRate" + ] }, { - "time": "2026-11-05T19:31:44.755480+00:00", - "monotonic": 43637.36218675, + "time": "2026-11-06T06:55:03.928237+00:00", + "monotonic": 51475.778157458, "phase": "burst_recovery", "counts": { "ok": 285, "failed": 36 }, - "firing": [] + "firing": [], + "all_firing": [ + "MessengerHighErrorRate" + ] }, { - "time": "2026-11-05T19:31:49.769677+00:00", - "monotonic": 43642.376390333, + "time": "2026-11-06T06:55:08.948107+00:00", + "monotonic": 51480.797979708, "phase": "burst_recovery", "counts": { "ok": 300, "failed": 36 }, - "firing": [] + "firing": [], + "all_firing": [ + "MessengerHighErrorRate" + ] }, { - "time": "2026-11-05T19:31:54.778133+00:00", - "monotonic": 43647.384860875, + "time": "2026-11-06T06:55:13.965736+00:00", + "monotonic": 51485.815554833, "phase": "burst_recovery", "counts": { "ok": 315, "failed": 36 }, - "firing": [] + "firing": [], + "all_firing": [ + "MessengerHighErrorRate" + ] }, { - "time": "2026-11-05T19:31:59.788234+00:00", - "monotonic": 43652.394963833, + "time": "2026-11-06T06:55:18.983097+00:00", + "monotonic": 51490.83286575, "phase": "burst_recovery", "counts": { "ok": 330, "failed": 36 }, - "firing": [] + "firing": [], + "all_firing": [] }, { - "time": "2026-11-05T19:32:04.797259+00:00", - "monotonic": 43657.403998, + "time": "2026-11-06T06:55:23.996671+00:00", + "monotonic": 51495.846385125, "phase": "burst_recovery", "counts": { "ok": 345, "failed": 36 }, - "firing": [] + "firing": [], + "all_firing": [] }, { - "time": "2026-11-05T19:32:09.813581+00:00", - "monotonic": 43662.42033325, + "time": "2026-11-06T06:55:29.010045+00:00", + "monotonic": 51500.859710625, "phase": "burst_recovery", "counts": { - "ok": 360, + "ok": 357, "failed": 36 }, - "firing": [] + "firing": [], + "all_firing": [] }, { - "time": "2026-11-05T19:32:14.823961+00:00", - "monotonic": 43667.430715458, + "time": "2026-11-06T06:55:34.032445+00:00", + "monotonic": 51505.882069958, "phase": "burst_recovery", "counts": { - "ok": 375, + "ok": 372, "failed": 36 }, - "firing": [] + "firing": [], + "all_firing": [] }, { - "time": "2026-11-05T19:32:19.834831+00:00", - "monotonic": 43672.441596833, + "time": "2026-11-06T06:55:39.048403+00:00", + "monotonic": 51510.897980041, "phase": "burst_recovery", "counts": { - "ok": 390, + "ok": 387, "failed": 36 }, - "firing": [] + "firing": [], + "all_firing": [] }, { - "time": "2026-11-05T19:32:24.847866+00:00", - "monotonic": 43677.454636958, + "time": "2026-11-06T06:55:44.065766+00:00", + "monotonic": 51515.915294375, "phase": "burst_recovery", "counts": { - "ok": 405, + "ok": 402, "failed": 36 }, - "firing": [] + "firing": [], + "all_firing": [] }, { - "time": "2026-11-05T19:32:29.862747+00:00", - "monotonic": 43682.469528375, + "time": "2026-11-06T06:55:49.081176+00:00", + "monotonic": 51520.930651458, "phase": "burst_recovery", "counts": { "ok": 417, "failed": 36 }, - "firing": [] + "firing": [], + "all_firing": [] }, { - "time": "2026-11-05T19:32:34.870564+00:00", - "monotonic": 43687.47735275, + "time": "2026-11-06T06:55:54.097495+00:00", + "monotonic": 51525.946920083, "phase": "burst_recovery", "counts": { "ok": 432, "failed": 36 }, - "firing": [] + "firing": [], + "all_firing": [] }, { - "time": "2026-11-05T19:32:39.880308+00:00", - "monotonic": 43692.487106, + "time": "2026-11-06T06:55:59.116926+00:00", + "monotonic": 51530.966305541, "phase": "burst_recovery", "counts": { "ok": 447, "failed": 36 }, - "firing": [] + "firing": [], + "all_firing": [] }, { - "time": "2026-11-05T19:32:44.899942+00:00", - "monotonic": 43697.506751, + "time": "2026-11-06T06:56:04.123010+00:00", + "monotonic": 51535.972316291, "phase": "burst_recovery", "counts": { "ok": 462, "failed": 36 }, - "firing": [] + "firing": [], + "all_firing": [] }, { - "time": "2026-11-05T19:32:49.916853+00:00", - "monotonic": 43702.523669458, + "time": "2026-11-06T06:56:09.139780+00:00", + "monotonic": 51540.989044958, + "phase": "burst_recovery", + "counts": { + "ok": 477, + "failed": 36 + }, + "firing": [], + "all_firing": [] + }, + { + "time": "2026-11-06T06:56:14.158912+00:00", + "monotonic": 51546.008146666, "phase": "sustained", "counts": { "ok": 477, "failed": 36 }, - "firing": [] + "firing": [], + "all_firing": [] }, { - "time": "2026-11-05T19:32:54.942437+00:00", - "monotonic": 43707.549288958, - "phase": "sustained", - "counts": { - "ok": 477, - "failed": 36 - }, - "firing": [] - }, - { - "time": "2026-11-05T19:32:59.957394+00:00", - "monotonic": 43712.564228083, + "time": "2026-11-06T06:56:19.170927+00:00", + "monotonic": 51551.020093583, "phase": "sustained", "counts": { "ok": 477, "failed": 39 }, - "firing": [] + "firing": [], + "all_firing": [] }, { - "time": "2026-11-05T19:33:04.970687+00:00", - "monotonic": 43717.577532333, + "time": "2026-11-06T06:56:24.181412+00:00", + "monotonic": 51556.030526708, "phase": "sustained", "counts": { "ok": 477, "failed": 42 }, - "firing": [] + "firing": [], + "all_firing": [] }, { - "time": "2026-11-05T19:33:09.993536+00:00", - "monotonic": 43722.600420875, + "time": "2026-11-06T06:56:29.189049+00:00", + "monotonic": 51561.038109333, "phase": "sustained", "counts": { "ok": 477, "failed": 45 }, - "firing": [] + "firing": [], + "all_firing": [] }, { - "time": "2026-11-05T19:33:15.009372+00:00", - "monotonic": 43727.616250583, + "time": "2026-11-06T06:56:34.205264+00:00", + "monotonic": 51566.054276125, "phase": "sustained", "counts": { "ok": 477, "failed": 48 }, - "firing": [] + "firing": [], + "all_firing": [] }, { - "time": "2026-11-05T19:33:20.031828+00:00", - "monotonic": 43732.638718, + "time": "2026-11-06T06:56:39.211225+00:00", + "monotonic": 51571.060184458, "phase": "sustained", "counts": { "ok": 477, "failed": 51 }, - "firing": [] + "firing": [], + "all_firing": [] }, { - "time": "2026-11-05T19:33:25.051997+00:00", - "monotonic": 43737.658880125, + "time": "2026-11-06T06:56:44.222128+00:00", + "monotonic": 51576.07104075, "phase": "sustained", "counts": { "ok": 477, "failed": 51 }, - "firing": [] + "firing": [], + "all_firing": [] }, { - "time": "2026-11-05T19:33:30.067799+00:00", - "monotonic": 43742.674685791, + "time": "2026-11-06T06:56:49.245180+00:00", + "monotonic": 51581.094060708, "phase": "sustained", "counts": { "ok": 477, "failed": 54 }, - "firing": [] + "firing": [], + "all_firing": [ + "MessengerHighErrorRate" + ] }, { - "time": "2026-11-05T19:33:35.075453+00:00", - "monotonic": 43747.682349166, + "time": "2026-11-06T06:56:54.262383+00:00", + "monotonic": 51586.11121325, "phase": "sustained", "counts": { "ok": 477, "failed": 57 }, - "firing": [] + "firing": [], + "all_firing": [ + "MessengerHighErrorRate" + ] }, { - "time": "2026-11-05T19:33:40.096081+00:00", - "monotonic": 43752.703001416, + "time": "2026-11-06T06:56:59.279143+00:00", + "monotonic": 51591.127925625, "phase": "sustained", "counts": { "ok": 477, @@ -627,11 +727,15 @@ }, "firing": [ "MessengerApiBurnFast" + ], + "all_firing": [ + "MessengerApiBurnFast", + "MessengerHighErrorRate" ] }, { - "time": "2026-11-05T19:33:45.119089+00:00", - "monotonic": 43757.7260235, + "time": "2026-11-06T06:57:04.297663+00:00", + "monotonic": 51596.146368833, "phase": "sustained", "counts": { "ok": 477, @@ -639,11 +743,15 @@ }, "firing": [ "MessengerApiBurnFast" + ], + "all_firing": [ + "MessengerApiBurnFast", + "MessengerHighErrorRate" ] }, { - "time": "2026-11-05T19:33:50.133709+00:00", - "monotonic": 43762.740644916, + "time": "2026-11-06T06:57:09.310113+00:00", + "monotonic": 51601.158780166, "phase": "sustained", "counts": { "ok": 477, @@ -651,11 +759,15 @@ }, "firing": [ "MessengerApiBurnFast" + ], + "all_firing": [ + "MessengerApiBurnFast", + "MessengerHighErrorRate" ] }, { - "time": "2026-11-05T19:33:55.161233+00:00", - "monotonic": 43767.768187291, + "time": "2026-11-06T06:57:14.323594+00:00", + "monotonic": 51606.172214833, "phase": "sustained", "counts": { "ok": 477, @@ -663,11 +775,15 @@ }, "firing": [ "MessengerApiBurnFast" + ], + "all_firing": [ + "MessengerApiBurnFast", + "MessengerHighErrorRate" ] }, { - "time": "2026-11-05T19:34:00.176785+00:00", - "monotonic": 43772.783745208, + "time": "2026-11-06T06:57:19.344779+00:00", + "monotonic": 51611.193360083, "phase": "sustained", "counts": { "ok": 477, @@ -675,11 +791,15 @@ }, "firing": [ "MessengerApiBurnFast" + ], + "all_firing": [ + "MessengerApiBurnFast", + "MessengerHighErrorRate" ] }, { - "time": "2026-11-05T19:34:05.196858+00:00", - "monotonic": 43777.803812291, + "time": "2026-11-06T06:57:24.364531+00:00", + "monotonic": 51616.213063416, "phase": "sustained", "counts": { "ok": 477, @@ -687,11 +807,15 @@ }, "firing": [ "MessengerApiBurnFast" + ], + "all_firing": [ + "MessengerApiBurnFast", + "MessengerHighErrorRate" ] }, { - "time": "2026-11-05T19:34:10.212935+00:00", - "monotonic": 43782.819901041, + "time": "2026-11-06T06:57:29.393320+00:00", + "monotonic": 51621.241828666, "phase": "sustained", "counts": { "ok": 477, @@ -699,11 +823,15 @@ }, "firing": [ "MessengerApiBurnFast" + ], + "all_firing": [ + "MessengerApiBurnFast", + "MessengerHighErrorRate" ] }, { - "time": "2026-11-05T19:34:15.231470+00:00", - "monotonic": 43787.838454541, + "time": "2026-11-06T06:57:34.412151+00:00", + "monotonic": 51626.260583541, "phase": "sustained", "counts": { "ok": 477, @@ -711,11 +839,15 @@ }, "firing": [ "MessengerApiBurnFast" + ], + "all_firing": [ + "MessengerApiBurnFast", + "MessengerHighErrorRate" ] }, { - "time": "2026-11-05T19:34:20.245337+00:00", - "monotonic": 43792.852329875, + "time": "2026-11-06T06:57:39.433399+00:00", + "monotonic": 51631.281782041, "phase": "sustained", "counts": { "ok": 477, @@ -724,11 +856,16 @@ "firing": [ "MessengerApiBurnFast", "MessengerApiBurnSlow" + ], + "all_firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow", + "MessengerHighErrorRate" ] }, { - "time": "2026-11-05T19:34:25.257984+00:00", - "monotonic": 43797.864980958, + "time": "2026-11-06T06:57:44.449034+00:00", + "monotonic": 51636.29735525, "phase": "sustained", "counts": { "ok": 477, @@ -737,11 +874,16 @@ "firing": [ "MessengerApiBurnFast", "MessengerApiBurnSlow" + ], + "all_firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow", + "MessengerHighErrorRate" ] }, { - "time": "2026-11-05T19:34:30.282024+00:00", - "monotonic": 43802.889036875, + "time": "2026-11-06T06:57:49.461390+00:00", + "monotonic": 51641.309667083, "phase": "sustained", "counts": { "ok": 477, @@ -750,11 +892,16 @@ "firing": [ "MessengerApiBurnFast", "MessengerApiBurnSlow" + ], + "all_firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow", + "MessengerHighErrorRate" ] }, { - "time": "2026-11-05T19:34:35.303328+00:00", - "monotonic": 43807.910348833, + "time": "2026-11-06T06:57:54.483943+00:00", + "monotonic": 51646.332173708, "phase": "sustained", "counts": { "ok": 477, @@ -763,11 +910,16 @@ "firing": [ "MessengerApiBurnFast", "MessengerApiBurnSlow" + ], + "all_firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow", + "MessengerHighErrorRate" ] }, { - "time": "2026-11-05T19:34:40.328342+00:00", - "monotonic": 43812.935370625, + "time": "2026-11-06T06:57:59.509458+00:00", + "monotonic": 51651.357645291, "phase": "sustained", "counts": { "ok": 477, @@ -776,11 +928,16 @@ "firing": [ "MessengerApiBurnFast", "MessengerApiBurnSlow" + ], + "all_firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow", + "MessengerHighErrorRate" ] }, { - "time": "2026-11-05T19:34:45.343604+00:00", - "monotonic": 43817.950633291, + "time": "2026-11-06T06:58:04.535029+00:00", + "monotonic": 51656.383169208, "phase": "sustained", "counts": { "ok": 477, @@ -789,11 +946,16 @@ "firing": [ "MessengerApiBurnFast", "MessengerApiBurnSlow" + ], + "all_firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow", + "MessengerHighErrorRate" ] }, { - "time": "2026-11-05T19:34:50.359471+00:00", - "monotonic": 43822.966518166, + "time": "2026-11-06T06:58:09.551012+00:00", + "monotonic": 51661.399097541, "phase": "sustained", "counts": { "ok": 477, @@ -802,11 +964,16 @@ "firing": [ "MessengerApiBurnFast", "MessengerApiBurnSlow" + ], + "all_firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow", + "MessengerHighErrorRate" ] }, { - "time": "2026-11-05T19:34:55.367578+00:00", - "monotonic": 43827.974616416, + "time": "2026-11-06T06:58:14.567487+00:00", + "monotonic": 51666.415528875, "phase": "sustained", "counts": { "ok": 477, @@ -815,11 +982,16 @@ "firing": [ "MessengerApiBurnFast", "MessengerApiBurnSlow" + ], + "all_firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow", + "MessengerHighErrorRate" ] }, { - "time": "2026-11-05T19:35:00.386215+00:00", - "monotonic": 43832.993284208, + "time": "2026-11-06T06:58:19.584767+00:00", + "monotonic": 51671.432743666, "phase": "sustained", "counts": { "ok": 477, @@ -828,11 +1000,16 @@ "firing": [ "MessengerApiBurnFast", "MessengerApiBurnSlow" + ], + "all_firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow", + "MessengerHighErrorRate" ] }, { - "time": "2026-11-05T19:35:05.400579+00:00", - "monotonic": 43838.0076475, + "time": "2026-11-06T06:58:24.610726+00:00", + "monotonic": 51676.458661375, "phase": "sustained", "counts": { "ok": 477, @@ -841,11 +1018,16 @@ "firing": [ "MessengerApiBurnFast", "MessengerApiBurnSlow" + ], + "all_firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow", + "MessengerHighErrorRate" ] }, { - "time": "2026-11-05T19:35:10.422157+00:00", - "monotonic": 43843.0292365, + "time": "2026-11-06T06:58:29.636000+00:00", + "monotonic": 51681.483882625, "phase": "sustained", "counts": { "ok": 477, @@ -854,11 +1036,16 @@ "firing": [ "MessengerApiBurnFast", "MessengerApiBurnSlow" + ], + "all_firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow", + "MessengerHighErrorRate" ] }, { - "time": "2026-11-05T19:35:15.442075+00:00", - "monotonic": 43848.049173416, + "time": "2026-11-06T06:58:34.651395+00:00", + "monotonic": 51686.499233166, "phase": "sustained", "counts": { "ok": 477, @@ -867,11 +1054,16 @@ "firing": [ "MessengerApiBurnFast", "MessengerApiBurnSlow" + ], + "all_firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow", + "MessengerHighErrorRate" ] }, { - "time": "2026-11-05T19:35:20.452042+00:00", - "monotonic": 43853.05914025, + "time": "2026-11-06T06:58:39.665021+00:00", + "monotonic": 51691.512806666, "phase": "sustained", "counts": { "ok": 477, @@ -880,11 +1072,16 @@ "firing": [ "MessengerApiBurnFast", "MessengerApiBurnSlow" + ], + "all_firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow", + "MessengerHighErrorRate" ] }, { - "time": "2026-11-05T19:35:25.461182+00:00", - "monotonic": 43858.068280916, + "time": "2026-11-06T06:58:44.671694+00:00", + "monotonic": 51696.519412875, "phase": "sustained", "counts": { "ok": 477, @@ -893,11 +1090,16 @@ "firing": [ "MessengerApiBurnFast", "MessengerApiBurnSlow" + ], + "all_firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow", + "MessengerHighErrorRate" ] }, { - "time": "2026-11-05T19:35:30.480197+00:00", - "monotonic": 43863.087333625, + "time": "2026-11-06T06:58:49.683388+00:00", + "monotonic": 51701.531061666, "phase": "sustained", "counts": { "ok": 477, @@ -906,11 +1108,16 @@ "firing": [ "MessengerApiBurnFast", "MessengerApiBurnSlow" + ], + "all_firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow", + "MessengerHighErrorRate" ] }, { - "time": "2026-11-05T19:35:35.504075+00:00", - "monotonic": 43868.111188291, + "time": "2026-11-06T06:58:54.694272+00:00", + "monotonic": 51706.541890833, "phase": "sustained", "counts": { "ok": 477, @@ -919,11 +1126,16 @@ "firing": [ "MessengerApiBurnFast", "MessengerApiBurnSlow" + ], + "all_firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow", + "MessengerHighErrorRate" ] }, { - "time": "2026-11-05T19:35:40.521486+00:00", - "monotonic": 43873.129961791, + "time": "2026-11-06T06:58:59.718693+00:00", + "monotonic": 51711.56628275, "phase": "sustained", "counts": { "ok": 477, @@ -932,11 +1144,16 @@ "firing": [ "MessengerApiBurnFast", "MessengerApiBurnSlow" + ], + "all_firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow", + "MessengerHighErrorRate" ] }, { - "time": "2026-11-05T19:35:55.348285+00:00", - "monotonic": 43878.158763833, + "time": "2026-11-06T06:59:04.743549+00:00", + "monotonic": 51716.59108925, "phase": "sustained", "counts": { "ok": 477, @@ -945,226 +1162,289 @@ "firing": [ "MessengerApiBurnFast", "MessengerApiBurnSlow" + ], + "all_firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow", + "MessengerHighErrorRate" ] }, { - "time": "2026-11-05T19:36:00.359177+00:00", - "monotonic": 43883.173501041, + "time": "2026-11-06T06:59:09.765218+00:00", + "monotonic": 51721.61271225, "phase": "sustained_recovery", "counts": { - "ok": 480, - "failed": 123 + "ok": 477, + "failed": 126 }, "firing": [ "MessengerApiBurnFast", "MessengerApiBurnSlow" + ], + "all_firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow", + "MessengerHighErrorRate" ] }, { - "time": "2026-11-05T19:36:05.367876+00:00", - "monotonic": 43888.184996, + "time": "2026-11-06T06:59:14.783393+00:00", + "monotonic": 51726.630834, "phase": "sustained_recovery", "counts": { - "ok": 495, - "failed": 123 + "ok": 492, + "failed": 126 }, "firing": [ "MessengerApiBurnFast", "MessengerApiBurnSlow" + ], + "all_firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow", + "MessengerHighErrorRate" ] }, { - "time": "2026-11-05T19:36:10.387240+00:00", - "monotonic": 43893.206441208, + "time": "2026-11-06T06:59:19.797030+00:00", + "monotonic": 51731.644447416, "phase": "sustained_recovery", "counts": { - "ok": 510, - "failed": 123 + "ok": 507, + "failed": 126 + }, + "firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow" + ], + "all_firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow", + "MessengerHighErrorRate" + ] + }, + { + "time": "2026-11-06T06:59:24.818695+00:00", + "monotonic": 51736.666055666, + "phase": "sustained_recovery", + "counts": { + "ok": 522, + "failed": 126 }, "firing": [ "MessengerApiBurnSlow" + ], + "all_firing": [ + "MessengerApiBurnSlow", + "MessengerHighErrorRate" ] }, { - "time": "2026-11-05T19:36:15.397936+00:00", - "monotonic": 43898.218563625, + "time": "2026-11-06T06:59:29.831731+00:00", + "monotonic": 51741.679002958, "phase": "sustained_recovery", "counts": { - "ok": 525, - "failed": 123 + "ok": 537, + "failed": 126 }, "firing": [ "MessengerApiBurnSlow" + ], + "all_firing": [ + "MessengerApiBurnSlow", + "MessengerHighErrorRate" ] }, { - "time": "2026-11-05T19:36:20.421727+00:00", - "monotonic": 43903.243449875, + "time": "2026-11-06T06:59:34.843659+00:00", + "monotonic": 51746.690912916, "phase": "sustained_recovery", "counts": { - "ok": 540, - "failed": 123 + "ok": 551, + "failed": 126 }, "firing": [ "MessengerApiBurnSlow" + ], + "all_firing": [ + "MessengerApiBurnSlow", + "MessengerHighErrorRate" ] }, { - "time": "2026-11-05T19:37:48.510428+00:00", - "monotonic": 43908.257572375, + "time": "2026-11-06T06:59:39.857744+00:00", + "monotonic": 51751.704937458, "phase": "sustained_recovery", "counts": { - "ok": 555, - "failed": 123 + "ok": 564, + "failed": 126 }, "firing": [ "MessengerApiBurnSlow" + ], + "all_firing": [ + "MessengerApiBurnSlow", + "MessengerHighErrorRate" ] }, { - "time": "2026-11-05T19:37:53.534744+00:00", - "monotonic": 43913.282484708, + "time": "2026-11-06T06:59:44.874968+00:00", + "monotonic": 51756.722113666, "phase": "sustained_recovery", "counts": { - "ok": 570, - "failed": 123 + "ok": 579, + "failed": 126 }, - "firing": [] + "firing": [ + "MessengerApiBurnSlow" + ], + "all_firing": [ + "MessengerApiBurnSlow", + "MessengerHighErrorRate" + ] }, { - "time": "2026-11-05T19:37:58.549296+00:00", - "monotonic": 43918.29743975, + "time": "2026-11-06T06:59:49.911576+00:00", + "monotonic": 51761.75867025, "phase": "sustained_recovery", "counts": { - "ok": 585, - "failed": 123 + "ok": 594, + "failed": 126 }, - "firing": [] + "firing": [], + "all_firing": [ + "MessengerHighErrorRate" + ] }, { - "time": "2026-11-05T19:38:03.567430+00:00", - "monotonic": 43923.315891, + "time": "2026-11-06T06:59:54.935006+00:00", + "monotonic": 51766.782047625, "phase": "sustained_recovery", "counts": { - "ok": 600, - "failed": 123 + "ok": 609, + "failed": 126 }, - "firing": [] + "firing": [], + "all_firing": [ + "MessengerHighErrorRate" + ] }, { - "time": "2026-11-05T19:38:08.593041+00:00", - "monotonic": 43928.341731291, + "time": "2026-11-06T06:59:59.946639+00:00", + "monotonic": 51771.793639375, "phase": "sustained_recovery", "counts": { - "ok": 615, - "failed": 123 + "ok": 624, + "failed": 126 }, - "firing": [] + "firing": [], + "all_firing": [ + "MessengerHighErrorRate" + ] }, { - "time": "2026-11-05T19:38:13.607305+00:00", - "monotonic": 43933.356177, + "time": "2026-11-06T07:00:04.962978+00:00", + "monotonic": 51776.809903958, "phase": "sustained_recovery", "counts": { - "ok": 630, - "failed": 123 + "ok": 639, + "failed": 126 }, - "firing": [] + "firing": [], + "all_firing": [] }, { - "time": "2026-11-05T19:38:18.617266+00:00", - "monotonic": 43938.366246166, + "time": "2026-11-06T07:00:09.975033+00:00", + "monotonic": 51781.821923208, "phase": "sustained_recovery", "counts": { - "ok": 642, - "failed": 123 + "ok": 654, + "failed": 126 }, - "firing": [] + "firing": [], + "all_firing": [] }, { - "time": "2026-11-05T19:38:23.636630+00:00", - "monotonic": 43943.385724875, + "time": "2026-11-06T07:00:14.986604+00:00", + "monotonic": 51786.833433416, "phase": "sustained_recovery", "counts": { - "ok": 657, - "failed": 123 + "ok": 669, + "failed": 126 }, - "firing": [] + "firing": [], + "all_firing": [] }, { - "time": "2026-11-05T19:38:28.654169+00:00", - "monotonic": 43948.403359125, + "time": "2026-11-06T07:00:20.002499+00:00", + "monotonic": 51791.849318, "phase": "sustained_recovery", "counts": { - "ok": 672, - "failed": 123 + "ok": 684, + "failed": 126 }, - "firing": [] + "firing": [], + "all_firing": [] }, { - "time": "2026-11-05T19:38:33.670193+00:00", - "monotonic": 43953.419418, + "time": "2026-11-06T07:00:25.020127+00:00", + "monotonic": 51796.866889708, "phase": "sustained_recovery", "counts": { - "ok": 687, - "failed": 123 + "ok": 699, + "failed": 126 }, - "firing": [] + "firing": [], + "all_firing": [] }, { - "time": "2026-11-05T19:38:38.676034+00:00", - "monotonic": 43958.425318541, + "time": "2026-11-06T07:00:30.038727+00:00", + "monotonic": 51801.88541975, "phase": "sustained_recovery", "counts": { - "ok": 702, - "failed": 123 + "ok": 714, + "failed": 126 }, - "firing": [] + "firing": [], + "all_firing": [] }, { - "time": "2026-11-05T19:38:43.699279+00:00", - "monotonic": 43963.448615416, + "time": "2026-11-06T07:00:35.054844+00:00", + "monotonic": 51806.901469125, "phase": "sustained_recovery", "counts": { - "ok": 717, - "failed": 123 + "ok": 729, + "failed": 126 }, - "firing": [] + "firing": [], + "all_firing": [] }, { - "time": "2026-11-05T19:38:48.718747+00:00", - "monotonic": 43968.468122083, + "time": "2026-11-06T07:00:40.071309+00:00", + "monotonic": 51811.917906208, "phase": "sustained_recovery", "counts": { - "ok": 732, - "failed": 123 + "ok": 744, + "failed": 126 }, - "firing": [] + "firing": [], + "all_firing": [] }, { - "time": "2026-11-05T19:38:53.742289+00:00", - "monotonic": 43973.491697375, + "time": "2026-11-06T07:00:45.095471+00:00", + "monotonic": 51816.942013541, "phase": "sustained_recovery", "counts": { - "ok": 747, - "failed": 123 + "ok": 757, + "failed": 126 }, - "firing": [] - }, - { - "time": "2026-11-05T19:38:58.757203+00:00", - "monotonic": 43978.50662725, - "phase": "sustained_recovery", - "counts": { - "ok": 762, - "failed": 123 - }, - "firing": [] + "firing": [], + "all_firing": [] } ], - "end_time": "2026-11-05T19:39:02.820853+00:00", + "end_time": "2026-11-06T07:00:49.528862+00:00", "final_counts": { - "ok": 774, - "failed": 123 + "ok": 771, + "failed": 126 } } diff --git a/evidence/slo-ready-incident.json b/evidence/slo-ready-incident.json index 5eff503..a14cd2c 100644 --- a/evidence/slo-ready-incident.json +++ b/evidence/slo-ready-incident.json @@ -1,211 +1,302 @@ { "samples": [ { - "time": "2026-11-06T06:18:43.857552+00:00", - "monotonic": 49295.73000975, + "time": "2026-11-06T07:05:52.042304+00:00", + "monotonic": 52123.885794541, "phase": "baseline", "firing": [], - "eligible": 9.0, - "bad": 4.0 + "all_firing": [], + "queue_depth": 0.0, + "queue_age": 0.0, + "eligible": 17.0, + "bad": 16.0 }, { - "time": "2026-11-06T06:18:49.195942+00:00", - "monotonic": 49301.068345333, + "time": "2026-11-06T07:05:57.341477+00:00", + "monotonic": 52129.184913625, "phase": "worker_stopped", "firing": [], - "eligible": 9.0, - "bad": 4.0 + "all_firing": [], + "queue_depth": 0.0, + "queue_age": 0.0, + "eligible": 17.0, + "bad": 16.0 }, { - "time": "2026-11-06T06:18:54.211236+00:00", - "monotonic": 49306.083585125, + "time": "2026-11-06T07:06:02.378900+00:00", + "monotonic": 52134.222306833, "phase": "worker_stopped", "firing": [], - "eligible": 9.0, - "bad": 4.0 + "all_firing": [], + "queue_depth": 0.0, + "queue_age": 0.0, + "eligible": 17.0, + "bad": 16.0 }, { - "time": "2026-11-06T06:18:59.238528+00:00", - "monotonic": 49311.110847333, + "time": "2026-11-06T07:06:07.432741+00:00", + "monotonic": 52139.276099166, "phase": "worker_stopped", "firing": [], - "eligible": 9.0, - "bad": 4.0 + "all_firing": [], + "queue_depth": 8.0, + "queue_age": 1.943177, + "eligible": 17.0, + "bad": 16.0 }, { - "time": "2026-11-06T06:19:04.276851+00:00", - "monotonic": 49316.149107833, + "time": "2026-11-06T07:06:12.488438+00:00", + "monotonic": 52144.331749416, "phase": "worker_stopped", "firing": [], - "eligible": 9.0, - "bad": 4.0 + "all_firing": [], + "queue_depth": 8.0, + "queue_age": 6.984557, + "eligible": 17.0, + "bad": 16.0 }, { - "time": "2026-11-06T06:19:09.316299+00:00", - "monotonic": 49321.18853025, + "time": "2026-11-06T07:06:17.540313+00:00", + "monotonic": 52149.383570875, "phase": "worker_stopped", "firing": [], - "eligible": 9.0, - "bad": 4.0 + "all_firing": [], + "queue_depth": 8.0, + "queue_age": 12.016421, + "eligible": 17.0, + "bad": 16.0 }, { - "time": "2026-11-06T06:19:14.361014+00:00", - "monotonic": 49326.233190833, + "time": "2026-11-06T07:06:22.591885+00:00", + "monotonic": 52154.435094083, "phase": "worker_stopped", "firing": [], - "eligible": 9.0, - "bad": 4.0 + "all_firing": [], + "queue_depth": 8.0, + "queue_age": 17.061549, + "eligible": 17.0, + "bad": 16.0 }, { - "time": "2026-11-06T06:19:19.399292+00:00", - "monotonic": 49331.271419, + "time": "2026-11-06T07:06:27.629967+00:00", + "monotonic": 52159.473106666, "phase": "worker_stopped", "firing": [], - "eligible": 9.0, - "bad": 4.0 + "all_firing": [], + "queue_depth": 8.0, + "queue_age": 22.089736, + "eligible": 17.0, + "bad": 16.0 }, { - "time": "2026-11-06T06:19:24.439389+00:00", - "monotonic": 49336.31146675, + "time": "2026-11-06T07:06:32.659343+00:00", + "monotonic": 52164.502452166, "phase": "worker_stopped", "firing": [], - "eligible": 9.0, - "bad": 4.0 + "all_firing": [], + "queue_depth": 8.0, + "queue_age": 27.133281, + "eligible": 17.0, + "bad": 16.0 }, { - "time": "2026-11-06T06:19:29.469853+00:00", - "monotonic": 49341.341864875, + "time": "2026-11-06T07:06:37.707157+00:00", + "monotonic": 52169.550216041, "phase": "worker_stopped", "firing": [ "MessengerReadyBurnFast", "MessengerReadyBurnSlow" ], - "eligible": 17.0, - "bad": 12.0 + "all_firing": [ + "MessengerReadyBurnFast", + "MessengerReadyBurnSlow" + ], + "queue_depth": 8.0, + "queue_age": 32.172568, + "eligible": 25.0, + "bad": 24.0 }, { - "time": "2026-11-06T06:19:35.732760+00:00", - "monotonic": 49347.604713666, - "phase": "recovery", + "time": "2026-11-06T07:06:42.757662+00:00", + "monotonic": 52174.600673666, + "phase": "worker_stopped", "firing": [ "MessengerReadyBurnFast", "MessengerReadyBurnSlow" ], - "eligible": 17.0, - "bad": 12.0 + "all_firing": [ + "MessengerReadyBurnFast", + "MessengerReadyBurnSlow" + ], + "queue_depth": 8.0, + "queue_age": 37.20728, + "eligible": 25.0, + "bad": 24.0 }, { - "time": "2026-11-06T06:19:40.766904+00:00", - "monotonic": 49352.638829583, - "phase": "recovery", + "time": "2026-11-06T07:06:47.807056+00:00", + "monotonic": 52179.650017583, + "phase": "worker_stopped", "firing": [ "MessengerReadyBurnFast", "MessengerReadyBurnSlow" ], - "eligible": 17.0, - "bad": 12.0 + "all_firing": [ + "MessengerReadyBurnFast", + "MessengerReadyBurnSlow" + ], + "queue_depth": 8.0, + "queue_age": 42.242111, + "eligible": 25.0, + "bad": 24.0 }, { - "time": "2026-11-06T06:19:45.800827+00:00", - "monotonic": 49357.672704333, + "time": "2026-11-06T07:06:52.863166+00:00", + "monotonic": 52184.706071666, + "phase": "worker_stopped", + "firing": [ + "MessengerReadyBurnSlow" + ], + "all_firing": [ + "MessengerQueueLag", + "MessengerReadyBurnSlow" + ], + "queue_depth": 8.0, + "queue_age": 47.286266, + "eligible": 25.0, + "bad": 24.0 + }, + { + "time": "2026-11-06T07:06:57.078846+00:00", + "monotonic": 52188.921692166, "phase": "recovery", "firing": [ "MessengerReadyBurnSlow" ], - "eligible": 17.0, - "bad": 12.0 + "all_firing": [ + "MessengerQueueLag", + "MessengerReadyBurnSlow" + ], + "queue_depth": 8.0, + "queue_age": 52.325419, + "eligible": 25.0, + "bad": 24.0 }, { - "time": "2026-11-06T06:19:50.839444+00:00", - "monotonic": 49362.711278166, + "time": "2026-11-06T07:07:02.119205+00:00", + "monotonic": 52193.962019291, "phase": "recovery", "firing": [ "MessengerReadyBurnSlow" ], - "eligible": 17.0, - "bad": 12.0 + "all_firing": [ + "MessengerQueueLag", + "MessengerReadyBurnSlow" + ], + "queue_depth": 6.0, + "queue_age": 57.330259, + "eligible": 25.0, + "bad": 24.0 }, { - "time": "2026-11-06T06:19:55.882108+00:00", - "monotonic": 49367.753864791, + "time": "2026-11-06T07:07:07.168689+00:00", + "monotonic": 52199.011455875, "phase": "recovery", "firing": [ "MessengerReadyBurnSlow" ], - "eligible": 17.0, - "bad": 12.0 + "all_firing": [ + "MessengerReadyBurnSlow" + ], + "queue_depth": 0.0, + "queue_age": 0.0, + "eligible": 25.0, + "bad": 24.0 }, { - "time": "2026-11-06T06:20:00.935125+00:00", - "monotonic": 49372.806871, + "time": "2026-11-06T07:07:12.218862+00:00", + "monotonic": 52204.061583875, "phase": "recovery", "firing": [ "MessengerReadyBurnSlow" ], - "eligible": 17.0, - "bad": 12.0 + "all_firing": [ + "MessengerReadyBurnSlow" + ], + "queue_depth": 0.0, + "queue_age": 0.0, + "eligible": 25.0, + "bad": 24.0 }, { - "time": "2026-11-06T06:20:05.976123+00:00", - "monotonic": 49377.847798625, + "time": "2026-11-06T07:07:17.266525+00:00", + "monotonic": 52209.109195083, "phase": "recovery", "firing": [ "MessengerReadyBurnSlow" ], - "eligible": 17.0, - "bad": 12.0 + "all_firing": [ + "MessengerReadyBurnSlow" + ], + "queue_depth": 0.0, + "queue_age": 0.0, + "eligible": 25.0, + "bad": 24.0 }, { - "time": "2026-11-06T06:20:11.006201+00:00", - "monotonic": 49382.877811166, + "time": "2026-11-06T07:07:22.324719+00:00", + "monotonic": 52214.16734425, "phase": "recovery", "firing": [ "MessengerReadyBurnSlow" ], - "eligible": 17.0, - "bad": 12.0 + "all_firing": [ + "MessengerReadyBurnSlow" + ], + "queue_depth": 0.0, + "queue_age": 0.0, + "eligible": 25.0, + "bad": 24.0 }, { - "time": "2026-11-06T06:20:16.034665+00:00", - "monotonic": 49387.906272875, + "time": "2026-11-06T07:07:27.378443+00:00", + "monotonic": 52219.221010083, "phase": "recovery", "firing": [ "MessengerReadyBurnSlow" ], - "eligible": 17.0, - "bad": 12.0 - }, - { - "time": "2026-11-06T06:20:21.073601+00:00", - "monotonic": 49392.945128916, - "phase": "recovery", - "firing": [ + "all_firing": [ "MessengerReadyBurnSlow" ], - "eligible": 17.0, - "bad": 12.0 + "queue_depth": 0.0, + "queue_age": 0.0, + "eligible": 25.0, + "bad": 24.0 }, { - "time": "2026-11-06T06:20:26.112334+00:00", - "monotonic": 49397.983810916, + "time": "2026-11-06T07:07:32.422103+00:00", + "monotonic": 52224.264601875, "phase": "recovery", "firing": [], - "eligible": 17.0, - "bad": 12.0 + "all_firing": [], + "queue_depth": 0.0, + "queue_age": 0.0, + "eligible": 25.0, + "bad": 24.0 } ], "jobs": 8, "accepted_at": { - "time": "2026-11-06T06:18:49.190891+00:00", - "monotonic": 49301.063291333 + "time": "2026-11-06T07:05:57.337134+00:00", + "monotonic": 52129.180572833 }, "worker_started": { - "time": "2026-11-06T06:19:31.951591+00:00", - "monotonic": 49343.823643916 + "time": "2026-11-06T07:06:53.291419+00:00", + "monotonic": 52185.134359875 }, "all_ready": { - "time": "2026-11-06T06:19:35.725614+00:00", - "monotonic": 49347.597564333 + "time": "2026-11-06T07:06:57.073151+00:00", + "monotonic": 52188.915998458 }, "result": "PASS" } diff --git a/evidence/slo-track.md b/evidence/slo-track.md index a24838f..6d2c20e 100644 --- a/evidence/slo-track.md +++ b/evidence/slo-track.md @@ -8,27 +8,31 @@ Испытания на существующем локальном стенде Docker Compose. API: три клиентских потока создавали синтетические сообщения, каждый делал паузу 1 секунду между завершившимися запросами; до воздействия — 60 секунд baseline. На время воздействия таблица `users` блокировалась транзакцией на 75 и 180 секунд, между опытами было 100 секунд восстановления. Это воспроизводимая искусственная блокировка, а не реальный пользовательский трафик. Каждые примерно 5 секунд сценарий запрашивал firing alerts через Grafana/Prometheus и сохранял монотонное время. Начало — момент, когда скрипт увидел lock, конец — возврат транзакции. Эти точки и опрос имеют собственную задержку; полученные длительности — наблюдательные оценки, не точные времена переключения Prometheus. -Исходные агрегаты и временные отметки: [slo-incidents.json](slo-incidents.json). Пересчёт: `python3 student-tests/slo_analyze.py` → [slo-analysis.json](slo-analysis.json). Ошибок опроса не зафиксировано. В ходе длительного опыта наблюдался скачок системных часов UTC: разность некоторых соседних UTC-отметок не равна разности монотонных отметок. Длительности ниже рассчитаны по `monotonic`; скачок также мог повлиять на временные окна Prometheus, поэтому эксперимент следует повторить на системе со стабильными часами. +Повторный опыт 2026-11-06: [сырые отсчёты](slo-incidents.json), [расчёт](slo-analysis.json). Анализатор сравнивает UTC и monotonic; максимальное расхождение при округлении до 0,001 с равно 0,000 с, максимальный фактический интервал опроса — 5,037 с. Ошибок опроса — 0. Прежние результаты с расхождением до 83,075 с сохранены в [pre-review](pre-review/slo-analysis.json) и не используются для выводов о длительностях; причина расхождения не установлена. ## Результаты API -| Воздействие | Наблюдаемая длительность блокировки | Fast: первое firing / снятие после конца | Slow: первое firing / снятие после конца | +Все интервалы ниже — разности `monotonic` из JSON; это времена наблюдения, а не точные моменты вычисления Prometheus. Погрешность опроса ограничивается фактическими соседними пробами, до 5,037 с в этом опыте. + +| Воздействие | Наблюдаемая длительность | Fast: обнаружение / снятие после конца | Slow: обнаружение / снятие после конца | | --- | ---: | ---: | ---: | -| Короткое | 75,033 с | 55,088 с / 15,181 с | 80,179 с / 15,181 с | -| Длительное | 180,134 с | 50,278 с / 10,648 с | 90,428 с / 30,724 с | +| Короткое | 75,001 с | 54,676 с / 14,760 с | Не наблюдался | +| Длительное | 180,097 с | 49,860 с / 15,301 с | 90,013 с / 40,394 с | -В коротком сценарии slow впервые появился уже через 5,145 с после окончания воздействия и исчез вместе с fast. По строгой классификации «блокировка уже завершилась — система восстановлена» зафиксирована одна новая тревога вне воздействия; по смыслу окна это запоздалый сигнал о предыдущем расходе бюджета. При длительном сценарии оба сигнала впервые появились во время блокировки. За один baseline получено 13 проб опроса и 0 проб с burn alert: наблюдаемых ложных срабатываний на этом коротком интервале нет, но данных недостаточно для оценки долгосрочной частоты ложных тревог. +В 13 пробах baseline не было burn alerts; новые срабатывания после окончания воздействия в этом повторе не наблюдались. Это ограниченная проверка ложных тревог, не долгосрочная частота. В предыдущем опыте slow впервые появился в восстановлении; повтор не даёт основания обещать отсутствие таких запоздалых сигналов при другом положении окон. -Разность клиентских накопленных счётчиков между первой и последней пробами фазы: baseline 179 успешных / 0 ошибок; короткая блокировка 0 / 33; длительная 0 / 87. Они не включают полные пограничные интервалы и могут включать timeout; это не числитель HTTP 5xx SLI. Срабатывания рассчитаны Prometheus по серверным метрикам. В Loki найден синтетический HTTP 503 с trace `47caf6293f35aabf09f71f307ed860a3`, где span `postgres.user_for_token` занял 5,004 с: это проверка пути диагностики в runbook. Проверка выполнена автором сценария; прохождение другим студентом пока не зафиксировано. +Для диагностики сохранён [лог 503 и связанный trace](review-db-trace.json): `postgres.user_for_token` занимает 5,006 с, HTTP span — 5,009 с (разности `endTimeUnixNano-startTimeUnixNano`, делённые на 10⁹, округлены до 0,001 с). Это подтверждает задержку БД, а не только медленный HTTP handler. Проверку выполнял автор сценария; независимое прохождение runbook другим студентом не подтверждено. ## Проверка SLO вложений `python3 student-tests/slo_ready_incident.py` убедился, что незавершённых задач нет, остановил worker, принял восемь синтетических вложений и дождался alert. Затем worker запущен, все вложения достигли ready. Исходные измерения: [slo-ready-incident.json](slo-ready-incident.json). -До воздействия счётчики matured/плохих заданий были 9/4. В первой пробе firing — 17/12: прирост обоих счётчиков равен 8, то есть все восемь заданий попали в знаменатель и нарушили 30-секундный срок. Fast и slow стали firing через 40,279 с после последнего принятия. Все задания были ready через 3,774 с после команды восстановления worker. Fast исчез через 13,849 с, slow — через 54,160 с после команды запуска. Значения опросные, включая задержку запуска Docker и опроса. При снятии slow новых заданий не поступало: условие минимального числа событий перестало выполняться. Это не доказывает, что 24-часовой SLO снова выполнен; факт просрочки остаётся в счётчиках. +Повтор после исправления методики: baseline не имел firing alerts, queue depth/age были 0. Счётчики matured/bad — 17/16 до опыта и 25/24 после: прирост каждого `25−17 = 24−16 = 8`, все восемь новых заданий нарушили срок. Fast и slow впервые наблюдались через 40,370 с после последнего принятия; QueueLag — через 55,525 с. Все файлы стали ready через 3,782 с после команды запуска worker. QueueLag снялся через 13,877 с, slow — через 39,130 с после запуска. Fast уже отсутствовал в последней пробе до запуска worker (за 0,428 с до отметки запуска): прекращение firing не означало устранения причины. Финальные depth/age = 0, bad остался 24. Все интервалы получены вычитанием сохранённых `monotonic`, округлением до 0,001 с. Максимальный фактический шаг опроса в ready-опыте — 5,299 с; максимальное расхождение приращений UTC/monotonic меньше 0,001 с. + +[Промежуточный неудачный прогон](pre-review/ready-scenario-failed.json) выявил ошибки теста: он требовал одновременного firing Fast и QueueLag и выбирал первый series старого outbox. Исправленный сценарий фиксирует каждый alert отдельно, проверяет восстановление обоих и использует max абсолютного счётчика общей DB-когорты по экземплярам outbox. Неудачный прогон не скрыт и не считается PASS. ## Дополнительная проверка правил и ограничения `promtool test rules` на [burn_rules.test.yaml](../student-tests/burn_rules.test.yaml) проверил нормальную нагрузку без burn alerts, тяжёлый краткий всплеск с последующим снятием fast, устойчивую 20%-ную долю ошибок с firing slow без fast, а также поведение при малом числе событий и fast alert просроченных вложений. Это синтетические ряды, не замена эксперименту с реальными данными. -Границы: API SLI не включает ошибки LB до приложения. Счётчики готовности вычисляются из состояний PostgreSQL раз в цикл outbox; при недоступном outbox данные устаревают, а события во время разрыва наблюдения могут не войти в `increase` после перезапуска процесса. Начало и конец срока фиксируются внутри DB-транзакций, поэтому приближённо описывают пользовательские границы durable acceptance и ready. Возможные manual replay/редактирование БД, смена или удаление заданий могут нарушить монотонность DB-счётчиков; учебный API не удаляет задания. Нет оценки ложных срабатываний за длительный период или проверки runbook другим студентом. Ошибки, произошедшие до восстановления, учитываются в бюджетном окне до истечения окна и могут вызвать задержанный alert. +Границы: API SLI не включает ошибки LB до приложения. Ready-когорта после миграции 009 использует сохранённые commit timestamps и неизменный результат; старые задания исключены. При недоступном outbox данные устаревают; stale/unknown alerts показывают неполноту измерения. События во время разрыва наблюдения могут не войти в `increase` после смены процесса. Короткий опыт не доказывает выполнение 24-часового SLO. Проверка другим студентом отсутствует; протокол описан в runbook. diff --git a/infra/prometheus/alerts.yaml b/infra/prometheus/alerts.yaml index d7a700b..af62567 100644 --- a/infra/prometheus/alerts.yaml +++ b/infra/prometheus/alerts.yaml @@ -1,6 +1,16 @@ groups: - name: messenger rules: + - alert: MessengerQueueTelemetryStale + expr: time() - max(messenger_queue_sample_time_seconds{service_name="messenger-outbox"}) > 20 or sum(absent(messenger_queue_sample_time_seconds{service_name="messenger-outbox"})) + for: 15s + labels: {severity: warning} + annotations: {summary: 'Queue and ready SLO telemetry is stale', runbook: 'See docs/slo-runbook.md; check outbox and DB before interpreting ready SLO.'} + - alert: MessengerReadyTimestampUnknown + expr: max(messenger_slo_unknown{service_name="messenger-outbox"}) > 0 + for: 15s + labels: {severity: warning} + annotations: {summary: 'Ready SLO commit timestamps unavailable', runbook: 'Check track_commit_timestamp and outbox; unknown events are not proof of SLO compliance.'} - alert: MessengerHighErrorRate expr: (sum(rate(messenger_http_requests_total{status_class="5xx",route!~"/health/.*"}[1m])) or vector(0)) / clamp_min(sum(rate(messenger_http_requests_total{route!~"/health/.*"}[1m])), 0.001) > 0.05 for: 10s diff --git a/migrations/009_slo_commits.sql b/migrations/009_slo_commits.sql new file mode 100644 index 0000000..dd5f37c --- /dev/null +++ b/migrations/009_slo_commits.sql @@ -0,0 +1,11 @@ +-- A separate immutable cohort: replay and job cleanup must not rewrite SLO history. +-- Existing jobs have no recoverable acceptance commit and are deliberately not backfilled. +CREATE TABLE attachment_slo ( + attachment_id uuid PRIMARY KEY, + accepted_xid xid NOT NULL DEFAULT pg_current_xact_id()::xid, + ready_xid xid, + accepted_at timestamptz, + ready_at timestamptz, + bad boolean +); +CREATE INDEX attachment_slo_unresolved_idx ON attachment_slo(accepted_at) WHERE bad IS NULL; diff --git a/pyproject.toml b/pyproject.toml index 6d3ce68..df9aa92 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] -name = "mtusi2026lab4" -version = "0.1.0" -description = "MTUSI high-load messenger, laboratory 4" +name = "mtusi2026lab6" +version = "1.6.0" +description = "MTUSI high-load messenger, laboratory 6" readme = "README.md" requires-python = ">=3.12" dependencies = [ diff --git a/src/broker.py b/src/broker.py index fadd839..ed306b5 100644 --- a/src/broker.py +++ b/src/broker.py @@ -48,7 +48,11 @@ async def publish(payload: dict, context=None) -> None: try: with tel.tracer.start_as_current_span("attachment.publish", context=tel.extract(context), kind=tel.SpanKind.PRODUCER, record_exception=False, set_status_on_exception=False) as span: span.set_attribute("messaging.retry.attempt", int(payload.get("attempt", 1))) - await run_in_threadpool(_publish, payload, tel.carrier()) + try: + await run_in_threadpool(_publish, payload, tel.carrier()) + except Exception: + span.set_status(tel.Status(tel.StatusCode.ERROR)) + raise except (pika.exceptions.AMQPError, OSError, ValueError) as error: raise ApiError(503, "unavailable", "Брокер недоступен") from error diff --git a/src/main.py b/src/main.py index a8cb62d..004b3e6 100644 --- a/src/main.py +++ b/src/main.py @@ -75,7 +75,7 @@ async def lifespan(application: FastAPI): app = FastAPI( title="MTUSI Messenger", - version="1.4.0", + version="1.6.0", redoc_url=None, lifespan=lifespan, ) @@ -97,14 +97,15 @@ async def add_request_id(request: Request, call_next) -> Response: return response finally: route = getattr(request.scope.get("route"), "path", "unmatched") - attrs = {"method": request.method, "route": route, "status_class": f"{status_code // 100}xx"} - span.update_name(f"{request.method} {route}") - span.set_attributes({"http.request.method": request.method, "http.route": route, "http.response.status_code": status_code}) + method = tel.http_method(request.method) + attrs = {"method": method, "route": route, "status_class": f"{status_code // 100}xx"} + span.update_name(f"{method} {route}") + span.set_attributes({"http.request.method": method, "http.route": route, "http.response.status_code": status_code}) if status_code >= 500: span.set_status(tel.Status(tel.StatusCode.ERROR)) tel.http_requests.add(1, attrs) tel.http_duration.record(time.monotonic() - started_at, attrs) - tel.log.info("http.request", extra={"request_id": request.state.request_id, "method": request.method, "route": route, "status": status_code}) + tel.log.info("http.request", extra={"request_id": request.state.request_id, "method": method, "route": route, "status": status_code}) def error_response( diff --git a/src/outbox.py b/src/outbox.py index 79522aa..a8ac14d 100644 --- a/src/outbox.py +++ b/src/outbox.py @@ -3,6 +3,7 @@ import asyncio import logging import os import json +import time from .broker import publish from .config import load_settings @@ -26,20 +27,16 @@ async def run() -> None: context = event['trace_context'] await publish(event["payload"], json.loads(context) if isinstance(context, str) else context) await store.mark_outbox_published(event["id"]) - try: - queue = await store.pool.fetchrow("SELECT count(*) AS depth, COALESCE(EXTRACT(EPOCH FROM now()-min(created_at)), 0)::float8 AS age FROM attachment_jobs WHERE status='queued'") - slo = await store.pool.fetchrow(""" - SELECT count(*) FILTER (WHERE created_at <= now() - interval '30 seconds') AS eligible, - count(*) FILTER (WHERE created_at <= now() - interval '30 seconds' - AND (status <> 'ready' OR updated_at > created_at + interval '30 seconds')) AS bad - FROM attachment_jobs - """) - tel.state.update(queue_depth=queue['depth'], queue_age=queue['age'], - slo_ready_eligible=slo['eligible'], slo_ready_bad=slo['bad']) - except Exception: - logging.warning("queue telemetry unavailable") except Exception: logging.exception("outbox dispatch failed; durable event remains pending") + try: + queue = await store.pool.fetchrow("SELECT count(*) AS depth, COALESCE(EXTRACT(EPOCH FROM now()-min(created_at)), 0)::float8 AS age FROM attachment_jobs WHERE status='queued'") + slo = await store.sample_ready_slo() + tel.state.update(queue_depth=queue['depth'], queue_age=queue['age'], + slo_ready_eligible=slo['eligible'], slo_ready_bad=slo['bad'], + slo_unknown=slo['unknown'], queue_sample_time=time.time()) + except Exception: + logging.warning("queue telemetry unavailable") await asyncio.sleep(interval) finally: await store.close() diff --git a/src/storage/postgres.py b/src/storage/postgres.py index b4cedd9..9e6edfe 100644 --- a/src/storage/postgres.py +++ b/src/storage/postgres.py @@ -341,6 +341,9 @@ class PostgresStore: await (connection or self.pool).execute( "INSERT INTO attachment_jobs (job_id, attachment_id, trace_context) VALUES ($1, $2, $3::jsonb)", job_id, attachment_id, json.dumps(tel.carrier()) ) + await (connection or self.pool).execute( + "INSERT INTO attachment_slo (attachment_id) VALUES ($1)", attachment_id + ) payload = { "schema_version": 1, "job_id": str(job_id), "type": "attachment.process.v1", "attachment_id": str(attachment_id), "pipeline_version": 1, "attempt": 1, @@ -438,10 +441,44 @@ class PostgresStore: attachment_id, original_key, ) await connection.execute("UPDATE attachment_jobs SET status = 'ready', lease_until = NULL, updated_at = now() WHERE attachment_id = $1", attachment_id) + await connection.execute( + "UPDATE attachment_slo SET ready_xid=pg_current_xact_id()::xid " + "WHERE attachment_id=$1 AND ready_xid IS NULL", attachment_id + ) return True except DB_ERRORS as error: raise RuntimeError("PostgreSQL is unavailable") from error + async def ready_seconds(self, attachment_id: UUID): + # Both ends are DB commit timestamps, not transaction start or a worker's clock. + return await self.pool.fetchval(""" + SELECT EXTRACT(EPOCH FROM ( + COALESCE(ready_at, pg_xact_commit_timestamp(ready_xid)) - + COALESCE(accepted_at, pg_xact_commit_timestamp(accepted_xid))))::float8 + FROM attachment_slo WHERE attachment_id=$1 + """, attachment_id) + + async def sample_ready_slo(self): + # Materialize promptly: PostgreSQL does not retain commit timestamps forever. + async with self.pool.acquire() as connection, connection.transaction(): + await connection.execute(""" + UPDATE attachment_slo SET + accepted_at=COALESCE(accepted_at, pg_xact_commit_timestamp(accepted_xid)), + ready_at=COALESCE(ready_at, pg_xact_commit_timestamp(ready_xid)) + WHERE accepted_at IS NULL OR (ready_xid IS NOT NULL AND ready_at IS NULL) + """) + await connection.execute(""" + UPDATE attachment_slo SET bad=(ready_at IS NULL OR ready_at>accepted_at+interval '30 seconds') + WHERE bad IS NULL AND accepted_at<=now()-interval '30 seconds' + AND (ready_xid IS NULL OR ready_at IS NOT NULL) + """) + return await connection.fetchrow(""" + SELECT count(bad) AS eligible, count(*) FILTER (WHERE bad) AS bad, + count(*) FILTER (WHERE accepted_at IS NULL OR + (ready_xid IS NOT NULL AND ready_at IS NULL)) AS unknown + FROM attachment_slo + """) + async def retry_attachment_job(self, attachment_id: UUID, max_attempts: int, delay_seconds: float, claim_time, permanent_code=None) -> bool: """Return true only when a bounded at-least-once retry was recorded.""" try: diff --git a/src/telemetry.py b/src/telemetry.py index c8c2969..082661e 100644 --- a/src/telemetry.py +++ b/src/telemetry.py @@ -22,9 +22,9 @@ ready_duration = meter.create_histogram("messenger.job.ready.duration", unit="s" busy = meter.create_up_down_counter("messenger.worker.busy", unit="{worker}") capacity = meter.create_up_down_counter("messenger.worker.capacity", unit="{worker}") state = {"queue_depth": 0, "queue_age": 0, "pool_used": 0, "pool_capacity": 0, "pool_waiting": 0, - "slo_ready_eligible": 0, "slo_ready_bad": 0} -for name, unit in (("queue_depth", "{job}"), ("queue_age", "s"), ("pool_used", "{connection}"), ("pool_capacity", "{connection}"), ("pool_waiting", "{request}")): - meter.create_observable_gauge("messenger." + name, callbacks=[lambda options, key=name: [metrics.Observation(state[key])]], unit=unit) + "slo_ready_eligible": 0, "slo_ready_bad": 0, "slo_unknown": 0, "queue_sample_time": 0} +for name, unit in (("queue_sample_time", "s"), ("slo_unknown", "{job}"), ("queue_depth", "{job}"), ("queue_age", "s"), ("pool_used", "{connection}"), ("pool_capacity", "{connection}"), ("pool_waiting", "{request}")): + meter.create_observable_gauge("messenger." + name, callbacks=[lambda options, key=name: [metrics.Observation(state[key], {"queue": "attachment.process.v1"} if key.startswith("queue_") else {})]], unit=unit) for name in ("slo_ready_eligible", "slo_ready_bad"): meter.create_observable_counter("messenger." + name, callbacks=[lambda options, key=name: [metrics.Observation(state[key])]], unit="{job}") @@ -46,6 +46,29 @@ log = logging.getLogger("messenger.telemetry") log.setLevel(logging.INFO) log.propagate = False +LOG_FIELDS = ("request_id", "method", "route", "status", "outcome") +LOG_EVENTS = {"http.request", "job.processed", "telemetry.unavailable"} +STANDARD_LOG_FIELDS = frozenset(logging.makeLogRecord({}).__dict__) + + +def http_method(value): + return value if value in {"GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS", "CONNECT", "TRACE"} else "_OTHER" + + +class SafeLogFilter(logging.Filter): + """Sanitize the shared record BEFORE both stdout and OTLP handlers.""" + def filter(self, record): + for key in list(record.__dict__): + if key not in STANDARD_LOG_FIELDS and key not in LOG_FIELDS: + del record.__dict__[key] + record.msg = record.msg if isinstance(record.msg, str) and record.msg in LOG_EVENTS else "telemetry.unavailable" + record.args = () + record.exc_info = record.exc_text = record.stack_info = None + return True + + +log.addFilter(SafeLogFilter()) + class JsonFormatter(logging.Formatter): def format(self, record): ctx = trace.get_current_span().get_span_context() @@ -53,7 +76,7 @@ class JsonFormatter(logging.Formatter): "service.name": os.environ.get("OTEL_SERVICE_NAME", "messenger"), "event": record.getMessage()} if ctx.is_valid: obj.update(trace_id=f"{ctx.trace_id:032x}", span_id=f"{ctx.span_id:016x}") - for field in ("request_id", "method", "route", "status", "outcome"): + for field in LOG_FIELDS: if hasattr(record, field): obj[field] = getattr(record, field) return json.dumps(obj, ensure_ascii=False) diff --git a/src/worker.py b/src/worker.py index ef82c21..6aa4f8b 100644 --- a/src/worker.py +++ b/src/worker.py @@ -94,17 +94,17 @@ def variants(data: bytes, max_pixels: int = 40_000_000) -> list[tuple[str, bytes ] -async def process(store: PostgresStore, s3: S3Store, payload: dict, lease_seconds: int) -> None: +async def process(store: PostgresStore, s3: S3Store, payload: dict, lease_seconds: int) -> str: # Excludes cleanup for the whole lifetime of S3 effects, including stale attempts. async with store.attachment_transaction(): - await process_locked(store, s3, payload, lease_seconds) + return await process_locked(store, s3, payload, lease_seconds) -async def process_locked(store: PostgresStore, s3: S3Store, payload: dict, lease_seconds: int) -> None: +async def process_locked(store: PostgresStore, s3: S3Store, payload: dict, lease_seconds: int) -> str: attachment_id = UUID(payload["attachment_id"]) attachment = await store.claim_attachment_job(attachment_id, lease_seconds) if attachment is None: - return + return "skipped" payload['_claim_time'] = attachment['claim_time'] payload['attempt'] = attachment.get('attempt', payload.get('attempt', 1)) attempt_key = uuid4().hex @@ -133,13 +133,18 @@ async def process_locked(store: PostgresStore, s3: S3Store, payload: dict, lease os._exit(70) committed = await store.complete_attachment_job(attachment_id, original_key, metadata, attachment['claim_time']) if committed: - if attachment.get('created_at') is not None: - tel.ready_duration.record(max(0, (datetime.now(timezone.utc) - attachment['created_at']).total_seconds()), {"job_type": "attachment"}) + try: + duration = await store.ready_seconds(attachment_id) + if duration is not None: + tel.ready_duration.record(duration, {"job_type": "attachment"}) + except Exception: + tel.log.warning("telemetry.unavailable") try: await s3.delete(attachment["staging_key"]) except Exception: logging.warning('staging cleanup deferred attachment_id=%s', attachment_id) + return "success" if committed else "skipped" def settle_delivery(channel, delivery_tag, future): """Only a successful durable disposition permits acknowledgement.""" @@ -169,7 +174,7 @@ def main() -> None: s3 = S3Store(S3Settings.load()) await store.connect() try: - await process(store, s3, payload, max(1, int(os.environ.get("JOB_TIMEOUT_SECONDS", "60")))) + return await process(store, s3, payload, max(1, int(os.environ.get("JOB_TIMEOUT_SECONDS", "60")))) finally: s3.client.close() await store.close() @@ -198,15 +203,15 @@ def main() -> None: except (ValueError, KeyError, TypeError): try: asyncio.run(publish_dlq({'failure': 'invalid_job'})) - return True, False + return True, "failed" except Exception: - return False, False + return False, "failed" try: - asyncio.run(handle_async(payload)) + outcome = asyncio.run(handle_async(payload)) except Exception as error: logging.warning("job failed attachment_id=%s", payload['attachment_id']) if '_claim_time' not in payload: - return False, False + return False, "failed" try: attachment_id = UUID(payload["attachment_id"]) attempt = int(payload.get("attempt", 1)) @@ -214,17 +219,16 @@ def main() -> None: asyncio.run(schedule_retry(attachment_id, delay, payload['_claim_time'], str(error) if isinstance(error, PermanentFailure) else None)) except Exception: - return False, False - return True, False - return True, True + return False, "failed" + return True, "failed" + return True, outcome def handle(body, headers): started = time.monotonic() tel.busy.add(1) with tel.tracer.start_as_current_span("attachment.consume", context=tel.extract(headers), kind=tel.SpanKind.CONSUMER, record_exception=False, set_status_on_exception=False) as span: try: - acknowledged, succeeded = handle_job(body) - outcome = "success" if succeeded else "failed" + acknowledged, outcome = handle_job(body) return acknowledged except Exception: outcome = "failed" diff --git a/student-tests/capture_observability.py b/student-tests/capture_observability.py new file mode 100644 index 0000000..f0d066c --- /dev/null +++ b/student-tests/capture_observability.py @@ -0,0 +1,41 @@ +"""Export synthetic trace/log evidence through Grafana; credentials never enter artifacts.""" +import argparse +import base64 +from datetime import datetime, timezone +import json +from pathlib import Path +import urllib.parse +import urllib.request + +ROOT = Path(__file__).resolve().parents[1] +auth = 'Basic ' + base64.b64encode(('admin:' + (ROOT/'secrets/grafana_password').read_text().strip()).encode()).decode() +def get(source, path, params=None): + url = f'http://127.0.0.1:3000/api/datasources/proxy/uid/{source}/' + path + if params: url += '?' + urllib.parse.urlencode(params) + with urllib.request.urlopen(urllib.request.Request(url,headers={'Authorization':auth}),timeout=15) as r: + return json.load(r) + +def capture(trace_id=None, query=None): + query = query or '{service_name="messenger-api"} | status="503"' + if trace_id: + query = '{service_name=~"messenger-api|messenger-worker"} | trace_id="' + trace_id + '"' + logs = get('loki','loki/api/v1/query_range',{'query':query,'since':'30m','limit':20,'direction':'backward'}) + if not trace_id: + for stream in logs['data']['result']: + for value in stream['values']: + metadata = dict(stream['stream']) + if len(value)>2: metadata.update(value[2]) + trace_id = metadata.get('trace_id') + if trace_id: break + if trace_id: break + if not trace_id: + raise RuntimeError('No correlated log/trace found') + trace = get('tempo','api/traces/'+trace_id) + return {'captured_at':datetime.now(timezone.utc).isoformat(),'query':query,'trace_id':trace_id,'logs':logs,'trace':trace} + +if __name__=='__main__': + parser=argparse.ArgumentParser();parser.add_argument('--trace-id');parser.add_argument('--output',required=True) + args=parser.parse_args() + result=capture(args.trace_id) + (ROOT/args.output).write_text(json.dumps(result,ensure_ascii=False,indent=2)+'\n') + print('Saved correlated evidence:',args.output,'trace_id:',result['trace_id']) diff --git a/student-tests/check_dashboard.py b/student-tests/check_dashboard.py new file mode 100644 index 0000000..cef28a8 --- /dev/null +++ b/student-tests/check_dashboard.py @@ -0,0 +1,18 @@ +"""Execute every provisioned PromQL target against the running Grafana datasource.""" +import importlib.util +import json +from pathlib import Path +ROOT=Path(__file__).resolve().parents[1] +spec=importlib.util.spec_from_file_location('capture',ROOT/'student-tests/capture_observability.py') +capture=importlib.util.module_from_spec(spec);spec.loader.exec_module(capture) +dashboard=json.loads((ROOT/'infra/grafana/dashboards/messenger.json').read_text()) +results=[] +for panel in dashboard['panels']: + for target in panel.get('targets',[]): + if 'expr' not in target:continue + query=target['expr'].replace('$__rate_interval','1m').replace('$__interval','5s') + result=capture.get('prometheus','api/v1/query',{'query':query}) + assert result['status']=='success',result + results.append({'panel':panel['title'],'query':query,'series':len(result['data']['result'])}) +(ROOT/'evidence/review-dashboard.json').write_text(json.dumps(results,ensure_ascii=False,indent=2)+'\n') +print('PASS:',len(results),'dashboard queries; empty targets:',sum(r['series']==0 for r in results)) diff --git a/student-tests/check_retry_trace.py b/student-tests/check_retry_trace.py new file mode 100644 index 0000000..1f75474 --- /dev/null +++ b/student-tests/check_retry_trace.py @@ -0,0 +1,31 @@ +"""Verify exported parent relationships and the completed retry, not just trace existence.""" +import base64 +import json +from collections import Counter +from pathlib import Path +ROOT=Path(__file__).resolve().parents[1] +data=json.loads((ROOT/'evidence/review-retry-trace.json').read_text()) +spans=[s for batch in data['trace']['batches'] for scope in batch.get('scopeSpans',[]) for s in scope.get('spans',[])] +by_id={s['spanId']:s for s in spans} +counts=Counter(s['name'] for s in spans) +producers=[s for s in spans if s['kind']=='SPAN_KIND_PRODUCER'] +consumers=[s for s in spans if s['kind']=='SPAN_KIND_CONSUMER'] +assert len(producers)>=2 and consumers,counts +for producer in producers: + ancestor = producer + for _ in range(len(spans)): + ancestor = by_id[ancestor['parentSpanId']] + if ancestor['kind']=='SPAN_KIND_SERVER': break + else: raise AssertionError('producer has no API ancestor') +for consumer in consumers: + assert by_id[consumer['parentSpanId']]['kind']=='SPAN_KIND_PRODUCER' +assert counts['postgres.complete_attachment_job']>=1,counts +assert any(s['name'].startswith('s3.') for s in spans),counts +assert all(base64.b64decode(s['traceId']).hex()==data['trace_id'] for s in spans) +services={stream['stream']['service_name'] for stream in data['logs']['data']['result']} +assert {'messenger-api','messenger-worker'}<=services,services +result={'trace_id':data['trace_id'],'span_count':len(spans),'span_names':dict(counts), + 'producer_count':len(producers),'consumer_count':len(consumers),'log_services':sorted(services), + 'parent_relationships':'PASS','note':'Abrupt os._exit may lose unfinished spans of the crashed attempt; redelivery and committed retry remain visible.'} +(ROOT/'evidence/review-retry-summary.json').write_text(json.dumps(result,indent=2)+'\n') +print('PASS:',json.dumps(result)) diff --git a/student-tests/pipeline_critical.py b/student-tests/pipeline_critical.py index dfb716f..3750ba9 100644 --- a/student-tests/pipeline_critical.py +++ b/student-tests/pipeline_critical.py @@ -53,12 +53,14 @@ class CriticalTests(unittest.IsolatedAsyncioTestCase): 'content_type': 'image/png', 'size_bytes': 1, 'sha256': 'unused', } store.complete_attachment_job.return_value = committed + store.ready_seconds.return_value = 0.25 s3.settings = SimpleNamespace(max_pixels=40000000) s3.download_key.return_value = b'fixture' s3.delete.side_effect = RuntimeError('storage temporarily unavailable') payload = {'attachment_id': str(uuid4())} with patch('src.worker.render_isolated', return_value=[('thumbnail', b'jpeg', 'image/jpeg', 1, 1)]): - await process(store, s3, payload, 60) + outcome = await process(store, s3, payload, 60) + self.assertEqual(outcome, "success" if committed else "skipped") self.assertEqual(store.complete_attachment_job.call_args.args[-1], stamp) return store, s3 @@ -71,6 +73,14 @@ class CriticalTests(unittest.IsolatedAsyncioTestCase): _, s3 = await self.run_processing(False) s3.delete.assert_not_awaited() + async def test_duplicate_is_skipped_not_successful(self): + store, s3 = AsyncMock(), AsyncMock() + store.claim_attachment_job.return_value = None + result = await process(store, s3, {'attachment_id': str(uuid4())}, 60) + self.assertEqual(result, 'skipped') + store.complete_attachment_job.assert_not_awaited() + s3.copy.assert_not_awaited() + async def test_stale_claim_cannot_publish_metadata(self): store = PostgresStore('unused') connection = AsyncMock() diff --git a/student-tests/retry_evidence.py b/student-tests/retry_evidence.py new file mode 100644 index 0000000..f3b5b01 --- /dev/null +++ b/student-tests/retry_evidence.py @@ -0,0 +1,30 @@ +"""Crash one idle lab worker after S3 writes; restore normal configuration in finally.""" +import json +import os +from pathlib import Path +import subprocess +import sys +import time +ROOT=Path(__file__).resolve().parents[1] +def compose(*args, env=None): + return subprocess.run(['docker','compose',*args],cwd=ROOT,env=env,check=True,capture_output=True,text=True).stdout +count=compose('exec','-T','postgres','psql','-U','messenger','-d','messenger','-Atc',"SELECT count(*) FROM attachment_jobs WHERE status IN ('queued','processing')") +if int(count.strip()):raise RuntimeError('Pending work; refusing crash injection') +try: + compose('stop','-t','5','worker') + compose('up','-d','--no-deps','worker',env={**os.environ,'WORKER_FAULT_AFTER_S3_PUT':'1'}) + subprocess.run([sys.executable,'student-tests/trace_retry.py'],cwd=ROOT,check=True,timeout=180) + out=json.loads((ROOT/'evidence/review-retry-job.json').read_text()) + from uuid import UUID + identity=str(UUID(out['attachment_id'])) + out['db_attempt']=int(compose('exec','-T','postgres','psql','-U','messenger','-d','messenger','-Atc',f"SELECT attempt FROM attachment_jobs WHERE attachment_id='{identity}'").strip()) + cid=compose('ps','-q','worker').strip() + out['restart_count']=int(subprocess.run(['docker','inspect','--format','{{.RestartCount}}',cid],check=True,capture_output=True,text=True).stdout.strip()) + assert out['db_attempt']==2 and out['restart_count']==1,out + (ROOT/'evidence/review-retry-job.json').write_text(json.dumps(out,indent=2)+'\n') + time.sleep(10) + subprocess.run([sys.executable,'student-tests/capture_observability.py','--trace-id',out['trace_id'],'--output','evidence/review-retry-trace.json'],cwd=ROOT,check=True) +finally: + compose('stop','-t','5','worker') + compose('up','-d','--no-deps','worker',env={**os.environ,'WORKER_FAULT_AFTER_S3_PUT':'0'}) +print('PASS: attempt 2, one restart, normal worker restored') diff --git a/student-tests/review_checks.py b/student-tests/review_checks.py new file mode 100644 index 0000000..a22b16c --- /dev/null +++ b/student-tests/review_checks.py @@ -0,0 +1,22 @@ +"""Repeat short review checks; experiments are separate and intentionally disruptive.""" +from datetime import datetime, timezone +import json +from pathlib import Path +import subprocess +ROOT=Path(__file__).resolve().parents[1] +checks=[['make','check'],['make','validate','PYTHON=.venv/bin/python'], + ['docker','compose','config','--quiet'], + ['docker','run','--rm','-v',f'{ROOT}:/workspace:ro','--entrypoint','promtool','prom/prometheus:v3.2.1','test','rules', + '/workspace/student-tests/alerts.test.yaml','/workspace/student-tests/burn_rules.test.yaml','/workspace/student-tests/telemetry_alerts.test.yaml'], + ['make','test'],['python3','student-tests/check_dashboard.py']] +results=[] +for command in checks: + proc=subprocess.run(command,cwd=ROOT,capture_output=True,text=True,timeout=180) + results.append({'command':command,'exit_code':proc.returncode,'output':proc.stdout+proc.stderr}) +for filename in ('telemetry_unit.py','pipeline_critical.py','slo_commit_integration.py'): + command=['docker','compose','exec','-T','api-a','/app/.venv/bin/python','-'] + proc=subprocess.run(command,input=(ROOT/'student-tests'/filename).read_text(),cwd=ROOT,capture_output=True,text=True,timeout=60) + results.append({'command':command,'stdin':f'student-tests/{filename}','exit_code':proc.returncode,'output':proc.stdout+proc.stderr}) +(ROOT/'evidence/review-checks.json').write_text(json.dumps({'utc':datetime.now(timezone.utc).isoformat(),'checks':results},ensure_ascii=False,indent=2)+'\n') +for r in results:print('PASS' if r['exit_code']==0 else 'FAIL',r.get('stdin',r['command'])) +raise SystemExit(any(r['exit_code'] for r in results)) diff --git a/student-tests/review_queries.py b/student-tests/review_queries.py new file mode 100644 index 0000000..1c4d9ae --- /dev/null +++ b/student-tests/review_queries.py @@ -0,0 +1,13 @@ +"""Read current health of telemetry, without printing access credentials.""" +import importlib.util +import json +from pathlib import Path +ROOT=Path(__file__).resolve().parents[1] +spec=importlib.util.spec_from_file_location('capture',ROOT/'student-tests/capture_observability.py') +c=importlib.util.module_from_spec(spec);spec.loader.exec_module(c) +queries=['messenger_slo_unknown{service_name="messenger-outbox"}', + 'time()-max(messenger_queue_sample_time_seconds{service_name="messenger-outbox"})', + 'messenger_queue_depth{service_name="messenger-outbox"}', + 'messenger_job_ready_duration_seconds_count{service_name="messenger-worker"}'] +for q in queries: + print(q,json.dumps(c.get('prometheus','api/v1/query',{'query':q})['data']['result'])) diff --git a/student-tests/review_resilience.py b/student-tests/review_resilience.py new file mode 100644 index 0000000..cfc49e1 --- /dev/null +++ b/student-tests/review_resilience.py @@ -0,0 +1,20 @@ +"""Run inherited public durability/failover checks and restore both API replicas.""" +from datetime import datetime, timezone +import json +from pathlib import Path +import subprocess +ROOT=Path(__file__).resolve().parents[1] +results=[] +try: + for command in [ + ['make','test-persistence','ACTION_SCRIPT=scripts/restart.sh'], + ['make','test-failover','ACTION_SCRIPT=scripts/stop-one.sh']]: + result=subprocess.run(command,cwd=ROOT,capture_output=True,text=True,timeout=240) + results.append({'command':command,'exit_code':result.returncode,'output':result.stdout+result.stderr}) + print('PASS' if result.returncode==0 else 'FAIL',command,flush=True) + if result.returncode:break +finally: + restore=subprocess.run(['docker','compose','up','-d','--no-deps','--wait','api-a','api-b','worker','outbox'],cwd=ROOT,capture_output=True,text=True,timeout=90) + results.append({'restore_exit_code':restore.returncode,'output':restore.stdout+restore.stderr}) + (ROOT/'evidence/review-resilience.json').write_text(json.dumps({'utc':datetime.now(timezone.utc).isoformat(),'checks':results},ensure_ascii=False,indent=2)+'\n') +raise SystemExit(any(r.get('exit_code',r.get('restore_exit_code')) for r in results)) diff --git a/student-tests/slo_analyze.py b/student-tests/slo_analyze.py index e4dfb87..d01f1d4 100644 --- a/student-tests/slo_analyze.py +++ b/student-tests/slo_analyze.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 """Summarize observed alert transitions; sampling bounds detection precision.""" import json +from datetime import datetime from pathlib import Path root = Path(__file__).resolve().parents[1] @@ -8,6 +9,13 @@ data = json.loads((root / 'evidence/slo-incidents.json').read_text()) samples = sorted((s for s in data['samples'] if 'firing' in s), key=lambda s: s['monotonic']) alerts = ['MessengerApiBurnFast', 'MessengerApiBurnSlow'] output = {'sampling_interval_seconds': 5, 'monitor_errors': sum('monitor_error' in s for s in data['samples']), 'phases': {}} +drifts = [abs((datetime.fromisoformat(b['time']) - datetime.fromisoformat(a['time'])).total_seconds() + - (b['monotonic'] - a['monotonic'])) for a, b in zip(samples, samples[1:])] +gaps = [b['monotonic'] - a['monotonic'] for a, b in zip(samples, samples[1:])] +output['timing_quality'] = {'max_wall_monotonic_disagreement_seconds': round(max(drifts, default=0), 3), + 'max_observed_sampling_gap_seconds': round(max(gaps, default=0), 3), + 'valid_for_elapsed_time_claims': max(drifts, default=0) <= 1, + 'note': 'Disagreement does not identify its cause; rerun if invalid. Precision is bounded by actual gaps, not the configured sleep.'} base = [s for s in samples if s['phase'] == 'baseline'] output['baseline'] = {'samples': len(base), 'samples_with_burn_alert': sum(bool(s['firing']) for s in base), 'scope': 'one observed healthy baseline, not a long-term false-positive rate'} for name, period in data['periods'].items(): diff --git a/student-tests/slo_commit_integration.py b/student-tests/slo_commit_integration.py new file mode 100644 index 0000000..acc1dd0 --- /dev/null +++ b/student-tests/slo_commit_integration.py @@ -0,0 +1,45 @@ +"""Run inside api-a; session-local table shadows real SLO data (no business mutations).""" +import asyncio +from datetime import datetime, timezone +from uuid import uuid4 +import asyncpg +from src.config import load_settings +from src.storage.postgres import PostgresStore + +async def main(): + store = PostgresStore(load_settings().database_url) + store._pool = await asyncpg.create_pool(load_settings().database_url, min_size=1, max_size=1) + try: + async with store.pool.acquire() as conn: + await conn.execute('CREATE TEMP TABLE attachment_slo (LIKE public.attachment_slo INCLUDING DEFAULTS)') + identity = uuid4() + async with conn.transaction(): + await conn.execute('INSERT INTO attachment_slo (attachment_id) VALUES ($1)', identity) + await asyncio.sleep(.2) + before_accept_commit = await conn.fetchval('SELECT clock_timestamp()') + async with conn.transaction(): + await conn.execute('UPDATE attachment_slo SET ready_xid=pg_current_xact_id()::xid') + await asyncio.sleep(.2) + before_ready_commit = await conn.fetchval('SELECT clock_timestamp()') + await store.sample_ready_slo() + row = await store.pool.fetchrow('SELECT * FROM attachment_slo') + assert row['accepted_at'] >= before_accept_commit + assert row['ready_at'] >= before_ready_commit + elapsed = await store.ready_seconds(identity) + assert elapsed >= .2, elapsed + # Mature a successful synthetic cohort, then emulate replay metadata changes. + await store.pool.execute("UPDATE attachment_slo SET accepted_at=now()-interval '32 seconds',ready_at=now()-interval '31 seconds'") + first = dict(await store.sample_ready_slo()) + assert first == {'eligible': 1, 'bad': 0, 'unknown': 0}, first + await store.pool.execute("UPDATE attachment_slo SET ready_at=now()+interval '1 minute'") + assert dict(await store.sample_ready_slo()) == first + await store.pool.execute("INSERT INTO attachment_slo (attachment_id,accepted_at) VALUES ($1,now()-interval '31 seconds')", uuid4()) + late = dict(await store.sample_ready_slo()) + assert late == {'eligible': 2, 'bad': 1, 'unknown': 0}, late + await store.pool.execute('UPDATE attachment_slo SET ready_at=now(),ready_xid=pg_current_xact_id()::xid WHERE ready_at IS NULL') + assert dict(await store.sample_ready_slo()) == late + print('PASS: commit boundaries; immutable good/bad cohorts; late completion remains bad') + finally: + await store.close() + +asyncio.run(main()) diff --git a/student-tests/slo_incidents.py b/student-tests/slo_incidents.py index 241c504..09c2bcc 100644 --- a/student-tests/slo_incidents.py +++ b/student-tests/slo_incidents.py @@ -2,7 +2,7 @@ """Run two bounded DB-lock incidents against the local Compose project. Creates synthetic data, records only aggregate counts/alert states, and releases -all locks on exit. Run on the isolated lab6 stack, never a shared database. +bounded locks; interrupted server-side sleeps may persist until their deadline. Run on the isolated lab6 stack, never a shared database. """ import base64 from concurrent.futures import ThreadPoolExecutor @@ -43,7 +43,7 @@ def monitor(): try: alerts = get('alerts')['alerts'] firing = sorted(a['labels']['alertname'] for a in alerts if a['state'] == 'firing' and 'Burn' in a['labels']['alertname']) - with LOCK: sample = {'time': utc(), 'monotonic': time.monotonic(), 'phase': SCENARIO['phase'], 'counts': dict(COUNTS), 'firing': firing} + with LOCK: sample = {'time': utc(), 'monotonic': time.monotonic(), 'phase': SCENARIO['phase'], 'counts': dict(COUNTS), 'firing': firing, 'all_firing': sorted(a['labels']['alertname'] for a in alerts if a['state'] == 'firing')} SAMPLES.append(sample) except Exception as error: SAMPLES.append({'time': utc(), 'monotonic': time.monotonic(), 'phase': SCENARIO['phase'], 'monitor_error': type(error).__name__}) @@ -103,13 +103,15 @@ def main(): try: with ThreadPoolExecutor(max_workers=3) as pool: futures = [pool.submit(worker, actor, chat) for _ in range(3)] - phase('baseline'); time.sleep(60) - result['periods']['burst'] = incident('burst', 75) - time.sleep(100) - result['periods']['sustained'] = incident('sustained', 180) - time.sleep(100) - phase('complete') - STOP.set() + try: + phase('baseline'); time.sleep(60) + result['periods']['burst'] = incident('burst', 75) + time.sleep(100) + result['periods']['sustained'] = incident('sustained', 180) + time.sleep(100) + phase('complete') + finally: + STOP.set() for future in futures: future.result(timeout=15) finally: STOP.set(); monitor_thread.join(timeout=10) diff --git a/student-tests/slo_ready_incident.py b/student-tests/slo_ready_incident.py index c816668..f745af7 100644 --- a/student-tests/slo_ready_incident.py +++ b/student-tests/slo_ready_incident.py @@ -22,12 +22,14 @@ def prom(path, query=None): if query is not None: url += '?' + urllib.parse.urlencode({'query': query}) with urllib.request.urlopen(urllib.request.Request(url, headers={'Authorization': auth}), timeout=5) as r: return json.load(r)['data'] def metric(name): - result = prom('query', name + '{service_name="messenger-outbox"}')['result'] + result = prom('query', 'max(' + name + '{service_name="messenger-outbox"})')['result'] return float(result[0]['value'][1]) if result else None def sample(phase): alerts = prom('alerts')['alerts'] return {'time': utc(), 'monotonic': time.monotonic(), 'phase': phase, 'firing': sorted(a['labels']['alertname'] for a in alerts if a['state']=='firing' and a['labels']['alertname'].startswith('MessengerReadyBurn')), + 'all_firing': sorted(a['labels']['alertname'] for a in alerts if a['state']=='firing'), + 'queue_depth': metric('messenger_queue_depth'), 'queue_age': metric('messenger_queue_age_seconds'), 'eligible': metric('messenger_slo_ready_eligible_total'), 'bad': metric('messenger_slo_ready_bad_total')} def docker(*args): subprocess.run(['docker','compose',*args], cwd=ROOT, check=True, stdout=subprocess.DEVNULL) def run(): @@ -43,13 +45,15 @@ def run(): for _ in range(8): accepted.append(client.attachment(actor,chat)[0]) out['accepted_at']={'time':utc(),'monotonic':time.monotonic()} print('8 synthetic jobs accepted; waiting for burn alert', flush=True) + seen=set() deadline=time.monotonic()+100 while time.monotonic()=0 for value in out['counter_deltas'].values()),'counter reset' + out['load']=json.loads((ROOT/'evidence/review-tempo-load.json').read_text()) + assert out['load']['failed_requests']==0,out['load'] + out['result']='PASS' + finally: + if load is not None and load.poll() is None: + load.terminate();load.wait(timeout=15) + if stopped: docker('up','-d','--no-deps','tempo') + (ROOT/'evidence/review-tempo-outage.json').write_text(json.dumps(out,indent=2)+'\n') + print('PASS: Tempo outage; baseline/deltas/memory in evidence/review-tempo-outage.json') +if __name__=='__main__': main() diff --git a/student-tests/trace_retry.py b/student-tests/trace_retry.py index da858de..056143b 100644 --- a/student-tests/trace_retry.py +++ b/student-tests/trace_retry.py @@ -1,5 +1,7 @@ """Synthetic upload with a known W3C trace ID; wait through a worker restart.""" import secrets +import json +from datetime import datetime, timezone import sys import time from pathlib import Path @@ -18,3 +20,8 @@ print('trace_id:', trace_id, 'attachment_id:', attachment['id'], flush=True) started = time.monotonic() result = client.wait_ready(actor, attachment, wait_seconds=150) print('status:', result['status'], 'elapsed_seconds:', round(time.monotonic()-started, 3), flush=True) + +(Path(__file__).resolve().parents[1] / 'evidence/review-retry-job.json').write_text(json.dumps({ + 'utc': datetime.now(timezone.utc).isoformat(), 'trace_id': trace_id, + 'attachment_id': attachment['id'], 'status': result['status'], + 'elapsed_seconds': round(time.monotonic()-started, 3)}, indent=2)+'\n') diff --git a/uv.lock b/uv.lock index 359ef2c..33119d4 100644 --- a/uv.lock +++ b/uv.lock @@ -462,8 +462,8 @@ wheels = [ ] [[package]] -name = "mtusi2026lab4" -version = "0.1.0" +name = "mtusi2026lab6" +version = "1.6.0" source = { virtual = "." } dependencies = [ { name = "argon2-cffi" }, -- 2.54.0