diff --git a/.env.example b/.env.example index 2b32655..7d13cbe 100644 --- a/.env.example +++ b/.env.example @@ -19,12 +19,14 @@ S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key MAX_UPLOAD_BYTES=10485760 MAX_IMAGE_PIXELS=40000000 BROKER_URL_FILE=/run/secrets/broker_url -WORKER_CONCURRENCY=2 +WORKER_CONCURRENCY=1 JOB_MAX_ATTEMPTS=5 -JOB_TIMEOUT_SECONDS=60 +JOB_TIMEOUT_SECONDS=20 STAGING_RETENTION_SECONDS=86400 OTEL_SERVICE_NAME=messenger-api # Worker must set a distinct service.name. -OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318 -OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf +OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4317 +OTEL_EXPORTER_OTLP_PROTOCOL=grpc OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab + +PUBLIC_PORT=18083 diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..b5e60d3 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,60 @@ +# +# MTUSI Messenger, lab6 — PostgreSQL + Redis/Valkey HTTP API. +# Base image is pinned by tag *and* digest (multi-arch manifest list), so a rebuild +# on another machine with the same platform yields the same layers. +# +# Runtime dependencies: psycopg2-binary for PostgreSQL, redis for the session cache. +# Development-only tooling lives in requirements-dev.txt and never enters this image. +FROM python:3.12-slim-bookworm@sha256:392307d22300de8b5986851a12d9176dfc0fc073e65bf6523ebd7dcbeb23564e + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PYTHONPATH=/app/src + +# Pinned runtime dependencies (versions fixed for reproducible builds). +RUN pip install --no-cache-dir \ + psycopg2-binary==2.9.11 \ + redis==7.0.1 \ + bcrypt==4.0.1 \ + boto3==1.42.97 \ + Pillow==12.3.0 \ + pika==1.3.2 \ + opentelemetry-sdk==1.38.0 \ + opentelemetry-exporter-otlp-proto-grpc==1.38.0 + +# Unprivileged runtime account; the image's default root user is never used at runtime. +RUN groupadd --gid 10001 app \ + && useradd --uid 10001 --gid 10001 --create-home --shell /usr/sbin/nologin app + +ARG APP_REVISION=local +ENV APP_REVISION=$APP_REVISION + +WORKDIR /app +COPY --chown=10001:10001 src/ /app/src/ +COPY --chown=10001:10001 migrations/ /app/migrations/ + +USER 10001:10001 + +# Configuration comes from the environment; secrets are read from files at runtime. +ENV APP_HOST=0.0.0.0 \ + APP_PORT=8080 \ + APP_ENV=lab \ + LOG_LEVEL=info \ + SHUTDOWN_GRACE_SECONDS=8 \ + SESSION_TTL_SECONDS=3600 \ + SESSION_STORE=redis \ + DB_POOL_MIN=2 \ + DB_POOL_MAX=10 \ + DB_CONNECT_TIMEOUT=5 \ + DB_STATEMENT_TIMEOUT=10 + +EXPOSE 8080 + +# Python (not curl) keeps the image slim; /health/ready is the synchronous-path probe. +HEALTHCHECK --interval=10s --timeout=3s --start-period=3s --retries=3 \ + CMD ["python", "-c", "import os, urllib.request; urllib.request.urlopen('http://127.0.0.1:%s/health/ready' % os.environ.get('APP_PORT', '8080'), timeout=2)"] + +STOPSIGNAL SIGTERM + +# The HTTP process is PID 1 and handles SIGTERM itself (graceful drain, exit code 0). +CMD ["python", "-m", "messenger"] diff --git a/Makefile b/Makefile index 5d86d17..a953aaa 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ PYTHON ?= python3 -BASE_URL ?= http://localhost:8080 +BASE_URL ?= http://localhost:18083 CA_FILE ?= ACTION_SCRIPT ?= TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) @@ -28,3 +28,7 @@ test-persistence: test-failover: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2) $(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) + +.PHONY: test-observability +test-observability: + $(PYTHON) student-tests/observability.py smoke diff --git a/REPORT.md b/REPORT.md index b97ae42..a745d16 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,168 @@ # Отчёт — лабораторная 6 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор — Любченко Никита, БПИ2303. +- Целевая оценка — 4. Трек 5 не заявлен. +- Использованы Python 3.12, OpenTelemetry SDK 1.38.0, semantic conventions 0.59b0, Collector contrib 0.123.0, Prometheus 3.2.1, Loki 3.4.2, Tempo 2.7.2, Grafana 11.6.0 и компоненты lab5. ИИ использован при разработке и проверке реализации. +- Ссылку на PR добавляет автор после публикации. Результаты проверок приведены в evidence/. -## Запуск из чистого clone +## Запуск из чистого клона -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Требуются Docker Engine с Compose, Make и Python 3.9+ для публичных проверок. Первая сборка обращается к реестрам образов, PyPI и исходникам MinIO. Приложение работает на Python 3.12. Изображения обрабатываются Pillow. Go используется при сборке MinIO и mc из исходников. + +```sh +cp -n .env.example .env +python3 scripts/bootstrap.py +docker compose --env-file .env config --quiet +docker compose --env-file .env build +docker compose --env-file .env up -d --no-build +python3 scripts/manage.py status +python3 scripts/wait_ready.py +make check +make test +python3 -m venv .venv +.venv/bin/python -m pip install -r requirements-dev.txt +make validate PYTHON=.venv/bin/python +make test-observability +``` + +API доступен через Nginx на http://localhost:18083. Grafana — http://localhost:13000, логин admin. Пароль читается из secrets/grafana_password. Prometheus — http://localhost:19090. Loki и Tempo имеют loopback HTTP-порты 13100 и 13200 для воспроизводимых проверок. OTLP и внутренние API-порты на хост не опубликованы. Dashboard Messenger и три источника данных подключаются provisioning автоматически. + +Bootstrap создаёт локальные secrets с ограниченными правами, сохраняет уже существующие значения и не печатает их. `.env`, `secrets/` и `.venv/` исключены из Git. Безопасная остановка — `docker compose down`. Намеренное удаление всех учебных данных текущего проекта — `docker compose down -v`. ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +Перенесены Nginx, две API-реплики, PostgreSQL, Redis, приватный MinIO, RabbitMQ, publisher, worker и cleaner из lab5. PostgreSQL остаётся источником истины. API сохраняет исходник в S3 и фиксирует queued, job и outbox одной транзакцией перед ответом 202. Издатель использует publisher confirms. Воркер сохраняет три варианта, фиксирует ready, затем подтверждает сообщение брокеру. Lease, retry/backoff, fencing по attempt, replay и очистка сохранены. -## Доказательства критериев +Добавлен путь API/publisher/worker → OTLP/gRPC → Collector → Prometheus/Loki/Tempo → Grafana. Prometheus хранит числовые временные ряды, Loki — структурные логи, Tempo — трассы. Grafana выполняет запросы к этим источникам. У хранилищ собственные volumes, retention сигналов составляет 72 часа. Collector обрабатывает и экспортирует данные. -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +Миграция 0006 добавляет trace_context jsonb в attachment_jobs. Контекст сохраняется вместе с заданием. Издатель создаёт producer span с родителем из сохранённого контекста и передаёт traceparent/tracestate в AMQP headers. Consumer продолжает этот контекст. Повторные попытки имеют общий исходный trace и отдельные producer/consumer spans. Старые задания получают пустой контекст и обрабатываются обычным способом. -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +Ресурсы содержат service.name (messenger-api, messenger-publisher, messenger-worker), service.version, service.instance.id и deployment.environment.name=lab. Логи включают timestamp, severity, event, request_id и trace_id/span_id при активном span. HTTP возвращает X-Request-Id и дополнительный X-Trace-Id. SQL-тексты, значения параметров, Authorization, пароли, signed URL, тела сообщений и изображения в инструментацию не передаются. OTLP bridge отбрасывает полный текст исключений и traceback. Технический job.id допустим для синтетических traces/logs. В metric labels его нет. -## Эксперименты и отказы +## Сигналы и запросы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +| Сигнал | Реальное имя в Prometheus | Единица | +| --- | --- | --- | +| HTTP count | messenger_http_requests_total | запросы | +| HTTP duration | messenger_http_duration_seconds_bucket | секунды, histogram | +| Jobs | messenger_jobs_processed_total | попытки, outcome success/error/stale | +| Processing duration | messenger_job_duration_seconds_bucket | секунды одной попытки | +| Accepted-to-ready | messenger_attachment_time_to_ready_seconds_bucket | секунды до фиксации ready | +| Backlog | messenger_queue_depth | queued + processing, включая outbox/retry | +| Broker ready | messenger_queue_broker_ready | ожидающие доставки сообщения RabbitMQ | +| Oldest age | messenger_queue_oldest_age_seconds | возраст старейшего незавершённого задания | +| Backlog freshness | messenger_queue_observed_at_seconds | Unix timestamp последнего опроса | +| DB pool | messenger_db_pool_used, messenger_db_pool_waiting, messenger_db_pool_capacity | соединения / запросы на получение | +| Pool exhaustion | messenger_db_pool_exhausted_total | отказы получения соединения | +| Pool acquisition | messenger_db_pool_acquire_seconds_bucket | секунды | +| Worker | messenger_worker_busy, messenger_worker_capacity | занятые / доступные обработчики | +| Memory | messenger_process_memory_bytes | текущий RSS процесса, байты | +| CPU | messenger_process_cpu_seconds_total | накопленное процессорное время, секунды | -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +HTTP labels содержат method, route template и status_class. Health исключён из бизнес-счётчиков. Неизвестные маршруты сводятся к unmatched. Histogram boundaries — 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2, 5, 10, 30, 60, 120 секунд и +Inf. Pool waiting измеряет поток во время вызова getconn. Пул psycopg2 немедленно отказывает при исчерпании ёмкости, длительного ожидания свободного соединения в нём нет. -## Ограничения и следующие шаги +Time to ready начинается с clock_timestamp при регистрации задания после PUT исходника, непосредственно перед COMMIT принятия, и заканчивается после COMMIT ready. Начальная точка приближает границу durable acceptance с точностью до оставшейся части транзакции. Метрика записывается только после успешного COMMIT результата. При аварии процесса сразу после COMMIT отдельное наблюдение может отсутствовать. Бизнес-состояние восстанавливается из PostgreSQL. -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +Запросы dashboard сохранены в infra/grafana/dashboards/messenger.json. Например, p95 всех реплик вычисляется из объединённых buckets. + +```promql +sum by (method, route) (rate(messenger_http_requests_total[1m])) +histogram_quantile(0.95, sum by (le) (rate(messenger_http_duration_seconds_bucket[1m]))) +rate(messenger_process_cpu_seconds_total[1m]) +messenger_queue_oldest_age_seconds +``` + +p50 и p99 отличаются аргументом histogram_quantile. Метрика CPU rate выражена в занятых ядрах. Backlog, processing duration и end-to-end time показывают разные части обработки. Логи ищутся в Loki запросом `{service_name=~"messenger-.*"} |= ""`. Tempo принимает поиск по trace ID и TraceQL `{resource.service.name = "messenger-worker"}`. + +## SLI, SLO и алерты + +SLI — доля бизнес-запросов без 5xx за скользящие 5 минут. SLO ≥99%. 4xx входят в знаменатель и не считаются отказом сервера. При отсутствии запросов SLI не оценивается. Этот SLI отражает запросы, дошедшие до API. Ошибки самого Nginx и отсутствие входящего трафика требуют внешнего probe. + +HighApiErrorRate срабатывает при доле 5xx >1% за 1 минуту, for 6 секунд. QueueLagGrowing срабатывает при oldest age >15 секунд и backlog >0, for 6 секунд. QueueObservationStale показывает, что publisher перестал обновлять состояние очереди. Правила сохранены в infra/prometheus/alerts.yaml. Runbook находится в infra/runbook.md. Локальные firing/resolved проверяются через Prometheus API. + +## Ограничение памяти и отказ телеметрии + +У SDK очереди до 256 spans и 256 logs на процесс, batch 64, export timeout 1 секунда. Метрики экспортируются каждые 3 секунды. SDK передаёт телеметрию фоновыми потоками. + +Collector применяет memory_limiter 128 MiB со spike 32 MiB, batch до 512 записей, очереди Loki/Tempo до 128 batches каждая, два consumer на экспортёр. Retry ограничен 10 секундами, timeout отдельного экспорта 2 секунды. Контейнер ограничен 192 MiB. При недоступности backend часть наблюдений может теряться. Prometheus забирает данные pull-запросами, остановка Prometheus создаёт пробел в истории scrape. Для логов и трасс проверяется отказ push backend Tempo. + +Внутренние accepted/sent/send_failed/enqueue_failed counters Collector позволяют увидеть деградацию экспорта. send_failed показывает отказ экспорта. Потери в контролируемом окне дополнительно подсчитываются по dropped_items в журнале Collector. Точное число уникальных потерь во всей цепочке по одному counter не вычисляется. + +## Проверки и эксперименты + +Файлы stage1-* относятся к переносу основы до включения полного стека наблюдаемости. Итоговые доказательства приведены ниже. + +| Критерий | Сценарий / результат | Доказательство | +| --- | --- | --- | +| Структура и схема API | make check, make validate | evidence/check.txt, evidence/validate.txt | +| Бизнес-API с телеметрией | make test | evidence/smoke.txt | +| Метрики, логи и реальные traces API/worker | make test-observability, проверены имена сигналов и логи трёх процессов | evidence/observability-smoke.json | +| W3C propagation и связь S3/DB | Parent-child spans, сохранён tracestate, внешний traceparent принят API | evidence/observability-smoke.json | +| Retry в исходном trace | Две consumer-попытки, ready после восстановления S3 | evidence/observability-retry.json | +| Dashboard и datasources | Provisioning 14 панелей, проверены все PromQL и поиск Tempo через Grafana | evidence/observability-smoke.json, infra/grafana | +| Percentiles, backlog, oldest age, DB pool, worker saturation, CPU/RAM | Histogram buckets и gauge/counters, единицы указаны выше | infra/grafana/dashboards/messenger.json | +| SLI/SLO и локальные alerts | promtool, firing/resolved для error rate и queue lag | evidence/alert-config.txt, evidence/observability-worker-stop.json, evidence/observability-db-outage.json | +| Остановка worker | Алерт firing через 30,1 с, после восстановления задание ready, алерт resolved | evidence/observability-worker-stop.json | +| Задержка PostgreSQL | 8 запросов получили 503, client span показывает ошибку БД, алерт firing/resolved | evidence/observability-db-outage.json | +| Отказ Tempo | 138 успешных сообщений, вложение ready, потеря 248 spans по dropped_items, без OOM/рестартов | evidence/observability-backend-outage.json | +| История сигналов после перезапуска | Сохранены исходная trace, логи трёх процессов и historical metric sample | evidence/telemetry-persistence.json | +| Сохранность бизнес-данных и отказ реплики | Публичные persistence/failover | evidence/persistence.txt, evidence/failover.txt | +| Полная HTTP duration | Регрессионная проверка включает отправку тела | evidence/http-timing.txt | + +### Нагрузка + +Docker Engine 29.5.3, Compose 5.1.4, aarch64, 10 доступных CPU и около 7,75 GiB RAM в VM. Лимиты API — по 0,5 CPU/256 MiB, worker — 1 CPU/768 MiB, Collector — 0,5 CPU/192 MiB. Параметры среды сохранены в evidence/environment.json. + +`make load` выполнял closed-loop POST сообщений 30 секунд, concurrency 4, без автоматических повторов. Выполнено 14 048 успешных запросов, ошибок 0, 467,941 RPS. Клиентские latency успешных запросов — p50 5,119 ms, p95 33,476 ms, p99 81,147 ms. Это один прогон, отдельного прогрева не было. Percentiles клиента охватывают только успехи и подвержены coordinated omission. Они дополняют histogram dashboard. Результат не заявляется как максимальная пропускная способность. Raw summary — evidence/load-output.txt. + +### Инцидент 1. Остановленный worker + +При продолжающемся приёме upload возраст незавершённого задания превысил 15 секунд, backlog оставался положительным. QueueLagGrowing перешёл в firing примерно через 30,1 секунды после загрузки. По метрикам видно ожидание обработки, producer span опубликован, consumer появляется после запуска worker. После восстановления job достигла ready, backlog уменьшился, алерт resolved. Диагностика выполняется по сохранённым возрасту/глубине очереди, состоянию alert и связанной trace. Сценарий — `python3 student-tests/observability.py worker-stop`. + +### Инцидент 2. Ожидание PostgreSQL + +Сценарий удерживает ACCESS EXCLUSIVE lock таблицы users 35 секунд. Восемь GET /users/me завершились 503 вследствие statement_timeout. В trace длительный postgresql.transaction имеет error status, что локализует задержку в БД. HighApiErrorRate перешёл в firing, после снятия блокировки запросы восстановились, alert resolved. Сценарий — `python3 student-tests/observability.py db-outage`. + +Во время разработки выявлено, что retry HTTP 503 в Nginx временно исключал обе реплики при общем тайм-ауте БД и приводил к 502. Теперь Nginx передаёт 503 клиенту. Переключение при сетевых ошибках и active readiness сохранено и проверено публичным failover. Старая и новая серии gauge после рестарта publisher различаются по service.instance.id. Dashboard и alerts используют freshness, чтобы устаревший экземпляр не создавал ложный queue alert. + +### Недоступность telemetry backend + +Tempo был остановлен на время последовательной записи сообщений и обработки изображения. SDK/Collector продолжили ограниченный экспорт с retry и drops. Успешно записано 138 сообщений, изображение стало ready. Collector зарегистрировал 248 dropped spans. Это число явно отброшенных записей данного exporter за окно, общая оценка уникальных потерь по всей системе не заявляется. Send-failed сравнивался до/после отказа. + +Наблюдаемый RSS Collector находился примерно в пределах 92–97 MiB при лимите 192 MiB, API — около 74–76 MiB, worker — около 63 MiB, publisher — около 34 MiB. Контейнеры работали без OOM и автоматических перезапусков. Измерение длилось около 40 секунд и подтверждает поведение данного сценария. Конфигурация задаёт конечные очереди и memory limiter для более длительной недоступности. После запуска Tempo найдены новые связанные trace и логи. Сценарий — `python3 student-tests/observability.py backend-outage`. + +### Воспроизведение + +```sh +make test-observability +docker compose exec -T api-a python - < student-tests/http_timing.py +python3 student-tests/observability.py worker-stop +python3 student-tests/observability.py db-outage +python3 student-tests/observability.py retry +python3 student-tests/observability.py backend-outage +python3 student-tests/telemetry_persistence.py +make test-persistence ACTION_SCRIPT=scripts/restart.sh +make test-failover ACTION_SCRIPT=scripts/stop-one.sh +python3 scripts/manage.py restore +``` + +Fault-сценарии выполняются по одному. Для проверки retry worker запускается через docker start, чтобы Compose не поднял остановленный S3 как dependency. PostgreSQL lock освобождается по завершении ограниченного pg_sleep. Все проверки используют синтетические данные. + +## Ограничения + +Стенд учебный, single-node. PostgreSQL, MinIO, RabbitMQ и telemetry backend представлены одиночными экземплярами. Авторизация Grafana локальная, TLS относится к следующей лабораторной. Sampling по умолчанию 100%, исследовательские треки на 5 не заявлены. Старые вспомогательные student-tests предыдущих лабораторных могут требовать специальных условий, например остановленного worker. Для lab6 использовать make test и student-tests/observability.py. + +## Источники + +- https://opentelemetry.io/docs/collector/configuration/ +- https://opentelemetry.io/docs/concepts/context-propagation/ +- https://www.w3.org/TR/trace-context/ +- https://grafana.com/docs/loki/latest/send-data/otel/ +- https://prometheus.io/docs/practices/histograms/ +- https://grafana.com/docs/grafana/latest/administration/provisioning/ + +## Соответствие переменных окружения + +Compose задаёт служебные адреса и пути secrets внутри контейнеров. PUBLIC_PORT и JOB_MAX_ATTEMPTS читаются из локального .env. API использует SESSION_STORE=redis. DATABASE_URL_FILE, CACHE_URL_FILE, CURSOR_SECRET_FILE и S3_*_FILE указывают на secrets. Издатель и worker получают RABBITMQ_PASSWORD_FILE, подключаются к rabbitmq:5672 под локальным пользователем lab6. BROKER_URL_FILE из исходной заготовки заменён этой настройкой. + +В данной реализации один worker-процесс обрабатывает одно задание с prefetch 1. WORKER_CONCURRENCY=1 описывает эту ёмкость. Тайм-аут попытки фиксирован на 20 секундах, lease на 60 секундах. MAX_UPLOAD_BYTES, MAX_IMAGE_PIXELS и STAGING_RETENTION_SECONDS соответствуют константам перенесённого приложения. Изменять эти поля .env отдельно от кода не следует. OTEL_SERVICE_NAME и endpoint задаются Compose отдельно для каждого процесса, используется grpc на 4317. Параметры SDK/Collector приведены выше и закреплены в коде и YAML. diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..c1d6c2e --- /dev/null +++ b/compose.yaml @@ -0,0 +1,341 @@ +name: mtusi-lab6 + +x-api: &api + build: + context: . + args: + APP_REVISION: ${API_IMAGE_TAG:-local} + image: mtusi-lab6-api:${API_IMAGE_TAG:-local} + environment: &api-env + APP_HOST: 0.0.0.0 + APP_PORT: '8080' + APP_ENV: lab6 + OTEL_SERVICE_NAME: messenger-api + OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4317 + OTEL_EXPORTER_OTLP_PROTOCOL: grpc + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + CURSOR_SECRET_FILE: /run/secrets/cursor_secret + DATABASE_URL_FILE: /run/secrets/database_url + CACHE_URL_FILE: /run/secrets/cache_url + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + SESSION_STORE: redis + SHUTDOWN_GRACE_SECONDS: '8' + DB_POOL_MIN: '2' + DB_POOL_MAX: '16' + DB_CONNECT_TIMEOUT: '1' + DB_STATEMENT_TIMEOUT: '4' + S3_ENDPOINT: http://s3:9000 + S3_REGION: us-east-1 + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + secrets: [database_url, cache_url, cursor_secret, s3_access_key_id, s3_secret_access_key] + depends_on: + s3-bootstrap: + condition: service_completed_successfully + migrate: + condition: service_completed_successfully + redis: + condition: service_healthy + networks: [backend] + cpus: 0.50 + mem_limit: 256m + stop_grace_period: 9s + healthcheck: + test: [CMD, python, -c, "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=1)"] + interval: 2s + timeout: 2s + retries: 2 + start_period: 3s + # Deliberately no auto-restart: failover hook must leave its target stopped. + restart: 'no' + +services: + otel-collector: + image: otel/opentelemetry-collector-contrib:0.123.0 + command: [--config=/etc/otel/collector.yaml] + volumes: [./infra/otel/collector.yaml:/etc/otel/collector.yaml:ro] + networks: [backend, telemetry] + mem_limit: 192m + cpus: 0.50 + restart: unless-stopped + + prometheus: + image: prom/prometheus:v3.2.1 + command: [--config.file=/etc/prometheus/prometheus.yaml, --storage.tsdb.retention.time=72h] + volumes: + - ./infra/prometheus:/etc/prometheus:ro + - prometheus_data:/prometheus + networks: [telemetry, frontend] + ports: ['127.0.0.1:19090:9090'] + mem_limit: 384m + restart: unless-stopped + + loki: + image: grafana/loki:3.4.2 + command: [-config.file=/etc/loki/loki.yaml] + volumes: + - ./infra/loki/loki.yaml:/etc/loki/loki.yaml:ro + - loki_data:/loki + networks: [telemetry, frontend] + ports: ['127.0.0.1:13100:3100'] + mem_limit: 384m + restart: unless-stopped + + tempo: + image: grafana/tempo:2.7.2 + command: [-config.file=/etc/tempo/tempo.yaml] + volumes: + - ./infra/tempo/tempo.yaml:/etc/tempo/tempo.yaml:ro + - tempo_data:/var/tempo + networks: [telemetry, frontend] + ports: ['127.0.0.1:13200:3200'] + mem_limit: 384m + restart: unless-stopped + + grafana: + image: grafana/grafana:11.6.0 + environment: + GF_SECURITY_ADMIN_USER: admin + GF_SECURITY_ADMIN_PASSWORD__FILE: /run/secrets/grafana_password + GF_USERS_ALLOW_SIGN_UP: 'false' + GF_ANALYTICS_REPORTING_ENABLED: 'false' + GF_ANALYTICS_CHECK_FOR_UPDATES: 'false' + secrets: [grafana_password] + volumes: + - grafana_data:/var/lib/grafana + - ./infra/grafana/provisioning:/etc/grafana/provisioning:ro + - ./infra/grafana/dashboards:/var/lib/grafana/dashboards:ro + networks: [telemetry, frontend] + ports: ['127.0.0.1:13000:3000'] + mem_limit: 256m + restart: unless-stopped + + rabbitmq: + image: rabbitmq:4.1.4-alpine + hostname: rabbitmq + entrypoint: [/bin/sh, -ec] + command: + - | + export RABBITMQ_DEFAULT_USER=lab6 + RABBITMQ_DEFAULT_PASS=$$(cat /run/secrets/rabbitmq_password) + export RABBITMQ_DEFAULT_PASS + exec docker-entrypoint.sh rabbitmq-server + secrets: [rabbitmq_password] + volumes: [rabbitmq_data:/var/lib/rabbitmq] + networks: [backend] + healthcheck: + test: [CMD, rabbitmq-diagnostics, -q, check_port_connectivity] + interval: 5s + timeout: 5s + retries: 12 + + outbox-publisher: + image: mtusi-lab6-api:${API_IMAGE_TAG:-local} + command: [python, -m, messenger.publish_jobs] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + RABBITMQ_PASSWORD_FILE: /run/secrets/rabbitmq_password + OTEL_SERVICE_NAME: messenger-publisher + OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4317 + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + DB_POOL_MIN: '1' + DB_POOL_MAX: '2' + secrets: [database_url, rabbitmq_password] + depends_on: + migrate: + condition: service_completed_successfully + rabbitmq: + condition: service_healthy + networks: [backend] + restart: unless-stopped + healthcheck: + disable: true + + s3: + image: mtusi-lab6-s3:2025-09-07 + build: &s3-build + context: . + dockerfile: infra/Dockerfile.s3 + entrypoint: [/bin/sh, -ec] + command: + - | + MINIO_ROOT_USER=$$(cat /run/secrets/s3_root_user) + MINIO_ROOT_PASSWORD=$$(cat /run/secrets/s3_root_password) + export MINIO_ROOT_USER MINIO_ROOT_PASSWORD + exec minio server /data + environment: + MINIO_BROWSER: "off" + secrets: [s3_root_user, s3_root_password] + volumes: [s3_data:/data] + networks: [backend] + + s3-bootstrap: + image: mtusi-lab6-s3:2025-09-07 + build: *s3-build + entrypoint: [/bin/sh, /bootstrap.sh] + command: [] + environment: + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + secrets: [s3_root_user, s3_root_password, s3_access_key_id, s3_secret_access_key] + volumes: [./infra/s3-bootstrap.sh:/bootstrap.sh:ro] + depends_on: [s3] + networks: [backend] + + postgres: + image: postgres:15-alpine@sha256:54e0dd8e8cddcdbb9156c2486eed5a43eccb7214d7c6828097d41ce049fb3984 + environment: + POSTGRES_DB: messenger + POSTGRES_USER: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + secrets: [postgres_password] + volumes: [pg_data:/var/lib/postgresql/data] + networks: [backend] + cpus: 1.0 + mem_limit: 512m + healthcheck: + test: [CMD-SHELL, 'pg_isready -U messenger -d messenger'] + interval: 2s + timeout: 2s + retries: 20 + + redis: + image: redis:7-alpine@sha256:8da19c7399dca900c6822a1cf30f9d2285585e2c4b457ff4095b0f4500054d3f + command: [redis-server, --appendonly, 'yes', --maxmemory, 96mb, --maxmemory-policy, allkeys-lru] + volumes: [redis_data:/data] + networks: [backend] + cpus: 0.25 + mem_limit: 128m + healthcheck: + test: [CMD, redis-cli, ping] + interval: 2s + timeout: 1s + retries: 10 + + migrate: + build: + context: . + args: + APP_REVISION: ${API_IMAGE_TAG:-local} + image: mtusi-lab6-api:${API_IMAGE_TAG:-local} + command: [python, -m, messenger.migrate] + environment: *api-env + secrets: [database_url, cache_url, cursor_secret] + depends_on: + postgres: + condition: service_healthy + networks: [backend] + cpus: 0.50 + mem_limit: 256m + restart: 'no' + healthcheck: + disable: true + + attachment-cleaner: + image: mtusi-lab6-api:${API_IMAGE_TAG:-local} + build: + context: . + args: + APP_REVISION: ${API_IMAGE_TAG:-local} + command: [python, -m, messenger.clean_attachments] + environment: *api-env + secrets: [database_url, s3_access_key_id, s3_secret_access_key] + depends_on: + s3-bootstrap: + condition: service_completed_successfully + migrate: + condition: service_completed_successfully + networks: [backend] + cpus: 0.25 + mem_limit: 128m + restart: unless-stopped + healthcheck: + disable: true + + worker: + image: mtusi-lab6-api:${API_IMAGE_TAG:-local} + command: [python, -m, messenger.worker] + environment: + <<: *api-env + OTEL_SERVICE_NAME: messenger-worker + RABBITMQ_PASSWORD_FILE: /run/secrets/rabbitmq_password + DB_POOL_MIN: '1' + DB_POOL_MAX: '2' + secrets: [database_url, s3_access_key_id, s3_secret_access_key, rabbitmq_password] + depends_on: + outbox-publisher: + condition: service_started + s3-bootstrap: + condition: service_completed_successfully + networks: [backend] + cpus: 1.0 + mem_limit: 768m + restart: unless-stopped + healthcheck: + disable: true + + api-a: + <<: *api + api-b: + <<: *api + + nginx: + image: nginx:1.28.2-alpine@sha256:5b4900b042ccfa8b0a73df622c3a60f2322faeb2be800cbee5aa7b44d241649e + entrypoint: [/bin/sh, -ec] + command: + - 'cp /etc/nginx/lab/upstreams.conf /etc/nginx/upstreams.conf; /bin/sh /etc/nginx/lab/healthwatch.sh & exec nginx -g "daemon off;"' + volumes: + - ./infra/nginx.conf:/etc/nginx/nginx.conf:ro + - ./infra:/etc/nginx/lab:ro + ports: ['127.0.0.1:${PUBLIC_PORT:-18083}:8080'] + networks: [frontend, backend] + depends_on: + api-a: + condition: service_healthy + api-b: + condition: service_healthy + cpus: 0.50 + mem_limit: 128m + stop_grace_period: 10s + healthcheck: + test: [CMD-SHELL, 'kill -0 $$(cat /tmp/healthwatch.pid) && wget -T 1 -q -O /dev/null http://127.0.0.1:8080/health/ready'] + interval: 2s + timeout: 2s + retries: 3 + +networks: + telemetry: + internal: true + frontend: + backend: + internal: true +volumes: + prometheus_data: + loki_data: + tempo_data: + grafana_data: + pg_data: + redis_data: + s3_data: + rabbitmq_data: +secrets: + grafana_password: + file: ./secrets/grafana_password + postgres_password: + file: ./secrets/postgres_password + database_url: + file: ./secrets/database_url + cache_url: + file: ./secrets/cache_url + cursor_secret: + file: ./secrets/cursor_secret + s3_root_user: + file: ./secrets/s3_root_user + s3_root_password: + file: ./secrets/s3_root_password + s3_access_key_id: + file: ./secrets/s3_access_key_id + s3_secret_access_key: + file: ./secrets/s3_secret_access_key + rabbitmq_password: + file: ./secrets/rabbitmq_password diff --git a/evidence/alert-config.txt b/evidence/alert-config.txt new file mode 100644 index 0000000..fe93f24 --- /dev/null +++ b/evidence/alert-config.txt @@ -0,0 +1,7 @@ +Checking /etc/prometheus/prometheus.yaml + SUCCESS: 1 rule files found + SUCCESS: /etc/prometheus/prometheus.yaml is valid prometheus config file syntax + +Checking /etc/prometheus/alerts.yaml + SUCCESS: 3 rules found + diff --git a/evidence/application-state.json b/evidence/application-state.json new file mode 100644 index 0000000..b1e3400 --- /dev/null +++ b/evidence/application-state.json @@ -0,0 +1,65 @@ +[ + { + "name": "/mtusi-lab6-postgres-1", + "started_at": "2026-11-20T17:27:24.629073504Z", + "running": true, + "restart_count": 0, + "memory_limit": 536870912 + }, + { + "name": "/mtusi-lab6-redis-1", + "started_at": "2026-11-20T17:27:24.710887337Z", + "running": true, + "restart_count": 0, + "memory_limit": 134217728 + }, + { + "name": "/mtusi-lab6-s3-1", + "started_at": "2026-11-20T17:27:24.605233629Z", + "running": true, + "restart_count": 0, + "memory_limit": 0 + }, + { + "name": "/mtusi-lab6-rabbitmq-1", + "started_at": "2026-11-20T17:27:25.649083629Z", + "running": true, + "restart_count": 0, + "memory_limit": 0 + }, + { + "name": "/mtusi-lab6-api-a-1", + "started_at": "2026-11-20T17:27:47.844003625Z", + "running": true, + "restart_count": 0, + "memory_limit": 268435456 + }, + { + "name": "/mtusi-lab6-api-b-1", + "started_at": "2026-11-20T17:27:25.985424213Z", + "running": true, + "restart_count": 0, + "memory_limit": 268435456 + }, + { + "name": "/mtusi-lab6-nginx-1", + "started_at": "2026-11-20T17:27:24.766067837Z", + "running": true, + "restart_count": 0, + "memory_limit": 134217728 + }, + { + "name": "/mtusi-lab6-worker-1", + "started_at": "2026-11-20T17:27:28.139978422Z", + "running": true, + "restart_count": 2, + "memory_limit": 805306368 + }, + { + "name": "/mtusi-lab6-outbox-publisher-1", + "started_at": "2026-11-20T17:27:25.619581296Z", + "running": true, + "restart_count": 0, + "memory_limit": 0 + } +] diff --git a/evidence/application-state.txt b/evidence/application-state.txt new file mode 100644 index 0000000..b9a1795 --- /dev/null +++ b/evidence/application-state.txt @@ -0,0 +1,5 @@ +{"service": "postgres", "id": "914b5dae50c3", "instance": "914b5dae50c3", "running": true, "started_at": "2026-11-21T08:17:27.913170257Z", "health": "healthy", "image_id": "sha256:54e0dd8e8cddcdbb9156c2486eed5a43eccb7214d7c6828097d41ce049fb3984", "cpu_limit": 1.0, "memory_limit_bytes": 536870912, "published_ports": {}} +{"service": "redis", "id": "331192624ebd", "instance": "331192624ebd", "running": true, "started_at": "2026-11-21T08:17:27.721573173Z", "health": "healthy", "image_id": "sha256:8da19c7399dca900c6822a1cf30f9d2285585e2c4b457ff4095b0f4500054d3f", "cpu_limit": 0.25, "memory_limit_bytes": 134217728, "published_ports": {}} +{"service": "api-a", "id": "e305493d2118", "instance": "e305493d2118", "running": true, "started_at": "2026-11-21T08:17:28.125710257Z", "health": "healthy", "image_id": "sha256:f1b5f50a8470841ce56c6b33e182594568c228e3c541e5b8b945f7d4eee1aa76", "cpu_limit": 0.5, "memory_limit_bytes": 268435456, "published_ports": {}} +{"service": "api-b", "id": "e46ce208e44b", "instance": "e46ce208e44b", "running": true, "started_at": "2026-11-21T08:17:50.183420711Z", "health": "healthy", "image_id": "sha256:f1b5f50a8470841ce56c6b33e182594568c228e3c541e5b8b945f7d4eee1aa76", "cpu_limit": 0.5, "memory_limit_bytes": 268435456, "published_ports": {}} +{"service": "nginx", "id": "7c8111b7fbdb", "instance": "7c8111b7fbdb", "running": true, "started_at": "2026-11-21T08:17:27.962736673Z", "health": "healthy", "image_id": "sha256:5b4900b042ccfa8b0a73df622c3a60f2322faeb2be800cbee5aa7b44d241649e", "cpu_limit": 0.5, "memory_limit_bytes": 134217728, "published_ports": {"8080/tcp": [{"HostIp": "127.0.0.1", "HostPort": "18083"}]}} diff --git a/evidence/check.txt b/evidence/check.txt new file mode 100644 index 0000000..9a55330 --- /dev/null +++ b/evidence/check.txt @@ -0,0 +1,2 @@ +python3 scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. diff --git a/evidence/dashboard-validation.json b/evidence/dashboard-validation.json new file mode 100644 index 0000000..a3d14c4 --- /dev/null +++ b/evidence/dashboard-validation.json @@ -0,0 +1,5 @@ +{ + "promql_panels_valid": 12, + "tempo_search_via_grafana_proxy": true, + "traces_found": 20 +} diff --git a/evidence/environment.json b/evidence/environment.json new file mode 100644 index 0000000..b7f0c54 --- /dev/null +++ b/evidence/environment.json @@ -0,0 +1,8 @@ +{ + "host_architecture": "arm64", + "docker_version": "29.5.3", + "docker_architecture": "aarch64", + "docker_cpus": 10, + "docker_memory_bytes": 8321515520, + "compose_version": "5.1.4" +} diff --git a/evidence/failover.txt b/evidence/failover.txt new file mode 100644 index 0000000..9724377 --- /dev/null +++ b/evidence/failover.txt @@ -0,0 +1,4 @@ +python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:18083" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 130, "failed_attempts": 0, "first_recovery_seconds": 0.78} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/final-services.txt b/evidence/final-services.txt new file mode 100644 index 0000000..5c83fac --- /dev/null +++ b/evidence/final-services.txt @@ -0,0 +1,16 @@ +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +mtusi-lab6-api-a-1 mtusi-lab6-api:local "python -m messenger" api-a 7 minutes ago Up About a minute (healthy) 8080/tcp +mtusi-lab6-api-b-1 mtusi-lab6-api:local "python -m messenger" api-b 7 minutes ago Up About a minute (healthy) 8080/tcp +mtusi-lab6-attachment-cleaner-1 mtusi-lab6-api:local "python -m messenger…" attachment-cleaner 7 minutes ago Up 6 minutes 8080/tcp +mtusi-lab6-grafana-1 grafana/grafana:11.6.0 "/run.sh" grafana 6 minutes ago Up 2 minutes 127.0.0.1:13000->3000/tcp +mtusi-lab6-loki-1 grafana/loki:3.4.2 "/usr/bin/loki -conf…" loki 15 hours ago Up 2 minutes 127.0.0.1:13100->3100/tcp +mtusi-lab6-nginx-1 nginx:1.28.2-alpine@sha256:5b4900b042ccfa8b0a73df622c3a60f2322faeb2be800cbee5aa7b44d241649e "/bin/sh -ec 'cp /et…" nginx 15 hours ago Up About a minute (healthy) 127.0.0.1:18083->8080/tcp +mtusi-lab6-otel-collector-1 otel/opentelemetry-collector-contrib:0.123.0 "/otelcol-contrib --…" otel-collector 15 hours ago Up 2 minutes 4317-4318/tcp, 55678-55679/tcp +mtusi-lab6-outbox-publisher-1 mtusi-lab6-api:local "python -m messenger…" outbox-publisher 7 minutes ago Up About a minute 8080/tcp +mtusi-lab6-postgres-1 postgres:15-alpine@sha256:54e0dd8e8cddcdbb9156c2486eed5a43eccb7214d7c6828097d41ce049fb3984 "docker-entrypoint.s…" postgres 15 hours ago Up About a minute (healthy) 5432/tcp +mtusi-lab6-prometheus-1 prom/prometheus:v3.2.1 "/bin/prometheus --c…" prometheus 15 hours ago Up 2 minutes 127.0.0.1:19090->9090/tcp +mtusi-lab6-rabbitmq-1 rabbitmq:4.1.4-alpine "/bin/sh -ec 'export…" rabbitmq 15 hours ago Up About a minute (healthy) 4369/tcp, 5671-5672/tcp, 15691-15692/tcp, 25672/tcp +mtusi-lab6-redis-1 redis:7-alpine@sha256:8da19c7399dca900c6822a1cf30f9d2285585e2c4b457ff4095b0f4500054d3f "docker-entrypoint.s…" redis 15 hours ago Up About a minute (healthy) 6379/tcp +mtusi-lab6-s3-1 mtusi-lab6-s3:2025-09-07 "/bin/sh -ec 'MINIO_…" s3 15 hours ago Up About a minute +mtusi-lab6-tempo-1 grafana/tempo:2.7.2 "/tempo -config.file…" tempo 15 hours ago Up 2 minutes 127.0.0.1:13200->3200/tcp +mtusi-lab6-worker-1 mtusi-lab6-api:local "python -m messenger…" worker 7 minutes ago Up About a minute 8080/tcp diff --git a/evidence/http-timing.txt b/evidence/http-timing.txt new file mode 100644 index 0000000..6f9398d --- /dev/null +++ b/evidence/http-timing.txt @@ -0,0 +1 @@ +PASS: HTTP histogram includes response transmission diff --git a/evidence/load-output.txt b/evidence/load-output.txt new file mode 100644 index 0000000..d10f004 --- /dev/null +++ b/evidence/load-output.txt @@ -0,0 +1,18 @@ +python3 tests/load.py --base-url "http://localhost:18083" +{ + "lab": 6, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30, + "elapsed_seconds": 30.021, + "concurrency": 4, + "successful_requests": 14048, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 467.941, + "successful_latency_ms": { + "p50": 5.119, + "p95": 33.476, + "p99": 81.147 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/observability-backend-outage.json b/evidence/observability-backend-outage.json new file mode 100644 index 0000000..9880d7e --- /dev/null +++ b/evidence/observability-backend-outage.json @@ -0,0 +1,522 @@ +{ + "successful_messages": 138, + "attachment_ready_during_outage": true, + "collector_reported_dropped_spans": 248, + "export_failures_before": [ + { + "metric": {}, + "value": [ + 1795248850.545, + "0" + ] + } + ], + "export_failures_after": [ + { + "metric": {}, + "value": [ + 1795248892.504, + "248" + ] + } + ], + "application_and_collector_stats": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "15.83%", + "Container": "mtusi-lab6-api-a-1", + "ID": "e305493d2118", + "MemPerc": "28.91%", + "MemUsage": "74.02MiB / 256MiB", + "Name": "mtusi-lab6-api-a-1", + "NetIO": "430kB / 941kB", + "PIDs": "16" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.04%", + "Container": "mtusi-lab6-api-b-1", + "ID": "e46ce208e44b", + "MemPerc": "28.88%", + "MemUsage": "73.93MiB / 256MiB", + "Name": "mtusi-lab6-api-b-1", + "NetIO": "546kB / 1.04MB", + "PIDs": "16" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.31%", + "Container": "mtusi-lab6-worker-1", + "ID": "23d809845172", + "MemPerc": "8.23%", + "MemUsage": "63.24MiB / 768MiB", + "Name": "mtusi-lab6-worker-1", + "NetIO": "130kB / 152kB", + "PIDs": "16" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.54%", + "Container": "mtusi-lab6-outbox-publisher-1", + "ID": "ed59d06f4e97", + "MemPerc": "0.42%", + "MemUsage": "33.36MiB / 7.75GiB", + "Name": "mtusi-lab6-outbox-publisher-1", + "NetIO": "448kB / 794kB", + "PIDs": "16" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.65%", + "Container": "mtusi-lab6-otel-collector-1", + "ID": "1a2ff6617b62", + "MemPerc": "47.69%", + "MemUsage": "91.57MiB / 192MiB", + "Name": "mtusi-lab6-otel-collector-1", + "NetIO": "25.1MB / 13.7MB", + "PIDs": "15" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "14.84%", + "Container": "mtusi-lab6-api-a-1", + "ID": "e305493d2118", + "MemPerc": "28.92%", + "MemUsage": "74.03MiB / 256MiB", + "Name": "mtusi-lab6-api-a-1", + "NetIO": "467kB / 1.01MB", + "PIDs": "16" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "13.70%", + "Container": "mtusi-lab6-api-b-1", + "ID": "e46ce208e44b", + "MemPerc": "28.89%", + "MemUsage": "73.96MiB / 256MiB", + "Name": "mtusi-lab6-api-b-1", + "NetIO": "583kB / 1.11MB", + "PIDs": "16" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.10%", + "Container": "mtusi-lab6-worker-1", + "ID": "23d809845172", + "MemPerc": "8.24%", + "MemUsage": "63.25MiB / 768MiB", + "Name": "mtusi-lab6-worker-1", + "NetIO": "131kB / 155kB", + "PIDs": "16" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.89%", + "Container": "mtusi-lab6-outbox-publisher-1", + "ID": "ed59d06f4e97", + "MemPerc": "0.42%", + "MemUsage": "33.36MiB / 7.75GiB", + "Name": "mtusi-lab6-outbox-publisher-1", + "NetIO": "462kB / 820kB", + "PIDs": "16" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.85%", + "Container": "mtusi-lab6-otel-collector-1", + "ID": "1a2ff6617b62", + "MemPerc": "47.69%", + "MemUsage": "91.57MiB / 192MiB", + "Name": "mtusi-lab6-otel-collector-1", + "NetIO": "25.2MB / 13.7MB", + "PIDs": "15" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "16.36%", + "Container": "mtusi-lab6-api-a-1", + "ID": "e305493d2118", + "MemPerc": "28.95%", + "MemUsage": "74.1MiB / 256MiB", + "Name": "mtusi-lab6-api-a-1", + "NetIO": "503kB / 1.09MB", + "PIDs": "16" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "14.02%", + "Container": "mtusi-lab6-api-b-1", + "ID": "e46ce208e44b", + "MemPerc": "28.89%", + "MemUsage": "73.96MiB / 256MiB", + "Name": "mtusi-lab6-api-b-1", + "NetIO": "624kB / 1.2MB", + "PIDs": "16" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.20%", + "Container": "mtusi-lab6-worker-1", + "ID": "23d809845172", + "MemPerc": "8.24%", + "MemUsage": "63.25MiB / 768MiB", + "Name": "mtusi-lab6-worker-1", + "NetIO": "131kB / 158kB", + "PIDs": "16" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.25%", + "Container": "mtusi-lab6-outbox-publisher-1", + "ID": "ed59d06f4e97", + "MemPerc": "0.42%", + "MemUsage": "33.36MiB / 7.75GiB", + "Name": "mtusi-lab6-outbox-publisher-1", + "NetIO": "478kB / 849kB", + "PIDs": "16" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.13%", + "Container": "mtusi-lab6-otel-collector-1", + "ID": "1a2ff6617b62", + "MemPerc": "47.69%", + "MemUsage": "91.57MiB / 192MiB", + "Name": "mtusi-lab6-otel-collector-1", + "NetIO": "25.3MB / 13.7MB", + "PIDs": "15" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "15.94%", + "Container": "mtusi-lab6-api-a-1", + "ID": "e305493d2118", + "MemPerc": "28.95%", + "MemUsage": "74.12MiB / 256MiB", + "Name": "mtusi-lab6-api-a-1", + "NetIO": "540kB / 1.16MB", + "PIDs": "16" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "14.14%", + "Container": "mtusi-lab6-api-b-1", + "ID": "e46ce208e44b", + "MemPerc": "28.89%", + "MemUsage": "73.96MiB / 256MiB", + "Name": "mtusi-lab6-api-b-1", + "NetIO": "663kB / 1.27MB", + "PIDs": "16" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.20%", + "Container": "mtusi-lab6-worker-1", + "ID": "23d809845172", + "MemPerc": "8.24%", + "MemUsage": "63.25MiB / 768MiB", + "Name": "mtusi-lab6-worker-1", + "NetIO": "132kB / 164kB", + "PIDs": "16" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.34%", + "Container": "mtusi-lab6-outbox-publisher-1", + "ID": "ed59d06f4e97", + "MemPerc": "0.42%", + "MemUsage": "33.35MiB / 7.75GiB", + "Name": "mtusi-lab6-outbox-publisher-1", + "NetIO": "496kB / 882kB", + "PIDs": "16" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "2.08%", + "Container": "mtusi-lab6-otel-collector-1", + "ID": "1a2ff6617b62", + "MemPerc": "47.69%", + "MemUsage": "91.57MiB / 192MiB", + "Name": "mtusi-lab6-otel-collector-1", + "NetIO": "25.4MB / 13.8MB", + "PIDs": "15" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "16.21%", + "Container": "mtusi-lab6-api-a-1", + "ID": "e305493d2118", + "MemPerc": "29.27%", + "MemUsage": "74.93MiB / 256MiB", + "Name": "mtusi-lab6-api-a-1", + "NetIO": "578kB / 1.24MB", + "PIDs": "16" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "14.39%", + "Container": "mtusi-lab6-api-b-1", + "ID": "e46ce208e44b", + "MemPerc": "28.90%", + "MemUsage": "73.98MiB / 256MiB", + "Name": "mtusi-lab6-api-b-1", + "NetIO": "703kB / 1.35MB", + "PIDs": "16" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.27%", + "Container": "mtusi-lab6-worker-1", + "ID": "23d809845172", + "MemPerc": "8.24%", + "MemUsage": "63.25MiB / 768MiB", + "Name": "mtusi-lab6-worker-1", + "NetIO": "133kB / 170kB", + "PIDs": "16" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "2.09%", + "Container": "mtusi-lab6-outbox-publisher-1", + "ID": "ed59d06f4e97", + "MemPerc": "0.42%", + "MemUsage": "33.6MiB / 7.75GiB", + "Name": "mtusi-lab6-outbox-publisher-1", + "NetIO": "508kB / 905kB", + "PIDs": "16" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.30%", + "Container": "mtusi-lab6-otel-collector-1", + "ID": "1a2ff6617b62", + "MemPerc": "47.82%", + "MemUsage": "91.82MiB / 192MiB", + "Name": "mtusi-lab6-otel-collector-1", + "NetIO": "25.5MB / 13.8MB", + "PIDs": "15" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.52%", + "Container": "mtusi-lab6-api-a-1", + "ID": "e305493d2118", + "MemPerc": "29.27%", + "MemUsage": "74.93MiB / 256MiB", + "Name": "mtusi-lab6-api-a-1", + "NetIO": "612kB / 1.3MB", + "PIDs": "16" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.44%", + "Container": "mtusi-lab6-api-b-1", + "ID": "e46ce208e44b", + "MemPerc": "28.91%", + "MemUsage": "74MiB / 256MiB", + "Name": "mtusi-lab6-api-b-1", + "NetIO": "740kB / 1.43MB", + "PIDs": "16" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.16%", + "Container": "mtusi-lab6-worker-1", + "ID": "23d809845172", + "MemPerc": "8.24%", + "MemUsage": "63.25MiB / 768MiB", + "Name": "mtusi-lab6-worker-1", + "NetIO": "133kB / 172kB", + "PIDs": "16" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.96%", + "Container": "mtusi-lab6-outbox-publisher-1", + "ID": "ed59d06f4e97", + "MemPerc": "0.42%", + "MemUsage": "33.36MiB / 7.75GiB", + "Name": "mtusi-lab6-outbox-publisher-1", + "NetIO": "530kB / 942kB", + "PIDs": "16" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.63%", + "Container": "mtusi-lab6-otel-collector-1", + "ID": "1a2ff6617b62", + "MemPerc": "47.82%", + "MemUsage": "91.82MiB / 192MiB", + "Name": "mtusi-lab6-otel-collector-1", + "NetIO": "25.6MB / 13.8MB", + "PIDs": "15" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "15.56%", + "Container": "mtusi-lab6-api-a-1", + "ID": "e305493d2118", + "MemPerc": "29.39%", + "MemUsage": "75.23MiB / 256MiB", + "Name": "mtusi-lab6-api-a-1", + "NetIO": "651kB / 1.39MB", + "PIDs": "16" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "15.54%", + "Container": "mtusi-lab6-api-b-1", + "ID": "e46ce208e44b", + "MemPerc": "28.90%", + "MemUsage": "73.99MiB / 256MiB", + "Name": "mtusi-lab6-api-b-1", + "NetIO": "782kB / 1.51MB", + "PIDs": "16" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.28%", + "Container": "mtusi-lab6-worker-1", + "ID": "23d809845172", + "MemPerc": "8.24%", + "MemUsage": "63.26MiB / 768MiB", + "Name": "mtusi-lab6-worker-1", + "NetIO": "134kB / 178kB", + "PIDs": "16" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.94%", + "Container": "mtusi-lab6-outbox-publisher-1", + "ID": "ed59d06f4e97", + "MemPerc": "0.43%", + "MemUsage": "33.77MiB / 7.75GiB", + "Name": "mtusi-lab6-outbox-publisher-1", + "NetIO": "546kB / 971kB", + "PIDs": "16" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.24%", + "Container": "mtusi-lab6-otel-collector-1", + "ID": "1a2ff6617b62", + "MemPerc": "47.95%", + "MemUsage": "92.06MiB / 192MiB", + "Name": "mtusi-lab6-otel-collector-1", + "NetIO": "25.7MB / 13.9MB", + "PIDs": "15" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.96%", + "Container": "mtusi-lab6-api-a-1", + "ID": "e305493d2118", + "MemPerc": "29.39%", + "MemUsage": "75.24MiB / 256MiB", + "Name": "mtusi-lab6-api-a-1", + "NetIO": "685kB / 1.45MB", + "PIDs": "16" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "15.35%", + "Container": "mtusi-lab6-api-b-1", + "ID": "e46ce208e44b", + "MemPerc": "28.91%", + "MemUsage": "74MiB / 256MiB", + "Name": "mtusi-lab6-api-b-1", + "NetIO": "822kB / 1.59MB", + "PIDs": "16" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.24%", + "Container": "mtusi-lab6-worker-1", + "ID": "23d809845172", + "MemPerc": "8.24%", + "MemUsage": "63.26MiB / 768MiB", + "Name": "mtusi-lab6-worker-1", + "NetIO": "134kB / 181kB", + "PIDs": "16" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.87%", + "Container": "mtusi-lab6-outbox-publisher-1", + "ID": "ed59d06f4e97", + "MemPerc": "0.43%", + "MemUsage": "33.77MiB / 7.75GiB", + "Name": "mtusi-lab6-outbox-publisher-1", + "NetIO": "561kB / 1MB", + "PIDs": "16" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.58%", + "Container": "mtusi-lab6-otel-collector-1", + "ID": "1a2ff6617b62", + "MemPerc": "48.85%", + "MemUsage": "93.8MiB / 192MiB", + "Name": "mtusi-lab6-otel-collector-1", + "NetIO": "25.8MB / 13.9MB", + "PIDs": "15" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "15.39%", + "Container": "mtusi-lab6-api-a-1", + "ID": "e305493d2118", + "MemPerc": "29.44%", + "MemUsage": "75.37MiB / 256MiB", + "Name": "mtusi-lab6-api-a-1", + "NetIO": "726kB / 1.53MB", + "PIDs": "16" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "13.65%", + "Container": "mtusi-lab6-api-b-1", + "ID": "e46ce208e44b", + "MemPerc": "28.92%", + "MemUsage": "74.03MiB / 256MiB", + "Name": "mtusi-lab6-api-b-1", + "NetIO": "857kB / 1.66MB", + "PIDs": "16" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.20%", + "Container": "mtusi-lab6-worker-1", + "ID": "23d809845172", + "MemPerc": "8.24%", + "MemUsage": "63.26MiB / 768MiB", + "Name": "mtusi-lab6-worker-1", + "NetIO": "135kB / 187kB", + "PIDs": "16" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.49%", + "Container": "mtusi-lab6-outbox-publisher-1", + "ID": "ed59d06f4e97", + "MemPerc": "0.43%", + "MemUsage": "33.77MiB / 7.75GiB", + "Name": "mtusi-lab6-outbox-publisher-1", + "NetIO": "576kB / 1.03MB", + "PIDs": "16" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.73%", + "Container": "mtusi-lab6-otel-collector-1", + "ID": "1a2ff6617b62", + "MemPerc": "50.17%", + "MemUsage": "96.33MiB / 192MiB", + "Name": "mtusi-lab6-otel-collector-1", + "NetIO": "25.9MB / 13.9MB", + "PIDs": "15" + } + ], + "no_oom_or_restarts": true, + "telemetry_restored_trace": "c485b3e4b21d427a94f1cd345789c843" +} diff --git a/evidence/observability-db-outage.json b/evidence/observability-db-outage.json new file mode 100644 index 0000000..8595c66 --- /dev/null +++ b/evidence/observability-db-outage.json @@ -0,0 +1,82 @@ +{ + "firing": true, + "failed_requests": 8, + "trace_id": "9a03429d57e1b78a8363efb6292dfd5b", + "db_error_spans": 1, + "alerts_firing": [ + { + "labels": { + "alertname": "HighApiErrorRate", + "severity": "warning" + }, + "annotations": { + "description": "SLO is 99% non-5xx business requests over 5 minutes. Inspect PostgreSQL client spans and pool metrics.", + "runbook_url": "http://localhost:13000/d/messenger/messenger?viewPanel=5", + "summary": "API 5xx exceeds 1% over 1 minute" + }, + "state": "firing", + "activeAt": "2026-11-20T17:41:07.478517494Z", + "value": "3.6363636363636365e-01" + }, + { + "labels": { + "alertname": "QueueObservationStale", + "deployment_environment_name": "lab", + "exported_instance": "459bbaf48a82", + "exported_job": "messenger-publisher", + "instance": "otel-collector:8889", + "job": "application", + "queue": "attachment.process.v1", + "service_instance_id": "459bbaf48a82", + "service_name": "messenger-publisher", + "service_version": "local", + "severity": "warning", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.38.0" + }, + "annotations": { + "description": "Check publisher and PostgreSQL availability before trusting queue gauges.", + "summary": "Publisher backlog observations are stale" + }, + "state": "firing", + "activeAt": "2026-11-20T17:39:31.478517494Z", + "value": "1.1822987508773804e+02" + }, + { + "labels": { + "alertname": "QueueObservationStale", + "deployment_environment_name": "lab", + "exported_instance": "af26b1841208", + "exported_job": "messenger-publisher", + "instance": "otel-collector:8889", + "job": "application", + "queue": "attachment.process.v1", + "service_instance_id": "af26b1841208", + "service_name": "messenger-publisher", + "service_version": "local", + "severity": "warning", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.38.0" + }, + "annotations": { + "description": "Check publisher and PostgreSQL availability before trusting queue gauges.", + "summary": "Publisher backlog observations are stale" + }, + "state": "firing", + "activeAt": "2026-11-20T17:38:01.478517494Z", + "value": "2.077088599205017e+02" + } + ], + "error_ratio": [ + { + "metric": {}, + "value": [ + 1795196476.495, + "0.4615384615384615" + ] + } + ], + "resolved": true +} diff --git a/evidence/observability-retry.json b/evidence/observability-retry.json new file mode 100644 index 0000000..f8429f0 --- /dev/null +++ b/evidence/observability-retry.json @@ -0,0 +1,28 @@ +{ + "trace_id": "8e9d1c9daddb49778c887f0cc63f470a", + "consumer_attempts": 2, + "span_names": [ + "POST /api/v1/chats/{chat_id}/attachments", + "postgresql.transaction", + "postgresql.transaction", + "postgresql.transaction", + "postgresql.transaction", + "postgresql.transaction", + "s3.put", + "attachment.publish", + "attachment.consume", + "postgresql.transaction", + "s3.download", + "postgresql.transaction", + "attachment.publish", + "attachment.consume", + "postgresql.transaction", + "s3.download", + "s3.put", + "s3.put", + "s3.put", + "s3.put", + "postgresql.transaction" + ], + "ready_after_retry": true +} diff --git a/evidence/observability-smoke.json b/evidence/observability-smoke.json new file mode 100644 index 0000000..7bc6f72 --- /dev/null +++ b/evidence/observability-smoke.json @@ -0,0 +1,42 @@ +{ + "trace_id": "f1aa109fe9fd47f284eaafd2c34a8d96", + "span_names": [ + "postgresql.transaction", + "postgresql.transaction", + "postgresql.transaction", + "postgresql.transaction", + "s3.put", + "postgresql.transaction", + "POST /api/v1/chats/{chat_id}/attachments", + "postgresql.transaction", + "s3.download", + "s3.put", + "s3.put", + "s3.put", + "s3.put", + "postgresql.transaction", + "attachment.consume", + "attachment.publish" + ], + "log_services": [ + "messenger-api", + "messenger-publisher", + "messenger-worker" + ], + "metric_names_checked": [ + "messenger_http_requests_total", + "messenger_http_duration_seconds_bucket", + "messenger_queue_depth", + "messenger_queue_oldest_age_seconds", + "messenger_db_pool_used", + "messenger_worker_busy", + "messenger_job_duration_seconds_bucket", + "messenger_attachment_time_to_ready_seconds_bucket", + "messenger_process_memory_bytes", + "messenger_process_cpu_seconds_total" + ], + "dashboard_panels": 14, + "dashboard_queries_valid": true, + "incoming_w3c_context": true, + "body_and_token_absent_from_logs": true +} diff --git a/evidence/observability-worker-stop.json b/evidence/observability-worker-stop.json new file mode 100644 index 0000000..4810ee6 --- /dev/null +++ b/evidence/observability-worker-stop.json @@ -0,0 +1,173 @@ +{ + "seconds_to_firing": 30.1, + "oldest_job_age": [ + { + "metric": { + "__name__": "messenger_queue_oldest_age_seconds", + "deployment_environment_name": "lab", + "exported_instance": "af26b1841208", + "exported_job": "messenger-publisher", + "instance": "otel-collector:8889", + "job": "application", + "queue": "attachment.process.v1", + "service_instance_id": "af26b1841208", + "service_name": "messenger-publisher", + "service_version": "local", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.38.0" + }, + "value": [ + 1795196321.26, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_queue_oldest_age_seconds", + "deployment_environment_name": "lab", + "exported_instance": "459bbaf48a82", + "exported_job": "messenger-publisher", + "instance": "otel-collector:8889", + "job": "application", + "queue": "attachment.process.v1", + "service_instance_id": "459bbaf48a82", + "service_name": "messenger-publisher", + "service_version": "local", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.38.0" + }, + "value": [ + 1795196321.26, + "25.465688" + ] + } + ], + "backlog": [ + { + "metric": { + "__name__": "messenger_queue_depth", + "deployment_environment_name": "lab", + "exported_instance": "af26b1841208", + "exported_job": "messenger-publisher", + "instance": "otel-collector:8889", + "job": "application", + "queue": "attachment.process.v1", + "service_instance_id": "af26b1841208", + "service_name": "messenger-publisher", + "service_version": "local", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.38.0" + }, + "value": [ + 1795196321.262, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_queue_depth", + "deployment_environment_name": "lab", + "exported_instance": "459bbaf48a82", + "exported_job": "messenger-publisher", + "instance": "otel-collector:8889", + "job": "application", + "queue": "attachment.process.v1", + "service_instance_id": "459bbaf48a82", + "service_name": "messenger-publisher", + "service_version": "local", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.38.0" + }, + "value": [ + 1795196321.262, + "1" + ] + } + ], + "alerts_firing": [ + { + "labels": { + "alertname": "QueueLagGrowing", + "deployment_environment_name": "lab", + "exported_instance": "459bbaf48a82", + "exported_job": "messenger-publisher", + "instance": "otel-collector:8889", + "job": "application", + "queue": "attachment.process.v1", + "service_instance_id": "459bbaf48a82", + "service_name": "messenger-publisher", + "service_version": "local", + "severity": "warning", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.38.0" + }, + "annotations": { + "description": "Inspect worker capacity, busy gauge and consumer traces. Restore worker and verify backlog drains. See infra/runbook.md.", + "summary": "Oldest unfinished job exceeds 15 seconds" + }, + "state": "firing", + "activeAt": "2026-11-20T17:38:31.478517494Z", + "value": "2.2414185e+01" + }, + { + "labels": { + "alertname": "QueueObservationStale", + "deployment_environment_name": "lab", + "exported_instance": "af26b1841208", + "exported_job": "messenger-publisher", + "instance": "otel-collector:8889", + "job": "application", + "queue": "attachment.process.v1", + "service_instance_id": "af26b1841208", + "service_name": "messenger-publisher", + "service_version": "local", + "severity": "warning", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.38.0" + }, + "annotations": { + "description": "Check publisher and PostgreSQL availability before trusting queue gauges.", + "summary": "Publisher backlog observations are stale" + }, + "state": "firing", + "activeAt": "2026-11-20T17:38:01.478517494Z", + "value": "5.170885992050171e+01" + } + ], + "resolved": true, + "trace_id": "a0d4df4afbe44a30999830b9d563817c", + "span_count": 16, + "alerts_after": [ + { + "labels": { + "alertname": "QueueObservationStale", + "deployment_environment_name": "lab", + "exported_instance": "af26b1841208", + "exported_job": "messenger-publisher", + "instance": "otel-collector:8889", + "job": "application", + "queue": "attachment.process.v1", + "service_instance_id": "af26b1841208", + "service_name": "messenger-publisher", + "service_version": "local", + "severity": "warning", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.38.0" + }, + "annotations": { + "description": "Check publisher and PostgreSQL availability before trusting queue gauges.", + "summary": "Publisher backlog observations are stale" + }, + "state": "firing", + "activeAt": "2026-11-20T17:38:01.478517494Z", + "value": "6.370885992050171e+01" + } + ] +} diff --git a/evidence/persistence.txt b/evidence/persistence.txt new file mode 100644 index 0000000..8625fed --- /dev/null +++ b/evidence/persistence.txt @@ -0,0 +1,3 @@ +python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://localhost:18083" +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/restore.txt b/evidence/restore.txt new file mode 100644 index 0000000..a657870 --- /dev/null +++ b/evidence/restore.txt @@ -0,0 +1,2 @@ +e305493d2118fcf5b65beba48547258a269d2d353cf2a20fc9b841af2ab5f153 +e46ce208e44b8b48f2f15dedca018e0c4d780d1e96a88858fdcd9d0eb4da139d diff --git a/evidence/secrets-check.txt b/evidence/secrets-check.txt new file mode 100644 index 0000000..c2b562e --- /dev/null +++ b/evidence/secrets-check.txt @@ -0,0 +1 @@ +PASS: generated private credentials absent from application, configs and evidence. Redis URL without credentials is public configuration. diff --git a/evidence/smoke.txt b/evidence/smoke.txt new file mode 100644 index 0000000..85f8a82 --- /dev/null +++ b/evidence/smoke.txt @@ -0,0 +1,2 @@ +python3 tests/smoke.py --base-url "http://localhost:18083" +PASS: lab6 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/stage1-check.txt b/evidence/stage1-check.txt new file mode 100644 index 0000000..9a55330 --- /dev/null +++ b/evidence/stage1-check.txt @@ -0,0 +1,2 @@ +python3 scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. diff --git a/evidence/stage1-pipeline.txt b/evidence/stage1-pipeline.txt new file mode 100644 index 0000000..f31ba87 --- /dev/null +++ b/evidence/stage1-pipeline.txt @@ -0,0 +1,2 @@ +PASS: HTTP 202 queued -> RabbitMQ/worker -> ready +PASS: original unchanged; thumbnail, crop, optimized downloaded with matching size/SHA-256 diff --git a/evidence/stage1-services.txt b/evidence/stage1-services.txt new file mode 100644 index 0000000..165263d --- /dev/null +++ b/evidence/stage1-services.txt @@ -0,0 +1,11 @@ +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +mtusi-lab6-api-a-1 mtusi-lab6-api:local "python -m messenger" api-a 27 seconds ago Up 23 seconds (healthy) 8080/tcp +mtusi-lab6-api-b-1 mtusi-lab6-api:local "python -m messenger" api-b 27 seconds ago Up 23 seconds (healthy) 8080/tcp +mtusi-lab6-attachment-cleaner-1 mtusi-lab6-api:local "python -m messenger…" attachment-cleaner 27 seconds ago Up 23 seconds 8080/tcp +mtusi-lab6-nginx-1 nginx:1.28.2-alpine@sha256:5b4900b042ccfa8b0a73df622c3a60f2322faeb2be800cbee5aa7b44d241649e "/bin/sh -ec 'cp /et…" nginx 26 seconds ago Up 17 seconds (healthy) 127.0.0.1:18083->8080/tcp +mtusi-lab6-outbox-publisher-1 mtusi-lab6-api:local "python -m messenger…" outbox-publisher 27 seconds ago Up 21 seconds 8080/tcp +mtusi-lab6-postgres-1 postgres:15-alpine@sha256:54e0dd8e8cddcdbb9156c2486eed5a43eccb7214d7c6828097d41ce049fb3984 "docker-entrypoint.s…" postgres 27 seconds ago Up 26 seconds (healthy) 5432/tcp +mtusi-lab6-rabbitmq-1 rabbitmq:4.1.4-alpine "/bin/sh -ec 'export…" rabbitmq 27 seconds ago Up 26 seconds (healthy) 4369/tcp, 5671-5672/tcp, 15691-15692/tcp, 25672/tcp +mtusi-lab6-redis-1 redis:7-alpine@sha256:8da19c7399dca900c6822a1cf30f9d2285585e2c4b457ff4095b0f4500054d3f "docker-entrypoint.s…" redis 27 seconds ago Up 26 seconds (healthy) 6379/tcp +mtusi-lab6-s3-1 mtusi-lab6-s3:2025-09-07 "/bin/sh -ec 'MINIO_…" s3 27 seconds ago Up 26 seconds +mtusi-lab6-worker-1 mtusi-lab6-api:local "python -m messenger…" worker 26 seconds ago Up 20 seconds 8080/tcp diff --git a/evidence/stage1-smoke.txt b/evidence/stage1-smoke.txt new file mode 100644 index 0000000..85f8a82 --- /dev/null +++ b/evidence/stage1-smoke.txt @@ -0,0 +1,2 @@ +python3 tests/smoke.py --base-url "http://localhost:18083" +PASS: lab6 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/telemetry-persistence.json b/evidence/telemetry-persistence.json new file mode 100644 index 0000000..740370e --- /dev/null +++ b/evidence/telemetry-persistence.json @@ -0,0 +1,13 @@ +{ + "trace_id": "9efec3dd76d0440a902d19838f8e4c5b", + "trace_spans_preserved": 16, + "api_worker_publisher_logs_preserved": true, + "historical_metric_preserved": true, + "restarted": [ + "otel-collector", + "prometheus", + "loki", + "tempo", + "grafana" + ] +} diff --git a/evidence/validate.txt b/evidence/validate.txt new file mode 100644 index 0000000..37e8188 --- /dev/null +++ b/evidence/validate.txt @@ -0,0 +1,2 @@ +.venv/bin/python scripts/validate_openapi.py +PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested. diff --git a/infra/Dockerfile.s3 b/infra/Dockerfile.s3 new file mode 100644 index 0000000..feccc38 --- /dev/null +++ b/infra/Dockerfile.s3 @@ -0,0 +1,12 @@ +FROM public.ecr.aws/docker/library/golang:1.24.2-alpine@sha256:7772cb5322baa875edd74705556d08f0eeca7b9c4b5367754ce3f2f00041ccee AS build +ENV CGO_ENABLED=0 GOPROXY="https://proxy.golang.org|https://goproxy.io|direct" +WORKDIR /src/minio +ADD https://codeload.github.com/minio/minio/tar.gz/refs/tags/RELEASE.2025-09-07T16-13-09Z /tmp/minio.tar.gz +RUN --mount=type=cache,target=/go/pkg/mod --mount=type=cache,target=/root/.cache/go-build tar -xzf /tmp/minio.tar.gz --strip-components=1 && go build -o /out/minio . +WORKDIR /src/mc +ADD https://codeload.github.com/minio/mc/tar.gz/refs/tags/RELEASE.2025-08-13T08-35-41Z /tmp/mc.tar.gz +RUN --mount=type=cache,target=/go/pkg/mod --mount=type=cache,target=/root/.cache/go-build tar -xzf /tmp/mc.tar.gz --strip-components=1 && go build -o /out/mc . + +FROM python:3.12-slim-bookworm@sha256:392307d22300de8b5986851a12d9176dfc0fc073e65bf6523ebd7dcbeb23564e +COPY --from=build /out/minio /out/mc /usr/local/bin/ +ENTRYPOINT ["minio"] diff --git a/infra/README.md b/infra/README.md index 5a39e86..5b5da5b 100644 --- a/infra/README.md +++ b/infra/README.md @@ -1,7 +1,22 @@ -# Инфраструктура студента +# Инфраструктура лабораторной 6 -Здесь разместите конфигурации выбранных сервисов; основную топологию — в корневом compose.yaml/stack.yaml. `compose.example.yaml` — только worksheet ролей и намеренно не запускается. `contracts/infrastructure.json` задаёт минимальные роли, не готовые образы/пароли. +Топология и ограничения ресурсов заданы в compose.yaml. Проект mtusi-lab6 использует собственные named volumes. API доступен через Nginx на localhost:18083, Grafana — localhost:13000. Секреты создаёт scripts/bootstrap.py. -В REPORT.md сопоставьте роль → service name → версия/digest → health/readiness → volume → сеть/порт → способ передачи credentials. Приведите bootstrap (schema, bucket, broker topology, а с lab6 provisioning) и безопасное повторение этих команд. +| Роль | Сервисы | Хранение / проверка | +| --- | --- | --- | +| API | api-a, api-b | PostgreSQL, readiness /health/ready | +| Балансировщик | nginx | healthwatch проверяет готовность двух реплик | +| База данных | postgres | pg_data, pg_isready | +| Кэш сессий | redis | redis_data, AOF, redis-cli ping | +| Объекты | s3, s3-bootstrap | s3_data, приватный бакет и отдельная учётная запись API | +| Очередь | rabbitmq | rabbitmq_data, durable queue, publisher confirms | +| Фоновая обработка | outbox-publisher, worker, attachment-cleaner | Job/outbox/lease в PostgreSQL | +| Сбор сигналов | otel-collector | OTLP 4317/4318, внутренний health endpoint 13133 | +| Метрики | prometheus | prometheus_data, /-/ready, локальные alert rules | +| Логи | loki | loki_data, /ready, OTLP/HTTP ingestion | +| Трассы | tempo | tempo_data, /ready, OTLP/gRPC ingestion | +| Dashboard | grafana | grafana_data, /api/health, provisioning в infra/grafana | -Приложение строится из исходников. Не используйте заранее существующий образ преподавательского сервера вместо своей реализации. Restart hook перезапускает все соответствующие stores с volumes; failover hook останавливает только заданную API-реплику. +Backend и telemetry используют внутренние сети. Nginx и HTTP-интерфейсы наблюдаемости также подключены к frontend для публикации loopback-портов. OTLP на хост не опубликован. Все версии закреплены в Compose/Dockerfile. Запуск, сигналы, ограничения и результаты приведены в REPORT.md. Действия при сбое описаны в runbook.md. + +scripts/restart.sh перезапускает бизнес-хранилища и приложения с существующими volumes. scripts/stop-one.sh останавливает API-реплику по TARGET_INSTANCE_ID. Историю наблюдений проверяет student-tests/telemetry_persistence.py. Сценарии student-tests/observability.py воспроизводят остановку worker, блокировку БД, retry после отказа S3 и недоступность Tempo. Каждый fault-сценарий выполняется отдельно и восстанавливает остановленные сервисы. diff --git a/infra/grafana/dashboards/messenger.json b/infra/grafana/dashboards/messenger.json new file mode 100644 index 0000000..24cdcf3 --- /dev/null +++ b/infra/grafana/dashboards/messenger.json @@ -0,0 +1,446 @@ +{ + "uid": "messenger", + "title": "Messenger", + "schemaVersion": 40, + "version": 1, + "refresh": "5s", + "time": { + "from": "now-15m", + "to": "now" + }, + "tags": [ + "lab6" + ], + "panels": [ + { + "id": 1, + "title": "API requests per second", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "reqps" + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "expr": "sum by (method,route)(rate(messenger_http_requests_total[1m]))", + "legendFormat": "{{method}} {{route}}" + } + ] + }, + { + "id": 2, + "title": "API latency p50 / p95 / p99", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "expr": "histogram_quantile(0.5, sum by (le)(rate(messenger_http_duration_seconds_bucket[1m])))", + "legendFormat": "p50" + }, + { + "refId": "B", + "expr": "histogram_quantile(0.95, sum by (le)(rate(messenger_http_duration_seconds_bucket[1m])))", + "legendFormat": "p95" + }, + { + "refId": "C", + "expr": "histogram_quantile(0.99, sum by (le)(rate(messenger_http_duration_seconds_bucket[1m])))", + "legendFormat": "p99" + } + ] + }, + { + "id": 3, + "title": "API 5xx ratio and 5-minute SLI", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit" + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "expr": "(sum(rate(messenger_http_requests_total{status_class=\"5xx\"}[1m])) or vector(0)) / clamp_min(sum(rate(messenger_http_requests_total[1m])),0.001)", + "legendFormat": "5xx ratio" + }, + { + "refId": "B", + "expr": "(1 - ((sum(increase(messenger_http_requests_total{status_class=\"5xx\"}[5m])) or vector(0)) / clamp_min(sum(increase(messenger_http_requests_total[5m])),1))) and (sum(increase(messenger_http_requests_total[5m])) > 0)", + "legendFormat": "SLI 5m" + } + ] + }, + { + "id": 4, + "title": "Unfinished jobs and RabbitMQ ready messages", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "expr": "max((messenger_queue_depth) and on(service_instance_id) (time() - messenger_queue_observed_at_seconds < 10))", + "legendFormat": "unfinished jobs" + }, + { + "refId": "B", + "expr": "max((messenger_queue_broker_ready) and on(service_instance_id) (time() - messenger_queue_observed_at_seconds < 10))", + "legendFormat": "broker ready" + } + ] + }, + { + "id": 5, + "title": "PostgreSQL pool use / capacity / acquisition", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "expr": "(messenger_db_pool_used) and on(service_instance_id) (time() - messenger_process_observed_at_seconds < 10)", + "legendFormat": "used {{service_name}} {{service_instance_id}}" + }, + { + "refId": "B", + "expr": "(messenger_db_pool_capacity) and on(service_instance_id) (time() - messenger_process_observed_at_seconds < 10)", + "legendFormat": "capacity {{service_name}} {{service_instance_id}}" + }, + { + "refId": "C", + "expr": "(messenger_db_pool_waiting) and on(service_instance_id) (time() - messenger_process_observed_at_seconds < 10)", + "legendFormat": "acquiring {{service_name}} {{service_instance_id}}" + } + ] + }, + { + "id": 6, + "title": "Oldest unfinished job age", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "expr": "max((messenger_queue_oldest_age_seconds) and on(service_instance_id) (time() - messenger_queue_observed_at_seconds < 10))", + "legendFormat": "age" + } + ] + }, + { + "id": 7, + "title": "Worker busy / capacity", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "expr": "sum((messenger_worker_busy{service_name=\"messenger-worker\"}) and on(service_instance_id) (time() - messenger_process_observed_at_seconds < 10)) or vector(0)", + "legendFormat": "busy" + }, + { + "refId": "B", + "expr": "sum((messenger_worker_capacity{service_name=\"messenger-worker\"}) and on(service_instance_id) (time() - messenger_process_observed_at_seconds < 10)) or vector(0)", + "legendFormat": "capacity" + } + ] + }, + { + "id": 8, + "title": "Worker attempts by outcome", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "ops" + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "expr": "sum by (outcome)(rate(messenger_jobs_processed_total[1m]))", + "legendFormat": "{{outcome}}" + } + ] + }, + { + "id": 9, + "title": "Worker processing p95 and accepted-to-ready p95", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 32 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "expr": "histogram_quantile(0.95,sum by(le)(rate(messenger_job_duration_seconds_bucket[1m])))", + "legendFormat": "processing p95" + }, + { + "refId": "B", + "expr": "histogram_quantile(0.95,sum by(le)(rate(messenger_attachment_time_to_ready_seconds_bucket[1m])))", + "legendFormat": "time to ready p95" + } + ] + }, + { + "id": 10, + "title": "Process resident memory", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 32 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "bytes" + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "expr": "(messenger_process_memory_bytes) and on(service_instance_id) (time() - messenger_process_observed_at_seconds < 10)", + "legendFormat": "{{service_name}} {{service_instance_id}}" + } + ] + }, + { + "id": 11, + "title": "Process CPU cores", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 40 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "expr": "(rate(messenger_process_cpu_seconds_total[1m])) and on(service_instance_id) (time() - messenger_process_observed_at_seconds < 10)", + "legendFormat": "{{service_name}} {{service_instance_id}}" + } + ] + }, + { + "id": 12, + "title": "Collector exporter failures / lost records", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 40 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "expr": "{__name__=~\"otelcol_exporter_(send_failed|enqueue_failed)_(spans|log_records|metric_points).*\"}", + "legendFormat": "{{__name__}} {{exporter}}" + } + ] + }, + { + "id": 13, + "title": "Application logs with trace IDs", + "type": "logs", + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 48 + }, + "datasource": { + "type": "loki", + "uid": "loki" + }, + "targets": [ + { + "refId": "A", + "expr": "{service_name=~\"messenger-.*\"}" + } + ], + "options": { + "showTime": true, + "wrapLogMessage": true + } + }, + { + "id": 14, + "title": "Recent application traces", + "type": "table", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 57 + }, + "datasource": { + "type": "tempo", + "uid": "tempo" + }, + "targets": [ + { + "refId": "A", + "queryType": "traceql", + "query": "{resource.service.name =~ \"messenger-.*\"}", + "tableType": "traces" + } + ] + } + ], + "annotations": { + "list": [] + }, + "templating": { + "list": [] + } +} diff --git a/infra/grafana/provisioning/dashboards/dashboards.yaml b/infra/grafana/provisioning/dashboards/dashboards.yaml new file mode 100644 index 0000000..b0237a0 --- /dev/null +++ b/infra/grafana/provisioning/dashboards/dashboards.yaml @@ -0,0 +1,10 @@ +apiVersion: 1 +providers: + - name: messenger + orgId: 1 + folder: Messenger + type: file + disableDeletion: true + updateIntervalSeconds: 10 + options: + path: /var/lib/grafana/dashboards diff --git a/infra/grafana/provisioning/datasources/datasources.yaml b/infra/grafana/provisioning/datasources/datasources.yaml new file mode 100644 index 0000000..2fc6f2b --- /dev/null +++ b/infra/grafana/provisioning/datasources/datasources.yaml @@ -0,0 +1,33 @@ +apiVersion: 1 +datasources: + - name: Prometheus + uid: prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + jsonData: + timeInterval: 3s + - name: Loki + uid: loki + type: loki + access: proxy + url: http://loki:3100 + jsonData: + derivedFields: + - name: TraceID + datasourceUid: tempo + matcherRegex: '"trace_id":\s*"([0-9a-f]{32})"' + url: '$${__value.raw}' + - name: Tempo + uid: tempo + type: tempo + access: proxy + url: http://tempo:3200 + jsonData: + tracesToLogsV2: + datasourceUid: loki + spanStartTimeShift: '-1m' + spanEndTimeShift: '1m' + tags: [{key: service.name, value: service_name}] + filterByTraceID: true diff --git a/infra/healthwatch.sh b/infra/healthwatch.sh new file mode 100755 index 0000000..7c048eb --- /dev/null +++ b/infra/healthwatch.sh @@ -0,0 +1,53 @@ +#!/bin/sh +# Active readiness bridge for open-source Nginx. No Docker socket or credentials. +set -eu +probe() { + if wget -T 1 -q -O /dev/null "http://$1:8080/health/ready" 2>/dev/null; then + echo up > "/tmp/$1.health" + else + echo down > "/tmp/$1.health" + fi +} +control() { cat "/tmp/control-$1" 2>/dev/null || echo serve:boot; } +echo $$ > /tmp/healthwatch.pid +while :; do + sleep 1 + probe api-a & + first=$! + probe api-b & + second=$! + wait "$first" + wait "$second" + # Read desired state AFTER probes. A generation is acknowledged only after + # the matching upstream configuration has been applied, never from a stale probe. + a_control=$(control api-a) + b_control=$(control api-b) + a_state=$(cat /tmp/api-a.health) + b_state=$(cat /tmp/api-b.health) + case "$a_control" in drain:*) a_state=down;; esac + case "$b_control" in drain:*) b_state=down;; esac + : > /tmp/upstreams.next + for service in api-a api-b; do + if [ "$service" = api-a ]; then state=$a_state; else state=$b_state; fi + suffix='' + [ "$state" = up ] || suffix=' down' + printf 'server %s:8080 resolve max_fails=1 fail_timeout=2s%s;\n' "$service" "$suffix" >> /tmp/upstreams.next + done + if ! cmp -s /tmp/upstreams.next /etc/nginx/upstreams.conf; then + cp /etc/nginx/upstreams.conf /tmp/upstreams.previous + # Rename on the same filesystem: readers never see missing/partial config. + cp /tmp/upstreams.next /etc/nginx/upstreams.next + mv -f /etc/nginx/upstreams.next /etc/nginx/upstreams.conf + if nginx -t && nginx -s reload; then + echo 'readiness upstream set updated' + else + cp /tmp/upstreams.previous /etc/nginx/upstreams.next + mv -f /etc/nginx/upstreams.next /etc/nginx/upstreams.conf + exit 1 + fi + fi + printf '%s %s\n' "$a_control" "$a_state" > /tmp/applied-api-a.next + mv -f /tmp/applied-api-a.next /tmp/applied-api-a + printf '%s %s\n' "$b_control" "$b_state" > /tmp/applied-api-b.next + mv -f /tmp/applied-api-b.next /tmp/applied-api-b +done diff --git a/infra/loki/loki.yaml b/infra/loki/loki.yaml new file mode 100644 index 0000000..3b14979 --- /dev/null +++ b/infra/loki/loki.yaml @@ -0,0 +1,31 @@ +auth_enabled: false +server: + http_listen_port: 3100 +common: + path_prefix: /loki + replication_factor: 1 + ring: + kvstore: + store: inmemory + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules +schema_config: + configs: + - from: 2024-01-01 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h +limits_config: + allow_structured_metadata: true + retention_period: 72h +compactor: + working_directory: /loki/compactor + retention_enabled: true + delete_request_store: filesystem +analytics: + reporting_enabled: false diff --git a/infra/nginx.conf b/infra/nginx.conf new file mode 100644 index 0000000..d785057 --- /dev/null +++ b/infra/nginx.conf @@ -0,0 +1,49 @@ +worker_processes 1; +worker_shutdown_timeout 6s; +events { worker_connections 1024; } +http { + resolver 127.0.0.11 valid=1s ipv6=off; + resolver_timeout 1s; + log_format lab '$time_iso8601 $request_method $uri $status upstream=$upstream_addr ' + 'instance=$upstream_http_x_instance_id time=$request_time'; + access_log /dev/stdout lab; + error_log /dev/stderr warn; + upstream api { + zone api 64k; + include /etc/nginx/upstreams.conf; + } + server { + listen 8080; + client_max_body_size 1m; + client_body_timeout 5s; + proxy_http_version 1.1; + proxy_set_header Connection close; + proxy_set_header Host $host; + proxy_set_header X-Request-Id $http_x_request_id; + proxy_connect_timeout 1s; + proxy_read_timeout 5s; + proxy_send_timeout 5s; + proxy_next_upstream error timeout http_502 http_504; + proxy_next_upstream_tries 2; + proxy_next_upstream_timeout 6s; + + location ~ ^/api/v1/chats/[^/]+/attachments$ { + # Keep a proxy ceiling above the API limit so the API can return its JSON 413. + client_max_body_size 11m; + proxy_read_timeout 45s; + proxy_send_timeout 30s; + proxy_next_upstream off; + proxy_pass http://api; + } + location ~ ^/api/v1/attachments/[^/]+/content$ { + proxy_buffering off; + proxy_read_timeout 10s; + # A shared S3 outage returns 503 without marking the API itself down. + proxy_next_upstream error timeout http_502 http_504; + proxy_pass http://api; + } + location / { + proxy_pass http://api; + } + } +} diff --git a/infra/otel/README.md b/infra/otel/README.md index c46fe89..4b90390 100644 --- a/infra/otel/README.md +++ b/infra/otel/README.md @@ -1,7 +1,11 @@ -# Конфигурация наблюдаемости — реализовать +# OpenTelemetry -Добавьте `collector.yaml` с OTLP receivers, processors и exporters выбранной версии Collector. Укажите явные bind endpoints, внутренние сети, memory limits/batch и поведение при backpressure по уровню. Сопоставьте все три pipeline с реально развёрнутыми backend. +API, publisher и worker используют SDK 1.38.0 и OTLP/gRPC на otel-collector:4317. HTTP receiver 4318 также доступен внутри сети. Ресурсы содержат service.name, service.version, service.instance.id и deployment.environment.name=lab. Используются ручные spans и собственные имена метрик, соответствие приведено в REPORT.md. -Добавьте конфигурации storage, Grafana provisioning/datasources, provisioning/dashboards и импортируемые dashboard JSON. На 4 — alert rules и runbooks. Пустой dashboard или debug exporter, который только печатает данные в stdout, не заменяют хранилища/доказательства из `contracts/observability.md`. +Collector contrib 0.123.0 применяет memory_limiter и batch ко всем трём сигналам. Метрики доступны Prometheus на порту 8889, внутренние метрики Collector — на 8888. Логи отправляются OTLP/HTTP на Loki /otlp, трассы OTLP/gRPC в Tempo. Для Loki включены structured metadata и schema v13. -Не публикуйте OTLP/metrics/административные ports в общую сеть без необходимости. В lab7 реализуйте TLS/mTLS по trust matrix; экспорт Authorization и message bodies запрещён. +Очереди SDK ограничены 256 spans и logs на процесс. Collector имеет по 128 batches для Loki и Tempo, до 512 записей в batch. Retry ограничен 10 секундами. После заполнения/исчерпания retry возможны потери телеметрии, бизнес-транзакции не ждут успешного экспорта. Контейнер Collector ограничен 192 MiB, memory_limiter — 128 MiB. Критерии проверяются сценарием student-tests/observability.py backend-outage. + +Grafana, Prometheus, Loki и Tempo имеют собственные volumes и retention 72 часа для сигналов. Административные HTTP-порты опубликованы только на loopback. Collector не публикует OTLP на хост. + +Первичные материалы — https://opentelemetry.io/docs/collector/configuration/ и https://grafana.com/docs/loki/latest/send-data/otel/. diff --git a/infra/otel/collector.yaml b/infra/otel/collector.yaml new file mode 100644 index 0000000..5c067cd --- /dev/null +++ b/infra/otel/collector.yaml @@ -0,0 +1,73 @@ +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 +processors: + memory_limiter: + check_interval: 1s + limit_mib: 128 + spike_limit_mib: 32 + batch: + timeout: 1s + send_batch_size: 256 + send_batch_max_size: 512 +exporters: + prometheus: + endpoint: 0.0.0.0:8889 + resource_to_telemetry_conversion: + enabled: true + otlphttp/loki: + endpoint: http://loki:3100/otlp + timeout: 2s + sending_queue: + enabled: true + num_consumers: 2 + queue_size: 128 + retry_on_failure: + enabled: true + initial_interval: 1s + max_interval: 3s + max_elapsed_time: 10s + otlp/tempo: + endpoint: tempo:4317 + tls: + insecure: true + timeout: 2s + sending_queue: + enabled: true + num_consumers: 2 + queue_size: 128 + retry_on_failure: + enabled: true + initial_interval: 1s + max_interval: 3s + max_elapsed_time: 10s +extensions: + health_check: + endpoint: 0.0.0.0:13133 +service: + extensions: [health_check] + telemetry: + metrics: + readers: + - pull: + exporter: + prometheus: + host: 0.0.0.0 + port: 8888 + pipelines: + traces: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlp/tempo] + metrics: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [prometheus] + logs: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlphttp/loki] diff --git a/infra/prometheus/alerts.yaml b/infra/prometheus/alerts.yaml new file mode 100644 index 0000000..edaecac --- /dev/null +++ b/infra/prometheus/alerts.yaml @@ -0,0 +1,28 @@ +groups: + - name: messenger + rules: + - alert: HighApiErrorRate + expr: sum(rate(messenger_http_requests_total{status_class="5xx"}[1m])) / clamp_min(sum(rate(messenger_http_requests_total[1m])), 0.001) > 0.01 + for: 6s + labels: + severity: warning + annotations: + summary: API 5xx exceeds 1% over 1 minute + description: SLO is 99% non-5xx business requests over 5 minutes. Inspect PostgreSQL client spans and pool metrics. + runbook_url: http://localhost:13000/d/messenger/messenger?viewPanel=5 + - alert: QueueLagGrowing + expr: messenger_queue_oldest_age_seconds > 15 and messenger_queue_depth > 0 and on(service_instance_id) (time() - messenger_queue_observed_at_seconds < 10) + for: 6s + labels: + severity: warning + annotations: + summary: Oldest unfinished job exceeds 15 seconds + description: Inspect worker capacity, busy gauge and consumer traces. Restore worker and verify backlog drains. See infra/runbook.md. + - alert: QueueObservationStale + expr: time() - max(messenger_queue_observed_at_seconds) > 15 + for: 6s + labels: + severity: warning + annotations: + summary: Publisher backlog observations are stale + description: Check publisher and PostgreSQL availability before trusting queue gauges. diff --git a/infra/prometheus/prometheus.yaml b/infra/prometheus/prometheus.yaml new file mode 100644 index 0000000..61e4593 --- /dev/null +++ b/infra/prometheus/prometheus.yaml @@ -0,0 +1,11 @@ +global: + scrape_interval: 3s + evaluation_interval: 3s +rule_files: [/etc/prometheus/alerts.yaml] +scrape_configs: + - job_name: application + static_configs: + - targets: [otel-collector:8889] + - job_name: collector + static_configs: + - targets: [otel-collector:8888] diff --git a/infra/runbook.md b/infra/runbook.md new file mode 100644 index 0000000..2b5542d --- /dev/null +++ b/infra/runbook.md @@ -0,0 +1,30 @@ +# Диагностика лабораторной 6 + +Grafana — http://localhost:13000, dashboard Messenger. Prometheus — http://localhost:19090/alerts. Учётная запись Grafana admin, пароль в локальном secrets/grafana_password. + +## Ошибки API + +1. Посмотреть RPS, долю 5xx и p95/p99 за интервал алерта HighApiErrorRate. +2. В Tempo найти server span с ошибкой, раскрыть postgresql.transaction и s3 spans. HTTP server span включает шаблон маршрута и статус. SQL-параметры и содержимое объектов не экспортируются. +3. В Loki найти логи по service_name и trace_id. X-Request-Id из ответа помогает найти HTTP-событие. +4. При ошибках БД проверить здоровье postgres и число занятых соединений. getconn в psycopg2 fail-fast, waiting показывает только текущие попытки получения соединения. Exhausted и histogram acquire дополняют этот gauge. +5. Восстановить postgres. Проверить успешные запросы и исчезновение firing после выхода ошибки из минутного окна. + +## Рост возраста очереди + +1. Сопоставить messenger_queue_depth, messenger_queue_oldest_age_seconds, broker_ready и worker busy/capacity. +2. Backlog включает queued/processing в PostgreSQL, в том числе outbox и отложенные retry. Broker ready показывает только ожидающие доставки RabbitMQ сообщения. Processing duration измеряет одну попытку. Time to ready включает ожидание и retry. +3. Если нет свежих worker метрик, проверить worker. Gauge после остановки процесса временно сохраняется в exporter, время последнего получения данных проверять отдельно. +4. Найти trace upload по X-Trace-Id, затем producer, consumer и операции S3/DB. Попытки имеют job.attempt в traces, без job_id в metric labels. +5. Запустить worker. Убедиться, что backlog и oldest age равны нулю, алерт resolved. При QueueObservationStale проверить publisher/БД. + +## Хранилище телеметрии недоступно + +1. Посмотреть собственные метрики Collector, exporter send_failed и enqueue_failed, а также docker stats. +2. SDK ограничены 256 spans/logs на процесс, batch 64 и timeout 1 секунда. Collector ограничен memory_limiter 128 MiB, контейнером 192 MiB, batch максимум 512 записей и экспортными очередями по 128 batches. При заполнении/исчерпании retry телеметрия может теряться. Prometheus экспортируется pull endpoint, его остановка оставляет пробел в scrape-истории. +3. Восстановить Loki/Tempo, убедиться, что свежий trace и лог снова доступны. Подтвердить, что принятые сообщения и вложения доступны через API. +4. Разница accepted/sent и exporter failure counters помогает оценивать проблему. send_failed учитывает попытки отправки, поэтому не является точным числом уникальных потерянных spans. Без sequence IDs точные потери не заявляются. + +## SLI/SLO + +SLI = 1 − число 5xx / число всех бизнес-запросов за скользящие 5 минут. Health исключён при записи HTTP-метрик. SLO ≥ 99%. При отсутствии запросов окно не даёт оценки доступности. 4xx остаются в знаменателе и не считаются отказами сервера. Алерт ошибок использует более короткое окно 1 минута, порог 1%, for 6 секунд. QueueLagGrowing — возраст старейшего незавершённого задания >15 секунд, backlog >0, for 6 секунд. Это учебные пороги для воспроизводимой проверки. diff --git a/infra/s3-bootstrap.sh b/infra/s3-bootstrap.sh new file mode 100644 index 0000000..cce888a --- /dev/null +++ b/infra/s3-bootstrap.sh @@ -0,0 +1,73 @@ +#!/bin/sh +set -eu +umask 077 + +bucket=${S3_BUCKET:-messenger-attachments} +case "$bucket" in + ''|*[!a-z0-9.-]*) echo 'Invalid S3 bucket name' >&2; exit 1 ;; +esac + +config=$(mktemp -d) +trap 'rm -rf "$config"' EXIT +export MC_CONFIG_DIR="$config" +root_user=$(cat /run/secrets/s3_root_user) +root_password=$(cat /run/secrets/s3_root_password) +api_user=$(cat /run/secrets/s3_access_key_id) +api_password=$(cat /run/secrets/s3_secret_access_key) +endpoint=${S3_ENDPOINT:-http://s3:9000} + +if [ "${1:-}" = verify ]; then + export MC_HOST_api="http://$api_user:$api_password@${endpoint#http://}" + key="attachments/probe-${config##*/}" + object="api/$bucket/$key" + trap 'mc rm "$object" >/dev/null 2>&1 || true; rm -rf "$config"' EXIT + printf '%s' 'lab6-private-bucket-probe' | mc pipe "$object" >/dev/null + test "$(mc cat "$object")" = 'lab6-private-bucket-probe' + python3 - "$endpoint/$bucket/$key" <<'PYTEST' +import sys +from urllib.error import HTTPError +from urllib.request import urlopen + +try: + with urlopen(sys.argv[1], timeout=5): + raise SystemExit("FAIL: anonymous GET succeeded") +except HTTPError as exc: + if exc.code != 403: + raise +print("PASS: authenticated round trip; anonymous GET returned 403") +PYTEST + mc rm "$object" >/dev/null + exit 0 +fi + +attempt=0 +until mc alias set storage "$endpoint" "$root_user" "$root_password" >/dev/null 2>&1 && + mc admin info storage >/dev/null 2>&1; do + attempt=$((attempt + 1)) + if [ "$attempt" -ge 30 ]; then + echo 'S3 bootstrap could not connect or authenticate' >&2 + exit 1 + fi + sleep 1 +done + +mc mb --ignore-existing "storage/$bucket" +mc anonymous set none "storage/$bucket" +cat > "$config/api-policy.json" </dev/null +mc admin policy attach storage lab6-attachments --user "$api_user" +echo 'PASS: private S3 bucket and scoped API policy configured' diff --git a/infra/tempo/tempo.yaml b/infra/tempo/tempo.yaml new file mode 100644 index 0000000..bf255a3 --- /dev/null +++ b/infra/tempo/tempo.yaml @@ -0,0 +1,22 @@ +server: + http_listen_port: 3200 +distributor: + receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 +storage: + trace: + backend: local + wal: + path: /var/tempo/wal + local: + path: /var/tempo/blocks +compactor: + compaction: + block_retention: 72h +usage_report: + reporting_enabled: false diff --git a/infra/upstreams.conf b/infra/upstreams.conf new file mode 100644 index 0000000..ad9aa43 --- /dev/null +++ b/infra/upstreams.conf @@ -0,0 +1,2 @@ +server api-a:8080 resolve max_fails=1 fail_timeout=2s; +server api-b:8080 resolve max_fails=1 fail_timeout=2s; diff --git a/migrations/0001_init.sql b/migrations/0001_init.sql new file mode 100644 index 0000000..2d9ef8e --- /dev/null +++ b/migrations/0001_init.sql @@ -0,0 +1,51 @@ +-- Migration 0001: initial schema for lab2. +-- Idempotent: safe to re-run on an existing database. + +CREATE TABLE IF NOT EXISTS users ( + id uuid PRIMARY KEY, + username text NOT NULL UNIQUE, + password_hash text NOT NULL, + display_name text NOT NULL, + created_at timestamptz NOT NULL DEFAULT now() +); + +CREATE TABLE IF NOT EXISTS chats ( + id uuid PRIMARY KEY, + title text NOT NULL, + created_at timestamptz NOT NULL DEFAULT now() +); + +CREATE TABLE IF NOT EXISTS chat_members ( + chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE, + PRIMARY KEY (chat_id, user_id) +); + +CREATE TABLE IF NOT EXISTS messages ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + sender_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE, + text text NOT NULL, + created_at timestamptz NOT NULL DEFAULT now() +); + +-- Keyset pagination index: history by (chat_id, created_at, id) avoids +-- full table scan on every page read. +CREATE INDEX IF NOT EXISTS idx_messages_chat_created_id + ON messages (chat_id, created_at, id); + +CREATE INDEX IF NOT EXISTS idx_messages_chat_id + ON messages (chat_id); + +CREATE TABLE IF NOT EXISTS sessions ( + token_hash text PRIMARY KEY, + user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE, + expires_at timestamptz NOT NULL, + created_at timestamptz NOT NULL DEFAULT now() +); + +CREATE INDEX IF NOT EXISTS idx_sessions_user_id + ON sessions (user_id); + +CREATE INDEX IF NOT EXISTS idx_sessions_expires_at + ON sessions (expires_at); diff --git a/migrations/0002_idempotency.sql b/migrations/0002_idempotency.sql new file mode 100644 index 0000000..394cb57 --- /dev/null +++ b/migrations/0002_idempotency.sql @@ -0,0 +1,14 @@ +-- Additive migration: compatible with the lab2 schema and existing messages. +CREATE TABLE IF NOT EXISTS message_idempotency ( + user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE, + method text NOT NULL, + path text NOT NULL, + key text NOT NULL, + request_hash text NOT NULL, + response_status integer NOT NULL, + response_body jsonb NOT NULL, + created_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (user_id, method, path, key) +); +-- Results are retained indefinitely in this lab (therefore at least 24 hours). +-- There is intentionally no independent cache expiry that could allow duplicates. diff --git a/migrations/0003_attachments.sql b/migrations/0003_attachments.sql new file mode 100644 index 0000000..83061da --- /dev/null +++ b/migrations/0003_attachments.sql @@ -0,0 +1,37 @@ +-- Additive migration. Existing text messages have no attachment links. +ALTER TABLE messages + ADD CONSTRAINT messages_id_chat_id_key UNIQUE (id, chat_id); + +CREATE TABLE attachments ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + owner_id uuid NOT NULL REFERENCES users(id), + filename text NOT NULL CHECK (char_length(filename) BETWEEN 1 AND 255), + content_type text NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg')), + size_bytes bigint NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760), + sha256 text NOT NULL CHECK (sha256 ~ '^[0-9a-f]{64}$'), + object_key text NOT NULL UNIQUE CHECK (object_key LIKE 'attachments/%'), + -- pending is internal. The upload API exposes ready after committing metadata. + status text NOT NULL CHECK (status IN ('pending', 'ready', 'failed')), + error_code text, + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + UNIQUE (id, chat_id), + CHECK ((status = 'failed') = (error_code IS NOT NULL)) +); + +CREATE INDEX attachments_pending_updated_idx + ON attachments (updated_at, id) WHERE status = 'pending'; + +CREATE TABLE message_attachments ( + message_id uuid NOT NULL, + attachment_id uuid NOT NULL, + chat_id uuid NOT NULL, + position smallint NOT NULL CHECK (position BETWEEN 0 AND 9), + PRIMARY KEY (message_id, position), + UNIQUE (message_id, attachment_id), + FOREIGN KEY (message_id, chat_id) REFERENCES messages(id, chat_id) ON DELETE CASCADE, + FOREIGN KEY (attachment_id, chat_id) REFERENCES attachments(id, chat_id) +); + +CREATE INDEX message_attachments_attachment_idx ON message_attachments (attachment_id); diff --git a/migrations/0004_attachment_jobs.sql b/migrations/0004_attachment_jobs.sql new file mode 100644 index 0000000..78307d5 --- /dev/null +++ b/migrations/0004_attachment_jobs.sql @@ -0,0 +1,33 @@ +ALTER TABLE attachments DROP CONSTRAINT attachments_status_check; +ALTER TABLE attachments ADD CHECK (status IN ('pending','queued','processing','ready','failed')); + +CREATE TABLE attachment_jobs ( + id uuid PRIMARY KEY, + attachment_id uuid NOT NULL REFERENCES attachments(id) ON DELETE CASCADE, + pipeline_version integer NOT NULL CHECK (pipeline_version > 0), + attempt integer NOT NULL DEFAULT 1 CHECK (attempt > 0), + created_at timestamptz NOT NULL DEFAULT now(), + UNIQUE (attachment_id, pipeline_version) +); + +CREATE TABLE attachment_outbox ( + job_id uuid NOT NULL REFERENCES attachment_jobs(id) ON DELETE CASCADE, + attempt integer NOT NULL CHECK (attempt > 0), + available_at timestamptz NOT NULL DEFAULT now(), + published_at timestamptz, + PRIMARY KEY (job_id, attempt) +); +CREATE INDEX attachment_outbox_due_idx ON attachment_outbox (available_at) WHERE published_at IS NULL; + +CREATE TABLE attachment_variants ( + attachment_id uuid NOT NULL REFERENCES attachments(id) ON DELETE CASCADE, + pipeline_version integer NOT NULL CHECK (pipeline_version > 0), + variant text NOT NULL CHECK (variant IN ('thumbnail','crop','optimized')), + object_key text NOT NULL UNIQUE, + content_type text NOT NULL CHECK (content_type IN ('image/jpeg','image/webp')), + size_bytes bigint NOT NULL CHECK (size_bytes > 0), + sha256 text NOT NULL CHECK (sha256 ~ '^[0-9a-f]{64}$'), + width integer NOT NULL CHECK (width > 0), + height integer NOT NULL CHECK (height > 0), + PRIMARY KEY (attachment_id, pipeline_version, variant) +); diff --git a/migrations/0005_job_leases.sql b/migrations/0005_job_leases.sql new file mode 100644 index 0000000..14110ee --- /dev/null +++ b/migrations/0005_job_leases.sql @@ -0,0 +1,12 @@ +ALTER TABLE attachment_jobs ADD COLUMN lease_until timestamptz; +ALTER TABLE attachment_jobs ADD COLUMN attempts_used integer NOT NULL DEFAULT 0; +ALTER TABLE attachment_jobs ADD COLUMN staging_cleaned_at timestamptz; +ALTER TABLE attachment_outbox ADD COLUMN objects_cleaned_at timestamptz; +CREATE INDEX attachment_jobs_lease_idx ON attachment_jobs (lease_until) WHERE lease_until IS NOT NULL; +CREATE TABLE attachment_replays ( + job_id uuid NOT NULL REFERENCES attachment_jobs(id) ON DELETE CASCADE, + attempt integer NOT NULL, + operator text NOT NULL, + created_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (job_id, attempt) +); diff --git a/migrations/0006_observability.sql b/migrations/0006_observability.sql new file mode 100644 index 0000000..abeeeef --- /dev/null +++ b/migrations/0006_observability.sql @@ -0,0 +1,2 @@ +-- Preserve causal context across outbox retries and publisher restarts. +ALTER TABLE attachment_jobs ADD COLUMN trace_context jsonb NOT NULL DEFAULT '{}'::jsonb; diff --git a/scripts/bootstrap.py b/scripts/bootstrap.py new file mode 100644 index 0000000..b3e154e --- /dev/null +++ b/scripts/bootstrap.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python3 +"""Generate local credentials once, never print them or overwrite existing files.""" +from pathlib import Path +import os +import secrets + +root = Path(__file__).resolve().parents[1] +folder = root / 'secrets' +folder.mkdir(mode=0o700, exist_ok=True) +password_file = folder / 'postgres_password' + +def save(name, value): + path = folder / name + if path.exists(): + return + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(fd, 'w') as handle: + handle.write(value + '\n') + +save('postgres_password', secrets.token_hex(24)) +password = password_file.read_text().strip() +save('database_url', 'postgresql://messenger:%s@postgres:5432/messenger' % password) +save('cache_url', 'redis://redis:6379/0') +save('cursor_secret', secrets.token_hex(32)) +save('s3_root_user', 'lab6-admin-' + secrets.token_hex(8)) +save('s3_root_password', secrets.token_hex(32)) +save('s3_access_key_id', 'lab6-api-' + secrets.token_hex(8)) +save('s3_secret_access_key', secrets.token_hex(24)) +save('rabbitmq_password', secrets.token_hex(24)) +save('grafana_password', secrets.token_hex(24)) +print('PASS: local secret files ready (existing values preserved)') diff --git a/scripts/manage.py b/scripts/manage.py new file mode 100644 index 0000000..4c15f23 --- /dev/null +++ b/scripts/manage.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 +"""Explicit, bounded operator actions for this lab's Compose project only.""" +import argparse +import json +import os +from pathlib import Path +import subprocess +import time +import uuid + +ROOT = Path(__file__).resolve().parents[1] +SERVICES = ('api-a', 'api-b') + + +def command(args, capture=False): + result = subprocess.run(args, cwd=ROOT, check=True, text=True, + stdout=subprocess.PIPE if capture else None) + return result.stdout.strip() if capture else None + + +def compose(*args, capture=False): + return command(['docker', 'compose', '--env-file', '.env', *args], capture) + + +def inspect(container): + return json.loads(command(['docker', 'inspect', container], True))[0] + + +def containers(service): + return compose('ps', '-a', '-q', service, capture=True).splitlines() + + +def start_services(*services): + for service in services: + for container in containers(service): + command(['docker', 'start', container]) + + +def wait_healthy(service, timeout=90): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + ids = containers(service) + if ids and all(inspect(c)['State'].get('Health', {}).get('Status') == 'healthy' for c in ids): + return + time.sleep(.5) + raise RuntimeError('%s did not become healthy' % service) + + +def target_service(target): + if not target: + raise RuntimeError('TARGET_INSTANCE_ID is required') + for service in SERVICES: + for container in containers(service): + data = inspect(container) + if data['Config']['Hostname'] == target: + return service, container + raise RuntimeError('Unknown API instance alias') + + +def set_backend(service, enabled): + # Fixed allowlist; shell receives no untrusted instance or credential input. + if service not in SERVICES: + raise ValueError('Unknown service') + token = ('serve:' if enabled else 'drain:') + uuid.uuid4().hex + path = '/tmp/control-' + service + compose('exec', '-T', 'nginx', 'sh', '-ec', + "printf '%s\\n' '" + token + "' > " + path + '.next; mv -f ' + path + '.next ' + path) + deadline = time.monotonic() + 10 + expected = token + (' up' if enabled else ' down') + while time.monotonic() < deadline: + ack = compose('exec', '-T', 'nginx', 'sh', '-c', + 'cat /tmp/applied-' + service + ' 2>/dev/null || true', capture=True) + if ack == expected: + return + time.sleep(.2) + raise RuntimeError('Readiness controller did not acknowledge backend state for ' + service) + + +def stop_one(): + service, container = target_service(os.environ.get('TARGET_INSTANCE_ID')) + running = [c for name in SERVICES for c in containers(name) + if inspect(c)['State'].get('Health', {}).get('Status') == 'healthy' + and inspect(c)['State']['Running']] + if len(running) < 2: + raise RuntimeError('Restore the other replica before this experiment') + print('Stopping API replica:', service, container[:12], flush=True) + command(['docker', 'stop', '--time', '8', container]) + if inspect(container)['State']['Running']: + raise RuntimeError('Target is still running') + + +def restart(): + # Never start the one-shot migration service as part of persistence testing. + compose('restart', 'postgres', 'redis', 's3', 'rabbitmq', *SERVICES, 'nginx', 'outbox-publisher', 'worker') + for service in (*SERVICES, 'nginx'): + wait_healthy(service) + + +def rolling_update(): + # Run additive migrations once before replacing either API version. + compose('run', '--rm', '--no-deps', 'migrate') + for service in SERVICES: + other = SERVICES[1] if service == SERVICES[0] else SERVICES[0] + wait_healthy(other) + set_backend(other, True) + before = containers(service) + set_backend(service, False) + # The old nginx worker can retain the old upstream table until it exits. + # worker_shutdown_timeout=6s bounds that drain; no new work then reaches it. + time.sleep(6.5) + compose('up', '-d', '--no-deps', '--force-recreate', '--no-build', service) + wait_healthy(service) + set_backend(service, True) + time.sleep(2) # DNS valid=1s + readiness before moving to the next replica. + after = containers(service) + if before == after: + raise RuntimeError('Replica was not replaced') + print(json.dumps({'service': service, 'before': before, 'after': after}), flush=True) + + +def restore(): + start_services(*SERVICES) + for service in SERVICES: + wait_healthy(service) + for service in SERVICES: + set_backend(service, True) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('action', choices=('stop-one', 'restart', 'restore', 'rolling-update', 'status')) + args = parser.parse_args() + if args.action == 'stop-one': + stop_one() + elif args.action == 'restart': + restart() + elif args.action == 'rolling-update': + rolling_update() + elif args.action == 'restore': + restore() + else: + for service in ('postgres', 'redis', *SERVICES, 'nginx'): + for container in containers(service): + data = inspect(container) + print(json.dumps({'service': service, 'id': data['Id'][:12], + 'instance': data['Config']['Hostname'], + 'running': data['State']['Running'], + 'started_at': data['State']['StartedAt'], + 'health': data['State'].get('Health', {}).get('Status'), + 'image_id': data['Image'], + 'cpu_limit': data['HostConfig']['NanoCpus'] / 1e9, + 'memory_limit_bytes': data['HostConfig']['Memory'], + 'published_ports': data['HostConfig'].get('PortBindings')})) + + +if __name__ == '__main__': + main() diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 0000000..d5f557b --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu +cd "$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)" +exec python3 scripts/manage.py restart diff --git a/scripts/rolling-update.sh b/scripts/rolling-update.sh new file mode 100755 index 0000000..ca0c4bb --- /dev/null +++ b/scripts/rolling-update.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu +cd "$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)" +exec python3 scripts/manage.py rolling-update diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 0000000..c98f51b --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu +cd "$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)" +exec python3 scripts/manage.py stop-one diff --git a/scripts/wait_ready.py b/scripts/wait_ready.py new file mode 100644 index 0000000..5ef85a6 --- /dev/null +++ b/scripts/wait_ready.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python3 +"""Wait for the public API and local telemetry backends, with a shared deadline.""" +import time +from urllib.error import URLError +from urllib.request import urlopen + +pending = { + 'api': 'http://localhost:18083/health/ready', + 'prometheus': 'http://localhost:19090/-/ready', + 'loki': 'http://localhost:13100/ready', + 'tempo': 'http://localhost:13200/ready', + 'grafana': 'http://localhost:13000/api/health', +} +deadline = time.monotonic() + 120 +while pending and time.monotonic() < deadline: + for name, url in list(pending.items()): + try: + with urlopen(url, timeout=2) as response: + if response.status == 200: + del pending[name] + except (URLError, TimeoutError, OSError): + pass + if pending: + time.sleep(1) +if pending: + raise SystemExit('Readiness timeout: ' + ', '.join(sorted(pending))) +print('PASS: API and telemetry backends ready') diff --git a/src/messenger/__init__.py b/src/messenger/__init__.py new file mode 100644 index 0000000..6852455 --- /dev/null +++ b/src/messenger/__init__.py @@ -0,0 +1,3 @@ +"""MTUSI Messenger lab1: in-memory HTTP API (standard library only).""" + +__version__ = '1.0.0' diff --git a/src/messenger/__main__.py b/src/messenger/__main__.py new file mode 100644 index 0000000..1d64403 --- /dev/null +++ b/src/messenger/__main__.py @@ -0,0 +1,7 @@ +"""Entry point: python -m messenger.""" +import sys + +from .server import main + +if __name__ == '__main__': + sys.exit(main()) diff --git a/src/messenger/api.py b/src/messenger/api.py new file mode 100644 index 0000000..8d83784 --- /dev/null +++ b/src/messenger/api.py @@ -0,0 +1,299 @@ +"""Domain logic and routing for the lab2 API: PostgreSQL state, Bearer sessions.""" +from __future__ import annotations + +import base64 +import datetime as dt +import json +import re +import uuid + +from . import cursor as cursor_codec +from . import security +from . import validation +from .attachments import AttachmentStorage, filename_from, upload_file +from .db import Database +from .errors import ApiError, Conflict, InvalidRequest, NotFound, Unauthorized, Unavailable, UnsupportedMediaType +from .store import Store, utc_now, rfc3339 + +SCOPE_CHATS = 'chats' +SCOPE_MESSAGES = 'chat/{chat_id}/messages' + + +class Request: + """One parsed HTTP request; the API layer never touches sockets.""" + + __slots__ = ('method', 'path', 'query', 'headers', 'body', 'request_id', 'client') + + def __init__(self, method, path, query, headers, body, request_id, client): + self.method = method + self.path = path + self.query = query + self.headers = headers + self.body = body + self.request_id = request_id + self.client = client + + +class Api: + UPLOAD_PATH = re.compile(r'^/api/v1/chats/(?P[^/]+)/attachments$') + ROUTES = ( + ('POST', UPLOAD_PATH, 'upload_attachment'), + ('GET', re.compile(r'^/api/v1/attachments/(?P[^/]+)$'), 'get_attachment'), + ('GET', re.compile(r'^/api/v1/attachments/(?P[^/]+)/content$'), 'download_attachment'), + ('GET', re.compile(r'^/health/live$'), 'health_live'), + ('GET', re.compile(r'^/health/ready$'), 'health_ready'), + ('POST', re.compile(r'^/api/v1/users$'), 'create_user'), + ('GET', re.compile(r'^/api/v1/users/me$'), 'current_user'), + ('GET', re.compile(r'^/api/v1/chats$'), 'list_chats'), + ('POST', re.compile(r'^/api/v1/chats$'), 'create_chat'), + ('GET', re.compile(r'^/api/v1/chats/(?P[^/]+)$'), 'get_chat'), + ('GET', re.compile(r'^/api/v1/chats/(?P[^/]+)/messages$'), 'list_messages'), + ('POST', re.compile(r'^/api/v1/chats/(?P[^/]+)/messages$'), 'post_message'), + ('POST', re.compile(r'^/api/v1/auth/sessions$'), 'create_session'), + ('DELETE', re.compile(r'^/api/v1/auth/sessions/current$'), 'revoke_session'), + ) + + def __init__(self, config, store: Store, database: Database): + self.config = config + self.store = store + self.database = database + self.attachments = AttachmentStorage() + + def dispatch(self, request): + """Return (status, payload). Only ApiError and payloads leave this method.""" + path_matched = False + for method, pattern, handler in self.ROUTES: + match = pattern.match(request.path) + if match: + path_matched = True + if method == request.method: + return getattr(self, handler)(request, match.groupdict()) + raise NotFound('Маршрут не найден' if not path_matched else 'Метод не поддерживается') + + # ------------------------------------------------------------- health + def health_live(self, request, path_params): + return 200, {'status': 'ok'} + + def health_ready(self, request, path_params): + try: + self.database.ping() + except Exception: + raise Unavailable('База данных недоступна') + return 200, {'status': 'ok'} + + # -------------------------------------------------------------- users + def create_user(self, request, path_params): + payload = self._json_object(request) + validation.require_fields(payload, ('username', 'display_name', 'password'), + ('username', 'display_name', 'password')) + username = validation.require_username(payload) + display_name = validation.require_text_field( + payload, 'display_name', 1, self.config.max_display_name_code_points) + password = validation.require_text_field( + payload, 'password', 12, 128, strip_check=False) + password_hash = security.hash_password(password) + return 201, self.store.create_user(username, display_name, password_hash) + + def current_user(self, request, path_params): + return 200, self._authenticate(request) + + # -------------------------------------------------------------- chats + def create_chat(self, request, path_params): + actor = self._authenticate(request) + payload = self._json_object(request) + validation.require_fields(payload, ('title', 'member_ids'), ('title', 'member_ids')) + title = validation.require_text_field(payload, 'title', 1, + self.config.max_title_code_points) + members = validation.require_member_ids(payload, 100) + if actor['id'] not in members: + raise InvalidRequest('Создатель чата должен быть указан в member_ids') + if not self.store.users_exist(members): + raise InvalidRequest('member_ids содержит неизвестных пользователей') + return 201, self.store.create_chat(title, members) + + def list_chats(self, request, path_params): + actor = self._authenticate(request) + limit, after_key = self._pagination(request, SCOPE_CHATS) + items, has_more = self.store.page_chats(actor['id'], after_key, limit) + for item in items: + item['member_ids'] = self._chat_member_ids(item['id']) + return 200, {'items': items, 'next_cursor': self._next_cursor(SCOPE_CHATS, items, has_more)} + + def get_chat(self, request, path_params): + actor = self._authenticate(request) + chat = self._member_chat(actor, path_params['chat_id']) + chat['member_ids'] = self._chat_member_ids(chat['id']) + return 200, chat + + # ----------------------------------------------------------- messages + def post_message(self, request, path_params): + actor = self._authenticate(request) + chat = self._member_chat(actor, path_params['chat_id']) + key = validation.require_idempotency_key(request.headers.get('Idempotency-Key')) + payload = self._json_object(request) + validation.require_fields(payload, ('text',), ('text', 'attachment_ids')) + attachment_ids = validation.require_attachment_ids(payload) + text = validation.require_text_field(payload, 'text', 0 if attachment_ids else 1, + self.config.max_text_code_points, + strip_check=not attachment_ids) + return 201, self.store.add_message(chat['id'], actor['id'], text, key, attachment_ids) + + def list_messages(self, request, path_params): + actor = self._authenticate(request) + chat = self._member_chat(actor, path_params['chat_id']) + scope = SCOPE_MESSAGES.format(chat_id=chat['id']) + limit, after_key = self._pagination(request, scope) + items, has_more = self.store.page_messages(chat['id'], after_key, limit) + return 200, {'items': items, 'next_cursor': self._next_cursor(scope, items, has_more)} + + def upload_attachment(self, request, path_params): + actor = self._authenticate(request) + chat = self._member_chat(actor, path_params['chat_id']) + filename = filename_from(request.query) + mime = request.headers.get('Content-Type', '').split(';', 1)[0].strip().lower() + with upload_file(request.body, mime) as (file, size, digest): + aid = str(uuid.uuid4()) + key = 'attachments/staging/' + aid + self.store.reserve_attachment(aid, chat['id'], actor['id'], filename, + mime, size, digest, key) + row = self.store.queue_attachment( + aid, lambda object_key: self.attachments.put(object_key, file, size, mime)) + return 202, self.store.public_attachment(row) + + def _visible_attachment(self, request, path_params): + actor = self._authenticate(request) + aid = validation.require_uuid_string(path_params['id'], 'attachment_id') + row = self.store.visible_attachment(aid, actor['id']) + if row is None: + raise NotFound('Вложение не найдено') + return row + + def get_attachment(self, request, path_params): + row = self._visible_attachment(request, path_params) + variants = self.store.attachment_variants(row['id']) if row['status'] == 'ready' else () + return 200, self.store.public_attachment(row, variants) + + def download_attachment(self, request, path_params): + row = self._visible_attachment(request, path_params) + if row['status'] != 'ready': + raise Conflict('Вложение не готово') + variant = request.query.get('variant', ['original']) + if variant != ['original']: + row = next((item for item in self.store.attachment_variants(row['id']) + if [item['name']] == variant), None) + if row is None: + raise NotFound('Вариант вложения не найден') + return 200, self.attachments.download(row) + + # ----------------------------------------------------------- sessions + def create_session(self, request, path_params): + """Login: only Authorization: Basic username:password, no JSON body.""" + authorization = request.headers.get('Authorization') or '' + if not authorization.lower().startswith('basic '): + raise Unauthorized('Требуется авторизация Basic') + try: + raw = base64.b64decode(authorization[len('basic '):].strip()).decode('utf-8') + except Exception: + raise Unauthorized('Некорректные учётные данные') from None + if ':' not in raw: + raise Unauthorized('Некорректные учётные данные') + username, password = raw.split(':', 1) + row = self.store.user_by_username(username) + if row is None or not security.verify_password(password, row['password_hash']): + raise Unauthorized('Неверное имя пользователя или пароль', www_authenticate='Basic realm="messenger", charset="UTF-8"') + token = security.generate_token() + token_hash = security.hash_token(token) + expires_at = utc_now() + dt.timedelta(seconds=self.config.session_ttl_seconds) + self.store.create_session(token_hash, row['id'], expires_at) + user = self.store._public_user({ + 'id': row['id'], + 'username': row['username'], + 'display_name': row['display_name'], + 'created_at': row['created_at'], + }) + return 201, { + 'token': token, + 'token_type': 'Bearer', + 'expires_at': rfc3339(expires_at), + 'user': user, + } + + def revoke_session(self, request, path_params): + actor = self._authenticate(request) + token = self._bearer_token(request) + self.store.delete_session(security.hash_token(token)) + return 204, None + + # ------------------------------------------------------------- helpers + def _authenticate(self, request): + """Lab2 identity: only Authorization: Bearer identifies the actor. + + X-User-Id is ignored: it does not change the session owner and alone + grants no access. Basic on business endpoints is 401. + """ + token = self._bearer_token(request) + if token is None: + raise Unauthorized('Требуется идентификация пользователя', + www_authenticate='Bearer realm="messenger"') + user_id = self.store.get_session(security.hash_token(token)) + if user_id is None: + raise Unauthorized('Сессия недействительна или истекла', + www_authenticate='Bearer realm="messenger"') + user = self.store.user(user_id) + if user is None: + raise Unauthorized('Пользователь не найден', + www_authenticate='Bearer realm="messenger"') + return user + + def _bearer_token(self, request): + authorization = request.headers.get('Authorization') or '' + if not authorization.lower().startswith('bearer '): + return None + return authorization[len('bearer '):].strip() + + def _chat_member_ids(self, chat_id): + with self.database.connection() as conn: + with conn.cursor() as cur: + cur.execute( + 'SELECT user_id FROM chat_members WHERE chat_id = %s', (str(chat_id),)) + return [str(row['user_id']) for row in cur.fetchall()] + + def _member_chat(self, actor, raw_chat_id): + """Foreign and missing chats are indistinguishable: both are 404.""" + chat_id = validation.require_uuid_string(raw_chat_id, 'chat_id') + chat = self.store.chat(chat_id) + if chat is None or not self.store.is_member(chat, actor['id']): + raise NotFound('Чат не найден') + return chat + + def _json_object(self, request): + content_type = request.headers.get('Content-Type') or '' + if content_type.split(';')[0].strip().lower() != 'application/json': + raise UnsupportedMediaType('Ожидается Content-Type: application/json') + if not request.body: + raise InvalidRequest('Требуется JSON-тело запроса') + try: + payload = json.loads(request.body.decode('utf-8')) + except (UnicodeError, ValueError): + raise InvalidRequest('Тело запроса не является корректным JSON') from None + return validation.require_object(payload) + + def _pagination(self, request, scope): + raw_limit = (request.query.get('limit') or [None])[0] + limit = validation.parse_limit(raw_limit, self.config.default_page_limit, + self.config.max_page_limit) + raw_cursor = validation.parse_cursor((request.query.get('cursor') or [None])[0]) + if raw_cursor is None: + return limit, None + try: + stamp, identifier = cursor_codec.decode(raw_cursor, scope, self.config.cursor_secret) + except cursor_codec.InvalidCursor: + raise InvalidRequest('cursor некорректен или относится к другому ресурсу') from None + return limit, (stamp, identifier) + + def _next_cursor(self, scope, items, has_more): + if not has_more or not items: + return None + last = items[-1] + return cursor_codec.encode(scope, last['created_at'], last['id'], + self.config.cursor_secret) diff --git a/src/messenger/attachments.py b/src/messenger/attachments.py new file mode 100644 index 0000000..4bcaf9d --- /dev/null +++ b/src/messenger/attachments.py @@ -0,0 +1,147 @@ +"""Bounded image validation and private S3 storage.""" +from contextlib import contextmanager +import hashlib +import os +import struct +import tempfile +import threading +import unicodedata +from functools import cached_property + +from . import telemetry + +import boto3 +from botocore.config import Config +from botocore.exceptions import BotoCoreError, ClientError +from PIL import Image + +from .errors import InvalidRequest, NotFound, PayloadTooLarge, Unavailable, UnsupportedMediaType + +MAX_UPLOAD_BYTES = 10 * 1024 * 1024 +MAX_IMAGE_PIXELS = 40_000_000 +CHUNK_SIZE = 64 * 1024 +FORMATS = {'PNG': 'image/png', 'JPEG': 'image/jpeg'} +# Pillow can allocate a full decoded image. Allow only one decode per API process. +_decode_slot = threading.Lock() +_upload_slots = threading.BoundedSemaphore(10) + + +def filename_from(query): + names = query.get('filename', []) + if len(names) != 1: + raise InvalidRequest('Укажите одно имя файла') + name = names[0] + if (not 1 <= len(name) <= 255 or not name.strip() + or any(c in '/\\' or unicodedata.category(c) in {'Cc', 'Cs'} for c in name)): + raise InvalidRequest('Некорректное имя файла') + return name + + +def validate_image(file, declared_type): + try: + with _decode_slot: + file.seek(0) + with Image.open(file, formats=list(FORMATS)) as image: + if image.width * image.height > MAX_IMAGE_PIXELS: + raise PayloadTooLarge('Изображение превышает 40 млн пикселей') + if FORMATS[image.format] != declared_type: + raise UnsupportedMediaType('Тип изображения не совпадает с Content-Type') + image.verify() + file.seek(0) + with Image.open(file, formats=list(FORMATS)) as image: + image.load() + except (Image.DecompressionBombError, Image.DecompressionBombWarning): + raise PayloadTooLarge('Изображение превышает 40 млн пикселей') from None + except (OSError, SyntaxError, ValueError, OverflowError, EOFError, struct.error): + raise UnsupportedMediaType('Повреждённое или неподдерживаемое изображение') from None + + +@contextmanager +def upload_file(read_chunks, content_type): + if content_type not in FORMATS.values(): + raise UnsupportedMediaType('Ожидается PNG или JPEG') + if not _upload_slots.acquire(blocking=False): + raise Unavailable('Слишком много одновременных загрузок') + try: + with tempfile.TemporaryFile() as file: + digest = hashlib.sha256() + size = 0 + for chunk in read_chunks(): + size += len(chunk) + if size > MAX_UPLOAD_BYTES: + raise PayloadTooLarge() + digest.update(chunk) + file.write(chunk) + if size == 0: + raise UnsupportedMediaType('Пустое изображение') + validate_image(file, content_type) + file.seek(0) + yield file, size, digest.hexdigest() + finally: + _upload_slots.release() + + +class BinaryResponse: + def __init__(self, stream, content_type, size): + self.stream = stream + self.content_type = content_type + self.size = size + + def chunks(self): + try: + yield from self.stream.iter_chunks(CHUNK_SIZE) + except (BotoCoreError, OSError) as exc: + raise OSError('S3 response interrupted') from exc + + def close(self): + self.stream.close() + + +class AttachmentStorage: + @cached_property + def client(self): + def secret(name): + with open(os.environ[name + '_FILE'], encoding='utf-8') as file: + return file.read().strip() + return boto3.session.Session().client( + 's3', endpoint_url=os.environ['S3_ENDPOINT'], + region_name=os.environ.get('S3_REGION', 'us-east-1'), + aws_access_key_id=secret('S3_ACCESS_KEY_ID'), + aws_secret_access_key=secret('S3_SECRET_ACCESS_KEY'), + config=Config(connect_timeout=2, read_timeout=5, retries={'total_max_attempts': 1}, + request_checksum_calculation="when_required", + s3={"addressing_style": "path"})) + + @property + def bucket(self): + return os.environ['S3_BUCKET'] + + @telemetry.operation("s3.put") + def put(self, key, file, size, content_type): + try: + self.client.put_object(Bucket=self.bucket, Key=key, Body=file, + ContentLength=size, ContentType=content_type) + except (BotoCoreError, ClientError, OSError, KeyError): + raise Unavailable('Хранилище вложений недоступно') from None + + @telemetry.operation("s3.delete") + def delete(self, key): + try: + self.client.delete_object(Bucket=self.bucket, Key=key) + except (BotoCoreError, ClientError, OSError, KeyError): + raise Unavailable('Хранилище вложений недоступно') from None + + @telemetry.operation("s3.download") + def download(self, row): + try: + obj = self.client.get_object(Bucket=self.bucket, Key=row['object_key']) + except ClientError as exc: + if exc.response.get('Error', {}).get('Code') in {'NoSuchKey', '404'}: + raise NotFound('Оригинал вложения не найден') from None + raise Unavailable('Хранилище вложений недоступно') from None + except (BotoCoreError, OSError, KeyError): + raise Unavailable('Хранилище вложений недоступно') from None + if obj['ContentLength'] != row['size_bytes']: + obj['Body'].close() + raise Unavailable('Размер оригинала не совпадает с метаданными') + return BinaryResponse(obj['Body'], row['content_type'], row['size_bytes']) diff --git a/src/messenger/cache.py b/src/messenger/cache.py new file mode 100644 index 0000000..93131f9 --- /dev/null +++ b/src/messenger/cache.py @@ -0,0 +1,95 @@ +"""Redis session cache with TTL and graceful degradation. + +PostgreSQL is the source of truth for sessions. Redis is a look-aside cache: +- write: session is persisted in PostgreSQL first, then copied to Redis with TTL +- read: Redis first (fast); on miss, read PostgreSQL and backfill Redis +- when Redis is unavailable: fall back to PostgreSQL only, never treat an + anonymous request as trusted +""" +from __future__ import annotations + +import logging + +import redis + +from .config import Config + +logger = logging.getLogger('messenger.cache') + + +class SessionCache: + """Look-aside cache for session tokens in Redis/Valkey.""" + + def __init__(self, config: Config): + self._config = config + self._client = None + self._available = False + + def open(self): + if not self._config.cache_url: + self._available = False + return + try: + self._client = redis.Redis.from_url( + self._config.cache_url, + socket_connect_timeout=2, + socket_timeout=2, + socket_keepalive=True, + health_check_interval=30, + decode_responses=True, + ) + self._client.ping() + self._available = True + logger.info('cache_connected', extra={'event': 'cache_connected'}) + except Exception as exc: # noqa: BLE001 - Redis is optional, degrade safely + self._available = False + self._client = None + logger.warning('cache_unavailable', extra={ + 'event': 'cache_unavailable', 'detail': 'falling back to PostgreSQL'}) + + def close(self): + if self._client is not None: + try: + self._client.close() + except Exception: + pass + self._client = None + self._available = False + + @property + def available(self): + return self._available and self._client is not None + + def get(self, token_hash): + """Return cached user_id or None (miss or unavailable).""" + if not self.available: + return None + try: + value = self._client.get('session:%s' % token_hash) + return value + except Exception as exc: # noqa: BLE001 + logger.warning('cache_read_failed', extra={ + 'event': 'cache_read_failed', 'detail': 'falling back to PostgreSQL'}) + self._available = False + return None + + def set(self, token_hash, user_id, ttl_seconds): + """Cache a session with TTL. Failure is non-fatal.""" + if not self.available: + return + try: + self._client.setex('session:%s' % token_hash, ttl_seconds, user_id) + except Exception as exc: # noqa: BLE001 + logger.warning('cache_write_failed', extra={ + 'event': 'cache_write_failed', 'detail': 'continuing without cache'}) + self._available = False + + def delete(self, token_hash): + """Remove a session from the cache (logout). Failure is non-fatal.""" + if not self.available: + return + try: + self._client.delete('session:%s' % token_hash) + except Exception as exc: # noqa: BLE001 + logger.warning('cache_delete_failed', extra={ + 'event': 'cache_delete_failed'}) diff --git a/src/messenger/clean_attachments.py b/src/messenger/clean_attachments.py new file mode 100644 index 0000000..a4222eb --- /dev/null +++ b/src/messenger/clean_attachments.py @@ -0,0 +1,58 @@ +"""Remove expired upload reservations and their S3 objects.""" +import argparse +import logging +import time + +from .attachments import AttachmentStorage +from .config import Config +from .db import Database +from .errors import Unavailable +from .store import Store +from .job_lifecycle import clean_staging, clean_abandoned_outputs + +GRACE_SECONDS = 15 * 60 +INTERVAL_SECONDS = 60 +BATCH_SIZE = 25 + + +def clean_batch(store, storage): + count = 0 + while count < BATCH_SIZE and store.clean_pending_attachment(GRACE_SECONDS, storage.delete): + count += 1 + return count + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--once', action='store_true', help='Process one bounded batch and exit') + args = parser.parse_args() + logging.basicConfig(level=logging.INFO) + config = Config.from_env() + database = Database(config) + store = Store(config, database, None) + storage = AttachmentStorage() + try: + database.open() + while True: + try: + deleted = clean_batch(store, storage) + for cleanup in (clean_staging, clean_abandoned_outputs): + for _ in range(BATCH_SIZE): + if not cleanup(database, storage.delete): + break + deleted += 1 + if deleted or args.once: + logging.info('Expired attachment reservations removed: %s', deleted) + except Unavailable: + logging.warning('Attachment cleanup deferred because a dependency is unavailable') + if args.once: + return 1 + if args.once: + return 0 + time.sleep(INTERVAL_SECONDS) + finally: + database.close() + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/src/messenger/config.py b/src/messenger/config.py new file mode 100644 index 0000000..168a0e9 --- /dev/null +++ b/src/messenger/config.py @@ -0,0 +1,149 @@ +"""Runtime configuration. Every value is read from the environment.""" +from __future__ import annotations + +import os +import secrets + +# Contract limits: they cannot be raised by configuration. +CONTRACT_MAX_TEXT_CODE_POINTS = 4000 +CONTRACT_MAX_PAGE_LIMIT = 100 +CONTRACT_SESSION_TTL_MIN = 1 +CONTRACT_SESSION_TTL_MAX = 86400 + + +class ConfigError(ValueError): + """Raised when an environment value cannot be used.""" + + +def _str(environ, name, default): + value = environ.get(name) + if value is None or value == '': + return default + return value + + +def _int(environ, name, default, minimum, maximum): + raw = environ.get(name) + if raw is None or raw == '': + return default + try: + value = int(raw) + except ValueError: + raise ConfigError('%s must be an integer, got %r' % (name, raw)) from None + if not minimum <= value <= maximum: + raise ConfigError('%s must be in %d..%d, got %d' % (name, minimum, maximum, value)) + return value + + +def _log_level(environ, name, default): + value = _str(environ, name, default).upper() + if value not in ('DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'): + raise ConfigError('%s must be one of DEBUG, INFO, WARNING, ERROR, CRITICAL' % name) + return value + + +def _secret_from_file(environ, name, default=None): + """Читает секрет из файла, путь к которому задаёт переменная окружения. + + Секреты не лежат в переменных окружения напрямую: контейнер монтирует + файл (например, Docker secret), а приложение читает его содержимое. + Если файл не задан или не существует, используется значение переменной + окружения без суффикса _FILE (удобно для локального запуска без секретов). + """ + path_name = name + '_FILE' + path = environ.get(path_name) + if path: + try: + with open(path, 'r', encoding='utf-8') as handle: + return handle.read().strip() + except OSError: + pass + return environ.get(name) or default + + +class Config: + """Effective configuration of one process instance.""" + + def __init__(self, host, port, env, log_level, cursor_secret, shutdown_grace_seconds, + max_text_code_points, max_display_name_code_points, max_title_code_points, + default_page_limit, max_page_limit, max_body_bytes, + database_url, cache_url, session_ttl_seconds, session_store, + pool_min, pool_max, connect_timeout, statement_timeout): + self.host = host + self.port = port + self.env = env + self.log_level = log_level + self.cursor_secret = cursor_secret + self.shutdown_grace_seconds = shutdown_grace_seconds + self.max_text_code_points = max_text_code_points + self.max_display_name_code_points = max_display_name_code_points + self.max_title_code_points = max_title_code_points + self.default_page_limit = default_page_limit + self.max_page_limit = max_page_limit + self.max_body_bytes = max_body_bytes + self.database_url = database_url + self.cache_url = cache_url + self.session_ttl_seconds = session_ttl_seconds + self.session_store = session_store + self.pool_min = pool_min + self.pool_max = pool_max + self.connect_timeout = connect_timeout + self.statement_timeout = statement_timeout + + @classmethod + def from_env(cls, environ=None): + environ = os.environ if environ is None else environ + secret = _secret_from_file(environ, 'CURSOR_SECRET', '') + if not secret: + secret = secrets.token_hex(32) + session_store = _str(environ, 'SESSION_STORE', 'postgres') + if session_store not in ('postgres', 'redis'): + raise ConfigError('SESSION_STORE должен быть postgres или redis, got %r' % session_store) + return cls( + host=_str(environ, 'APP_HOST', '0.0.0.0'), + port=_int(environ, 'APP_PORT', 8080, 1, 65535), + env=_str(environ, 'APP_ENV', 'lab'), + log_level=_log_level(environ, 'LOG_LEVEL', 'INFO'), + cursor_secret=secret.encode('utf-8'), + shutdown_grace_seconds=_int(environ, 'SHUTDOWN_GRACE_SECONDS', 8, 1, 10), + max_text_code_points=_int(environ, 'MAX_TEXT_CODE_POINTS', CONTRACT_MAX_TEXT_CODE_POINTS, + 1, CONTRACT_MAX_TEXT_CODE_POINTS), + max_display_name_code_points=_int(environ, 'MAX_DISPLAY_NAME_CODE_POINTS', 100, 1, 100), + max_title_code_points=_int(environ, 'MAX_TITLE_CODE_POINTS', 100, 1, 100), + default_page_limit=_int(environ, 'DEFAULT_PAGE_LIMIT', 50, 1, CONTRACT_MAX_PAGE_LIMIT), + max_page_limit=_int(environ, 'MAX_PAGE_LIMIT', CONTRACT_MAX_PAGE_LIMIT, 1, + CONTRACT_MAX_PAGE_LIMIT), + max_body_bytes=_int(environ, 'MAX_BODY_BYTES', 1048576, 1024, 16 * 1048576), + database_url=_secret_from_file(environ, 'DATABASE_URL', None), + cache_url=_secret_from_file(environ, 'CACHE_URL', None), + session_ttl_seconds=_int(environ, 'SESSION_TTL_SECONDS', 3600, + CONTRACT_SESSION_TTL_MIN, CONTRACT_SESSION_TTL_MAX), + session_store=session_store, + pool_min=_int(environ, 'DB_POOL_MIN', 2, 1, 50), + pool_max=_int(environ, 'DB_POOL_MAX', 10, 1, 100), + connect_timeout=_int(environ, 'DB_CONNECT_TIMEOUT', 5, 1, 60), + statement_timeout=_int(environ, 'DB_STATEMENT_TIMEOUT', 10, 1, 300), + ) + + def as_log_fields(self): + """Configuration summary safe for logs (no secrets).""" + return { + 'app_env': self.env, + 'host': self.host, + 'port': self.port, + 'log_level': self.log_level, + 'shutdown_grace_seconds': self.shutdown_grace_seconds, + 'default_page_limit': self.default_page_limit, + 'max_page_limit': self.max_page_limit, + 'max_text_code_points': self.max_text_code_points, + 'max_body_bytes': self.max_body_bytes, + 'cursor_secret_configured': bool(os.environ.get('CURSOR_SECRET')), + 'database_url_configured': bool(self.database_url), + 'cache_url_configured': bool(self.cache_url), + 'session_ttl_seconds': self.session_ttl_seconds, + 'session_store': self.session_store, + 'db_pool_min': self.pool_min, + 'db_pool_max': self.pool_max, + 'db_connect_timeout': self.connect_timeout, + 'db_statement_timeout': self.statement_timeout, + } diff --git a/src/messenger/cursor.py b/src/messenger/cursor.py new file mode 100644 index 0000000..4f017d6 --- /dev/null +++ b/src/messenger/cursor.py @@ -0,0 +1,66 @@ +"""Opaque keyset cursors: HMAC-protected (created_at, id) plus a resource scope.""" +from __future__ import annotations + +import base64 +import binascii +import datetime as dt +import hashlib +import hmac +import json +import uuid + +VERSION = 1 +MAC_BYTES = 16 + + +class InvalidCursor(ValueError): + """Cursor is malformed, forged, expired or belongs to another resource.""" + + +def _key(secret): + return hashlib.sha256(b'mtusi-messenger-cursor-v1' + secret).digest() + + +def encode(scope, created_at, object_id, secret): + """Serialize a keyset position bound to one resource scope.""" + payload = json.dumps( + {'v': VERSION, 'scope': scope, 'created_at': created_at, 'id': str(object_id)}, + separators=(',', ':'), sort_keys=True).encode('utf-8') + mac = hmac.new(_key(secret), payload, hashlib.sha256).digest()[:MAC_BYTES] + return base64.urlsafe_b64encode(payload + mac).decode('ascii').rstrip('=') + + +def decode(token, scope, secret): + """Return (created_at datetime, uuid) or raise InvalidCursor.""" + if not isinstance(token, str) or not token: + raise InvalidCursor('empty cursor') + padding = '=' * (-len(token) % 4) + try: + raw = base64.urlsafe_b64decode(token + padding) + except (binascii.Error, ValueError): + raise InvalidCursor('cursor is not base64url') from None + if len(raw) <= MAC_BYTES: + raise InvalidCursor('cursor is too short') + payload, mac = raw[:-MAC_BYTES], raw[-MAC_BYTES:] + if not hmac.compare_digest(mac, hmac.new(_key(secret), payload, hashlib.sha256).digest()[:MAC_BYTES]): + raise InvalidCursor('cursor signature mismatch') + try: + data = json.loads(payload.decode('utf-8')) + except (ValueError, UnicodeError): + raise InvalidCursor('cursor payload is not JSON') from None + if not isinstance(data, dict) or data.get('v') != VERSION: + raise InvalidCursor('unsupported cursor version') + if data.get('scope') != scope: + raise InvalidCursor('cursor belongs to another resource') + created_at = data.get('created_at') + object_id = data.get('id') + if not isinstance(created_at, str) or not isinstance(object_id, str): + raise InvalidCursor('cursor payload types') + try: + stamp = dt.datetime.fromisoformat(created_at.replace('Z', '+00:00')) + identifier = uuid.UUID(object_id) + except ValueError: + raise InvalidCursor('cursor payload values') from None + if stamp.tzinfo is None: + raise InvalidCursor('cursor timestamp has no timezone') + return stamp.astimezone(dt.timezone.utc), identifier diff --git a/src/messenger/db.py b/src/messenger/db.py new file mode 100644 index 0000000..36f061a --- /dev/null +++ b/src/messenger/db.py @@ -0,0 +1,114 @@ +"""PostgreSQL connection pool and schema migrations.""" +from __future__ import annotations + +import os +import time +from . import telemetry +from contextlib import contextmanager +from pathlib import Path + +from psycopg2 import pool, OperationalError, InterfaceError +from psycopg2.extras import RealDictCursor + +from .config import Config +from .errors import Unavailable + +_ROOT = Path(__file__).resolve().parents[2] +_MIGRATIONS_DIR = _ROOT / 'migrations' + + +class Database: + """Thread-safe connection pool with prepared statement timeouts. + + All queries are parameterised (%s). Connections are returned to the pool + after every operation; the pool itself is closed on shutdown. + """ + + def __init__(self, config: Config): + self._config = config + self._pool = None + + def open(self): + if self._pool is not None: + return + try: + self._pool = pool.ThreadedConnectionPool( + minconn=self._config.pool_min, + maxconn=self._config.pool_max, + dsn=self._config.database_url, + connect_timeout=self._config.connect_timeout, + cursor_factory=RealDictCursor, + ) + except Exception as exc: # psycopg2.Error or configuration error + raise Unavailable('Не удалось подключиться к PostgreSQL: %s' % exc) from None + + def close(self): + if self._pool is not None: + self._pool.closeall() + self._pool = None + + @contextmanager + def connection(self): + with telemetry.span('postgresql.transaction', telemetry.trace.SpanKind.CLIENT, + attributes={'db.system.name': 'postgresql'}): + with self._connection() as conn: + yield conn + + @contextmanager + def _connection(self): + """Yield a connection; rollback on exception, return to pool on exit.""" + if self._pool is None: + raise Unavailable('База данных не инициализирована') + started = time.monotonic() + telemetry.pool_waiting.add(1) + try: + conn = self._pool.getconn() + except (pool.PoolError, OperationalError, InterfaceError): + telemetry.pool_exhausted.add(1) + raise Unavailable('Пул соединений PostgreSQL недоступен') from None + finally: + telemetry.pool_waiting.add(-1) + telemetry.pool_acquire.record(time.monotonic() - started) + telemetry.pool_used.add(1) + try: + with conn.cursor() as cur: + cur.execute('SET statement_timeout = %s', + (self._config.statement_timeout * 1000,)) + yield conn + except Exception as exc: + try: + conn.rollback() + except Exception: + pass + if isinstance(exc, (OperationalError, InterfaceError)): + raise Unavailable('PostgreSQL недоступен') from None + raise + finally: + telemetry.pool_used.add(-1) + self._pool.putconn(conn, close=bool(conn.closed)) + + @contextmanager + def cursor(self): + """Yield a cursor inside a connection; commit on success, rollback on failure.""" + with self.connection() as conn: + with conn.cursor() as cur: + yield cur + conn.commit() + + def ping(self): + with self.cursor() as cur: + cur.execute('SELECT 1') + cur.fetchone() + + def run_migrations(self): + """Apply versioned migrations once, under a transaction advisory lock.""" + with self.cursor() as cur: + cur.execute('SELECT pg_advisory_xact_lock(3072026)') + cur.execute('CREATE TABLE IF NOT EXISTS schema_migrations ' + '(name text PRIMARY KEY, applied_at timestamptz NOT NULL DEFAULT now())') + for path in sorted(_MIGRATIONS_DIR.glob('*.sql')): + cur.execute('SELECT 1 FROM schema_migrations WHERE name = %s', (path.name,)) + if cur.fetchone() is not None: + continue + cur.execute(path.read_text(encoding='utf-8')) + cur.execute('INSERT INTO schema_migrations(name) VALUES (%s)', (path.name,)) diff --git a/src/messenger/errors.py b/src/messenger/errors.py new file mode 100644 index 0000000..553d0fd --- /dev/null +++ b/src/messenger/errors.py @@ -0,0 +1,73 @@ +"""API error taxonomy; every message is safe to return to a client.""" + + +class ApiError(Exception): + """Base class for errors that map onto the public error payload.""" + + status = 500 + code = 'internal_error' + default_message = 'Внутренняя ошибка сервера' + + def __init__(self, message=None): + self.message = message or self.default_message + super().__init__(self.message) + + +class InvalidRequest(ApiError): + status = 400 + code = 'invalid_request' + default_message = 'Некорректный запрос' + + +class Unauthorized(ApiError): + status = 401 + code = 'unauthorized' + default_message = 'Требуется идентификация пользователя' + www_authenticate = None + + def __init__(self, message=None, www_authenticate=None): + self.message = message or self.default_message + self.www_authenticate = www_authenticate + super().__init__(self.message) + + +class NotFound(ApiError): + status = 404 + code = 'not_found' + default_message = 'Объект не найден' + + +class Conflict(ApiError): + status = 409 + code = 'conflict' + default_message = 'Конфликт состояния' + + +class PayloadTooLarge(ApiError): + status = 413 + code = 'payload_too_large' + default_message = 'Тело запроса слишком велико' + + +class UnsupportedMediaType(ApiError): + status = 415 + code = 'unsupported_media_type' + default_message = 'Поддерживается только application/json' + + +class RateLimited(ApiError): + status = 429 + code = 'rate_limited' + default_message = 'Слишком много запросов' + + +class Unavailable(ApiError): + status = 503 + code = 'unavailable' + default_message = 'Сервис временно недоступен' + + +class InternalError(ApiError): + status = 500 + code = 'internal_error' + default_message = 'Внутренняя ошибка сервера' diff --git a/src/messenger/failed_jobs.py b/src/messenger/failed_jobs.py new file mode 100644 index 0000000..5b8fed2 --- /dev/null +++ b/src/messenger/failed_jobs.py @@ -0,0 +1,39 @@ +"""List failed jobs and explicitly replay a failed job with retained staging.""" +import argparse +import json +from uuid import UUID + +from .config import Config +from .db import Database +from .errors import ApiError +from .job_lifecycle import replay + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + actions = parser.add_subparsers(dest='action', required=True) + actions.add_parser('list') + command = actions.add_parser('replay') + command.add_argument('job_id', type=UUID) + command.add_argument('--operator', required=True) + args = parser.parse_args() + database = Database(Config.from_env()) + database.open() + try: + if args.action == 'replay': + print(json.dumps({'job_id': str(args.job_id), 'attempt': replay(database, str(args.job_id), args.operator)})) + else: + with database.cursor() as cur: + cur.execute("SELECT j.id AS job_id,a.id AS attachment_id,a.error_code,j.attempt, " + "j.staging_cleaned_at IS NULL AS staging_retained " + "FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id " + "WHERE a.status='failed' ORDER BY a.updated_at DESC LIMIT 100") + print(json.dumps(cur.fetchall(), default=str)) + except ApiError as exc: + parser.exit(1, str(exc) + '\n') + finally: + database.close() + + +if __name__ == '__main__': + main() diff --git a/src/messenger/job_lifecycle.py b/src/messenger/job_lifecycle.py new file mode 100644 index 0000000..507223c --- /dev/null +++ b/src/messenger/job_lifecycle.py @@ -0,0 +1,140 @@ +"""Transactional claims, bounded retries and explicit failed-job recovery.""" +import os +import random +import datetime as dt + +from . import telemetry + +from .errors import Conflict, NotFound + +LEASE_SECONDS = 60 +STAGING_RETENTION_SECONDS = 86400 +MAX_ATTEMPTS = int(os.getenv('JOB_MAX_ATTEMPTS', '5')) +if not 1 <= MAX_ATTEMPTS <= 20: + raise ValueError('JOB_MAX_ATTEMPTS must be between 1 and 20') + + +def _locked_job(cur, job_id): + cur.execute( + 'SELECT j.*, a.status, a.updated_at, a.error_code, ' + 'j.lease_until > clock_timestamp() AS lease_valid FROM attachment_jobs j ' + 'JOIN attachments a ON a.id=j.attachment_id WHERE j.id=%s FOR UPDATE OF j,a', + (job_id,)) + return cur.fetchone() + + +def claim(db, delivery): + with db.cursor() as cur: + row = _locked_job(cur, delivery['job_id']) + if (not row or row['status'] != 'queued' or row['attempt'] != delivery['attempt'] + or str(row['attachment_id']) != delivery['attachment_id'] + or row['pipeline_version'] != delivery['pipeline_version']): + return None + cur.execute('UPDATE attachment_jobs SET lease_until=clock_timestamp()+make_interval(secs => %s), ' + 'attempts_used=attempts_used+1 WHERE id=%s', (LEASE_SECONDS, row['id'])) + cur.execute("UPDATE attachments SET status='processing', updated_at=now() " + "WHERE id=%s RETURNING *", (row['attachment_id'],)) + return cur.fetchone() + + +def _requeue(cur, row, delay, reset=False): + attempt = row['attempt'] + 1 + cur.execute('UPDATE attachment_jobs SET attempt=%s, lease_until=NULL, attempts_used=%s WHERE id=%s', + (attempt, 0 if reset else row['attempts_used'], row['id'])) + cur.execute("UPDATE attachments SET status='queued', error_code=NULL, updated_at=now() WHERE id=%s", + (row['attachment_id'],)) + cur.execute('INSERT INTO attachment_outbox (job_id,attempt,available_at) ' + 'VALUES (%s,%s,now()+make_interval(secs => %s))', (row['id'], attempt, delay)) + return attempt + + +def _fail_or_retry(cur, row, error_code): + if error_code is None and row['attempts_used'] < MAX_ATTEMPTS: + delay = min(60, 2 ** row['attempts_used']) * random.uniform(.75, 1.25) + _requeue(cur, row, delay) + return + cur.execute('UPDATE attachment_jobs SET lease_until=NULL WHERE id=%s', (row['id'],)) + cur.execute("UPDATE attachments SET status='failed', error_code=%s, updated_at=now() WHERE id=%s", + (error_code or 'retry_exhausted', row['attachment_id'])) + + +def complete(db, delivery, result=None, error_code=None): + with db.cursor() as cur: + row = _locked_job(cur, delivery['job_id']) + if (not row or row['status'] != 'processing' + or row['attempt'] != delivery['attempt'] or not row['lease_valid']): + return False + if result is None: + _fail_or_retry(cur, row, error_code) + return True + key, variants = result + cur.executemany( + 'INSERT INTO attachment_variants (attachment_id,pipeline_version,variant,object_key,' + 'content_type,size_bytes,sha256,width,height) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s)', + [(row['attachment_id'], row['pipeline_version'], *item) for item in variants]) + cur.execute("UPDATE attachments SET object_key=%s,status='ready',updated_at=now() WHERE id=%s", + (key, row['attachment_id'])) + cur.execute('UPDATE attachment_jobs SET lease_until=NULL WHERE id=%s', (row['id'],)) + elapsed = (dt.datetime.now(dt.timezone.utc) - row['created_at']).total_seconds() + telemetry.time_to_ready.record(max(0, elapsed), {'job_type': 'attachment.process.v1'}) + return True + + +def recover_expired(db): + with db.cursor() as cur: + cur.execute( + "SELECT j.* FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id " + "WHERE a.status='processing' AND j.lease_until <= now() " + "ORDER BY j.lease_until LIMIT 10 FOR UPDATE OF j,a SKIP LOCKED") + for row in cur.fetchall(): + _fail_or_retry(cur, row, None) + + +def replay(db, job_id, operator): + with db.cursor() as cur: + row = _locked_job(cur, job_id) + if row is None: + raise NotFound('Задача не найдена') + if row['status'] != 'failed' or row['staging_cleaned_at'] is not None: + raise Conflict('Повтор доступен только для failed с сохранённым исходником') + attempt = _requeue(cur, row, 0, reset=True) + cur.execute('INSERT INTO attachment_replays (job_id,attempt,operator) VALUES (%s,%s,%s)', + (job_id, attempt, operator)) + return attempt + + +def clean_staging(db, remove): + with db.cursor() as cur: + cur.execute( + "SELECT j.id,j.attachment_id FROM attachment_jobs j " + "JOIN attachments a ON a.id=j.attachment_id " + "WHERE j.staging_cleaned_at IS NULL AND (a.status='ready' OR " + "(a.status='failed' AND a.updated_at < now()-make_interval(secs => %s))) " + "LIMIT 1 FOR UPDATE OF j,a SKIP LOCKED", (STAGING_RETENTION_SECONDS,)) + row = cur.fetchone() + if not row: + return False + remove('attachments/staging/' + str(row['attachment_id'])) + cur.execute('UPDATE attachment_jobs SET staging_cleaned_at=now() WHERE id=%s', (row['id'],)) + return True + + +def clean_abandoned_outputs(db, remove): + with db.cursor() as cur: + cur.execute( + "SELECT j.id,j.attachment_id,j.pipeline_version,o.attempt " + "FROM attachment_outbox o JOIN attachment_jobs j ON j.id=o.job_id " + "JOIN attachments a ON a.id=j.attachment_id " + "WHERE o.objects_cleaned_at IS NULL AND o.available_at < now()-interval '15 minutes' " + "AND (o.attempt < j.attempt OR (a.status='failed' AND " + "a.updated_at < now()-interval '15 minutes')) " + "ORDER BY o.available_at LIMIT 1 FOR UPDATE OF o SKIP LOCKED") + row = cur.fetchone() + if not row: + return False + prefix = 'attachments/final/{attachment_id}/v{pipeline_version}/a{attempt}/'.format(**row) + for name in ('original', 'thumbnail.jpeg', 'crop.jpeg', 'optimized.webp'): + remove(prefix + name) + cur.execute('UPDATE attachment_outbox SET objects_cleaned_at=now() WHERE job_id=%s AND attempt=%s', + (row['id'], row['attempt'])) + return True diff --git a/src/messenger/logging_utils.py b/src/messenger/logging_utils.py new file mode 100644 index 0000000..6b73e3c --- /dev/null +++ b/src/messenger/logging_utils.py @@ -0,0 +1,58 @@ +"""Structured stdout logging without payload contents.""" +from __future__ import annotations + +import json +import logging +import os +import sys +import time + +from opentelemetry import trace +from . import telemetry + +_FIELDS = ('event', 'request_id', 'method', 'route', 'status', 'duration_ms', 'actor_id', + 'job_id', 'attempt', 'error_code', 'client', 'outcome', 'count', 'detail', 'signal', 'version') + + +class JsonFormatter(logging.Formatter): + """One JSON object per line; message text of API payloads is never logged.""" + + def format(self, record): + payload = { + 'ts': time.strftime('%Y-%m-%dT%H:%M:%S', time.gmtime(record.created)) + + '.%03dZ' % int(record.msecs), + 'level': record.levelname.lower(), + 'logger': record.name, + 'message': record.getMessage(), + } + context = trace.get_current_span().get_span_context() + payload['service.name'] = os.getenv('OTEL_SERVICE_NAME', 'messenger-api') + if context.is_valid: + payload['trace_id'] = format(context.trace_id, '032x') + payload['span_id'] = format(context.span_id, '016x') + for field in _FIELDS: + value = getattr(record, field, None) + if value is not None: + payload[field] = value + if record.exc_info and record.levelno >= logging.ERROR: + payload['exception'] = record.exc_info[0].__name__ + return json.dumps(payload, ensure_ascii=False, sort_keys=True) + + +def configure(level_name): + handler = logging.StreamHandler(sys.stdout) + handler.setFormatter(JsonFormatter()) + root = logging.getLogger() + for existing in list(root.handlers): + root.removeHandler(existing) + root.addHandler(handler) + root.setLevel(getattr(logging, level_name)) + telemetry.configure() + for exporter in root.handlers: + if exporter is not handler: + exporter.setFormatter(JsonFormatter()) + return logging.getLogger('messenger') + + +def access_logger(): + return logging.getLogger('messenger.access') diff --git a/src/messenger/migrate.py b/src/messenger/migrate.py new file mode 100644 index 0000000..6465db0 --- /dev/null +++ b/src/messenger/migrate.py @@ -0,0 +1,17 @@ +"""One-shot schema migration entry point; API processes never migrate.""" +from .config import Config +from .db import Database + + +def main(): + database = Database(Config.from_env()) + try: + database.open() + database.run_migrations() + print('PASS: migrations applied') + finally: + database.close() + + +if __name__ == '__main__': + main() diff --git a/src/messenger/publish_jobs.py b/src/messenger/publish_jobs.py new file mode 100644 index 0000000..05c12bf --- /dev/null +++ b/src/messenger/publish_jobs.py @@ -0,0 +1,97 @@ +"""Publish committed attachment jobs using RabbitMQ publisher confirms.""" +import json +import logging +import os +from pathlib import Path +import time + +import pika + +from . import telemetry, logging_utils +from .config import Config +from .db import Database +from .errors import Unavailable +from .store import rfc3339 +from .job_lifecycle import recover_expired + +QUEUE = 'attachment.process.v1' +log = logging.getLogger('messenger.publish_jobs') + + +def connect(): + password = Path(os.environ['RABBITMQ_PASSWORD_FILE']).read_text().strip() + return pika.BlockingConnection(pika.ConnectionParameters( + host='rabbitmq', credentials=pika.PlainCredentials('lab6', password), + heartbeat=30, socket_timeout=3, stack_timeout=5, blocked_connection_timeout=5)) + + +def publish_one(database, channel): + with database.cursor() as cur: + cur.execute( + 'SELECT j.id, j.attachment_id, j.pipeline_version, j.created_at, j.trace_context, o.attempt ' + 'FROM attachment_outbox o JOIN attachment_jobs j ON j.id=o.job_id ' + 'WHERE o.published_at IS NULL AND o.available_at <= now() ' + 'ORDER BY o.available_at, o.job_id LIMIT 1 FOR UPDATE OF o SKIP LOCKED') + row = cur.fetchone() + if not row: + return False + job = { + 'schema_version': 1, 'job_id': str(row['id']), 'type': QUEUE, + 'attachment_id': str(row['attachment_id']), + 'pipeline_version': row['pipeline_version'], 'attempt': row['attempt'], + 'created_at': rfc3339(row['created_at']), + } + with telemetry.span('attachment.publish', telemetry.trace.SpanKind.PRODUCER, + context=telemetry.extract(row['trace_context']), + attributes={'messaging.system': 'rabbitmq', + 'messaging.destination.name': QUEUE, + 'job.id': job['job_id'], 'job.attempt': job['attempt']}): + channel.basic_publish( + exchange='', routing_key=QUEUE, body=json.dumps(job).encode(), + properties=pika.BasicProperties(content_type='application/json', + delivery_mode=2, message_id=job['job_id'], + headers=telemetry.carrier()), mandatory=True) + log.info('job_published', extra={'event': 'job_published', + 'job_id': job['job_id'], 'attempt': job['attempt']}) + cur.execute('UPDATE attachment_outbox SET published_at=now() WHERE job_id=%s AND attempt=%s', + (row['id'], row['attempt'])) + return True + + +def main(): + logging_utils.configure("INFO") + database = Database(Config.from_env()) + database.open() + try: + while True: + connection = None + try: + connection = connect() + channel = connection.channel() + channel.queue_declare(queue=QUEUE, durable=True) + channel.confirm_delivery() + next_recovery = 0 + while True: + if time.monotonic() >= next_recovery: + recover_expired(database) + telemetry.observe_queue(database, channel) + next_recovery = time.monotonic() + 1 + if not publish_one(database, channel): + connection.sleep(.25) + else: + connection.process_data_events() + except (pika.exceptions.AMQPError, OSError, Unavailable) as exc: + log.warning('Publisher will retry: %s', type(exc).__name__) + time.sleep(2) + finally: + if connection and connection.is_open: + try: + connection.close() + except pika.exceptions.AMQPError: + pass + finally: + database.close() + + +if __name__ == '__main__': + main() diff --git a/src/messenger/security.py b/src/messenger/security.py new file mode 100644 index 0000000..15605f3 --- /dev/null +++ b/src/messenger/security.py @@ -0,0 +1,99 @@ +"""Password hashing and session tokens. + +Passwords are hashed with a library password KDF: scrypt from the standard +library when available, otherwise bcrypt with a SHA-256 pre-hash so that +passwords longer than bcrypt's 72-byte limit are still supported. +Session tokens are opaque random values (>= 256 bits of entropy); only their +SHA-256 is stored in the database. +""" +from __future__ import annotations + +import base64 +import hashlib +import hmac +import secrets + +import bcrypt + +try: # scrypt is part of hashlib on OpenSSL-enabled builds + hashlib.scrypt(b'x', salt=b's', n=16, r=8, p=1, dklen=16) + _HAS_SCRYPT = True +except Exception: + _HAS_SCRYPT = False + +if not _HAS_SCRYPT: + import bcrypt # type: ignore + +SCRYPT_N = 16384 +SCRYPT_R = 8 +SCRYPT_P = 1 +SCRYPT_DKLEN = 32 +SALT_BYTES = 16 + + +def hash_password(password: str) -> str: + """Hash a UTF-8 password with a library password KDF. + + Returns a single storable string: + scrypt:$n:$r:$p:$salt_b64:$hash_b64 + or + bcrypt:$salt_b64:$hash_b64 + """ + raw = password.encode('utf-8') + if _HAS_SCRYPT: + salt = secrets.token_bytes(SALT_BYTES) + dk = hashlib.scrypt( + raw, + salt=salt, + n=SCRYPT_N, + r=SCRYPT_R, + p=SCRYPT_P, + dklen=SCRYPT_DKLEN, + ) + return 'scrypt:%d:%d:%d:%s:%s' % ( + SCRYPT_N, SCRYPT_R, SCRYPT_P, + base64.b64encode(salt).decode('ascii'), + base64.b64encode(dk).decode('ascii'), + ) + # bcrypt limits input to 72 bytes; pre-hash with SHA-256 so long passwords + # (up to 128 code points per contract) are fully covered. + digest = hashlib.sha256(raw).digest() + hashed = bcrypt.hashpw(digest, bcrypt.gensalt(rounds=12)) + return 'bcrypt:%s' % hashed.decode('ascii') + + +def verify_password(password: str, password_hash: str) -> bool: + """Verify a password against a KDF hash string. Constant-time compare.""" + try: + raw = password.encode('utf-8') + if password_hash.startswith('scrypt:'): + parts = password_hash.split(':') + if len(parts) != 6 or parts[0] != 'scrypt': + return False + _, n, r, p, salt_b64, hash_b64 = parts + salt = base64.b64decode(salt_b64) + expected = base64.b64decode(hash_b64) + dk = hashlib.scrypt( + raw, + salt=salt, + n=int(n), r=int(r), p=int(p), + dklen=len(expected), + ) + return hmac.compare_digest(dk, expected) + if password_hash.startswith('bcrypt:'): + stored = password_hash[len('bcrypt:'):].encode('ascii') + digest = hashlib.sha256(raw).digest() + return bcrypt.checkpw(digest, stored) + return False + except Exception: + return False + + +def generate_token() -> str: + """Opaque random bearer token, at least 256 bits of entropy.""" + return secrets.token_urlsafe(32) + + +def hash_token(token: str) -> str: + """SHA-256 of a bearer token, stored in the database instead of the token itself.""" + return hashlib.sha256(token.encode('utf-8')).hexdigest() diff --git a/src/messenger/server.py b/src/messenger/server.py new file mode 100644 index 0000000..c8eee6c --- /dev/null +++ b/src/messenger/server.py @@ -0,0 +1,327 @@ +"""HTTP/1.1 front end: request ids, structured logs, graceful SIGTERM shutdown.""" +from __future__ import annotations + +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +import json +import os +import platform +import re +import signal +import sys +import threading +import time +import uuid +from urllib.parse import parse_qs, urlsplit + +from . import __version__, logging_utils, telemetry +from .api import Api, Request +from .attachments import BinaryResponse, CHUNK_SIZE, MAX_UPLOAD_BYTES +from .cache import SessionCache +from .config import Config, ConfigError +from .db import Database +from .errors import ApiError, InvalidRequest, PayloadTooLarge, Unavailable +from .store import Store + +MAX_REQUEST_ID_LENGTH = 128 +MAX_LOGGED_LINE = 200 + + +def sanitize_request_id(raw): + """Echo a safe client id, otherwise generate one. Never trust raw input.""" + if raw and len(raw) <= MAX_REQUEST_ID_LENGTH and all(0x21 <= ord(ch) <= 0x7E for ch in raw): + return raw + return uuid.uuid4().hex + + +def route_template(path): + for _method, pattern, _handler in Api.ROUTES: + if pattern.match(path): + return re.sub(r'\(\?P<([^>]+)>\[\^/\]\+\)', r'{\1}', pattern.pattern).strip('^$') + return 'unmatched' + + +def encode_payload(payload): + return json.dumps(payload, ensure_ascii=False, separators=(',', ':')).encode('utf-8') + + +class ApiServer(ThreadingHTTPServer): + """Threaded server that tracks in-flight requests and stops accepting on demand.""" + + daemon_threads = True + allow_reuse_address = True + request_queue_size = 128 + + def __init__(self, address, api, config, logger, database): + self.api = api + self.config = config + self.logger = logger + self.database = database + self.instance_id = os.environ.get("HOSTNAME") or uuid.uuid4().hex + self.shutting_down = False + self._active = 0 + self._idle = threading.Condition() + super().__init__(address, ApiHandler) + + def enter(self): + with self._idle: + self._active += 1 + + def leave(self): + with self._idle: + self._active -= 1 + if self._active == 0: + self._idle.notify_all() + + def wait_idle(self, timeout): + deadline = time.monotonic() + timeout + with self._idle: + while self._active > 0: + remaining = deadline - time.monotonic() + if remaining <= 0: + return False + self._idle.wait(remaining) + return True + + +class ApiHandler(BaseHTTPRequestHandler): + protocol_version = 'HTTP/1.1' + server_version = 'mtusi-messenger/' + __version__ + sys_version = '' + timeout = 30 + + # --------------------------------------------------------- entry points + def do_GET(self): + self._handle() + + do_HEAD = do_POST = do_PUT = do_PATCH = do_DELETE = do_OPTIONS = do_GET + + def log_message(self, fmt, *args): + """Base-class chatter stays debug-only; access logs are structured.""" + self.server.logger.debug('http_note', extra={ + 'event': 'http_note', 'detail': (fmt % args)[:MAX_LOGGED_LINE]}) + + # ------------------------------------------------------------- pipeline + def _handle(self): + route = route_template(urlsplit(self.path).path) + with telemetry.span(self.command + ' ' + route, telemetry.trace.SpanKind.SERVER, + context=telemetry.extract({k.lower(): v for k, v in self.headers.items()}), + attributes={'http.request.method': self.command, 'http.route': route}): + self._handle_request() + + def _handle_request(self): + started = time.monotonic() + server = self.server + request_id = sanitize_request_id(self.headers.get('X-Request-Id')) + head_only = self.command == 'HEAD' + path = urlsplit(self.path).path + status, error_code = 500, None + self._last_exc = None + server.enter() + try: + try: + if server.shutting_down: + raise Unavailable('Сервер завершает работу') + if self.command == 'POST' and Api.UPLOAD_PATH.fullmatch(path): + # Rejected uploads may leave unread bytes in the connection. + self.close_connection = True + body = self._upload_chunks + else: + body = self._read_body() + query = parse_qs(urlsplit(self.path).query, keep_blank_values=True) + request = Request('GET' if head_only else self.command, path, query, self.headers, + body, request_id, self.client_address[0]) + status, payload = server.api.dispatch(request) + encoded = payload if isinstance(payload, BinaryResponse) else encode_payload(payload) + except ApiError as exc: + self._last_exc = exc + status, error_code = exc.status, exc.code + encoded = self._error_payload(exc, request_id) + except Exception: # last resort: log privately, answer generically + server.logger.exception('unhandled_error', extra={ + 'event': 'unhandled_error', 'request_id': request_id, + 'method': self.command, 'route': route_template(path)}) + status, error_code = 500, 'internal_error' + encoded = self._error_payload(ApiError(), request_id) + try: + self._send(status, encoded, request_id, head_only) + finally: + if isinstance(encoded, BinaryResponse): + encoded.close() + finally: + duration = time.monotonic() - started + current = telemetry.trace.get_current_span() + current.set_attribute('http.response.status_code', status) + if status >= 500: + current.set_status(telemetry.trace.Status(telemetry.trace.StatusCode.ERROR)) + if not path.startswith('/health/'): + labels = {'method': self.command, 'route': route_template(path), + 'status_class': str(status // 100) + 'xx'} + telemetry.http_requests.add(1, labels) + telemetry.http_duration.record(duration, labels) + server.logger.info('http_request', extra={ + 'event': 'http_request', 'request_id': request_id, 'method': self.command, + 'route': route_template(path), 'status': status, + 'duration_ms': round(1000 * (time.monotonic() - started), 3), + 'error_code': error_code, 'client': self.client_address[0]}) + server.leave() + + def _read_body(self): + if self.headers.get('Transfer-Encoding'): + self.close_connection = True + raise InvalidRequest('Transfer-Encoding не поддерживается') + raw_length = self.headers.get('Content-Length') + if raw_length is None: + return b'' + try: + length = int(raw_length.strip()) + except ValueError: + raise InvalidRequest('Некорректный Content-Length') from None + if length < 0: + raise InvalidRequest('Некорректный Content-Length') + if length > self.server.config.max_body_bytes: + self.close_connection = True + raise PayloadTooLarge('Тело запроса больше %d байт' % self.server.config.max_body_bytes) + if length == 0: + return b'' + return self.rfile.read(length) + + def _upload_chunks(self): + if self.headers.get('Transfer-Encoding'): + raise InvalidRequest('Требуется Content-Length') + lengths = self.headers.get_all('Content-Length') or [] + try: + length = int(lengths[0]) if len(lengths) == 1 else -1 + except ValueError: + length = -1 + if length < 0: + raise InvalidRequest('Некорректный Content-Length') + if length > MAX_UPLOAD_BYTES: + raise PayloadTooLarge() + deadline = time.monotonic() + 30 + while length: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise InvalidRequest('Истекло время загрузки') + self.connection.settimeout(min(5, remaining)) + try: + chunk = self.rfile.read1(min(CHUNK_SIZE, length)) + except OSError: + raise InvalidRequest('Загрузка прервана') from None + finally: + self.connection.settimeout(self.timeout) + if not chunk: + raise InvalidRequest('Загрузка прервана') + length -= len(chunk) + yield chunk + + @staticmethod + def _error_payload(exc, request_id): + return encode_payload({'error': { + 'code': exc.code, 'message': exc.message, 'request_id': request_id}}) + + def _send(self, status, encoded, request_id, head_only): + try: + self.send_response(status) + binary = isinstance(encoded, BinaryResponse) + self.send_header('Content-Type', encoded.content_type if binary else 'application/json; charset=utf-8') + self.send_header('X-Request-Id', request_id) + ctx = telemetry.trace.get_current_span().get_span_context() + if ctx.is_valid: + self.send_header('X-Trace-Id', format(ctx.trace_id, '032x')) + self.send_header('X-Instance-Id', self.server.instance_id) + self.send_header('Cache-Control', 'no-store') + if status == 204: + self.send_header('Content-Length', '0') + if status == 503: + self.send_header('Retry-After', '1') + self.end_headers() + return + self.send_header('Content-Length', str(encoded.size if binary else len(encoded))) + if status == 503: + self.send_header('Retry-After', '1') + if status == 401 and self._last_exc is not None: + www_auth = getattr(self._last_exc, 'www_authenticate', None) + if www_auth: + self.send_header('WWW-Authenticate', www_auth) + if self.close_connection: + self.send_header('Connection', 'close') + self.end_headers() + if not head_only and encoded: + if binary: + for chunk in encoded.chunks(): + self.wfile.write(chunk) + else: + self.wfile.write(encoded) + except OSError: + self.close_connection = True + + +def build_server(config, logger, api, database): + """Create the listening server; raises OSError when the port is taken.""" + return ApiServer((config.host, config.port), api, config, logger, database) + + +def run(config, logger, database, cache): + """Serve until SIGTERM/SIGINT, then drain in-flight work within the grace period.""" + api = Api(config, Store(config, database, cache), database) + try: + server = build_server(config, logger, api, database) + except OSError as exc: + logger.error('listen_failed', extra={'event': 'listen_failed', 'detail': str(exc)}) + return 1 + stop = threading.Event() + + def handle_signal(signum, _frame): + if stop.is_set(): + return + stop.set() + server.shutting_down = True + logger.warning('shutdown_signal', extra={ + 'event': 'shutdown_signal', 'signal': signum, + 'detail': 'stop accepting new requests'}) + threading.Thread(target=server.shutdown, name='shutdown', daemon=True).start() + + for name in ('SIGTERM', 'SIGINT'): + if hasattr(signal, name): + signal.signal(getattr(signal, name), handle_signal) + + logger.info('server_started', extra=dict( + {'event': 'server_started', 'version': __version__, + 'detail': 'pid %d, python %s' % (os.getpid(), platform.python_version())}, + **config.as_log_fields())) + started = time.monotonic() + try: + server.serve_forever(poll_interval=0.2) + finally: + server.server_close() + drained = server.wait_idle(config.shutdown_grace_seconds) + database.close() + logger.info('server_stopped', extra={ + 'event': 'server_stopped', + 'outcome': 'graceful' if drained else 'grace_period_exceeded', + 'duration_ms': round(1000 * (time.monotonic() - started), 3)}) + return 0 + + +def main(argv=None): + try: + config = Config.from_env() + except ConfigError as exc: + print('configuration error: %s' % exc, file=sys.stderr) + return 2 + logger = logging_utils.configure(config.log_level) + database = Database(config) + cache = SessionCache(config) + try: + database.open() + cache.open() + except Exception as exc: # noqa: BLE001 - startup must fail loud and clear + logger.error('database_start_failed', extra={ + 'event': 'database_start_failed', 'detail': str(exc)}) + database.close() + cache.close() + return 1 + try: + return run(config, logger, database, cache) + finally: + cache.close() diff --git a/src/messenger/store.py b/src/messenger/store.py new file mode 100644 index 0000000..f1da788 --- /dev/null +++ b/src/messenger/store.py @@ -0,0 +1,392 @@ +"""PostgreSQL-backed state store. All writes go through parameterised queries +inside explicit transactions; reads use keyset pagination over indexed columns.""" +from __future__ import annotations + +from . import telemetry + +import datetime as dt +import hashlib +import json + +from psycopg2.extras import Json +from psycopg2.errors import UniqueViolation, ForeignKeyViolation +import uuid + +from . import cursor as cursor_codec +from . import security +from .cache import SessionCache +from .config import Config +from .db import Database +from .errors import Conflict, InvalidRequest, NotFound + + +def utc_now(): + return dt.datetime.now(dt.timezone.utc) + + +def rfc3339(stamp): + """RFC 3339 UTC with fixed microsecond precision and the Z suffix.""" + return stamp.astimezone(dt.timezone.utc).isoformat(timespec='microseconds').replace('+00:00', 'Z') + + +class Store: + """PostgreSQL state: users, chats, membership, messages, sessions.""" + + def __init__(self, config: Config, database: Database, cache: SessionCache): + self._config = config + self._db = database + self._cache = cache + + # ---------------------------------------------------------------- reads + def stats(self): + with self._db.cursor() as cur: + cur.execute('SELECT (SELECT count(*) FROM users) AS users,' + ' (SELECT count(*) FROM chats) AS chats,' + ' (SELECT count(*) FROM messages) AS messages') + row = cur.fetchone() + return { + 'users': row['users'], + 'chats': row['chats'], + 'messages': row['messages'], + } + + def user(self, user_id): + with self._db.cursor() as cur: + cur.execute( + 'SELECT id, username, display_name, created_at FROM users WHERE id = %s', + (str(user_id),)) + row = cur.fetchone() + return self._public_user(row) if row else None + + def user_by_username(self, username): + """Return user row including password_hash, or None.""" + with self._db.cursor() as cur: + cur.execute( + 'SELECT id, username, password_hash, display_name, created_at FROM users ' + 'WHERE username = %s', (username,)) + return cur.fetchone() + + def users_exist(self, identifiers): + if not identifiers: + return True + with self._db.cursor() as cur: + cur.execute( + 'SELECT count(*) AS c FROM users WHERE id = ANY(%s::uuid[])', + ([str(i) for i in identifiers],)) + row = cur.fetchone() + return row['c'] == len(identifiers) + + # --------------------------------------------------------------- writes + def create_user(self, username, display_name, password_hash): + with self._db.cursor() as cur: + try: + cur.execute( + 'INSERT INTO users (id, username, password_hash, display_name, created_at) ' + 'VALUES (%s, %s, %s, %s, %s) ' + 'RETURNING id, username, display_name, created_at', + (str(uuid.uuid4()), username, password_hash, display_name, utc_now())) + except UniqueViolation: + # Unique violation on username -> conflict, rolled back by cursor context. + raise Conflict('username уже занят') from None + row = cur.fetchone() + return self._public_user(row) + + def create_chat(self, title, member_ids): + """Atomically create a chat and its membership. Rollback on any failure.""" + identifier = str(uuid.uuid4()) + stamp = utc_now() + with self._db.cursor() as cur: + try: + cur.execute( + 'INSERT INTO chats (id, title, created_at) VALUES (%s, %s, %s)', + (identifier, title, stamp)) + for member_id in member_ids: + cur.execute( + 'INSERT INTO chat_members (chat_id, user_id) VALUES (%s, %s)', + (identifier, str(member_id))) + except ForeignKeyViolation: + raise InvalidRequest('member_ids содержит неизвестных пользователей') from None + return { + 'id': identifier, + 'title': title, + 'member_ids': list(member_ids), + 'created_at': rfc3339(stamp), + } + + def reserve_attachment(self, aid, chat_id, owner_id, filename, mime, size, digest, key): + with self._db.cursor() as cur: + cur.execute( + "INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, " + "size_bytes, sha256, object_key, status) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,'pending')", + (aid, chat_id, owner_id, filename, mime, size, digest, key)) + + def queue_attachment(self, aid, put_object): + """Keep a row lock from before PUT until staging metadata, job and outbox commit.""" + with self._db.cursor() as cur: + cur.execute("SELECT object_key FROM attachments WHERE id=%s " + "AND status='pending' FOR UPDATE", (aid,)) + pending = cur.fetchone() + if pending is None: + raise Conflict('Загрузка вложения не может быть завершена') + put_object(pending['object_key']) + job_id = str(uuid.uuid4()) + cur.execute('INSERT INTO attachment_jobs (id, attachment_id, pipeline_version, trace_context, created_at) ' + 'VALUES (%s,%s,1,%s::jsonb,clock_timestamp())', (job_id, aid, json.dumps(telemetry.carrier()))) + cur.execute('INSERT INTO attachment_outbox (job_id, attempt) VALUES (%s,1)', (job_id,)) + cur.execute("UPDATE attachments SET status='queued', updated_at=now() " + "WHERE id=%s RETURNING *", (aid,)) + return cur.fetchone() + + def clean_pending_attachment(self, grace_seconds, delete_object): + """Remove one expired reservation and its object, or leave it for retry.""" + with self._db.cursor() as cur: + cur.execute( + "SELECT id, object_key FROM attachments WHERE status='pending' " + "AND updated_at < now() - make_interval(secs => %s) " + "ORDER BY updated_at, id LIMIT 1 FOR UPDATE SKIP LOCKED", + (grace_seconds,)) + row = cur.fetchone() + if row is None: + return False + delete_object(row['object_key']) + cur.execute('DELETE FROM attachments WHERE id=%s', (row['id'],)) + return True + + def visible_attachment(self, aid, user_id): + with self._db.cursor() as cur: + cur.execute( + "SELECT a.* FROM attachments a JOIN chat_members m ON m.chat_id=a.chat_id " + "WHERE a.id=%s AND m.user_id=%s AND a.status<>'pending'", (aid, user_id)) + return cur.fetchone() + + def attachment_variants(self, aid): + with self._db.cursor() as cur: + cur.execute( + 'SELECT DISTINCT ON (variant) variant AS name, object_key, content_type, ' + 'size_bytes, sha256, width, height FROM attachment_variants ' + 'WHERE attachment_id=%s ORDER BY variant, pipeline_version DESC', (aid,)) + return cur.fetchall() + + @staticmethod + def public_attachment(row, variants=()): + return { + 'id': str(row['id']), 'chat_id': str(row['chat_id']), 'owner_id': str(row['owner_id']), + 'filename': row['filename'], 'content_type': row['content_type'], + 'size_bytes': row['size_bytes'], 'sha256': row['sha256'], 'status': row['status'], + 'error_code': row['error_code'], + 'variants': [{k: v for k, v in item.items() if k != 'object_key'} for item in variants], + 'created_at': rfc3339(row['created_at']), 'updated_at': rfc3339(row['updated_at']), + } + + def add_message(self, chat_id, sender_id, text, key, attachment_ids=()): + """One transaction owns both the message and its replayable result. + + The advisory lock serializes only the same user/method/path/key across + processes. Hash collisions serialize extra requests, never alias keys: + the complete key is still checked against the primary key in the table. + A crash before COMMIT rolls back the message, links and replay record. A lost reply after COMMIT + is recovered by replay. Transient failures are never recorded. + """ + path = '/api/v1/chats/%s/messages' % chat_id + body = {'text': text} + # Keep the lab3 digest for text-only requests, including explicit []. + if attachment_ids: + body['attachment_ids'] = list(attachment_ids) + body_hash = hashlib.sha256(json.dumps(body, ensure_ascii=False, + sort_keys=True).encode('utf-8')).hexdigest() + scope = json.dumps([str(sender_id), 'POST', path, key], separators=(',', ':')) + lock_id = int.from_bytes(hashlib.sha256(scope.encode()).digest()[:8], 'big', signed=True) + identifier = str(uuid.uuid4()) + stamp = utc_now() + with self._db.cursor() as cur: + cur.execute('SELECT pg_advisory_xact_lock(%s)', (lock_id,)) + cur.execute('SELECT request_hash, response_body FROM message_idempotency ' + 'WHERE user_id=%s AND method=%s AND path=%s AND key=%s', + (str(sender_id), 'POST', path, key)) + replay = cur.fetchone() + if replay is not None: + if replay['request_hash'] != body_hash: + raise Conflict('Idempotency-Key уже использован с другим телом запроса') + return {'attachment_ids': [], **replay['response_body']} + if attachment_ids: + cur.execute( + 'SELECT id, status FROM attachments WHERE chat_id=%s AND id=ANY(%s::uuid[]) FOR SHARE', + (str(chat_id), list(attachment_ids))) + attachments = cur.fetchall() + if len(attachments) != len(attachment_ids): + raise NotFound('Вложение не найдено') + if any(row['status'] not in ('ready', 'queued', 'processing') for row in attachments): + raise Conflict('Вложение не готово') + cur.execute( + 'INSERT INTO messages (id, chat_id, sender_id, text, created_at) ' + 'VALUES (%s, %s, %s, %s, %s) ' + 'RETURNING id, chat_id, sender_id, text, created_at', + (identifier, str(chat_id), str(sender_id), text, stamp)) + result = self._public_message(cur.fetchone()) + if attachment_ids: + cur.executemany( + 'INSERT INTO message_attachments (message_id, attachment_id, chat_id, position) ' + 'VALUES (%s,%s,%s,%s)', + [(identifier, aid, str(chat_id), pos) for pos, aid in enumerate(attachment_ids)]) + result['attachment_ids'] = list(attachment_ids) + cur.execute('INSERT INTO message_idempotency ' + '(user_id, method, path, key, request_hash, response_status, response_body) ' + 'VALUES (%s,%s,%s,%s,%s,201,%s)', + (str(sender_id), 'POST', path, key, body_hash, Json(result))) + return result + + # --------------------------------------------------------------- pages + def page_messages(self, chat_id, after_key, limit): + """Return (public messages, has_more) for a keyset window.""" + with self._db.cursor() as cur: + if after_key is None: + cur.execute( + 'SELECT id, chat_id, sender_id, text, created_at FROM messages ' + 'WHERE chat_id = %s ' + 'ORDER BY created_at, id ' + 'LIMIT %s', (str(chat_id), limit + 1)) + else: + stamp, identifier = after_key + cur.execute( + 'SELECT id, chat_id, sender_id, text, created_at FROM messages ' + 'WHERE chat_id = %s ' + 'AND (created_at, id) > (%s, %s) ' + 'ORDER BY created_at, id ' + 'LIMIT %s', + (str(chat_id), stamp, str(identifier), limit + 1)) + rows = cur.fetchall() + items = [self._public_message(row) for row in rows[:limit]] + if items: + by_id = {item['id']: item for item in items} + cur.execute( + 'SELECT message_id, attachment_id FROM message_attachments ' + 'WHERE message_id=ANY(%s::uuid[]) ORDER BY message_id, position', + (list(by_id),)) + for row in cur.fetchall(): + by_id[str(row['message_id'])]['attachment_ids'].append(str(row['attachment_id'])) + has_more = len(rows) > limit + return items, has_more + + def page_chats(self, user_id, after_key, limit): + """Return (chats visible to the user, has_more) ordered by (created_at, id).""" + with self._db.cursor() as cur: + if after_key is None: + cur.execute( + 'SELECT c.id, c.title, c.created_at FROM chats c ' + 'JOIN chat_members cm ON cm.chat_id = c.id ' + 'WHERE cm.user_id = %s ' + 'ORDER BY c.created_at, c.id ' + 'LIMIT %s', (str(user_id), limit + 1)) + else: + stamp, identifier = after_key + cur.execute( + 'SELECT c.id, c.title, c.created_at FROM chats c ' + 'JOIN chat_members cm ON cm.chat_id = c.id ' + 'WHERE cm.user_id = %s ' + 'AND (c.created_at, c.id) > (%s, %s) ' + 'ORDER BY c.created_at, c.id ' + 'LIMIT %s', + (str(user_id), stamp, str(identifier), limit + 1)) + rows = cur.fetchall() + has_more = len(rows) > limit + items = [self._public_chat(dict(row)) for row in rows[:limit]] + return items, has_more + + # ----------------------------------------------------------- projection + def chat(self, chat_id): + with self._db.cursor() as cur: + cur.execute( + 'SELECT id, title, created_at FROM chats WHERE id = %s', (str(chat_id),)) + row = cur.fetchone() + if row is None: + return None + chat = self._public_chat(dict(row)) + cur.execute( + 'SELECT user_id FROM chat_members WHERE chat_id = %s', (str(chat_id),)) + chat['member_ids'] = [str(r['user_id']) for r in cur.fetchall()] + return chat + + @staticmethod + def is_member(chat, user_id): + return str(user_id) in chat['member_ids'] + + @staticmethod + def _public_user(row): + return { + 'id': str(row['id']), + 'username': row['username'], + 'display_name': row['display_name'], + 'created_at': rfc3339(row['created_at']), + } + + @staticmethod + def _public_chat(row): + return { + 'id': str(row['id']), + 'title': row['title'], + 'member_ids': [], # filled by API when needed + 'created_at': rfc3339(row['created_at']), + } + + @staticmethod + def _public_message(row): + return { + 'id': str(row['id']), + 'chat_id': str(row['chat_id']), + 'sender_id': str(row['sender_id']), + 'text': row['text'], + 'attachment_ids': [], + 'created_at': rfc3339(row['created_at']), + } + + # --------------------------------------------------------------- sessions + def create_session(self, token_hash, user_id, expires_at): + with self._db.cursor() as cur: + cur.execute( + 'INSERT INTO sessions (token_hash, user_id, expires_at, created_at) ' + 'VALUES (%s, %s, %s, %s)', + (token_hash, str(user_id), expires_at, utc_now())) + ttl = max(1, int((expires_at - utc_now()).total_seconds())) + self._cache.set(token_hash, str(user_id), ttl) + + def get_session(self, token_hash): + """Return user_id for a valid non-expired session, or None. + + The cache is consulted first, but every cache hit is re-validated + against PostgreSQL so that a revoked or expired session is never + trusted from a stale Redis key. On miss, read PostgreSQL and + backfill the cache. Expired sessions found during lookup are deleted. + """ + cached = self._cache.get(token_hash) + if cached is not None: + with self._db.cursor() as cur: + cur.execute( + 'SELECT 1 FROM sessions WHERE token_hash = %s AND expires_at > %s', + (token_hash, utc_now())) + if cur.fetchone() is None: + self._cache.delete(token_hash) + return None + return cached + with self._db.cursor() as cur: + cur.execute( + 'SELECT user_id, expires_at FROM sessions WHERE token_hash = %s', + (token_hash,)) + row = cur.fetchone() + if row is None: + return None + if row['expires_at'] <= utc_now(): + cur.execute('DELETE FROM sessions WHERE token_hash = %s', (token_hash,)) + return None + user_id = str(row['user_id']) + ttl = max(1, int((row['expires_at'] - utc_now()).total_seconds())) + self._cache.set(token_hash, user_id, ttl) + return user_id + + def delete_session(self, token_hash): + self._cache.delete(token_hash) + with self._db.cursor() as cur: + cur.execute('DELETE FROM sessions WHERE token_hash = %s', (token_hash,)) + + def cleanup_expired_sessions(self): + with self._db.cursor() as cur: + cur.execute('DELETE FROM sessions WHERE expires_at <= %s', (utc_now(),)) diff --git a/src/messenger/telemetry.py b/src/messenger/telemetry.py new file mode 100644 index 0000000..af23540 --- /dev/null +++ b/src/messenger/telemetry.py @@ -0,0 +1,153 @@ +"""Bounded OTLP export and low-cardinality application signals.""" +from contextlib import contextmanager +from functools import wraps +import atexit +import copy +import logging +import os +from pathlib import Path +import resource +import time + +from opentelemetry import metrics, trace +from opentelemetry.propagate import extract, inject +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor +from opentelemetry.sdk.metrics import MeterProvider, Histogram +from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader +from opentelemetry.sdk.metrics.view import View, ExplicitBucketHistogramAggregation +from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler +from opentelemetry.sdk._logs.export import BatchLogRecordProcessor +from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter +from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import OTLPMetricExporter +from opentelemetry.exporter.otlp.proto.grpc._log_exporter import OTLPLogExporter + +tracer = trace.get_tracer('messenger', '1.0') +meter = metrics.get_meter('messenger', '1.0') +http_requests = meter.create_counter('messenger.http.requests', unit='{request}') +http_duration = meter.create_histogram('messenger.http.duration', unit='s') +jobs = meter.create_counter('messenger.jobs.processed', unit='{job}') +job_duration = meter.create_histogram('messenger.job.duration', unit='s') +time_to_ready = meter.create_histogram('messenger.attachment.time_to_ready', unit='s') +pool_used = meter.create_up_down_counter('messenger.db.pool.used', unit='{connection}') +pool_waiting = meter.create_up_down_counter('messenger.db.pool.waiting', unit='{request}') +pool_exhausted = meter.create_counter('messenger.db.pool.exhausted', unit='{event}') +pool_acquire = meter.create_histogram('messenger.db.pool.acquire', unit='s') +worker_busy = meter.create_up_down_counter('messenger.worker.busy', unit='{worker}') +_state = {'depth': 0, 'age': 0, 'broker_ready': 0, 'updated': 0} +_configured = False + + +def configure(): + global _configured + if _configured or not os.getenv('OTEL_EXPORTER_OTLP_ENDPOINT'): + return + if os.getenv('OTEL_SDK_DISABLED', '').lower() == 'true': + return + _configured = True + res = Resource.create({ + 'service.name': os.getenv('OTEL_SERVICE_NAME', 'messenger-api'), + 'service.version': os.getenv('APP_REVISION', 'local'), + 'service.instance.id': os.getenv('HOSTNAME', 'local'), + 'deployment.environment.name': 'lab', + }) + endpoint = os.environ['OTEL_EXPORTER_OTLP_ENDPOINT'] + options = {'endpoint': endpoint, 'insecure': True, 'timeout': 1, + 'channel_options': (('grpc.initial_reconnect_backoff_ms', 500), + ('grpc.min_reconnect_backoff_ms', 500), + ('grpc.max_reconnect_backoff_ms', 2000), + ('grpc.dns_min_time_between_resolutions_ms', 1000))} + traces = TracerProvider(resource=res) + traces.add_span_processor(BatchSpanProcessor( + OTLPSpanExporter(**options), max_queue_size=256, max_export_batch_size=64, + schedule_delay_millis=500, export_timeout_millis=1500)) + trace.set_tracer_provider(traces) + reader = PeriodicExportingMetricReader( + OTLPMetricExporter(**options), export_interval_millis=3000, export_timeout_millis=2000) + metric_provider = MeterProvider(resource=res, metric_readers=[reader], views=[View( + instrument_type=Histogram, + aggregation=ExplicitBucketHistogramAggregation( + boundaries=(.005, .01, .025, .05, .1, .25, .5, 1, 2, 5, 10, 30, 60, 120)))]) + metrics.set_meter_provider(metric_provider) + logs = LoggerProvider(resource=res) + logs.add_log_record_processor(BatchLogRecordProcessor( + OTLPLogExporter(**options), max_queue_size=256, max_export_batch_size=64, + schedule_delay_millis=500, export_timeout_millis=1500)) + handler = LoggingHandler(level=logging.INFO, logger_provider=logs) + def safe_record(record): + if not record.name.startswith('messenger'): + return False + safe = copy.copy(record) + # OTLP's logging bridge otherwise exports full exception text/stack traces. + if safe.exc_info: + safe.error_type = safe.exc_info[0].__name__ + safe.exc_info = None + safe.exc_text = None + return safe + handler.addFilter(safe_record) + logging.getLogger().addHandler(handler) + atexit.register(logs.shutdown) + # SDK providers install their own shutdown hooks. All exporters have finite timeouts. + meter.create_observable_gauge('messenger.process.observed_at', unit='s', callbacks=[ + lambda _: [metrics.Observation(time.time())]]) + meter.create_observable_gauge('messenger.process.memory', unit='By', callbacks=[ + lambda _: [metrics.Observation(int(Path('/proc/self/statm').read_text().split()[1]) * + os.sysconf('SC_PAGE_SIZE'))]]) + meter.create_observable_counter('messenger.process.cpu', unit='s', callbacks=[ + lambda _: [metrics.Observation(resource.getrusage(resource.RUSAGE_SELF).ru_utime + + resource.getrusage(resource.RUSAGE_SELF).ru_stime)]]) + meter.create_observable_gauge('messenger.worker.capacity', unit='{worker}', callbacks=[ + lambda _: [metrics.Observation(1 if os.getenv('OTEL_SERVICE_NAME') == 'messenger-worker' else 0)]]) + meter.create_observable_gauge('messenger.db.pool.capacity', unit='{connection}', callbacks=[ + lambda _: [metrics.Observation(int(os.getenv('DB_POOL_MAX', '10')))]]) + if os.getenv('OTEL_SERVICE_NAME') == 'messenger-publisher': + for name, key, unit in ( + ('messenger.queue.depth', 'depth', '{job}'), + ('messenger.queue.oldest_age', 'age', 's'), + ('messenger.queue.broker_ready', 'broker_ready', '{message}'), + ('messenger.queue.observed_at', 'updated', 's'), + ): + meter.create_observable_gauge(name, unit=unit, callbacks=[ + lambda _, k=key: [metrics.Observation(_state[k], {'queue': 'attachment.process.v1'})]]) + + +@contextmanager +def span(name, kind=trace.SpanKind.INTERNAL, context=None, attributes=None): + with tracer.start_as_current_span(name, kind=kind, context=context, + attributes=attributes, record_exception=False, + set_status_on_exception=False) as current: + try: + yield current + except BaseException as exc: + current.set_status(trace.Status(trace.StatusCode.ERROR, type(exc).__name__)) + current.set_attribute('error.type', type(exc).__name__) + raise + + +def operation(name): + def decorate(function): + @wraps(function) + def wrapped(*args, **kwargs): + with span(name, trace.SpanKind.CLIENT): + return function(*args, **kwargs) + return wrapped + return decorate + + +def carrier(): + value = {} + inject(value) + return value + + +def observe_queue(database, channel): + """Include outbox and active leases in backlog, using durable acceptance time.""" + with database.cursor() as cur: + cur.execute("SELECT count(*) AS depth, COALESCE(EXTRACT(EPOCH FROM " + "clock_timestamp()-min(j.created_at)),0) AS age FROM attachment_jobs j " + "JOIN attachments a ON a.id=j.attachment_id WHERE a.status IN ('queued','processing')") + row = cur.fetchone() + broker = channel.queue_declare(queue='attachment.process.v1', durable=True, passive=True) + _state.update(depth=int(row['depth']), age=float(row['age']), + broker_ready=broker.method.message_count, updated=time.time()) diff --git a/src/messenger/validation.py b/src/messenger/validation.py new file mode 100644 index 0000000..c57c005 --- /dev/null +++ b/src/messenger/validation.py @@ -0,0 +1,142 @@ +"""Strict request payload validation; any deviation is 400 invalid_request.""" +from __future__ import annotations + +import re +import uuid + +from .errors import InvalidRequest + +USERNAME_PATTERN = re.compile(r'[a-z0-9_]{3,32}') +DIGITS_PATTERN = re.compile(r'[0-9]+') + + +def safe_label(value): + """Render untrusted text for an error message without breaking its encoding. + + Field names come straight from the client JSON, so they may contain the same + lone surrogates that the payload itself is rejected for. + """ + text = value if isinstance(value, str) else repr(value) + return text.encode('utf-8', 'replace').decode('utf-8') + + +def require_encodable(value, name): + """Reject text that cannot survive a UTF-8 round trip. + + `json.loads` happily produces lone surrogates from `\ud800` escapes. Such a + string cannot be UTF-8 encoded, so it must be rejected before it is stored; + otherwise the write is committed and the response encoding fails afterwards. + """ + try: + value.encode('utf-8') + except UnicodeEncodeError: + raise InvalidRequest( + 'Поле %s содержит недопустимые символы Unicode' % name) from None + return value + + +def require_object(payload): + if not isinstance(payload, dict): + raise InvalidRequest('Тело запроса должно быть JSON-объектом') + return payload + + +def require_fields(payload, required, allowed): + """Reject missing required fields and any unknown field.""" + missing = [name for name in required if name not in payload] + if missing: + raise InvalidRequest('Отсутствуют обязательные поля: ' + ', '.join(sorted(missing))) + extra = [name for name in payload if name not in allowed] + if extra: + raise InvalidRequest('Неизвестные поля: ' + ', '.join(sorted(safe_label(n) for n in extra))) + return payload + + +def require_text_field(payload, name, minimum, maximum, strip_check=True): + value = payload.get(name) + if not isinstance(value, str): + raise InvalidRequest('Поле %s должно быть строкой' % name) + require_encodable(value, name) + if strip_check and not value.strip(): + raise InvalidRequest('Поле %s не может быть пустым или состоять из пробелов' % name) + if len(value) < minimum: + raise InvalidRequest('Поле %s короче %d символов' % (name, minimum)) + if len(value) > maximum: + raise InvalidRequest('Поле %s длиннее %d символов' % (name, maximum)) + return value + + +def require_username(payload): + username = payload.get('username') + # fullmatch, not match: a trailing "$" would also accept a final newline. + if not isinstance(username, str) or USERNAME_PATTERN.fullmatch(username) is None: + raise InvalidRequest('username должен соответствовать ^[a-z0-9_]{3,32}$') + return require_encodable(username, 'username') + + +def require_uuid_string(value, name): + if not isinstance(value, str): + raise InvalidRequest('Поле %s должно быть строкой UUID' % name) + try: + return str(uuid.UUID(value)) + except ValueError: + raise InvalidRequest('Поле %s не является корректным UUID' % name) from None + + +def require_attachment_ids(payload): + value = payload.get('attachment_ids', []) + if not isinstance(value, list) or len(value) > 10: + raise InvalidRequest('attachment_ids должен содержать от 0 до 10 UUID') + identifiers = [require_uuid_string(item, 'attachment_ids') for item in value] + if len(set(identifiers)) != len(identifiers): + raise InvalidRequest('attachment_ids содержит повторяющиеся UUID') + return identifiers + + +def require_member_ids(payload, maximum, minimum=1): + value = payload.get('member_ids') + if not isinstance(value, list): + raise InvalidRequest('member_ids должен быть массивом UUID') + if not minimum <= len(value) <= maximum: + raise InvalidRequest('member_ids должен содержать от %d до %d участников' % (minimum, maximum)) + identifiers = [require_uuid_string(item, 'member_ids') for item in value] + if len(set(identifiers)) != len(identifiers): + raise InvalidRequest('member_ids содержит повторяющихся участников') + return identifiers + + +def parse_limit(raw, default, maximum): + """Parse a limit query value strictly: absent means default, anything else must be a plain integer. + + An empty value is a client error rather than an absent parameter, and the + digit run is length-bounded before int() because Python refuses to convert + strings longer than its integer-string limit, which would surface as a 500. + """ + if raw is None: + return default + if not DIGITS_PATTERN.fullmatch(raw): + raise InvalidRequest('limit должен быть целым числом от 1 до %d' % maximum) + # int() refuses absurdly long digit strings (ValueError -> 500). Comparing the + # significant digits against the maximum first rejects them without converting. + significant = raw.lstrip('0') or '0' + if len(significant) > len(str(maximum)): + raise InvalidRequest('limit должен быть в диапазоне 1..%d' % maximum) + value = int(significant) + if not 1 <= value <= maximum: + raise InvalidRequest('limit должен быть в диапазоне 1..%d' % maximum) + return value + + +def parse_cursor(raw): + if raw is None: + return None + if not isinstance(raw, str) or not raw: + raise InvalidRequest('cursor должен быть непустой строкой') + return raw + + +def require_idempotency_key(value): + """Only the contract's bounded ASCII token is accepted.""" + if not isinstance(value, str) or re.fullmatch(r'[A-Za-z0-9._:-]{1,128}', value) is None: + raise InvalidRequest('Требуется корректный Idempotency-Key (1–128 ASCII символов)') + return value diff --git a/src/messenger/worker.py b/src/messenger/worker.py new file mode 100644 index 0000000..00aa16f --- /dev/null +++ b/src/messenger/worker.py @@ -0,0 +1,170 @@ +"""Process attachment jobs and acknowledge only committed results.""" +import hashlib +from io import BytesIO +import json +import logging +import signal +import tempfile +import time +from uuid import UUID + +from PIL import Image, ImageOps +import pika + +from . import telemetry, logging_utils +from .attachments import AttachmentStorage, MAX_UPLOAD_BYTES, validate_image +from .config import Config +from .db import Database +from .errors import NotFound, PayloadTooLarge, Unavailable, UnsupportedMediaType +from .job_lifecycle import claim, complete +from .publish_jobs import QUEUE, connect + +log = logging.getLogger('messenger.worker') + + +def fitted(image, bound): + width, height = image.size + longest = max(width, height) + if longest <= bound: + return image.copy() + return image.resize((max(1, width * bound // longest), + max(1, height * bound // longest)), Image.Resampling.LANCZOS) + + +def save_variants(source, storage, prefix): + with Image.open(source, formats=['PNG', 'JPEG']) as original: + with ImageOps.exif_transpose(original) as corrected: + with corrected.convert('RGBA') as rgba: + rgb = Image.new('RGB', rgba.size, 'white') + rgb.paste(rgba, mask=rgba.getchannel('A')) + with rgb: + specs = ( + ('thumbnail', 'jpeg', 85, lambda: fitted(rgb, 256)), + ('crop', 'jpeg', 85, lambda: ImageOps.fit(rgb, (128, 128), Image.Resampling.LANCZOS)), + ('optimized', 'webp', 82, lambda: fitted(rgb, 1280)), + ) + results = [] + for name, fmt, quality, render in specs: + with render() as image, BytesIO() as file: + image.save(file, format=fmt.upper(), quality=quality) + size = file.tell() + digest = hashlib.sha256(file.getbuffer()).hexdigest() + file.seek(0) + key = f'{prefix}/{name}.{fmt}' + mime = f'image/{fmt}' + storage.put(key, file, size, mime) + results.append((name, key, mime, size, digest, image.width, image.height)) + return results + + +def materialize(attachment, storage, version, attempt): + prefix = f"attachments/final/{attachment['id']}/v{version}/a{attempt}" + key = prefix + '/original' + stream = storage.download(attachment) + try: + with tempfile.TemporaryFile() as source: + digest, size = hashlib.sha256(), 0 + for chunk in stream.chunks(): + size += len(chunk) + if size > MAX_UPLOAD_BYTES: + raise PayloadTooLarge() + digest.update(chunk) + source.write(chunk) + if size != attachment['size_bytes'] or digest.hexdigest() != attachment['sha256']: + raise UnsupportedMediaType('Контрольная сумма оригинала не совпадает') + validate_image(source, attachment['content_type']) + source.seek(0) + variants = save_variants(source, storage, prefix) + source.seek(0) + storage.put(key, source, size, attachment['content_type']) + finally: + stream.close() + return key, variants + + +def process(database, storage, job): + attachment = claim(database, job) + if attachment is None: + log.info('job_skipped', extra={'event': 'job_skipped', 'job_id': job['job_id']}) + return + started = time.monotonic() + outcome = 'error' + telemetry.worker_busy.add(1) + try: + try: + result = materialize(attachment, storage, job['pipeline_version'], job['attempt']) + except (PayloadTooLarge, UnsupportedMediaType, NotFound): + complete(database, job, error_code='invalid_image') + except (Unavailable, OSError): + complete(database, job) + else: + outcome = 'success' if complete(database, job, result=result) else 'stale' + finally: + telemetry.worker_busy.add(-1) + labels = {'job_type': QUEUE, 'outcome': outcome} + telemetry.jobs.add(1, labels) + telemetry.job_duration.record(time.monotonic() - started, labels) + if outcome == 'error': + telemetry.trace.get_current_span().set_status( + telemetry.trace.Status(telemetry.trace.StatusCode.ERROR)) + log.info('job_processed', extra={'event': 'job_processed', 'outcome': outcome, + 'job_id': job['job_id'], 'attempt': job['attempt']}) + + +def parse_job(body): + job = json.loads(body) + if not isinstance(job, dict) or job.get('type') != QUEUE or job.get('schema_version') != 1: + raise ValueError('Unknown job format') + for field in ('job_id', 'attachment_id'): + UUID(job[field]) + for field in ('pipeline_version', 'attempt'): + if type(job[field]) is not int or job[field] < 1: + raise ValueError('Invalid job number') + return job + + +def main(): + logging_utils.configure("INFO") + database = Database(Config.from_env()) + database.open() + storage = AttachmentStorage() + connection = None + + def timeout(_signal, _frame): + raise TimeoutError('Attachment processing exceeded 20 seconds') + + signal.signal(signal.SIGALRM, timeout) + + def consume(channel, method, _properties, body): + try: + job = parse_job(body) + except (ValueError, KeyError, TypeError, AttributeError): + channel.basic_reject(method.delivery_tag, requeue=False) + return + with telemetry.span('attachment.consume', telemetry.trace.SpanKind.CONSUMER, + context=telemetry.extract(_properties.headers or {}), + attributes={'messaging.system': 'rabbitmq', + 'messaging.destination.name': QUEUE, + 'job.id': job['job_id'], 'job.attempt': job['attempt']}): + signal.alarm(20) + try: + process(database, storage, job) + finally: + signal.alarm(0) + channel.basic_ack(method.delivery_tag) + + try: + connection = connect() + channel = connection.channel() + channel.queue_declare(queue=QUEUE, durable=True) + channel.basic_qos(prefetch_count=1) + channel.basic_consume(queue=QUEUE, on_message_callback=consume) + channel.start_consuming() + finally: + if connection and connection.is_open: + connection.close() + database.close() + + +if __name__ == '__main__': + main() diff --git a/student-tests/attachment_cleanup.py b/student-tests/attachment_cleanup.py new file mode 100644 index 0000000..6ec554b --- /dev/null +++ b/student-tests/attachment_cleanup.py @@ -0,0 +1,106 @@ +"""Run cleanup scenarios against a disposable PostgreSQL database.""" +import os +import sys +from pathlib import Path +from uuid import uuid4 + +import psycopg2 +from psycopg2 import sql +from psycopg2.extensions import make_dsn + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'src')) +from messenger.config import Config +from messenger.db import Database +from messenger.errors import Unavailable +from messenger.store import Store + +GRACE_SECONDS = 900 + + +def unavailable(_key): + raise Unavailable('synthetic S3 failure') + + +def expect_unavailable(operation): + try: + operation() + except Unavailable: + return + raise AssertionError('Expected Unavailable') + + +def exercise(database): + store = Store(None, database, None) + user, chat = str(uuid4()), str(uuid4()) + with database.cursor() as cur: + cur.execute("INSERT INTO users VALUES (%s,'cleanup','unused','Cleanup')", (user,)) + cur.execute("INSERT INTO chats VALUES (%s,'Cleanup')", (chat,)) + + def reserve(expired=False): + aid = str(uuid4()) + key = 'attachments/' + aid + store.reserve_attachment(aid, chat, user, 'x.png', 'image/png', 1, 'a' * 64, key) + if expired: + with database.cursor() as cur: + cur.execute("UPDATE attachments SET updated_at=now()-interval '16 minutes' WHERE id=%s", (aid,)) + return aid, key + + objects = set() + fresh, fresh_key = reserve() + objects.add(fresh_key) + ready, ready_key = reserve(expired=True) + store.finish_attachment(ready, objects.add) + with database.cursor() as cur: + cur.execute("UPDATE attachments SET updated_at=now()-interval '16 minutes' WHERE id=%s", (ready,)) + + active, active_key = reserve(expired=True) + def active_put(key): + objects.add(key) + assert not store.clean_pending_attachment(GRACE_SECONDS, objects.discard) + store.finish_attachment(active, active_put) + + failed, failed_key = reserve(expired=True) + def put_then_fail(key): + objects.add(key) + # Inject a failure after the object exists and before the ready update. + raise Unavailable('synthetic failure after PUT') + expect_unavailable(lambda: store.finish_attachment(failed, put_then_fail)) + assert failed_key in objects + expect_unavailable(lambda: store.clean_pending_attachment(GRACE_SECONDS, unavailable)) + with database.cursor() as cur: + cur.execute('SELECT status FROM attachments WHERE id=%s', (failed,)) + assert cur.fetchone()['status'] == 'pending' + assert store.clean_pending_attachment(GRACE_SECONDS, objects.discard) + assert objects == {fresh_key, ready_key, active_key} + + reserve(expired=True) # Reservation whose PUT never created an object. + assert store.clean_pending_attachment(GRACE_SECONDS, objects.discard) + assert not store.clean_pending_attachment(GRACE_SECONDS, objects.discard) + with database.cursor() as cur: + cur.execute('SELECT id::text AS id, status FROM attachments') + assert {row['id']: row['status'] for row in cur.fetchall()} == { + fresh: 'pending', ready: 'ready', active: 'ready', + } + print('PASS: PostgreSQL row lock protects active PUT; fresh and ready attachments preserved') + print('PASS: interrupted PUT and failure after PUT cleaned; S3 failure retries safely') + + +def main(): + dsn = os.environ['LAB4_TEST_DATABASE_URL'] + schema = 'cleanup_' + uuid4().hex + with psycopg2.connect(dsn) as conn, conn.cursor() as cur: + cur.execute(sql.SQL('CREATE SCHEMA {}').format(sql.Identifier(schema))) + config = Config.from_env({'DATABASE_URL': make_dsn(dsn, options='-csearch_path=' + schema)}) + database = Database(config) + try: + database.open() + database.run_migrations() + exercise(database) + finally: + database.close() + with psycopg2.connect(dsn) as conn, conn.cursor() as cur: + cur.execute(sql.SQL('DROP SCHEMA {} CASCADE').format(sql.Identifier(schema))) + + +if __name__ == '__main__': + main() diff --git a/student-tests/attachment_errors.py b/student-tests/attachment_errors.py new file mode 100644 index 0000000..7e55ff7 --- /dev/null +++ b/student-tests/attachment_errors.py @@ -0,0 +1,68 @@ +"""Check upload errors through Nginx and exercise an actual S3 outage.""" +from io import BytesIO +from pathlib import Path +import subprocess +import sys +import time + +from PIL import Image + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client, Failure, arguments, check, run + + +def main(): + args = arguments(__doc__) + client = Client(args.base_url, args.ca_file, timeout=45) + alice, _, eve, chat = client.bootstrap() + data = (ROOT / 'tests/fixtures/sample.png').read_bytes() + path = f"/api/v1/chats/{chat['id']}/attachments" + good = path + '?filename=sample.png' + item, _ = client.attachment(alice, chat) + content = f"/api/v1/attachments/{item['id']}/content" + + def upload(url, payload, expected, mime='image/png', actor=alice): + return client.call('POST', url, expected, actor=actor, raw=payload, + headers={'Content-Type': mime}) + + huge = BytesIO() + Image.new('1', (8001, 5000)).save(huge, 'PNG') + for payload, expected, mime in [ + (b'x' * (10 * 1024 * 1024 + 1), 413, 'image/png'), + (huge.getvalue(), 413, 'image/png'), + (data[:30], 415, 'image/png'), + (data, 415, 'image/jpeg'), + ]: + upload(good, payload, expected, mime) + upload(path + '?filename=a%2Fb.png', data, 400) + upload(good, data, 404, actor=eve) + client.call('GET', content, 404, actor=eve) + print('PASS: size, pixel count, corruption, MIME, filename and chat ACL over Nginx') + + def s3_command(action): + subprocess.run(['docker', 'compose', action, 's3'], cwd=ROOT, check=True, timeout=30) + + try: + s3_command('stop') + upload(good, data, 503) + client.call('GET', content, 503, actor=alice) + client.message(alice, chat, 'Текст при недоступном S3') + finally: + s3_command('start') + + started = time.monotonic() + for _ in range(40): + try: + downloaded = client.call('GET', content, actor=alice) + break + except Failure: + time.sleep(.25) + else: + raise Failure('S3 download did not recover after restart') + check(downloaded['bytes'] == data, 'original changed after S3 restart') + print(f'PASS: S3 outage returned 503, text remained available, original recovered in {time.monotonic()-started:.2f}s') + + +if __name__ == '__main__': + run(main) diff --git a/student-tests/attachment_resilience.py b/student-tests/attachment_resilience.py new file mode 100644 index 0000000..1b89eea --- /dev/null +++ b/student-tests/attachment_resilience.py @@ -0,0 +1,54 @@ +"""Stop the actual upload replica and check original bytes after failover and restart.""" +from hashlib import sha256 +import os +from pathlib import Path +import subprocess +import sys +import time + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client, arguments, check, run + + +def main(): + args = arguments(__doc__) + client = Client(args.base_url, args.ca_file, timeout=55) + alice, bob, _, chat = client.bootstrap() + path = f"/api/v1/chats/{chat['id']}/attachments?filename=sample.png" + original = (ROOT / 'tests/fixtures/sample.png').read_bytes() + reply = client.call('POST', path, 201, actor=alice, raw=original, + headers={'Content-Type': 'image/png'}, schema='Attachment') + aid = reply['json']['id'] + upload_instance = reply['headers']['X-Instance-Id'] + digest = sha256(original).hexdigest() + content = '/api/v1/attachments/' + aid + '/content' + text = client.message(alice, chat, '', attachments=[aid])['json'] + + def verify(): + received = client.call('GET', content, actor=bob) + check(received['bytes'] == original and sha256(received['bytes']).hexdigest() == digest, + 'original changed') + history = client.call('GET', f"/api/v1/chats/{chat['id']}/messages", + actor=bob, schema='MessagePage')['json'] + check(text in history['items'], 'message attachment link lost') + return received['headers']['X-Instance-Id'] + + start = time.monotonic() + try: + subprocess.run(['scripts/stop-one.sh'], cwd=ROOT, check=True, timeout=30, + env={**os.environ, 'TARGET_INSTANCE_ID': upload_instance}) + instance = verify() + check(instance != upload_instance, 'download was served by the stopped uploader') + print(f'PASS: {upload_instance} -> {instance}, SHA-256 {digest}, {time.monotonic()-start:.2f}s') + finally: + subprocess.run([sys.executable, 'scripts/manage.py', 'restore'], cwd=ROOT, check=True, timeout=100) + + start = time.monotonic() + subprocess.run(['scripts/restart.sh'], cwd=ROOT, check=True, timeout=100) + instance = verify() + print(f'PASS: after full restart via {instance}, SHA-256 {digest}, {time.monotonic()-start:.2f}s') + + +if __name__ == '__main__': + run(main) diff --git a/student-tests/attachments_http.py b/student-tests/attachments_http.py new file mode 100644 index 0000000..018e712 --- /dev/null +++ b/student-tests/attachments_http.py @@ -0,0 +1,141 @@ +from datetime import datetime, timezone +from hashlib import sha256 +from io import BytesIO +import logging +import os +import http.client +import socket +from pathlib import Path +import threading +from urllib.error import HTTPError +from urllib.request import Request, urlopen +from uuid import uuid4 +import json +import sys + +from botocore.response import StreamingBody +from PIL import Image + +root = Path(__file__).resolve().parents[1] +os.environ['S3_BUCKET'] = 'test' +sys.path.insert(0, str(root / 'src')) +from messenger.api import Api +from messenger.config import Config +from messenger.security import hash_token +from messenger.server import build_server +from messenger.store import Store + +chat, member, outsider = [str(uuid4()) for _ in range(3)] + +class Metadata: + rows = {} + def get_session(self, token): + return {hash_token('member'): member, hash_token('outsider'): outsider}.get(token) + def user(self, uid): + return {'id': uid} + def chat(self, cid): + return {'id': cid} if cid == chat else None + def is_member(self, _chat, uid): + return uid == member + def reserve_attachment(self, aid, cid, uid, filename, mime, size, digest, key): + now = datetime.now(timezone.utc) + self.rows[aid] = dict(id=aid, chat_id=cid, owner_id=uid, filename=filename, + content_type=mime, size_bytes=size, sha256=digest, + object_key=key, status='pending', error_code=None, + created_at=now, updated_at=now) + def finish_attachment(self, aid, put_object): + put_object(self.rows[aid]['object_key']) + self.rows[aid]['status'] = 'ready' + return self.rows[aid] + def visible_attachment(self, aid, uid): + row = self.rows.get(aid) + if uid == member and row and row['status'] == 'ready': + return row + return None + public_attachment = staticmethod(Store.public_attachment) + +class S3: + def __init__(self): + self.objects = {} + self.fail = False + def put_object(self, **kw): + if self.fail: + from botocore.exceptions import EndpointConnectionError + raise EndpointConnectionError(endpoint_url='http://s3.invalid') + data = kw['Body'].read() + assert len(data) == kw['ContentLength'] + self.objects[kw['Key']] = data + def get_object(self, **kw): + data = self.objects[kw['Key']] + return {'ContentLength': len(data), 'Body': StreamingBody(BytesIO(data), len(data))} + +config = Config.from_env({'APP_HOST':'127.0.0.1','APP_PORT':'8080'}) +config.port = 0 +api = Api(config, Metadata(), None) +s3 = S3() +api.attachments.client = s3 +server = build_server(config, logging.getLogger('lab4-check'), api, None) +thread = threading.Thread(target=server.serve_forever, daemon=True) +thread.start() +base = 'http://127.0.0.1:' + str(server.server_address[1]) +upload = f'/api/v1/chats/{chat}/attachments?filename=x.png' + +def call(path, method='GET', data=None, mime='image/png', token='member'): + headers = {'Authorization': 'Bearer ' + token, 'Content-Type': mime} + try: + response = urlopen(Request(base + path, data=data, headers=headers, method=method), timeout=12) + except HTTPError as exc: + response = exc + with response: + return response.code, response.headers, response.read() + +try: + for image_format, mime in [('PNG', 'image/png'), ('JPEG', 'image/jpeg')]: + buf = BytesIO() + Image.new('RGB', (10, 10), 'green').save(buf, format=image_format) + data = buf.getvalue() + status, _, raw = call(upload, 'POST', data, mime) + assert status == 201, (status, raw) + item = json.loads(raw) + assert item['status'] == 'ready' and item['sha256'] == sha256(data).hexdigest() + assert item['size_bytes'] == len(data) and 'object_key' not in item + path = '/api/v1/attachments/' + item['id'] + assert json.loads(call(path)[2]) == item + status, headers, actual = call(path + '/content') + assert (status, headers['Content-Type'], actual) == (200, mime, data) + assert call(path, token='outsider')[0] == 404 + assert call(path + '/content', token='outsider')[0] == 404 + assert call(path + '/content?variant=thumbnail')[0] == 404 + png = (root / 'tests/fixtures/sample.png').read_bytes() + assert call(upload, 'POST', png)[0] == 201 + assert call(upload, 'POST', png, 'image/jpeg')[0] == 415 + assert call(upload, 'POST', png[:30])[0] == 415 + assert call(upload.replace('x.png','a%2Fb.png'), 'POST', png)[0] == 400 + before = len(api.store.rows) + for length, partial, expected in [(10 * 1024 * 1024 + 1, b'', 413), (64, b'abc', 400)]: + conn = http.client.HTTPConnection(*server.server_address, timeout=5) + try: + conn.request('POST', upload, body=partial, headers={ + 'Authorization': 'Bearer member', 'Content-Type': 'image/png', + 'Content-Length': str(length)}) + if partial: + conn.sock.shutdown(socket.SHUT_WR) + with conn.getresponse() as response: + assert response.status == expected, response.status + response.read() + finally: + conn.close() + assert len(api.store.rows) == before + buf = BytesIO() + Image.new('1', (8001, 5000)).save(buf, 'PNG') + assert call(upload, 'POST', buf.getvalue())[0] == 413 + s3.fail = True + assert call(upload, 'POST', png)[0] == 503 + assert list(api.store.rows.values())[-1]['status'] == 'pending' + print('PASS: local HTTP PNG/JPEG round trip, checksum, metadata, ACL and missing variant') + print('PASS: malformed image, mismatched MIME, unsafe filename, byte/pixel limits and S3 error mapping') + print('NOTE: metadata and S3 are test substitutes; real Compose integration not exercised') +finally: + server.shutdown() + server.server_close() + thread.join() diff --git a/student-tests/broker_resilience.py b/student-tests/broker_resilience.py new file mode 100644 index 0000000..dde4c0f --- /dev/null +++ b/student-tests/broker_resilience.py @@ -0,0 +1,98 @@ +"""Exercise broker persistence and kill a worker after S3 writes, before SQL commit/ACK.""" +from pathlib import Path +import subprocess +import sys +import time +from uuid import uuid4 + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client, arguments, check, run + + +def cmd(*args, capture=False): + return subprocess.run(args, cwd=ROOT, text=True, check=True, timeout=100, + stdout=subprocess.PIPE if capture else None).stdout + + +def compose(*args, capture=False): + return cmd('docker', 'compose', *args, capture=capture) + + +def wait_for(probe, timeout, label): + until = time.monotonic() + timeout + while time.monotonic() < until: + if probe(): + return + time.sleep(.25) + raise RuntimeError(label) + + +def confirmed(aid): + count = compose('exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', + '-At', '-c', f"SELECT count(*) FROM attachment_outbox o JOIN attachment_jobs j " + f"ON j.id=o.job_id WHERE j.attachment_id='{aid}' AND o.published_at IS NOT NULL", + capture=True) + return int(count.strip()) > 0 + + +def main(): + args = arguments(__doc__) + client = Client(args.base_url, args.ca_file) + marker = 'AFTER_S3_BEFORE_COMMIT' + fault_worker = 'lab5-fault-' + uuid4().hex[:10] + compose('stop', 'worker') + try: + alice, bob, _, chat = client.bootstrap() + attachment, original = client.attachment(alice, chat) + client.message(alice, chat, '', attachments=[attachment['id']]) + path = '/api/v1/attachments/' + attachment['id'] + client.call('GET', path + '/content', 409, actor=bob) + wait_for(lambda: confirmed(attachment['id']), 30, 'no publisher confirm') + compose('restart', 'rabbitmq', 'api-a', 'api-b') + compose('up', '-d', '--no-build', '--no-deps', '--wait', 'api-a', 'api-b') + queued = client.call('GET', path, actor=bob, schema='Attachment')['json'] + check(queued['status'] == 'queued', 'job lost during restart') + compose('start', 'worker') + client.wait_ready(bob, attachment) + check(client.call('GET', path + '/content', actor=bob)['bytes'] == original, 'original lost') + print('PASS: worker stopped; after API and RabbitMQ restart the same attachment became ready', flush=True) + + compose('stop', 'worker') + attachment, original = client.attachment(alice, chat) + path = '/api/v1/attachments/' + attachment['id'] + code = """ +import sys,time +from messenger import worker +commit = worker.complete +def pause(db, job, result=None, error_code=None): + if job['attachment_id'] == sys.argv[1] and result is not None: + print('AFTER_S3_BEFORE_COMMIT', flush=True) + time.sleep(120) + return commit(db, job, result, error_code) +worker.complete = pause +worker.main() +""" + compose('run', '-d', '--no-deps', '--name', fault_worker, 'worker', + 'python', '-c', code, attachment['id']) + wait_for(lambda: marker in cmd('docker', 'logs', fault_worker, capture=True), + 40, 'worker did not reach fault point') + cmd('docker', 'kill', fault_worker) + compose('start', 'worker') + start = time.monotonic() + def is_ready(): + state = client.call('GET', path, actor=bob, schema='Attachment')['json'] + check(state['status'] != 'failed', 'processing failed after crash') + return state if state['status'] == 'ready' else None + wait_for(is_ready, 90, 'expired processing lease did not recover') + ready = is_ready() + check(len(ready['variants']) == 3, 'expected exactly one set of variants') + check(client.call('GET', path + '/content', actor=bob)['bytes'] == original, 'original changed') + print(f'PASS: worker killed after S3 write before COMMIT/ACK; recovered in {time.monotonic()-start:.2f}s') + finally: + subprocess.run(['docker', 'rm', '-f', fault_worker], cwd=ROOT, capture_output=True, timeout=15) + compose('start', 'worker') + + +if __name__ == '__main__': + run(main) diff --git a/student-tests/http_timing.py b/student-tests/http_timing.py new file mode 100644 index 0000000..d5618ed --- /dev/null +++ b/student-tests/http_timing.py @@ -0,0 +1,24 @@ +"""Regression: HTTP histogram includes response transmission.""" +from email.message import Message +import logging +from types import SimpleNamespace +import time +from unittest.mock import Mock, patch +from messenger.server import ApiHandler +from messenger import telemetry + +handler = object.__new__(ApiHandler) +handler.command = 'GET' +handler.path = '/api/v1/users/me' +handler.headers = Message() +handler.client_address = ('127.0.0.1', 1) +handler.server = SimpleNamespace( + shutting_down=False, config=SimpleNamespace(), logger=logging.getLogger('test'), + api=SimpleNamespace(dispatch=lambda _: (200, {})), enter=Mock(), leave=Mock()) +handler._read_body = lambda: b'' +handler._send = lambda *_: time.sleep(.08) +with patch.object(telemetry.http_duration, 'record') as record: + handler._handle_request() + assert record.call_args.args[0] >= .075, 'histogram omits response transmission' +handler.server.leave.assert_called_once() +print('PASS: HTTP histogram includes response transmission') diff --git a/student-tests/job_lifecycle.py b/student-tests/job_lifecycle.py new file mode 100644 index 0000000..005f0dc --- /dev/null +++ b/student-tests/job_lifecycle.py @@ -0,0 +1,89 @@ +"""Exercise job fencing, failed replay and cleanup on the stage-5 stand.""" +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client, arguments, check, run + + +def main(): + args = arguments(__doc__) + client = Client(args.base_url, args.ca_file) + alice, bob, _, chat = client.bootstrap() + attachment, original = client.attachment(alice, chat) + code = """ +import sys +from messenger.attachments import AttachmentStorage +from messenger.config import Config +from messenger.db import Database +from messenger.errors import Conflict +import messenger.job_lifecycle as lifecycle +from messenger.worker import materialize, process +db = Database(Config.from_env()) +db.open() +s3 = AttachmentStorage() +aid = sys.argv[1] +def delivery(): + with db.cursor() as cur: + cur.execute('SELECT id AS job_id,attachment_id,pipeline_version,attempt FROM attachment_jobs WHERE attachment_id=%s', (aid,)) + return cur.fetchone() +try: + first = delivery() + row = lifecycle.claim(db, first) + assert row['status'] == 'processing' and lifecycle.claim(db, first) is None + stale_result = materialize(row, s3, first['pipeline_version'], first['attempt']) + with db.cursor() as cur: + cur.execute("UPDATE attachment_jobs SET lease_until=now()-interval '1 second' WHERE id=%s", (first['job_id'],)) + lifecycle.recover_expired(db) + second = delivery() + assert second['attempt'] == first['attempt'] + 1 + assert not lifecycle.complete(db, first, result=stale_result) + assert lifecycle.claim(db, second) + lifecycle.MAX_ATTEMPTS = 2 + assert lifecycle.complete(db, second) + with db.cursor() as cur: + cur.execute('SELECT status,error_code FROM attachments WHERE id=%s', (aid,)) + assert cur.fetchone() == {'status':'failed','error_code':'retry_exhausted'} + assert lifecycle.replay(db, first['job_id'], 'stage5-test') == 3 + third = delivery() + process(db, s3, third) + process(db, s3, third) + try: + lifecycle.replay(db, first['job_id'], 'stage5-test') + except Conflict: + pass + else: + raise AssertionError('ready job was replayed') + with db.cursor() as cur: + cur.execute('SELECT count(*) AS n FROM attachment_variants WHERE attachment_id=%s', (aid,)) + assert cur.fetchone()['n'] == 3 + cur.execute('SELECT count(*) AS n FROM attachment_replays WHERE job_id=%s', (first['job_id'],)) + assert cur.fetchone()['n'] == 1 + cur.execute("UPDATE attachment_outbox SET available_at=now()-interval '16 minutes' WHERE job_id=%s AND attempt=1", + (first['job_id'],)) + # Check that the cleaner can remove an obsolete attempt even after replay succeeded. + removed = [] + while lifecycle.clean_abandoned_outputs(db, lambda key: (s3.delete(key), removed.append(key))): + pass + assert stale_result[0] in removed, removed + while lifecycle.clean_staging(db, s3.delete): + pass + with db.cursor() as cur: + cur.execute('SELECT staging_cleaned_at,lease_until FROM attachment_jobs WHERE id=%s', (first['job_id'],)) + row = cur.fetchone() + assert row['staging_cleaned_at'] is not None and row['lease_until'] is None + print('PASS: expired claim fenced, retry exhausted, audited replay, stale objects and staging cleaned') +finally: + db.close() +""" + subprocess.run(['docker', 'compose', 'exec', '-T', 'api-a', 'python', '-c', + code, attachment['id']], cwd=ROOT, check=True, timeout=45) + path = f"/api/v1/attachments/{attachment['id']}/content" + check(client.call('GET', path, actor=bob)['bytes'] == original, 'cleanup damaged original') + print('PASS: ready original survives cleanup and replay requests') + + +if __name__ == '__main__': + run(main) diff --git a/student-tests/message_attachments.py b/student-tests/message_attachments.py new file mode 100644 index 0000000..2b681c4 --- /dev/null +++ b/student-tests/message_attachments.py @@ -0,0 +1,53 @@ +"""Exercise attachment links and message replay over the running lab4 API.""" +import sys +import uuid +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests')) +from client import Client, arguments, check, run + + +def main(): + args = arguments('Message attachment links and replay') + client = Client(args.base_url, args.ca_file) + alice, bob, eve, chat = client.bootstrap() + first, _ = client.attachment(alice, chat) + second, _ = client.attachment(bob, chat) + ids = [second['id'], first['id']] + key = uuid.uuid4().hex + sent = client.message(bob, chat, '', key=key, attachments=ids)['json'] + replay = client.message(bob, chat, '', key=key, attachments=ids)['json'] + check(sent == replay and sent['attachment_ids'] == ids, 'ordered attachment replay mismatch') + + path = f"/api/v1/chats/{chat['id']}/messages" + client.call('POST', path, 409, actor=bob, body={'text': '', 'attachment_ids': ids[::-1]}, + headers={'Idempotency-Key': key}) + for invalid in [[ids[0]] * 2, [str(uuid.uuid4()) for _ in range(11)], ['bad UUID'], None]: + client.call('POST', path, 400, actor=bob, body={'text': '', 'attachment_ids': invalid}, + headers={'Idempotency-Key': uuid.uuid4().hex}) + client.call('POST', path, 400, actor=bob, body={'text': ''}, + headers={'Idempotency-Key': uuid.uuid4().hex}) + client.call('POST', path, 404, actor=bob, + body={'text': '', 'attachment_ids': [str(uuid.uuid4())]}, + headers={'Idempotency-Key': uuid.uuid4().hex}) + + other = client.call('POST', '/api/v1/chats', 201, actor=alice, + body={'title': 'Другой чат', 'member_ids': [alice['id'], eve['id']]}, + schema='Chat')['json'] + foreign, _ = client.attachment(alice, other) + client.call('POST', path, 404, actor=alice, + body={'text': '', 'attachment_ids': [foreign['id']]}, + headers={'Idempotency-Key': uuid.uuid4().hex}) + + plain_key = uuid.uuid4().hex + plain = client.message(alice, chat, 'Обычный текст', key=plain_key)['json'] + check(client.message(alice, chat, 'Обычный текст', key=plain_key, attachments=[])['json'] == plain, + 'omitted and empty attachment_ids must replay identically') + page = client.call('GET', path, actor=alice, schema='MessagePage')['json'] + check([item for item in page['items'] if item['id'] == sent['id']] == [sent], + 'ordered attachment IDs missing or duplicated in history') + print('PASS: ordered attachments, replay, invalid input, cross-chat rejection and history') + + +if __name__ == '__main__': + run(main) diff --git a/student-tests/observability.py b/student-tests/observability.py new file mode 100644 index 0000000..0a0dcf5 --- /dev/null +++ b/student-tests/observability.py @@ -0,0 +1,297 @@ +"""Live lab6 checks. Fault modes restore every stopped service in finally.""" +import argparse +import base64 +from concurrent.futures import ThreadPoolExecutor +import json +import re +from pathlib import Path +import subprocess +import sys +import time +import uuid +from urllib.error import HTTPError, URLError +from urllib.parse import urlencode +from urllib.request import Request, urlopen + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client, check + +PROM = 'http://localhost:19090' +LOKI = 'http://localhost:13100' +TEMPO = 'http://localhost:13200' +BASE = 'http://localhost:18083' + + +def get(url, headers=None): + with urlopen(Request(url, headers=headers or {}), timeout=8) as response: + return json.load(response) + + +def prom(query): + return get(PROM + '/api/v1/query?' + urlencode({'query': query}))['data']['result'] + + +def eventually(function, timeout=75): + deadline = time.monotonic() + timeout + last = None + while time.monotonic() < deadline: + try: + value = function() + if value: + return value + except OSError as exc: + last = type(exc).__name__ + time.sleep(2) + raise AssertionError('condition timeout: ' + str(last or function.__name__)) + + +def compose(*args): + subprocess.run(['docker', 'compose', '--env-file', '.env', *args], cwd=ROOT, + check=True, timeout=100, stdout=subprocess.DEVNULL) + + +def alert_firing(name): + alerts = get(PROM + '/api/v1/alerts')['data']['alerts'] + return any(a['labels']['alertname'] == name and a['state'] == 'firing' for a in alerts) + + +def trace_spans(trace_id): + data = get(TEMPO + '/api/traces/' + trace_id, {'Accept': 'application/json'}) + spans = [] + def visit(item): + if isinstance(item, dict): + if 'spanId' in item and 'name' in item: + spans.append(item) + for child in item.values(): + visit(child) + elif isinstance(item, list): + for child in item: + visit(child) + visit(data) + return spans + + +def upload(client, actor, chat): + original = (ROOT / 'tests/fixtures/sample.png').read_bytes() + incoming = uuid.uuid4().hex + reply = client.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=sample.png', + 202, actor=actor, raw=original, + headers={'Content-Type': 'image/png', + 'traceparent': '00-' + incoming + '-1234567890abcdef-01', + 'tracestate': 'lab=lab6'}, schema='Attachment') + check(reply['headers']['X-Trace-Id'] == incoming, 'upload context lost') + return reply['json'], reply['headers']['X-Trace-Id'] + + +def linked_trace(trace_id, attempts=1): + def ready(): + spans = trace_spans(trace_id) + names = [s['name'] for s in spans] + if (any(n.startswith('POST /api/v1/chats/') for n in names) + and names.count('attachment.publish') >= attempts + and names.count('attachment.consume') >= attempts + and 's3.put' in names and 'postgresql.transaction' in names): + return spans + spans = eventually(ready) + ids = {s['spanId'] for s in spans} + for item in spans: + if item['name'] in ('attachment.publish', 'attachment.consume'): + check(item.get('parentSpanId') in ids, 'producer/consumer parent missing') + check(item.get('traceState') == 'lab=lab6', 'AMQP tracestate lost') + return spans + + +def correlated_logs(trace_id): + query = '{service_name=~"messenger-.*"} |= "' + trace_id + '"' + def available(): + response = get(LOKI + '/loki/api/v1/query_range?' + urlencode({ + 'query': query, 'since': '1h', 'limit': 1000})) + rows = response['data']['result'] + names = {r['stream'].get('service_name') for r in rows} + return rows if {'messenger-api', 'messenger-worker', 'messenger-publisher'} <= names else None + return eventually(available) + + +def smoke(client, alice, bob, chat): + att, tid = upload(client, alice, chat) + client.wait_ready(bob, att) + spans = linked_trace(tid) + logs = correlated_logs(tid) + required = ['messenger_http_requests_total', 'messenger_http_duration_seconds_bucket', + 'messenger_queue_depth', 'messenger_queue_oldest_age_seconds', + 'messenger_db_pool_used', 'messenger_worker_busy', + 'messenger_job_duration_seconds_bucket', 'messenger_attachment_time_to_ready_seconds_bucket', + 'messenger_process_memory_bytes', 'messenger_process_cpu_seconds_total'] + for name in required: + eventually(lambda: prom(name)) + # Validate propagation from an incoming W3C header, including header case normalization. + incoming_id = '1234567890abcdef1234567890abcdef' + reply = client.call('GET', '/api/v1/users/me', actor=alice, + headers={'traceparent': '00-' + incoming_id + '-1234567890abcdef-01', + 'tracestate': 'lab=propagation'}) + check(reply['headers']['X-Trace-Id'] == incoming_id, 'incoming trace context lost') + # Synthetic message and token must never appear in persisted telemetry. + canary = 'PRIVATE_BODY_' + tid + client.message(alice, chat, canary) + time.sleep(5) + for secret in (canary, alice['token']): + query = '{service_name=~"messenger-.*"} |= "' + secret + '"' + found = get(LOKI + '/loki/api/v1/query_range?' + urlencode({ + 'query': query, 'since': '1h', 'limit': 10}))['data']['result'] + check(not found, 'private content found in telemetry') + password = (ROOT / 'secrets/grafana_password').read_text().strip() + auth = 'Basic ' + base64.b64encode(('admin:' + password).encode()).decode() + dashboard = get('http://localhost:13000/api/dashboards/uid/messenger', {'Authorization': auth}) + check(len(dashboard['dashboard']['panels']) >= 12, 'dashboard not provisioned') + for panel in dashboard['dashboard']['panels']: + for target in panel.get('targets', []): + if 'expr' in target: + prom(target['expr']) + search = urlencode({'q': '{resource.service.name =~ "messenger-.*"}', + 'start': int(time.time() - 900), 'end': int(time.time()), 'limit': 20}) + traces = get('http://localhost:13000/api/datasources/proxy/uid/tempo/api/search?' + search, + {'Authorization': auth}) + check(traces.get('traces'), 'Tempo search through Grafana failed') + return {'trace_id': tid, 'span_names': [s['name'] for s in spans], + 'log_services': sorted({r['stream']['service_name'] for r in logs}), + 'metric_names_checked': required, 'dashboard_panels': len(dashboard['dashboard']['panels']), + 'dashboard_queries_valid': True, 'incoming_w3c_context': True, 'body_and_token_absent_from_logs': True} + + +def worker_stop(client, alice, bob, chat): + compose('stop', 'worker') + try: + att, tid = upload(client, alice, chat) + started = time.monotonic() + eventually(lambda: alert_firing('QueueLagGrowing')) + observation = {'seconds_to_firing': round(time.monotonic() - started, 2), + 'oldest_job_age': prom('messenger_queue_oldest_age_seconds'), + 'backlog': prom('messenger_queue_depth'), + 'alerts_firing': get(PROM + '/api/v1/alerts')['data']['alerts']} + finally: + compose('start', 'worker') + client.wait_ready(bob, att) + eventually(lambda: not alert_firing('QueueLagGrowing')) + spans = linked_trace(tid) + observation.update(resolved=True, trace_id=tid, span_count=len(spans), + alerts_after=get(PROM + '/api/v1/alerts')['data']['alerts']) + return observation + + +def db_outage(client, alice): + # A held table lock makes actual PostgreSQL operations time out. Readiness SELECT 1 stays healthy. + locker = subprocess.Popen(['docker', 'compose', 'exec', '-T', 'postgres', 'psql', + '-U', 'messenger', '-d', 'messenger', '-v', 'ON_ERROR_STOP=1', '-c', + 'BEGIN; LOCK TABLE users IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep(35); COMMIT;'], + cwd=ROOT, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + failures = [] + try: + time.sleep(2) + def request(_): + reply = client.call('GET', '/api/v1/users/me', 503, actor=alice) + return reply['headers']['X-Trace-Id'] + with ThreadPoolExecutor(max_workers=2) as executor: + failures = list(executor.map(request, range(8))) + eventually(lambda: alert_firing('HighApiErrorRate')) + error_spans = eventually(lambda: [s for s in trace_spans(failures[0]) + if s['name'] == 'postgresql.transaction' and + s.get('status', {}).get('code') in (2, 'STATUS_CODE_ERROR')]) + observation = {'firing': True, 'failed_requests': len(failures), + 'trace_id': failures[0], 'db_error_spans': len(error_spans), + 'alerts_firing': get(PROM + '/api/v1/alerts')['data']['alerts'], + 'error_ratio': prom('sum(rate(messenger_http_requests_total{status_class="5xx"}[1m])) / sum(rate(messenger_http_requests_total[1m]))')} + finally: + locker.wait(timeout=50) + eventually(lambda: client.call('GET', '/api/v1/users/me', actor=alice, schema='User')['json']) + eventually(lambda: not alert_firing('HighApiErrorRate'), timeout=100) + observation['resolved'] = True + return observation + + +def retry(client, alice, bob, chat): + compose('stop', 'worker') + try: + att, tid = upload(client, alice, chat) + compose('stop', 's3') + # Compose start also starts dependencies in this version. Keep S3 stopped here. + subprocess.run(['docker', 'start', 'mtusi-lab6-worker-1'], check=True, + stdout=subprocess.DEVNULL) + def retried(): + spans = trace_spans(tid) + return any(s['name'] == 'attachment.consume' and + s.get('status', {}).get('code') in (2, 'STATUS_CODE_ERROR') for s in spans) + eventually(retried, timeout=45) + finally: + compose('start', 's3', 'worker') + client.wait_ready(bob, att) + spans = linked_trace(tid, attempts=2) + return {'trace_id': tid, 'consumer_attempts': sum(s['name'] == 'attachment.consume' for s in spans), + 'span_names': [s['name'] for s in spans], 'ready_after_retry': True} + + +def backend_outage(client, alice, bob, chat): + before = prom('sum(otelcol_exporter_send_failed_spans_total)') + tracked = ['mtusi-lab6-' + name + '-1' for name in + ('api-a', 'api-b', 'worker', 'outbox-publisher', 'otel-collector')] + before_containers = json.loads(subprocess.check_output(['docker', 'inspect', *tracked], text=True)) + outage_start = str(int(time.time())) + compose('stop', 'tempo') + count = 0 + samples = [] + try: + started = time.monotonic() + while time.monotonic() - started < 40: + client.message(alice, chat, 'telemetry-outage') + count += 1 + if count % 15 == 0: + raw = subprocess.check_output(['docker', 'stats', '--no-stream', '--format', + '{{json .}}', *tracked], cwd=ROOT, text=True) + samples.extend(json.loads(line) for line in raw.splitlines()) + time.sleep(.15) + att, _ = upload(client, alice, chat) + client.wait_ready(bob, att) + failures = prom('sum(otelcol_exporter_send_failed_spans_total)') + before_count = float(before[0]['value'][1]) if before else 0 + check(failures and float(failures[0]['value'][1]) > before_count, 'no new exporter failures') + finally: + compose('start', 'tempo') + collector_logs = subprocess.check_output(['docker', 'compose', 'logs', '--since', outage_start, + 'otel-collector'], cwd=ROOT, text=True) + dropped = sum(int(value) for value in re.findall(r'"dropped_items"\s*:\s*(\d+)', collector_logs)) + after_containers = json.loads(subprocess.check_output(['docker', 'inspect', *tracked], text=True)) + for before_item, after_item in zip(before_containers, after_containers): + check(after_item['State']['Running'] and not after_item['State']['OOMKilled'], 'container died') + check(before_item['RestartCount'] == after_item['RestartCount'], 'container restarted') + restored_att, restored_trace = upload(client, alice, chat) + client.wait_ready(bob, restored_att) + linked_trace(restored_trace) + correlated_logs(restored_trace) + return {'successful_messages': count, 'attachment_ready_during_outage': True, + 'collector_reported_dropped_spans': dropped, 'export_failures_before': before, 'export_failures_after': failures, + 'application_and_collector_stats': samples, 'no_oom_or_restarts': True, 'telemetry_restored_trace': restored_trace} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('mode', choices=['smoke', 'worker-stop', 'db-outage', 'retry', 'backend-outage']) + args = parser.parse_args() + client = Client(BASE, None, timeout=20) + alice, bob, _, chat = client.bootstrap() + if args.mode == 'smoke': + result = smoke(client, alice, bob, chat) + elif args.mode == 'worker-stop': + result = worker_stop(client, alice, bob, chat) + elif args.mode == 'db-outage': + result = db_outage(client, alice) + elif args.mode == 'retry': + result = retry(client, alice, bob, chat) + else: + result = backend_outage(client, alice, bob, chat) + path = ROOT / 'evidence' / ('observability-' + args.mode + '.json') + path.write_text(json.dumps(result, ensure_ascii=False, indent=2) + '\n') + print('PASS', args.mode, path.name) + + +if __name__ == '__main__': + main() diff --git a/student-tests/outbox.sh b/student-tests/outbox.sh new file mode 100644 index 0000000..9fc74b5 --- /dev/null +++ b/student-tests/outbox.sh @@ -0,0 +1,27 @@ +#!/bin/sh +# Run against a lab5 stage-3 stand without a worker. +set -eu +cd "$(dirname "$0")/.." +base_url=${BASE_URL:-http://localhost:18082} +sql() { + docker compose exec -T postgres psql -U messenger -d messenger -At -v ON_ERROR_STOP=1 -c "$1" +} +docker compose stop outbox-publisher +trap 'docker compose start outbox-publisher >/dev/null' EXIT +upload=$(PYTHONDONTWRITEBYTECODE=1 python3 student-tests/queued_upload.py --base-url "$base_url") +printf '%s\n' "$upload" +aid=$(printf '%s\n' "$upload" | awk '/Attachment ID:/ {print $3}') +query="SELECT count(*) FROM attachment_outbox o JOIN attachment_jobs j ON j.id=o.job_id WHERE j.attachment_id='$aid' AND o.published_at IS" +test "$(sql "$query NULL")" = 1 +echo 'PASS: upload committed while publisher was stopped' +docker compose start outbox-publisher +trap - EXIT +for i in $(seq 1 40); do + if test "$(sql "$query NOT NULL")" = 1; then + echo 'PASS: same attachment job confirmed after publisher startup' + exit 0 + fi + sleep 1 +done +echo 'FAIL: outbox row not confirmed' >&2 +exit 1 diff --git a/student-tests/queued_upload.py b/student-tests/queued_upload.py new file mode 100644 index 0000000..3914777 --- /dev/null +++ b/student-tests/queued_upload.py @@ -0,0 +1,27 @@ +"""Verify durable handoff before a publisher or worker is enabled.""" +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests')) +from client import Client, arguments, check, run + + +def main(): + args = arguments(__doc__) + client = Client(args.base_url, args.ca_file) + alice, bob, eve, chat = client.bootstrap() + attachment, _ = client.attachment(alice, chat) + path = '/api/v1/attachments/' + attachment['id'] + check(attachment['status'] == 'queued', 'expected queued with worker disabled') + metadata = client.call('GET', path, actor=bob, schema='Attachment')['json'] + check(metadata == attachment, 'queued metadata differs') + client.call('GET', path, 404, actor=eve) + client.call('GET', path + '/content', 409, actor=bob) + sent = client.message(alice, chat, '', attachments=[attachment['id']])['json'] + check(sent['attachment_ids'] == [attachment['id']], 'queued attachment not linked') + print('PASS: 202 queued, metadata ACL, 409 before ready, and queued attachment linked') + print('Attachment ID:', attachment['id']) + + +if __name__ == '__main__': + run(main) diff --git a/student-tests/s3_cleanup_live.py b/student-tests/s3_cleanup_live.py new file mode 100644 index 0000000..cc845d9 --- /dev/null +++ b/student-tests/s3_cleanup_live.py @@ -0,0 +1,72 @@ +"""Run via docker compose exec -T api-a python - with this file on stdin.""" +from hashlib import sha256 +from io import BytesIO +from uuid import uuid4 + +from botocore.exceptions import ClientError +from PIL import Image + +from messenger.attachments import AttachmentStorage +from messenger.clean_attachments import GRACE_SECONDS, clean_batch +from messenger.config import Config +from messenger.db import Database +from messenger.errors import Unavailable +from messenger.store import Store + + +def main(): + config = Config.from_env() + db = Database(config) + db.open() + store = Store(config, db, None) + s3 = AttachmentStorage() + user, chat, aid = [str(uuid4()) for _ in range(3)] + key = 'attachments/' + aid + payload = BytesIO() + Image.new('RGB', (2, 2), 'green').save(payload, format='PNG') + data = payload.getvalue() + + try: + with db.cursor() as cur: + cur.execute('INSERT INTO users (id,username,password_hash,display_name) VALUES (%s,%s,%s,%s)', + (user, 'gc_' + user[:8], 'unused', 'Cleanup check')) + cur.execute("INSERT INTO chats (id,title) VALUES (%s,'Cleanup check')", (chat,)) + store.reserve_attachment(aid, chat, user, 'probe.png', 'image/png', + len(data), sha256(data).hexdigest(), key) + + def put_then_fail(object_key): + s3.put(object_key, BytesIO(data), len(data), 'image/png') + raise Unavailable('synthetic failure after successful PUT') + + try: + store.finish_attachment(aid, put_then_fail) + except Unavailable as exc: + assert str(exc) == 'synthetic failure after successful PUT' + else: + raise AssertionError('fault did not run') + obj = s3.client.get_object(Bucket=s3.bucket, Key=key) + with obj['Body'] as body: + assert body.read() == data + with db.cursor() as cur: + cur.execute('SELECT status FROM attachments WHERE id=%s', (aid,)) + assert cur.fetchone()['status'] == 'pending' + cur.execute("UPDATE attachments SET updated_at=now()-make_interval(secs => %s) WHERE id=%s", + (GRACE_SECONDS + 1, aid)) + clean_batch(store, s3) + with db.cursor() as cur: + cur.execute('SELECT 1 FROM attachments WHERE id=%s', (aid,)) + assert cur.fetchone() is None + try: + obj = s3.client.get_object(Bucket=s3.bucket, Key=key) + except ClientError as exc: + assert exc.response['ResponseMetadata']['HTTPStatusCode'] == 404, exc + else: + obj['Body'].close() + raise AssertionError('orphan object still exists') + print('PASS: actual MinIO object existed after failed commit and was removed by cleanup') + finally: + db.close() + + +if __name__ == '__main__': + main() diff --git a/student-tests/telemetry_persistence.py b/student-tests/telemetry_persistence.py new file mode 100644 index 0000000..d24f417 --- /dev/null +++ b/student-tests/telemetry_persistence.py @@ -0,0 +1,20 @@ +import sys,json,time,urllib.parse,subprocess +from pathlib import Path +sys.path.insert(0,'student-tests');import observability as o +trace_id=json.loads(Path('evidence/observability-smoke.json').read_text())['trace_id'] +before=o.trace_spans(trace_id) +o.correlated_logs(trace_id) +stamp=time.time()-10 +query=o.PROM+'/api/v1/query?'+urllib.parse.urlencode({'query':'sum(messenger_http_requests_total)','time':stamp}) +old=o.get(query)['data']['result'] +assert old +for service in ('otel-collector','prometheus','loki','tempo','grafana'): + o.compose('restart',service) +subprocess.run([sys.executable, 'scripts/wait_ready.py'], check=True) +o.eventually(lambda:o.trace_spans(trace_id),timeout=100) +o.correlated_logs(trace_id) +after=o.get(query)['data']['result'] +assert after==old,'historical metric changed after restart' +assert {s['spanId'] for s in before}=={s['spanId'] for s in o.trace_spans(trace_id)},'trace history changed' +Path('evidence/telemetry-persistence.json').write_text(json.dumps({'trace_id':trace_id,'trace_spans_preserved':len(before),'api_worker_publisher_logs_preserved':True,'historical_metric_preserved':True,'restarted':['otel-collector','prometheus','loki','tempo','grafana']},indent=2)+'\n') +print('PASS metrics/logs/traces history survives backend restart') diff --git a/student-tests/upload_load.py b/student-tests/upload_load.py new file mode 100644 index 0000000..5ecf081 --- /dev/null +++ b/student-tests/upload_load.py @@ -0,0 +1,104 @@ +"""Measure sampled cgroup memory during ten simultaneous HTTP uploads.""" +from concurrent.futures import ThreadPoolExecutor +import hashlib +from io import BytesIO +import json +import math +from pathlib import Path +import random +import subprocess +import sys +from threading import Barrier +import time + +from PIL import Image + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client, Failure, arguments, check, run + +COUNT = 10 +SAMPLER = """ +import select, sys, time +from pathlib import Path +usage = Path('/sys/fs/cgroup/memory.current') +limit = Path('/sys/fs/cgroup/memory.max') +cpu = Path('/sys/fs/cgroup/cpu.stat') +print(limit.read_text().strip(), flush=True) +while True: + stats = dict(line.split() for line in cpu.read_text().splitlines()) + print(usage.read_text().strip(), stats['usage_usec'], flush=True) + if select.select([sys.stdin], [], [], .05)[0]: + break +""" + + +def main(): + args = arguments(__doc__) + client = Client(args.base_url, args.ca_file, timeout=60) + alice, _, _, chat = client.bootstrap() + width = 1700 + source = Image.frombytes('RGB', (width, width), random.Random(2026).randbytes(width * width * 3)) + buffer = BytesIO() + source.save(buffer, format='PNG', compress_level=0) + payload = buffer.getvalue() + digest = hashlib.sha256(payload).hexdigest() + check(len(payload) <= 10 * 1024 * 1024, 'fixture exceeds upload size limit') + path = f"/api/v1/chats/{chat['id']}/attachments?filename=load.png" + barrier = Barrier(COUNT + 1) + samplers = {} + + def upload(): + local = Client(args.base_url, args.ca_file, timeout=60) + barrier.wait(timeout=30) + start = time.monotonic() + try: + reply = local.call('POST', path, 201, actor=alice, raw=payload, + headers={'Content-Type': 'image/png'}, schema='Attachment') + check(reply['json']['sha256'] == digest, 'upload hash mismatch') + return {'instance': reply['headers']['X-Instance-Id'], 'id': reply['json']['id'], + 'seconds': time.monotonic() - start} + except Failure as exc: + return {'error': str(exc), 'seconds': time.monotonic() - start} + + try: + for service in ('api-a', 'api-b'): + proc = subprocess.Popen( + ['docker', 'compose', 'exec', '-T', service, 'python', '-u', '-c', SAMPLER], + cwd=ROOT, stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True) + samplers[service] = proc + with ThreadPoolExecutor(max_workers=COUNT) as pool: + futures = [pool.submit(upload) for _ in range(COUNT)] + limits = {service: int(proc.stdout.readline()) for service, proc in samplers.items()} + initial = {service: tuple(map(int, proc.stdout.readline().split())) + for service, proc in samplers.items()} + start = time.monotonic() + barrier.wait(timeout=30) + requests = [future.result() for future in futures] + elapsed = time.monotonic() - start + finally: + for proc in samplers.values(): + if proc.poll() is None: + proc.stdin.close() + resources = {} + for service, proc in samplers.items(): + samples = [initial[service]] + [tuple(map(int, line.split())) for line in proc.stdout] + check(proc.wait(timeout=10) == 0 and len(samples) > 1, 'memory sampler failed') + resources[service] = { + 'baseline_bytes': samples[0][0], 'peak_sample_bytes': max(row[0] for row in samples), + 'limit_bytes': limits[service], 'samples': len(samples), + 'cpu_seconds': (samples[-1][1] - samples[0][1]) / 1e6, + } + successes = [row for row in requests if 'id' in row] + durations = sorted(row['seconds'] for row in successes) + percentiles = {f'p{p}': durations[math.ceil(p / 100 * len(durations)) - 1] + for p in (50, 95, 99)} if durations else {} + print(json.dumps({'concurrency': COUNT, 'payload_bytes': len(payload), 'sha256': digest, + 'elapsed_seconds': elapsed, 'successful_rps': len(successes) / elapsed, + 'latency_seconds': percentiles, 'requests': requests, + 'memory_sample_interval_seconds': .05, 'resources': resources}, indent=2)) + check(len({row['id'] for row in successes}) == COUNT, 'not all uploads succeeded uniquely') + + +if __name__ == '__main__': + run(main) diff --git a/student-tests/worker_variants.py b/student-tests/worker_variants.py new file mode 100644 index 0000000..4200623 --- /dev/null +++ b/student-tests/worker_variants.py @@ -0,0 +1,62 @@ +"""Verify worker effects through HTTP and MinIO with a direct job handoff.""" +import hashlib +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client, arguments, check, run +from image_probe import jpeg_size + + +def main(): + args = arguments(__doc__) + client = Client(args.base_url, args.ca_file) + alice, bob, _, chat = client.bootstrap() + attachment, original = client.attachment(alice, chat) + code = """ +from messenger.attachments import AttachmentStorage +from messenger.config import Config +from messenger.db import Database +from messenger.worker import process +import sys +db = Database(Config.from_env()) +db.open() +try: + with db.cursor() as cur: + cur.execute('SELECT id AS job_id, attachment_id, pipeline_version, attempt ' + 'FROM attachment_jobs WHERE attachment_id=%s', (sys.argv[1],)) + job = cur.fetchone() + storage = AttachmentStorage() + process(db, storage, job) + process(db, storage, job) +finally: + db.close() +""" + subprocess.run(['docker', 'compose', 'exec', '-T', 'api-a', 'python', '-c', + code, attachment['id']], cwd=ROOT, check=True, timeout=30) + path = '/api/v1/attachments/' + attachment['id'] + ready = client.call('GET', path, actor=bob, schema='Attachment')['json'] + check(ready['status'] == 'ready' and len(ready['variants']) == 3, 'expected three variants') + check(client.call('GET', path + '/content', actor=bob)['bytes'] == original, 'original changed') + expected = {'thumbnail': (256, 160, 'image/jpeg'), 'crop': (128, 128, 'image/jpeg'), + 'optimized': (320, 200, 'image/webp')} + for item in ready['variants']: + width, height, mime = expected[item['name']] + reply = client.call('GET', path + '/content?variant=' + item['name'], actor=bob) + data = reply['bytes'] + check((item['width'], item['height'], item['content_type']) == (width, height, mime), + 'incorrect variant metadata') + check(reply['headers'].get_content_type() == mime and len(data) == item['size_bytes'] + and hashlib.sha256(data).hexdigest() == item['sha256'], 'incorrect variant payload') + if mime == 'image/jpeg': + check(jpeg_size(data) == (width, height), 'invalid JPEG dimensions') + else: + check(data[:4] == b'RIFF' and data[8:12] == b'WEBP', 'invalid WebP signature') + print('PASS: original unchanged, thumbnail 256x160, crop 128x128, WebP 320x200') + print('PASS: processing the same committed job twice keeps one set of variants') + + +if __name__ == '__main__': + run(main)