94 changed files with 6746 additions and 35 deletions
+6 -4
View File
@@ -19,12 +19,14 @@ S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
MAX_UPLOAD_BYTES=10485760
MAX_IMAGE_PIXELS=40000000
BROKER_URL_FILE=/run/secrets/broker_url
WORKER_CONCURRENCY=2
WORKER_CONCURRENCY=1
JOB_MAX_ATTEMPTS=5
JOB_TIMEOUT_SECONDS=60
JOB_TIMEOUT_SECONDS=20
STAGING_RETENTION_SECONDS=86400
OTEL_SERVICE_NAME=messenger-api
# Worker must set a distinct service.name.
OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318
OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4317
OTEL_EXPORTER_OTLP_PROTOCOL=grpc
OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab
PUBLIC_PORT=18083
+60
View File
@@ -0,0 +1,60 @@
#
# MTUSI Messenger, lab6 — PostgreSQL + Redis/Valkey HTTP API.
# Base image is pinned by tag *and* digest (multi-arch manifest list), so a rebuild
# on another machine with the same platform yields the same layers.
#
# Runtime dependencies: psycopg2-binary for PostgreSQL, redis for the session cache.
# Development-only tooling lives in requirements-dev.txt and never enters this image.
FROM python:3.12-slim-bookworm@sha256:392307d22300de8b5986851a12d9176dfc0fc073e65bf6523ebd7dcbeb23564e
ENV PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1 \
PYTHONPATH=/app/src
# Pinned runtime dependencies (versions fixed for reproducible builds).
RUN pip install --no-cache-dir \
psycopg2-binary==2.9.11 \
redis==7.0.1 \
bcrypt==4.0.1 \
boto3==1.42.97 \
Pillow==12.3.0 \
pika==1.3.2 \
opentelemetry-sdk==1.38.0 \
opentelemetry-exporter-otlp-proto-grpc==1.38.0
# Unprivileged runtime account; the image's default root user is never used at runtime.
RUN groupadd --gid 10001 app \
&& useradd --uid 10001 --gid 10001 --create-home --shell /usr/sbin/nologin app
ARG APP_REVISION=local
ENV APP_REVISION=$APP_REVISION
WORKDIR /app
COPY --chown=10001:10001 src/ /app/src/
COPY --chown=10001:10001 migrations/ /app/migrations/
USER 10001:10001
# Configuration comes from the environment; secrets are read from files at runtime.
ENV APP_HOST=0.0.0.0 \
APP_PORT=8080 \
APP_ENV=lab \
LOG_LEVEL=info \
SHUTDOWN_GRACE_SECONDS=8 \
SESSION_TTL_SECONDS=3600 \
SESSION_STORE=redis \
DB_POOL_MIN=2 \
DB_POOL_MAX=10 \
DB_CONNECT_TIMEOUT=5 \
DB_STATEMENT_TIMEOUT=10
EXPOSE 8080
# Python (not curl) keeps the image slim; /health/ready is the synchronous-path probe.
HEALTHCHECK --interval=10s --timeout=3s --start-period=3s --retries=3 \
CMD ["python", "-c", "import os, urllib.request; urllib.request.urlopen('http://127.0.0.1:%s/health/ready' % os.environ.get('APP_PORT', '8080'), timeout=2)"]
STOPSIGNAL SIGTERM
# The HTTP process is PID 1 and handles SIGTERM itself (graceful drain, exit code 0).
CMD ["python", "-m", "messenger"]
+5 -1
View File
@@ -1,5 +1,5 @@
PYTHON ?= python3
BASE_URL ?= http://localhost:8080
BASE_URL ?= http://localhost:18083
CA_FILE ?=
ACTION_SCRIPT ?=
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
@@ -28,3 +28,7 @@ test-persistence:
test-failover:
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
.PHONY: test-observability
test-observability:
$(PYTHON) student-tests/observability.py smoke
+153 -22
View File
@@ -1,37 +1,168 @@
# Отчёт — лабораторная 6
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
- Автор — Любченко Никита, БПИ2303.
- Целевая оценка — 4. Трек 5 не заявлен.
- Использованы Python 3.12, OpenTelemetry SDK 1.38.0, semantic conventions 0.59b0, Collector contrib 0.123.0, Prometheus 3.2.1, Loki 3.4.2, Tempo 2.7.2, Grafana 11.6.0 и компоненты lab5. ИИ использован при разработке и проверке реализации.
- Ссылку на PR добавляет автор после публикации. Результаты проверок приведены в evidence/.
## Запуск из чистого clone
## Запуск из чистого клона
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
Требуются Docker Engine с Compose, Make и Python 3.9+ для публичных проверок. Первая сборка обращается к реестрам образов, PyPI и исходникам MinIO. Приложение работает на Python 3.12. Изображения обрабатываются Pillow. Go используется при сборке MinIO и mc из исходников.
```sh
cp -n .env.example .env
python3 scripts/bootstrap.py
docker compose --env-file .env config --quiet
docker compose --env-file .env build
docker compose --env-file .env up -d --no-build
python3 scripts/manage.py status
python3 scripts/wait_ready.py
make check
make test
python3 -m venv .venv
.venv/bin/python -m pip install -r requirements-dev.txt
make validate PYTHON=.venv/bin/python
make test-observability
```
API доступен через Nginx на http://localhost:18083. Grafana — http://localhost:13000, логин admin. Пароль читается из secrets/grafana_password. Prometheus — http://localhost:19090. Loki и Tempo имеют loopback HTTP-порты 13100 и 13200 для воспроизводимых проверок. OTLP и внутренние API-порты на хост не опубликованы. Dashboard Messenger и три источника данных подключаются provisioning автоматически.
Bootstrap создаёт локальные secrets с ограниченными правами, сохраняет уже существующие значения и не печатает их. `.env`, `secrets/` и `.venv/` исключены из Git. Безопасная остановка — `docker compose down`. Намеренное удаление всех учебных данных текущего проекта — `docker compose down -v`.
## Архитектура и решения
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
Перенесены Nginx, две API-реплики, PostgreSQL, Redis, приватный MinIO, RabbitMQ, publisher, worker и cleaner из lab5. PostgreSQL остаётся источником истины. API сохраняет исходник в S3 и фиксирует queued, job и outbox одной транзакцией перед ответом 202. Издатель использует publisher confirms. Воркер сохраняет три варианта, фиксирует ready, затем подтверждает сообщение брокеру. Lease, retry/backoff, fencing по attempt, replay и очистка сохранены.
## Доказательства критериев
Добавлен путь API/publisher/worker → OTLP/gRPC → Collector → Prometheus/Loki/Tempo → Grafana. Prometheus хранит числовые временные ряды, Loki — структурные логи, Tempo — трассы. Grafana выполняет запросы к этим источникам. У хранилищ собственные volumes, retention сигналов составляет 72 часа. Collector обрабатывает и экспортирует данные.
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
Миграция 0006 добавляет trace_context jsonb в attachment_jobs. Контекст сохраняется вместе с заданием. Издатель создаёт producer span с родителем из сохранённого контекста и передаёт traceparent/tracestate в AMQP headers. Consumer продолжает этот контекст. Повторные попытки имеют общий исходный trace и отдельные producer/consumer spans. Старые задания получают пустой контекст и обрабатываются обычным способом.
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
Ресурсы содержат service.name (messenger-api, messenger-publisher, messenger-worker), service.version, service.instance.id и deployment.environment.name=lab. Логи включают timestamp, severity, event, request_id и trace_id/span_id при активном span. HTTP возвращает X-Request-Id и дополнительный X-Trace-Id. SQL-тексты, значения параметров, Authorization, пароли, signed URL, тела сообщений и изображения в инструментацию не передаются. OTLP bridge отбрасывает полный текст исключений и traceback. Технический job.id допустим для синтетических traces/logs. В metric labels его нет.
## Эксперименты и отказы
## Сигналы и запросы
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
| Сигнал | Реальное имя в Prometheus | Единица |
| --- | --- | --- |
| HTTP count | messenger_http_requests_total | запросы |
| HTTP duration | messenger_http_duration_seconds_bucket | секунды, histogram |
| Jobs | messenger_jobs_processed_total | попытки, outcome success/error/stale |
| Processing duration | messenger_job_duration_seconds_bucket | секунды одной попытки |
| Accepted-to-ready | messenger_attachment_time_to_ready_seconds_bucket | секунды до фиксации ready |
| Backlog | messenger_queue_depth | queued + processing, включая outbox/retry |
| Broker ready | messenger_queue_broker_ready | ожидающие доставки сообщения RabbitMQ |
| Oldest age | messenger_queue_oldest_age_seconds | возраст старейшего незавершённого задания |
| Backlog freshness | messenger_queue_observed_at_seconds | Unix timestamp последнего опроса |
| DB pool | messenger_db_pool_used, messenger_db_pool_waiting, messenger_db_pool_capacity | соединения / запросы на получение |
| Pool exhaustion | messenger_db_pool_exhausted_total | отказы получения соединения |
| Pool acquisition | messenger_db_pool_acquire_seconds_bucket | секунды |
| Worker | messenger_worker_busy, messenger_worker_capacity | занятые / доступные обработчики |
| Memory | messenger_process_memory_bytes | текущий RSS процесса, байты |
| CPU | messenger_process_cpu_seconds_total | накопленное процессорное время, секунды |
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
HTTP labels содержат method, route template и status_class. Health исключён из бизнес-счётчиков. Неизвестные маршруты сводятся к unmatched. Histogram boundaries — 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2, 5, 10, 30, 60, 120 секунд и +Inf. Pool waiting измеряет поток во время вызова getconn. Пул psycopg2 немедленно отказывает при исчерпании ёмкости, длительного ожидания свободного соединения в нём нет.
## Ограничения и следующие шаги
Time to ready начинается с clock_timestamp при регистрации задания после PUT исходника, непосредственно перед COMMIT принятия, и заканчивается после COMMIT ready. Начальная точка приближает границу durable acceptance с точностью до оставшейся части транзакции. Метрика записывается только после успешного COMMIT результата. При аварии процесса сразу после COMMIT отдельное наблюдение может отсутствовать. Бизнес-состояние восстанавливается из PostgreSQL.
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
Запросы dashboard сохранены в infra/grafana/dashboards/messenger.json. Например, p95 всех реплик вычисляется из объединённых buckets.
```promql
sum by (method, route) (rate(messenger_http_requests_total[1m]))
histogram_quantile(0.95, sum by (le) (rate(messenger_http_duration_seconds_bucket[1m])))
rate(messenger_process_cpu_seconds_total[1m])
messenger_queue_oldest_age_seconds
```
p50 и p99 отличаются аргументом histogram_quantile. Метрика CPU rate выражена в занятых ядрах. Backlog, processing duration и end-to-end time показывают разные части обработки. Логи ищутся в Loki запросом `{service_name=~"messenger-.*"} |= "<trace_id>"`. Tempo принимает поиск по trace ID и TraceQL `{resource.service.name = "messenger-worker"}`.
## SLI, SLO и алерты
SLI — доля бизнес-запросов без 5xx за скользящие 5 минут. SLO ≥99%. 4xx входят в знаменатель и не считаются отказом сервера. При отсутствии запросов SLI не оценивается. Этот SLI отражает запросы, дошедшие до API. Ошибки самого Nginx и отсутствие входящего трафика требуют внешнего probe.
HighApiErrorRate срабатывает при доле 5xx >1% за 1 минуту, for 6 секунд. QueueLagGrowing срабатывает при oldest age >15 секунд и backlog >0, for 6 секунд. QueueObservationStale показывает, что publisher перестал обновлять состояние очереди. Правила сохранены в infra/prometheus/alerts.yaml. Runbook находится в infra/runbook.md. Локальные firing/resolved проверяются через Prometheus API.
## Ограничение памяти и отказ телеметрии
У SDK очереди до 256 spans и 256 logs на процесс, batch 64, export timeout 1 секунда. Метрики экспортируются каждые 3 секунды. SDK передаёт телеметрию фоновыми потоками.
Collector применяет memory_limiter 128 MiB со spike 32 MiB, batch до 512 записей, очереди Loki/Tempo до 128 batches каждая, два consumer на экспортёр. Retry ограничен 10 секундами, timeout отдельного экспорта 2 секунды. Контейнер ограничен 192 MiB. При недоступности backend часть наблюдений может теряться. Prometheus забирает данные pull-запросами, остановка Prometheus создаёт пробел в истории scrape. Для логов и трасс проверяется отказ push backend Tempo.
Внутренние accepted/sent/send_failed/enqueue_failed counters Collector позволяют увидеть деградацию экспорта. send_failed показывает отказ экспорта. Потери в контролируемом окне дополнительно подсчитываются по dropped_items в журнале Collector. Точное число уникальных потерь во всей цепочке по одному counter не вычисляется.
## Проверки и эксперименты
Файлы stage1-* относятся к переносу основы до включения полного стека наблюдаемости. Итоговые доказательства приведены ниже.
| Критерий | Сценарий / результат | Доказательство |
| --- | --- | --- |
| Структура и схема API | make check, make validate | evidence/check.txt, evidence/validate.txt |
| Бизнес-API с телеметрией | make test | evidence/smoke.txt |
| Метрики, логи и реальные traces API/worker | make test-observability, проверены имена сигналов и логи трёх процессов | evidence/observability-smoke.json |
| W3C propagation и связь S3/DB | Parent-child spans, сохранён tracestate, внешний traceparent принят API | evidence/observability-smoke.json |
| Retry в исходном trace | Две consumer-попытки, ready после восстановления S3 | evidence/observability-retry.json |
| Dashboard и datasources | Provisioning 14 панелей, проверены все PromQL и поиск Tempo через Grafana | evidence/observability-smoke.json, infra/grafana |
| Percentiles, backlog, oldest age, DB pool, worker saturation, CPU/RAM | Histogram buckets и gauge/counters, единицы указаны выше | infra/grafana/dashboards/messenger.json |
| SLI/SLO и локальные alerts | promtool, firing/resolved для error rate и queue lag | evidence/alert-config.txt, evidence/observability-worker-stop.json, evidence/observability-db-outage.json |
| Остановка worker | Алерт firing через 30,1 с, после восстановления задание ready, алерт resolved | evidence/observability-worker-stop.json |
| Задержка PostgreSQL | 8 запросов получили 503, client span показывает ошибку БД, алерт firing/resolved | evidence/observability-db-outage.json |
| Отказ Tempo | 138 успешных сообщений, вложение ready, потеря 248 spans по dropped_items, без OOM/рестартов | evidence/observability-backend-outage.json |
| История сигналов после перезапуска | Сохранены исходная trace, логи трёх процессов и historical metric sample | evidence/telemetry-persistence.json |
| Сохранность бизнес-данных и отказ реплики | Публичные persistence/failover | evidence/persistence.txt, evidence/failover.txt |
| Полная HTTP duration | Регрессионная проверка включает отправку тела | evidence/http-timing.txt |
### Нагрузка
Docker Engine 29.5.3, Compose 5.1.4, aarch64, 10 доступных CPU и около 7,75 GiB RAM в VM. Лимиты API — по 0,5 CPU/256 MiB, worker — 1 CPU/768 MiB, Collector — 0,5 CPU/192 MiB. Параметры среды сохранены в evidence/environment.json.
`make load` выполнял closed-loop POST сообщений 30 секунд, concurrency 4, без автоматических повторов. Выполнено 14 048 успешных запросов, ошибок 0, 467,941 RPS. Клиентские latency успешных запросов — p50 5,119 ms, p95 33,476 ms, p99 81,147 ms. Это один прогон, отдельного прогрева не было. Percentiles клиента охватывают только успехи и подвержены coordinated omission. Они дополняют histogram dashboard. Результат не заявляется как максимальная пропускная способность. Raw summary — evidence/load-output.txt.
### Инцидент 1. Остановленный worker
При продолжающемся приёме upload возраст незавершённого задания превысил 15 секунд, backlog оставался положительным. QueueLagGrowing перешёл в firing примерно через 30,1 секунды после загрузки. По метрикам видно ожидание обработки, producer span опубликован, consumer появляется после запуска worker. После восстановления job достигла ready, backlog уменьшился, алерт resolved. Диагностика выполняется по сохранённым возрасту/глубине очереди, состоянию alert и связанной trace. Сценарий — `python3 student-tests/observability.py worker-stop`.
### Инцидент 2. Ожидание PostgreSQL
Сценарий удерживает ACCESS EXCLUSIVE lock таблицы users 35 секунд. Восемь GET /users/me завершились 503 вследствие statement_timeout. В trace длительный postgresql.transaction имеет error status, что локализует задержку в БД. HighApiErrorRate перешёл в firing, после снятия блокировки запросы восстановились, alert resolved. Сценарий — `python3 student-tests/observability.py db-outage`.
Во время разработки выявлено, что retry HTTP 503 в Nginx временно исключал обе реплики при общем тайм-ауте БД и приводил к 502. Теперь Nginx передаёт 503 клиенту. Переключение при сетевых ошибках и active readiness сохранено и проверено публичным failover. Старая и новая серии gauge после рестарта publisher различаются по service.instance.id. Dashboard и alerts используют freshness, чтобы устаревший экземпляр не создавал ложный queue alert.
### Недоступность telemetry backend
Tempo был остановлен на время последовательной записи сообщений и обработки изображения. SDK/Collector продолжили ограниченный экспорт с retry и drops. Успешно записано 138 сообщений, изображение стало ready. Collector зарегистрировал 248 dropped spans. Это число явно отброшенных записей данного exporter за окно, общая оценка уникальных потерь по всей системе не заявляется. Send-failed сравнивался до/после отказа.
Наблюдаемый RSS Collector находился примерно в пределах 92–97 MiB при лимите 192 MiB, API — около 74–76 MiB, worker — около 63 MiB, publisher — около 34 MiB. Контейнеры работали без OOM и автоматических перезапусков. Измерение длилось около 40 секунд и подтверждает поведение данного сценария. Конфигурация задаёт конечные очереди и memory limiter для более длительной недоступности. После запуска Tempo найдены новые связанные trace и логи. Сценарий — `python3 student-tests/observability.py backend-outage`.
### Воспроизведение
```sh
make test-observability
docker compose exec -T api-a python - < student-tests/http_timing.py
python3 student-tests/observability.py worker-stop
python3 student-tests/observability.py db-outage
python3 student-tests/observability.py retry
python3 student-tests/observability.py backend-outage
python3 student-tests/telemetry_persistence.py
make test-persistence ACTION_SCRIPT=scripts/restart.sh
make test-failover ACTION_SCRIPT=scripts/stop-one.sh
python3 scripts/manage.py restore
```
Fault-сценарии выполняются по одному. Для проверки retry worker запускается через docker start, чтобы Compose не поднял остановленный S3 как dependency. PostgreSQL lock освобождается по завершении ограниченного pg_sleep. Все проверки используют синтетические данные.
## Ограничения
Стенд учебный, single-node. PostgreSQL, MinIO, RabbitMQ и telemetry backend представлены одиночными экземплярами. Авторизация Grafana локальная, TLS относится к следующей лабораторной. Sampling по умолчанию 100%, исследовательские треки на 5 не заявлены. Старые вспомогательные student-tests предыдущих лабораторных могут требовать специальных условий, например остановленного worker. Для lab6 использовать make test и student-tests/observability.py.
## Источники
- https://opentelemetry.io/docs/collector/configuration/
- https://opentelemetry.io/docs/concepts/context-propagation/
- https://www.w3.org/TR/trace-context/
- https://grafana.com/docs/loki/latest/send-data/otel/
- https://prometheus.io/docs/practices/histograms/
- https://grafana.com/docs/grafana/latest/administration/provisioning/
## Соответствие переменных окружения
Compose задаёт служебные адреса и пути secrets внутри контейнеров. PUBLIC_PORT и JOB_MAX_ATTEMPTS читаются из локального .env. API использует SESSION_STORE=redis. DATABASE_URL_FILE, CACHE_URL_FILE, CURSOR_SECRET_FILE и S3_*_FILE указывают на secrets. Издатель и worker получают RABBITMQ_PASSWORD_FILE, подключаются к rabbitmq:5672 под локальным пользователем lab6. BROKER_URL_FILE из исходной заготовки заменён этой настройкой.
В данной реализации один worker-процесс обрабатывает одно задание с prefetch 1. WORKER_CONCURRENCY=1 описывает эту ёмкость. Тайм-аут попытки фиксирован на 20 секундах, lease на 60 секундах. MAX_UPLOAD_BYTES, MAX_IMAGE_PIXELS и STAGING_RETENTION_SECONDS соответствуют константам перенесённого приложения. Изменять эти поля .env отдельно от кода не следует. OTEL_SERVICE_NAME и endpoint задаются Compose отдельно для каждого процесса, используется grpc на 4317. Параметры SDK/Collector приведены выше и закреплены в коде и YAML.
+341
View File
@@ -0,0 +1,341 @@
name: mtusi-lab6
x-api: &api
build:
context: .
args:
APP_REVISION: ${API_IMAGE_TAG:-local}
image: mtusi-lab6-api:${API_IMAGE_TAG:-local}
environment: &api-env
APP_HOST: 0.0.0.0
APP_PORT: '8080'
APP_ENV: lab6
OTEL_SERVICE_NAME: messenger-api
OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4317
OTEL_EXPORTER_OTLP_PROTOCOL: grpc
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
CURSOR_SECRET_FILE: /run/secrets/cursor_secret
DATABASE_URL_FILE: /run/secrets/database_url
CACHE_URL_FILE: /run/secrets/cache_url
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
SESSION_STORE: redis
SHUTDOWN_GRACE_SECONDS: '8'
DB_POOL_MIN: '2'
DB_POOL_MAX: '16'
DB_CONNECT_TIMEOUT: '1'
DB_STATEMENT_TIMEOUT: '4'
S3_ENDPOINT: http://s3:9000
S3_REGION: us-east-1
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
secrets: [database_url, cache_url, cursor_secret, s3_access_key_id, s3_secret_access_key]
depends_on:
s3-bootstrap:
condition: service_completed_successfully
migrate:
condition: service_completed_successfully
redis:
condition: service_healthy
networks: [backend]
cpus: 0.50
mem_limit: 256m
stop_grace_period: 9s
healthcheck:
test: [CMD, python, -c, "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=1)"]
interval: 2s
timeout: 2s
retries: 2
start_period: 3s
# Deliberately no auto-restart: failover hook must leave its target stopped.
restart: 'no'
services:
otel-collector:
image: otel/opentelemetry-collector-contrib:0.123.0
command: [--config=/etc/otel/collector.yaml]
volumes: [./infra/otel/collector.yaml:/etc/otel/collector.yaml:ro]
networks: [backend, telemetry]
mem_limit: 192m
cpus: 0.50
restart: unless-stopped
prometheus:
image: prom/prometheus:v3.2.1
command: [--config.file=/etc/prometheus/prometheus.yaml, --storage.tsdb.retention.time=72h]
volumes:
- ./infra/prometheus:/etc/prometheus:ro
- prometheus_data:/prometheus
networks: [telemetry, frontend]
ports: ['127.0.0.1:19090:9090']
mem_limit: 384m
restart: unless-stopped
loki:
image: grafana/loki:3.4.2
command: [-config.file=/etc/loki/loki.yaml]
volumes:
- ./infra/loki/loki.yaml:/etc/loki/loki.yaml:ro
- loki_data:/loki
networks: [telemetry, frontend]
ports: ['127.0.0.1:13100:3100']
mem_limit: 384m
restart: unless-stopped
tempo:
image: grafana/tempo:2.7.2
command: [-config.file=/etc/tempo/tempo.yaml]
volumes:
- ./infra/tempo/tempo.yaml:/etc/tempo/tempo.yaml:ro
- tempo_data:/var/tempo
networks: [telemetry, frontend]
ports: ['127.0.0.1:13200:3200']
mem_limit: 384m
restart: unless-stopped
grafana:
image: grafana/grafana:11.6.0
environment:
GF_SECURITY_ADMIN_USER: admin
GF_SECURITY_ADMIN_PASSWORD__FILE: /run/secrets/grafana_password
GF_USERS_ALLOW_SIGN_UP: 'false'
GF_ANALYTICS_REPORTING_ENABLED: 'false'
GF_ANALYTICS_CHECK_FOR_UPDATES: 'false'
secrets: [grafana_password]
volumes:
- grafana_data:/var/lib/grafana
- ./infra/grafana/provisioning:/etc/grafana/provisioning:ro
- ./infra/grafana/dashboards:/var/lib/grafana/dashboards:ro
networks: [telemetry, frontend]
ports: ['127.0.0.1:13000:3000']
mem_limit: 256m
restart: unless-stopped
rabbitmq:
image: rabbitmq:4.1.4-alpine
hostname: rabbitmq
entrypoint: [/bin/sh, -ec]
command:
- |
export RABBITMQ_DEFAULT_USER=lab6
RABBITMQ_DEFAULT_PASS=$$(cat /run/secrets/rabbitmq_password)
export RABBITMQ_DEFAULT_PASS
exec docker-entrypoint.sh rabbitmq-server
secrets: [rabbitmq_password]
volumes: [rabbitmq_data:/var/lib/rabbitmq]
networks: [backend]
healthcheck:
test: [CMD, rabbitmq-diagnostics, -q, check_port_connectivity]
interval: 5s
timeout: 5s
retries: 12
outbox-publisher:
image: mtusi-lab6-api:${API_IMAGE_TAG:-local}
command: [python, -m, messenger.publish_jobs]
environment:
DATABASE_URL_FILE: /run/secrets/database_url
RABBITMQ_PASSWORD_FILE: /run/secrets/rabbitmq_password
OTEL_SERVICE_NAME: messenger-publisher
OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4317
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
DB_POOL_MIN: '1'
DB_POOL_MAX: '2'
secrets: [database_url, rabbitmq_password]
depends_on:
migrate:
condition: service_completed_successfully
rabbitmq:
condition: service_healthy
networks: [backend]
restart: unless-stopped
healthcheck:
disable: true
s3:
image: mtusi-lab6-s3:2025-09-07
build: &s3-build
context: .
dockerfile: infra/Dockerfile.s3
entrypoint: [/bin/sh, -ec]
command:
- |
MINIO_ROOT_USER=$$(cat /run/secrets/s3_root_user)
MINIO_ROOT_PASSWORD=$$(cat /run/secrets/s3_root_password)
export MINIO_ROOT_USER MINIO_ROOT_PASSWORD
exec minio server /data
environment:
MINIO_BROWSER: "off"
secrets: [s3_root_user, s3_root_password]
volumes: [s3_data:/data]
networks: [backend]
s3-bootstrap:
image: mtusi-lab6-s3:2025-09-07
build: *s3-build
entrypoint: [/bin/sh, /bootstrap.sh]
command: []
environment:
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
secrets: [s3_root_user, s3_root_password, s3_access_key_id, s3_secret_access_key]
volumes: [./infra/s3-bootstrap.sh:/bootstrap.sh:ro]
depends_on: [s3]
networks: [backend]
postgres:
image: postgres:15-alpine@sha256:54e0dd8e8cddcdbb9156c2486eed5a43eccb7214d7c6828097d41ce049fb3984
environment:
POSTGRES_DB: messenger
POSTGRES_USER: messenger
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
secrets: [postgres_password]
volumes: [pg_data:/var/lib/postgresql/data]
networks: [backend]
cpus: 1.0
mem_limit: 512m
healthcheck:
test: [CMD-SHELL, 'pg_isready -U messenger -d messenger']
interval: 2s
timeout: 2s
retries: 20
redis:
image: redis:7-alpine@sha256:8da19c7399dca900c6822a1cf30f9d2285585e2c4b457ff4095b0f4500054d3f
command: [redis-server, --appendonly, 'yes', --maxmemory, 96mb, --maxmemory-policy, allkeys-lru]
volumes: [redis_data:/data]
networks: [backend]
cpus: 0.25
mem_limit: 128m
healthcheck:
test: [CMD, redis-cli, ping]
interval: 2s
timeout: 1s
retries: 10
migrate:
build:
context: .
args:
APP_REVISION: ${API_IMAGE_TAG:-local}
image: mtusi-lab6-api:${API_IMAGE_TAG:-local}
command: [python, -m, messenger.migrate]
environment: *api-env
secrets: [database_url, cache_url, cursor_secret]
depends_on:
postgres:
condition: service_healthy
networks: [backend]
cpus: 0.50
mem_limit: 256m
restart: 'no'
healthcheck:
disable: true
attachment-cleaner:
image: mtusi-lab6-api:${API_IMAGE_TAG:-local}
build:
context: .
args:
APP_REVISION: ${API_IMAGE_TAG:-local}
command: [python, -m, messenger.clean_attachments]
environment: *api-env
secrets: [database_url, s3_access_key_id, s3_secret_access_key]
depends_on:
s3-bootstrap:
condition: service_completed_successfully
migrate:
condition: service_completed_successfully
networks: [backend]
cpus: 0.25
mem_limit: 128m
restart: unless-stopped
healthcheck:
disable: true
worker:
image: mtusi-lab6-api:${API_IMAGE_TAG:-local}
command: [python, -m, messenger.worker]
environment:
<<: *api-env
OTEL_SERVICE_NAME: messenger-worker
RABBITMQ_PASSWORD_FILE: /run/secrets/rabbitmq_password
DB_POOL_MIN: '1'
DB_POOL_MAX: '2'
secrets: [database_url, s3_access_key_id, s3_secret_access_key, rabbitmq_password]
depends_on:
outbox-publisher:
condition: service_started
s3-bootstrap:
condition: service_completed_successfully
networks: [backend]
cpus: 1.0
mem_limit: 768m
restart: unless-stopped
healthcheck:
disable: true
api-a:
<<: *api
api-b:
<<: *api
nginx:
image: nginx:1.28.2-alpine@sha256:5b4900b042ccfa8b0a73df622c3a60f2322faeb2be800cbee5aa7b44d241649e
entrypoint: [/bin/sh, -ec]
command:
- 'cp /etc/nginx/lab/upstreams.conf /etc/nginx/upstreams.conf; /bin/sh /etc/nginx/lab/healthwatch.sh & exec nginx -g "daemon off;"'
volumes:
- ./infra/nginx.conf:/etc/nginx/nginx.conf:ro
- ./infra:/etc/nginx/lab:ro
ports: ['127.0.0.1:${PUBLIC_PORT:-18083}:8080']
networks: [frontend, backend]
depends_on:
api-a:
condition: service_healthy
api-b:
condition: service_healthy
cpus: 0.50
mem_limit: 128m
stop_grace_period: 10s
healthcheck:
test: [CMD-SHELL, 'kill -0 $$(cat /tmp/healthwatch.pid) && wget -T 1 -q -O /dev/null http://127.0.0.1:8080/health/ready']
interval: 2s
timeout: 2s
retries: 3
networks:
telemetry:
internal: true
frontend:
backend:
internal: true
volumes:
prometheus_data:
loki_data:
tempo_data:
grafana_data:
pg_data:
redis_data:
s3_data:
rabbitmq_data:
secrets:
grafana_password:
file: ./secrets/grafana_password
postgres_password:
file: ./secrets/postgres_password
database_url:
file: ./secrets/database_url
cache_url:
file: ./secrets/cache_url
cursor_secret:
file: ./secrets/cursor_secret
s3_root_user:
file: ./secrets/s3_root_user
s3_root_password:
file: ./secrets/s3_root_password
s3_access_key_id:
file: ./secrets/s3_access_key_id
s3_secret_access_key:
file: ./secrets/s3_secret_access_key
rabbitmq_password:
file: ./secrets/rabbitmq_password
+7
View File
@@ -0,0 +1,7 @@
Checking /etc/prometheus/prometheus.yaml
SUCCESS: 1 rule files found
SUCCESS: /etc/prometheus/prometheus.yaml is valid prometheus config file syntax
Checking /etc/prometheus/alerts.yaml
SUCCESS: 3 rules found
+65
View File
@@ -0,0 +1,65 @@
[
{
"name": "/mtusi-lab6-postgres-1",
"started_at": "2026-11-20T17:27:24.629073504Z",
"running": true,
"restart_count": 0,
"memory_limit": 536870912
},
{
"name": "/mtusi-lab6-redis-1",
"started_at": "2026-11-20T17:27:24.710887337Z",
"running": true,
"restart_count": 0,
"memory_limit": 134217728
},
{
"name": "/mtusi-lab6-s3-1",
"started_at": "2026-11-20T17:27:24.605233629Z",
"running": true,
"restart_count": 0,
"memory_limit": 0
},
{
"name": "/mtusi-lab6-rabbitmq-1",
"started_at": "2026-11-20T17:27:25.649083629Z",
"running": true,
"restart_count": 0,
"memory_limit": 0
},
{
"name": "/mtusi-lab6-api-a-1",
"started_at": "2026-11-20T17:27:47.844003625Z",
"running": true,
"restart_count": 0,
"memory_limit": 268435456
},
{
"name": "/mtusi-lab6-api-b-1",
"started_at": "2026-11-20T17:27:25.985424213Z",
"running": true,
"restart_count": 0,
"memory_limit": 268435456
},
{
"name": "/mtusi-lab6-nginx-1",
"started_at": "2026-11-20T17:27:24.766067837Z",
"running": true,
"restart_count": 0,
"memory_limit": 134217728
},
{
"name": "/mtusi-lab6-worker-1",
"started_at": "2026-11-20T17:27:28.139978422Z",
"running": true,
"restart_count": 2,
"memory_limit": 805306368
},
{
"name": "/mtusi-lab6-outbox-publisher-1",
"started_at": "2026-11-20T17:27:25.619581296Z",
"running": true,
"restart_count": 0,
"memory_limit": 0
}
]
+5
View File
@@ -0,0 +1,5 @@
{"service": "postgres", "id": "914b5dae50c3", "instance": "914b5dae50c3", "running": true, "started_at": "2026-11-21T08:17:27.913170257Z", "health": "healthy", "image_id": "sha256:54e0dd8e8cddcdbb9156c2486eed5a43eccb7214d7c6828097d41ce049fb3984", "cpu_limit": 1.0, "memory_limit_bytes": 536870912, "published_ports": {}}
{"service": "redis", "id": "331192624ebd", "instance": "331192624ebd", "running": true, "started_at": "2026-11-21T08:17:27.721573173Z", "health": "healthy", "image_id": "sha256:8da19c7399dca900c6822a1cf30f9d2285585e2c4b457ff4095b0f4500054d3f", "cpu_limit": 0.25, "memory_limit_bytes": 134217728, "published_ports": {}}
{"service": "api-a", "id": "e305493d2118", "instance": "e305493d2118", "running": true, "started_at": "2026-11-21T08:17:28.125710257Z", "health": "healthy", "image_id": "sha256:f1b5f50a8470841ce56c6b33e182594568c228e3c541e5b8b945f7d4eee1aa76", "cpu_limit": 0.5, "memory_limit_bytes": 268435456, "published_ports": {}}
{"service": "api-b", "id": "e46ce208e44b", "instance": "e46ce208e44b", "running": true, "started_at": "2026-11-21T08:17:50.183420711Z", "health": "healthy", "image_id": "sha256:f1b5f50a8470841ce56c6b33e182594568c228e3c541e5b8b945f7d4eee1aa76", "cpu_limit": 0.5, "memory_limit_bytes": 268435456, "published_ports": {}}
{"service": "nginx", "id": "7c8111b7fbdb", "instance": "7c8111b7fbdb", "running": true, "started_at": "2026-11-21T08:17:27.962736673Z", "health": "healthy", "image_id": "sha256:5b4900b042ccfa8b0a73df622c3a60f2322faeb2be800cbee5aa7b44d241649e", "cpu_limit": 0.5, "memory_limit_bytes": 134217728, "published_ports": {"8080/tcp": [{"HostIp": "127.0.0.1", "HostPort": "18083"}]}}
+2
View File
@@ -0,0 +1,2 @@
python3 scripts/check_contract.py
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
+5
View File
@@ -0,0 +1,5 @@
{
"promql_panels_valid": 12,
"tempo_search_via_grafana_proxy": true,
"traces_found": 20
}
+8
View File
@@ -0,0 +1,8 @@
{
"host_architecture": "arm64",
"docker_version": "29.5.3",
"docker_architecture": "aarch64",
"docker_cpus": 10,
"docker_memory_bytes": 8321515520,
"compose_version": "5.1.4"
}
+4
View File
@@ -0,0 +1,4 @@
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:18083"
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
{"observed_instances_before": 2, "successful_writes": 130, "failed_attempts": 0, "first_recovery_seconds": 0.78}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
+16
View File
@@ -0,0 +1,16 @@
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
mtusi-lab6-api-a-1 mtusi-lab6-api:local "python -m messenger" api-a 7 minutes ago Up About a minute (healthy) 8080/tcp
mtusi-lab6-api-b-1 mtusi-lab6-api:local "python -m messenger" api-b 7 minutes ago Up About a minute (healthy) 8080/tcp
mtusi-lab6-attachment-cleaner-1 mtusi-lab6-api:local "python -m messenger…" attachment-cleaner 7 minutes ago Up 6 minutes 8080/tcp
mtusi-lab6-grafana-1 grafana/grafana:11.6.0 "/run.sh" grafana 6 minutes ago Up 2 minutes 127.0.0.1:13000->3000/tcp
mtusi-lab6-loki-1 grafana/loki:3.4.2 "/usr/bin/loki -conf…" loki 15 hours ago Up 2 minutes 127.0.0.1:13100->3100/tcp
mtusi-lab6-nginx-1 nginx:1.28.2-alpine@sha256:5b4900b042ccfa8b0a73df622c3a60f2322faeb2be800cbee5aa7b44d241649e "/bin/sh -ec 'cp /et…" nginx 15 hours ago Up About a minute (healthy) 127.0.0.1:18083->8080/tcp
mtusi-lab6-otel-collector-1 otel/opentelemetry-collector-contrib:0.123.0 "/otelcol-contrib --…" otel-collector 15 hours ago Up 2 minutes 4317-4318/tcp, 55678-55679/tcp
mtusi-lab6-outbox-publisher-1 mtusi-lab6-api:local "python -m messenger…" outbox-publisher 7 minutes ago Up About a minute 8080/tcp
mtusi-lab6-postgres-1 postgres:15-alpine@sha256:54e0dd8e8cddcdbb9156c2486eed5a43eccb7214d7c6828097d41ce049fb3984 "docker-entrypoint.s…" postgres 15 hours ago Up About a minute (healthy) 5432/tcp
mtusi-lab6-prometheus-1 prom/prometheus:v3.2.1 "/bin/prometheus --c…" prometheus 15 hours ago Up 2 minutes 127.0.0.1:19090->9090/tcp
mtusi-lab6-rabbitmq-1 rabbitmq:4.1.4-alpine "/bin/sh -ec 'export…" rabbitmq 15 hours ago Up About a minute (healthy) 4369/tcp, 5671-5672/tcp, 15691-15692/tcp, 25672/tcp
mtusi-lab6-redis-1 redis:7-alpine@sha256:8da19c7399dca900c6822a1cf30f9d2285585e2c4b457ff4095b0f4500054d3f "docker-entrypoint.s…" redis 15 hours ago Up About a minute (healthy) 6379/tcp
mtusi-lab6-s3-1 mtusi-lab6-s3:2025-09-07 "/bin/sh -ec 'MINIO_…" s3 15 hours ago Up About a minute
mtusi-lab6-tempo-1 grafana/tempo:2.7.2 "/tempo -config.file…" tempo 15 hours ago Up 2 minutes 127.0.0.1:13200->3200/tcp
mtusi-lab6-worker-1 mtusi-lab6-api:local "python -m messenger…" worker 7 minutes ago Up About a minute 8080/tcp
+1
View File
@@ -0,0 +1 @@
PASS: HTTP histogram includes response transmission
+18
View File
@@ -0,0 +1,18 @@
python3 tests/load.py --base-url "http://localhost:18083"
{
"lab": 6,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 30,
"elapsed_seconds": 30.021,
"concurrency": 4,
"successful_requests": 14048,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 467.941,
"successful_latency_ms": {
"p50": 5.119,
"p95": 33.476,
"p99": 81.147
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
}
+522
View File
@@ -0,0 +1,522 @@
{
"successful_messages": 138,
"attachment_ready_during_outage": true,
"collector_reported_dropped_spans": 248,
"export_failures_before": [
{
"metric": {},
"value": [
1795248850.545,
"0"
]
}
],
"export_failures_after": [
{
"metric": {},
"value": [
1795248892.504,
"248"
]
}
],
"application_and_collector_stats": [
{
"BlockIO": "0B / 0B",
"CPUPerc": "15.83%",
"Container": "mtusi-lab6-api-a-1",
"ID": "e305493d2118",
"MemPerc": "28.91%",
"MemUsage": "74.02MiB / 256MiB",
"Name": "mtusi-lab6-api-a-1",
"NetIO": "430kB / 941kB",
"PIDs": "16"
},
{
"BlockIO": "0B / 0B",
"CPUPerc": "1.04%",
"Container": "mtusi-lab6-api-b-1",
"ID": "e46ce208e44b",
"MemPerc": "28.88%",
"MemUsage": "73.93MiB / 256MiB",
"Name": "mtusi-lab6-api-b-1",
"NetIO": "546kB / 1.04MB",
"PIDs": "16"
},
{
"BlockIO": "0B / 0B",
"CPUPerc": "0.31%",
"Container": "mtusi-lab6-worker-1",
"ID": "23d809845172",
"MemPerc": "8.23%",
"MemUsage": "63.24MiB / 768MiB",
"Name": "mtusi-lab6-worker-1",
"NetIO": "130kB / 152kB",
"PIDs": "16"
},
{
"BlockIO": "0B / 0B",
"CPUPerc": "1.54%",
"Container": "mtusi-lab6-outbox-publisher-1",
"ID": "ed59d06f4e97",
"MemPerc": "0.42%",
"MemUsage": "33.36MiB / 7.75GiB",
"Name": "mtusi-lab6-outbox-publisher-1",
"NetIO": "448kB / 794kB",
"PIDs": "16"
},
{
"BlockIO": "0B / 0B",
"CPUPerc": "1.65%",
"Container": "mtusi-lab6-otel-collector-1",
"ID": "1a2ff6617b62",
"MemPerc": "47.69%",
"MemUsage": "91.57MiB / 192MiB",
"Name": "mtusi-lab6-otel-collector-1",
"NetIO": "25.1MB / 13.7MB",
"PIDs": "15"
},
{
"BlockIO": "0B / 0B",
"CPUPerc": "14.84%",
"Container": "mtusi-lab6-api-a-1",
"ID": "e305493d2118",
"MemPerc": "28.92%",
"MemUsage": "74.03MiB / 256MiB",
"Name": "mtusi-lab6-api-a-1",
"NetIO": "467kB / 1.01MB",
"PIDs": "16"
},
{
"BlockIO": "0B / 0B",
"CPUPerc": "13.70%",
"Container": "mtusi-lab6-api-b-1",
"ID": "e46ce208e44b",
"MemPerc": "28.89%",
"MemUsage": "73.96MiB / 256MiB",
"Name": "mtusi-lab6-api-b-1",
"NetIO": "583kB / 1.11MB",
"PIDs": "16"
},
{
"BlockIO": "0B / 0B",
"CPUPerc": "0.10%",
"Container": "mtusi-lab6-worker-1",
"ID": "23d809845172",
"MemPerc": "8.24%",
"MemUsage": "63.25MiB / 768MiB",
"Name": "mtusi-lab6-worker-1",
"NetIO": "131kB / 155kB",
"PIDs": "16"
},
{
"BlockIO": "0B / 0B",
"CPUPerc": "0.89%",
"Container": "mtusi-lab6-outbox-publisher-1",
"ID": "ed59d06f4e97",
"MemPerc": "0.42%",
"MemUsage": "33.36MiB / 7.75GiB",
"Name": "mtusi-lab6-outbox-publisher-1",
"NetIO": "462kB / 820kB",
"PIDs": "16"
},
{
"BlockIO": "0B / 0B",
"CPUPerc": "0.85%",
"Container": "mtusi-lab6-otel-collector-1",
"ID": "1a2ff6617b62",
"MemPerc": "47.69%",
"MemUsage": "91.57MiB / 192MiB",
"Name": "mtusi-lab6-otel-collector-1",
"NetIO": "25.2MB / 13.7MB",
"PIDs": "15"
},
{
"BlockIO": "0B / 0B",
"CPUPerc": "16.36%",
"Container": "mtusi-lab6-api-a-1",
"ID": "e305493d2118",
"MemPerc": "28.95%",
"MemUsage": "74.1MiB / 256MiB",
"Name": "mtusi-lab6-api-a-1",
"NetIO": "503kB / 1.09MB",
"PIDs": "16"
},
{
"BlockIO": "0B / 0B",
"CPUPerc": "14.02%",
"Container": "mtusi-lab6-api-b-1",
"ID": "e46ce208e44b",
"MemPerc": "28.89%",
"MemUsage": "73.96MiB / 256MiB",
"Name": "mtusi-lab6-api-b-1",
"NetIO": "624kB / 1.2MB",
"PIDs": "16"
},
{
"BlockIO": "0B / 0B",
"CPUPerc": "0.20%",
"Container": "mtusi-lab6-worker-1",
"ID": "23d809845172",
"MemPerc": "8.24%",
"MemUsage": "63.25MiB / 768MiB",
"Name": "mtusi-lab6-worker-1",
"NetIO": "131kB / 158kB",
"PIDs": "16"
},
{
"BlockIO": "0B / 0B",
"CPUPerc": "1.25%",
"Container": "mtusi-lab6-outbox-publisher-1",
"ID": "ed59d06f4e97",
"MemPerc": "0.42%",
"MemUsage": "33.36MiB / 7.75GiB",
"Name": "mtusi-lab6-outbox-publisher-1",
"NetIO": "478kB / 849kB",
"PIDs": "16"
},
{
"BlockIO": "0B / 0B",
"CPUPerc": "1.13%",
"Container": "mtusi-lab6-otel-collector-1",
"ID": "1a2ff6617b62",
"MemPerc": "47.69%",
"MemUsage": "91.57MiB / 192MiB",
"Name": "mtusi-lab6-otel-collector-1",
"NetIO": "25.3MB / 13.7MB",
"PIDs": "15"
},
{
"BlockIO": "0B / 0B",
"CPUPerc": "15.94%",
"Container": "mtusi-lab6-api-a-1",
"ID": "e305493d2118",
"MemPerc": "28.95%",
"MemUsage": "74.12MiB / 256MiB",
"Name": "mtusi-lab6-api-a-1",
"NetIO": "540kB / 1.16MB",
"PIDs": "16"
},
{
"BlockIO": "0B / 0B",
"CPUPerc": "14.14%",
"Container": "mtusi-lab6-api-b-1",
"ID": "e46ce208e44b",
"MemPerc": "28.89%",
"MemUsage": "73.96MiB / 256MiB",
"Name": "mtusi-lab6-api-b-1",
"NetIO": "663kB / 1.27MB",
"PIDs": "16"
},
{
"BlockIO": "0B / 0B",
"CPUPerc": "0.20%",
"Container": "mtusi-lab6-worker-1",
"ID": "23d809845172",
"MemPerc": "8.24%",
"MemUsage": "63.25MiB / 768MiB",
"Name": "mtusi-lab6-worker-1",
"NetIO": "132kB / 164kB",
"PIDs": "16"
},
{
"BlockIO": "0B / 0B",
"CPUPerc": "1.34%",
"Container": "mtusi-lab6-outbox-publisher-1",
"ID": "ed59d06f4e97",
"MemPerc": "0.42%",
"MemUsage": "33.35MiB / 7.75GiB",
"Name": "mtusi-lab6-outbox-publisher-1",
"NetIO": "496kB / 882kB",
"PIDs": "16"
},
{
"BlockIO": "0B / 0B",
"CPUPerc": "2.08%",
"Container": "mtusi-lab6-otel-collector-1",
"ID": "1a2ff6617b62",
"MemPerc": "47.69%",
"MemUsage": "91.57MiB / 192MiB",
"Name": "mtusi-lab6-otel-collector-1",
"NetIO": "25.4MB / 13.8MB",
"PIDs": "15"
},
{
"BlockIO": "0B / 0B",
"CPUPerc": "16.21%",
"Container": "mtusi-lab6-api-a-1",
"ID": "e305493d2118",
"MemPerc": "29.27%",
"MemUsage": "74.93MiB / 256MiB",
"Name": "mtusi-lab6-api-a-1",
"NetIO": "578kB / 1.24MB",
"PIDs": "16"
},
{
"BlockIO": "0B / 0B",
"CPUPerc": "14.39%",
"Container": "mtusi-lab6-api-b-1",
"ID": "e46ce208e44b",
"MemPerc": "28.90%",
"MemUsage": "73.98MiB / 256MiB",
"Name": "mtusi-lab6-api-b-1",
"NetIO": "703kB / 1.35MB",
"PIDs": "16"
},
{
"BlockIO": "0B / 0B",
"CPUPerc": "0.27%",
"Container": "mtusi-lab6-worker-1",
"ID": "23d809845172",
"MemPerc": "8.24%",
"MemUsage": "63.25MiB / 768MiB",
"Name": "mtusi-lab6-worker-1",
"NetIO": "133kB / 170kB",
"PIDs": "16"
},
{
"BlockIO": "0B / 0B",
"CPUPerc": "2.09%",
"Container": "mtusi-lab6-outbox-publisher-1",
"ID": "ed59d06f4e97",
"MemPerc": "0.42%",
"MemUsage": "33.6MiB / 7.75GiB",
"Name": "mtusi-lab6-outbox-publisher-1",
"NetIO": "508kB / 905kB",
"PIDs": "16"
},
{
"BlockIO": "0B / 0B",
"CPUPerc": "1.30%",
"Container": "mtusi-lab6-otel-collector-1",
"ID": "1a2ff6617b62",
"MemPerc": "47.82%",
"MemUsage": "91.82MiB / 192MiB",
"Name": "mtusi-lab6-otel-collector-1",
"NetIO": "25.5MB / 13.8MB",
"PIDs": "15"
},
{
"BlockIO": "0B / 0B",
"CPUPerc": "1.52%",
"Container": "mtusi-lab6-api-a-1",
"ID": "e305493d2118",
"MemPerc": "29.27%",
"MemUsage": "74.93MiB / 256MiB",
"Name": "mtusi-lab6-api-a-1",
"NetIO": "612kB / 1.3MB",
"PIDs": "16"
},
{
"BlockIO": "0B / 0B",
"CPUPerc": "1.44%",
"Container": "mtusi-lab6-api-b-1",
"ID": "e46ce208e44b",
"MemPerc": "28.91%",
"MemUsage": "74MiB / 256MiB",
"Name": "mtusi-lab6-api-b-1",
"NetIO": "740kB / 1.43MB",
"PIDs": "16"
},
{
"BlockIO": "0B / 0B",
"CPUPerc": "0.16%",
"Container": "mtusi-lab6-worker-1",
"ID": "23d809845172",
"MemPerc": "8.24%",
"MemUsage": "63.25MiB / 768MiB",
"Name": "mtusi-lab6-worker-1",
"NetIO": "133kB / 172kB",
"PIDs": "16"
},
{
"BlockIO": "0B / 0B",
"CPUPerc": "0.96%",
"Container": "mtusi-lab6-outbox-publisher-1",
"ID": "ed59d06f4e97",
"MemPerc": "0.42%",
"MemUsage": "33.36MiB / 7.75GiB",
"Name": "mtusi-lab6-outbox-publisher-1",
"NetIO": "530kB / 942kB",
"PIDs": "16"
},
{
"BlockIO": "0B / 0B",
"CPUPerc": "0.63%",
"Container": "mtusi-lab6-otel-collector-1",
"ID": "1a2ff6617b62",
"MemPerc": "47.82%",
"MemUsage": "91.82MiB / 192MiB",
"Name": "mtusi-lab6-otel-collector-1",
"NetIO": "25.6MB / 13.8MB",
"PIDs": "15"
},
{
"BlockIO": "0B / 0B",
"CPUPerc": "15.56%",
"Container": "mtusi-lab6-api-a-1",
"ID": "e305493d2118",
"MemPerc": "29.39%",
"MemUsage": "75.23MiB / 256MiB",
"Name": "mtusi-lab6-api-a-1",
"NetIO": "651kB / 1.39MB",
"PIDs": "16"
},
{
"BlockIO": "0B / 0B",
"CPUPerc": "15.54%",
"Container": "mtusi-lab6-api-b-1",
"ID": "e46ce208e44b",
"MemPerc": "28.90%",
"MemUsage": "73.99MiB / 256MiB",
"Name": "mtusi-lab6-api-b-1",
"NetIO": "782kB / 1.51MB",
"PIDs": "16"
},
{
"BlockIO": "0B / 0B",
"CPUPerc": "0.28%",
"Container": "mtusi-lab6-worker-1",
"ID": "23d809845172",
"MemPerc": "8.24%",
"MemUsage": "63.26MiB / 768MiB",
"Name": "mtusi-lab6-worker-1",
"NetIO": "134kB / 178kB",
"PIDs": "16"
},
{
"BlockIO": "0B / 0B",
"CPUPerc": "0.94%",
"Container": "mtusi-lab6-outbox-publisher-1",
"ID": "ed59d06f4e97",
"MemPerc": "0.43%",
"MemUsage": "33.77MiB / 7.75GiB",
"Name": "mtusi-lab6-outbox-publisher-1",
"NetIO": "546kB / 971kB",
"PIDs": "16"
},
{
"BlockIO": "0B / 0B",
"CPUPerc": "1.24%",
"Container": "mtusi-lab6-otel-collector-1",
"ID": "1a2ff6617b62",
"MemPerc": "47.95%",
"MemUsage": "92.06MiB / 192MiB",
"Name": "mtusi-lab6-otel-collector-1",
"NetIO": "25.7MB / 13.9MB",
"PIDs": "15"
},
{
"BlockIO": "0B / 0B",
"CPUPerc": "0.96%",
"Container": "mtusi-lab6-api-a-1",
"ID": "e305493d2118",
"MemPerc": "29.39%",
"MemUsage": "75.24MiB / 256MiB",
"Name": "mtusi-lab6-api-a-1",
"NetIO": "685kB / 1.45MB",
"PIDs": "16"
},
{
"BlockIO": "0B / 0B",
"CPUPerc": "15.35%",
"Container": "mtusi-lab6-api-b-1",
"ID": "e46ce208e44b",
"MemPerc": "28.91%",
"MemUsage": "74MiB / 256MiB",
"Name": "mtusi-lab6-api-b-1",
"NetIO": "822kB / 1.59MB",
"PIDs": "16"
},
{
"BlockIO": "0B / 0B",
"CPUPerc": "0.24%",
"Container": "mtusi-lab6-worker-1",
"ID": "23d809845172",
"MemPerc": "8.24%",
"MemUsage": "63.26MiB / 768MiB",
"Name": "mtusi-lab6-worker-1",
"NetIO": "134kB / 181kB",
"PIDs": "16"
},
{
"BlockIO": "0B / 0B",
"CPUPerc": "0.87%",
"Container": "mtusi-lab6-outbox-publisher-1",
"ID": "ed59d06f4e97",
"MemPerc": "0.43%",
"MemUsage": "33.77MiB / 7.75GiB",
"Name": "mtusi-lab6-outbox-publisher-1",
"NetIO": "561kB / 1MB",
"PIDs": "16"
},
{
"BlockIO": "0B / 0B",
"CPUPerc": "0.58%",
"Container": "mtusi-lab6-otel-collector-1",
"ID": "1a2ff6617b62",
"MemPerc": "48.85%",
"MemUsage": "93.8MiB / 192MiB",
"Name": "mtusi-lab6-otel-collector-1",
"NetIO": "25.8MB / 13.9MB",
"PIDs": "15"
},
{
"BlockIO": "0B / 0B",
"CPUPerc": "15.39%",
"Container": "mtusi-lab6-api-a-1",
"ID": "e305493d2118",
"MemPerc": "29.44%",
"MemUsage": "75.37MiB / 256MiB",
"Name": "mtusi-lab6-api-a-1",
"NetIO": "726kB / 1.53MB",
"PIDs": "16"
},
{
"BlockIO": "0B / 0B",
"CPUPerc": "13.65%",
"Container": "mtusi-lab6-api-b-1",
"ID": "e46ce208e44b",
"MemPerc": "28.92%",
"MemUsage": "74.03MiB / 256MiB",
"Name": "mtusi-lab6-api-b-1",
"NetIO": "857kB / 1.66MB",
"PIDs": "16"
},
{
"BlockIO": "0B / 0B",
"CPUPerc": "0.20%",
"Container": "mtusi-lab6-worker-1",
"ID": "23d809845172",
"MemPerc": "8.24%",
"MemUsage": "63.26MiB / 768MiB",
"Name": "mtusi-lab6-worker-1",
"NetIO": "135kB / 187kB",
"PIDs": "16"
},
{
"BlockIO": "0B / 0B",
"CPUPerc": "0.49%",
"Container": "mtusi-lab6-outbox-publisher-1",
"ID": "ed59d06f4e97",
"MemPerc": "0.43%",
"MemUsage": "33.77MiB / 7.75GiB",
"Name": "mtusi-lab6-outbox-publisher-1",
"NetIO": "576kB / 1.03MB",
"PIDs": "16"
},
{
"BlockIO": "0B / 0B",
"CPUPerc": "0.73%",
"Container": "mtusi-lab6-otel-collector-1",
"ID": "1a2ff6617b62",
"MemPerc": "50.17%",
"MemUsage": "96.33MiB / 192MiB",
"Name": "mtusi-lab6-otel-collector-1",
"NetIO": "25.9MB / 13.9MB",
"PIDs": "15"
}
],
"no_oom_or_restarts": true,
"telemetry_restored_trace": "c485b3e4b21d427a94f1cd345789c843"
}
+82
View File
@@ -0,0 +1,82 @@
{
"firing": true,
"failed_requests": 8,
"trace_id": "9a03429d57e1b78a8363efb6292dfd5b",
"db_error_spans": 1,
"alerts_firing": [
{
"labels": {
"alertname": "HighApiErrorRate",
"severity": "warning"
},
"annotations": {
"description": "SLO is 99% non-5xx business requests over 5 minutes. Inspect PostgreSQL client spans and pool metrics.",
"runbook_url": "http://localhost:13000/d/messenger/messenger?viewPanel=5",
"summary": "API 5xx exceeds 1% over 1 minute"
},
"state": "firing",
"activeAt": "2026-11-20T17:41:07.478517494Z",
"value": "3.6363636363636365e-01"
},
{
"labels": {
"alertname": "QueueObservationStale",
"deployment_environment_name": "lab",
"exported_instance": "459bbaf48a82",
"exported_job": "messenger-publisher",
"instance": "otel-collector:8889",
"job": "application",
"queue": "attachment.process.v1",
"service_instance_id": "459bbaf48a82",
"service_name": "messenger-publisher",
"service_version": "local",
"severity": "warning",
"telemetry_sdk_language": "python",
"telemetry_sdk_name": "opentelemetry",
"telemetry_sdk_version": "1.38.0"
},
"annotations": {
"description": "Check publisher and PostgreSQL availability before trusting queue gauges.",
"summary": "Publisher backlog observations are stale"
},
"state": "firing",
"activeAt": "2026-11-20T17:39:31.478517494Z",
"value": "1.1822987508773804e+02"
},
{
"labels": {
"alertname": "QueueObservationStale",
"deployment_environment_name": "lab",
"exported_instance": "af26b1841208",
"exported_job": "messenger-publisher",
"instance": "otel-collector:8889",
"job": "application",
"queue": "attachment.process.v1",
"service_instance_id": "af26b1841208",
"service_name": "messenger-publisher",
"service_version": "local",
"severity": "warning",
"telemetry_sdk_language": "python",
"telemetry_sdk_name": "opentelemetry",
"telemetry_sdk_version": "1.38.0"
},
"annotations": {
"description": "Check publisher and PostgreSQL availability before trusting queue gauges.",
"summary": "Publisher backlog observations are stale"
},
"state": "firing",
"activeAt": "2026-11-20T17:38:01.478517494Z",
"value": "2.077088599205017e+02"
}
],
"error_ratio": [
{
"metric": {},
"value": [
1795196476.495,
"0.4615384615384615"
]
}
],
"resolved": true
}
+28
View File
@@ -0,0 +1,28 @@
{
"trace_id": "8e9d1c9daddb49778c887f0cc63f470a",
"consumer_attempts": 2,
"span_names": [
"POST /api/v1/chats/{chat_id}/attachments",
"postgresql.transaction",
"postgresql.transaction",
"postgresql.transaction",
"postgresql.transaction",
"postgresql.transaction",
"s3.put",
"attachment.publish",
"attachment.consume",
"postgresql.transaction",
"s3.download",
"postgresql.transaction",
"attachment.publish",
"attachment.consume",
"postgresql.transaction",
"s3.download",
"s3.put",
"s3.put",
"s3.put",
"s3.put",
"postgresql.transaction"
],
"ready_after_retry": true
}
+42
View File
@@ -0,0 +1,42 @@
{
"trace_id": "f1aa109fe9fd47f284eaafd2c34a8d96",
"span_names": [
"postgresql.transaction",
"postgresql.transaction",
"postgresql.transaction",
"postgresql.transaction",
"s3.put",
"postgresql.transaction",
"POST /api/v1/chats/{chat_id}/attachments",
"postgresql.transaction",
"s3.download",
"s3.put",
"s3.put",
"s3.put",
"s3.put",
"postgresql.transaction",
"attachment.consume",
"attachment.publish"
],
"log_services": [
"messenger-api",
"messenger-publisher",
"messenger-worker"
],
"metric_names_checked": [
"messenger_http_requests_total",
"messenger_http_duration_seconds_bucket",
"messenger_queue_depth",
"messenger_queue_oldest_age_seconds",
"messenger_db_pool_used",
"messenger_worker_busy",
"messenger_job_duration_seconds_bucket",
"messenger_attachment_time_to_ready_seconds_bucket",
"messenger_process_memory_bytes",
"messenger_process_cpu_seconds_total"
],
"dashboard_panels": 14,
"dashboard_queries_valid": true,
"incoming_w3c_context": true,
"body_and_token_absent_from_logs": true
}
+173
View File
@@ -0,0 +1,173 @@
{
"seconds_to_firing": 30.1,
"oldest_job_age": [
{
"metric": {
"__name__": "messenger_queue_oldest_age_seconds",
"deployment_environment_name": "lab",
"exported_instance": "af26b1841208",
"exported_job": "messenger-publisher",
"instance": "otel-collector:8889",
"job": "application",
"queue": "attachment.process.v1",
"service_instance_id": "af26b1841208",
"service_name": "messenger-publisher",
"service_version": "local",
"telemetry_sdk_language": "python",
"telemetry_sdk_name": "opentelemetry",
"telemetry_sdk_version": "1.38.0"
},
"value": [
1795196321.26,
"0"
]
},
{
"metric": {
"__name__": "messenger_queue_oldest_age_seconds",
"deployment_environment_name": "lab",
"exported_instance": "459bbaf48a82",
"exported_job": "messenger-publisher",
"instance": "otel-collector:8889",
"job": "application",
"queue": "attachment.process.v1",
"service_instance_id": "459bbaf48a82",
"service_name": "messenger-publisher",
"service_version": "local",
"telemetry_sdk_language": "python",
"telemetry_sdk_name": "opentelemetry",
"telemetry_sdk_version": "1.38.0"
},
"value": [
1795196321.26,
"25.465688"
]
}
],
"backlog": [
{
"metric": {
"__name__": "messenger_queue_depth",
"deployment_environment_name": "lab",
"exported_instance": "af26b1841208",
"exported_job": "messenger-publisher",
"instance": "otel-collector:8889",
"job": "application",
"queue": "attachment.process.v1",
"service_instance_id": "af26b1841208",
"service_name": "messenger-publisher",
"service_version": "local",
"telemetry_sdk_language": "python",
"telemetry_sdk_name": "opentelemetry",
"telemetry_sdk_version": "1.38.0"
},
"value": [
1795196321.262,
"0"
]
},
{
"metric": {
"__name__": "messenger_queue_depth",
"deployment_environment_name": "lab",
"exported_instance": "459bbaf48a82",
"exported_job": "messenger-publisher",
"instance": "otel-collector:8889",
"job": "application",
"queue": "attachment.process.v1",
"service_instance_id": "459bbaf48a82",
"service_name": "messenger-publisher",
"service_version": "local",
"telemetry_sdk_language": "python",
"telemetry_sdk_name": "opentelemetry",
"telemetry_sdk_version": "1.38.0"
},
"value": [
1795196321.262,
"1"
]
}
],
"alerts_firing": [
{
"labels": {
"alertname": "QueueLagGrowing",
"deployment_environment_name": "lab",
"exported_instance": "459bbaf48a82",
"exported_job": "messenger-publisher",
"instance": "otel-collector:8889",
"job": "application",
"queue": "attachment.process.v1",
"service_instance_id": "459bbaf48a82",
"service_name": "messenger-publisher",
"service_version": "local",
"severity": "warning",
"telemetry_sdk_language": "python",
"telemetry_sdk_name": "opentelemetry",
"telemetry_sdk_version": "1.38.0"
},
"annotations": {
"description": "Inspect worker capacity, busy gauge and consumer traces. Restore worker and verify backlog drains. See infra/runbook.md.",
"summary": "Oldest unfinished job exceeds 15 seconds"
},
"state": "firing",
"activeAt": "2026-11-20T17:38:31.478517494Z",
"value": "2.2414185e+01"
},
{
"labels": {
"alertname": "QueueObservationStale",
"deployment_environment_name": "lab",
"exported_instance": "af26b1841208",
"exported_job": "messenger-publisher",
"instance": "otel-collector:8889",
"job": "application",
"queue": "attachment.process.v1",
"service_instance_id": "af26b1841208",
"service_name": "messenger-publisher",
"service_version": "local",
"severity": "warning",
"telemetry_sdk_language": "python",
"telemetry_sdk_name": "opentelemetry",
"telemetry_sdk_version": "1.38.0"
},
"annotations": {
"description": "Check publisher and PostgreSQL availability before trusting queue gauges.",
"summary": "Publisher backlog observations are stale"
},
"state": "firing",
"activeAt": "2026-11-20T17:38:01.478517494Z",
"value": "5.170885992050171e+01"
}
],
"resolved": true,
"trace_id": "a0d4df4afbe44a30999830b9d563817c",
"span_count": 16,
"alerts_after": [
{
"labels": {
"alertname": "QueueObservationStale",
"deployment_environment_name": "lab",
"exported_instance": "af26b1841208",
"exported_job": "messenger-publisher",
"instance": "otel-collector:8889",
"job": "application",
"queue": "attachment.process.v1",
"service_instance_id": "af26b1841208",
"service_name": "messenger-publisher",
"service_version": "local",
"severity": "warning",
"telemetry_sdk_language": "python",
"telemetry_sdk_name": "opentelemetry",
"telemetry_sdk_version": "1.38.0"
},
"annotations": {
"description": "Check publisher and PostgreSQL availability before trusting queue gauges.",
"summary": "Publisher backlog observations are stale"
},
"state": "firing",
"activeAt": "2026-11-20T17:38:01.478517494Z",
"value": "6.370885992050171e+01"
}
]
}
+3
View File
@@ -0,0 +1,3 @@
python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://localhost:18083"
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
+2
View File
@@ -0,0 +1,2 @@
e305493d2118fcf5b65beba48547258a269d2d353cf2a20fc9b841af2ab5f153
e46ce208e44b8b48f2f15dedca018e0c4d780d1e96a88858fdcd9d0eb4da139d
+1
View File
@@ -0,0 +1 @@
PASS: generated private credentials absent from application, configs and evidence. Redis URL without credentials is public configuration.
+2
View File
@@ -0,0 +1,2 @@
python3 tests/smoke.py --base-url "http://localhost:18083"
PASS: lab6 public API scenario; infrastructure and grading evidence remain separate.
+2
View File
@@ -0,0 +1,2 @@
python3 scripts/check_contract.py
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
+2
View File
@@ -0,0 +1,2 @@
PASS: HTTP 202 queued -> RabbitMQ/worker -> ready
PASS: original unchanged; thumbnail, crop, optimized downloaded with matching size/SHA-256
+11
View File
@@ -0,0 +1,11 @@
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
mtusi-lab6-api-a-1 mtusi-lab6-api:local "python -m messenger" api-a 27 seconds ago Up 23 seconds (healthy) 8080/tcp
mtusi-lab6-api-b-1 mtusi-lab6-api:local "python -m messenger" api-b 27 seconds ago Up 23 seconds (healthy) 8080/tcp
mtusi-lab6-attachment-cleaner-1 mtusi-lab6-api:local "python -m messenger…" attachment-cleaner 27 seconds ago Up 23 seconds 8080/tcp
mtusi-lab6-nginx-1 nginx:1.28.2-alpine@sha256:5b4900b042ccfa8b0a73df622c3a60f2322faeb2be800cbee5aa7b44d241649e "/bin/sh -ec 'cp /et…" nginx 26 seconds ago Up 17 seconds (healthy) 127.0.0.1:18083->8080/tcp
mtusi-lab6-outbox-publisher-1 mtusi-lab6-api:local "python -m messenger…" outbox-publisher 27 seconds ago Up 21 seconds 8080/tcp
mtusi-lab6-postgres-1 postgres:15-alpine@sha256:54e0dd8e8cddcdbb9156c2486eed5a43eccb7214d7c6828097d41ce049fb3984 "docker-entrypoint.s…" postgres 27 seconds ago Up 26 seconds (healthy) 5432/tcp
mtusi-lab6-rabbitmq-1 rabbitmq:4.1.4-alpine "/bin/sh -ec 'export…" rabbitmq 27 seconds ago Up 26 seconds (healthy) 4369/tcp, 5671-5672/tcp, 15691-15692/tcp, 25672/tcp
mtusi-lab6-redis-1 redis:7-alpine@sha256:8da19c7399dca900c6822a1cf30f9d2285585e2c4b457ff4095b0f4500054d3f "docker-entrypoint.s…" redis 27 seconds ago Up 26 seconds (healthy) 6379/tcp
mtusi-lab6-s3-1 mtusi-lab6-s3:2025-09-07 "/bin/sh -ec 'MINIO_…" s3 27 seconds ago Up 26 seconds
mtusi-lab6-worker-1 mtusi-lab6-api:local "python -m messenger…" worker 26 seconds ago Up 20 seconds 8080/tcp
+2
View File
@@ -0,0 +1,2 @@
python3 tests/smoke.py --base-url "http://localhost:18083"
PASS: lab6 public API scenario; infrastructure and grading evidence remain separate.
+13
View File
@@ -0,0 +1,13 @@
{
"trace_id": "9efec3dd76d0440a902d19838f8e4c5b",
"trace_spans_preserved": 16,
"api_worker_publisher_logs_preserved": true,
"historical_metric_preserved": true,
"restarted": [
"otel-collector",
"prometheus",
"loki",
"tempo",
"grafana"
]
}
+2
View File
@@ -0,0 +1,2 @@
.venv/bin/python scripts/validate_openapi.py
PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.
+12
View File
@@ -0,0 +1,12 @@
FROM public.ecr.aws/docker/library/golang:1.24.2-alpine@sha256:7772cb5322baa875edd74705556d08f0eeca7b9c4b5367754ce3f2f00041ccee AS build
ENV CGO_ENABLED=0 GOPROXY="https://proxy.golang.org|https://goproxy.io|direct"
WORKDIR /src/minio
ADD https://codeload.github.com/minio/minio/tar.gz/refs/tags/RELEASE.2025-09-07T16-13-09Z /tmp/minio.tar.gz
RUN --mount=type=cache,target=/go/pkg/mod --mount=type=cache,target=/root/.cache/go-build tar -xzf /tmp/minio.tar.gz --strip-components=1 && go build -o /out/minio .
WORKDIR /src/mc
ADD https://codeload.github.com/minio/mc/tar.gz/refs/tags/RELEASE.2025-08-13T08-35-41Z /tmp/mc.tar.gz
RUN --mount=type=cache,target=/go/pkg/mod --mount=type=cache,target=/root/.cache/go-build tar -xzf /tmp/mc.tar.gz --strip-components=1 && go build -o /out/mc .
FROM python:3.12-slim-bookworm@sha256:392307d22300de8b5986851a12d9176dfc0fc073e65bf6523ebd7dcbeb23564e
COPY --from=build /out/minio /out/mc /usr/local/bin/
ENTRYPOINT ["minio"]
+19 -4
View File
@@ -1,7 +1,22 @@
# Инфраструктура студента
# Инфраструктура лабораторной 6
Здесь разместите конфигурации выбранных сервисов; основную топологию — в корневом compose.yaml/stack.yaml. `compose.example.yaml` — только worksheet ролей и намеренно не запускается. `contracts/infrastructure.json` задаёт минимальные роли, не готовые образы/пароли.
Топология и ограничения ресурсов заданы в compose.yaml. Проект mtusi-lab6 использует собственные named volumes. API доступен через Nginx на localhost:18083, Grafana — localhost:13000. Секреты создаёт scripts/bootstrap.py.
В REPORT.md сопоставьте роль → service name → версия/digest → health/readiness → volume → сеть/порт → способ передачи credentials. Приведите bootstrap (schema, bucket, broker topology, а с lab6 provisioning) и безопасное повторение этих команд.
| Роль | Сервисы | Хранение / проверка |
| --- | --- | --- |
| API | api-a, api-b | PostgreSQL, readiness /health/ready |
| Балансировщик | nginx | healthwatch проверяет готовность двух реплик |
| База данных | postgres | pg_data, pg_isready |
| Кэш сессий | redis | redis_data, AOF, redis-cli ping |
| Объекты | s3, s3-bootstrap | s3_data, приватный бакет и отдельная учётная запись API |
| Очередь | rabbitmq | rabbitmq_data, durable queue, publisher confirms |
| Фоновая обработка | outbox-publisher, worker, attachment-cleaner | Job/outbox/lease в PostgreSQL |
| Сбор сигналов | otel-collector | OTLP 4317/4318, внутренний health endpoint 13133 |
| Метрики | prometheus | prometheus_data, /-/ready, локальные alert rules |
| Логи | loki | loki_data, /ready, OTLP/HTTP ingestion |
| Трассы | tempo | tempo_data, /ready, OTLP/gRPC ingestion |
| Dashboard | grafana | grafana_data, /api/health, provisioning в infra/grafana |
Приложение строится из исходников. Не используйте заранее существующий образ преподавательского сервера вместо своей реализации. Restart hook перезапускает все соответствующие stores с volumes; failover hook останавливает только заданную API-реплику.
Backend и telemetry используют внутренние сети. Nginx и HTTP-интерфейсы наблюдаемости также подключены к frontend для публикации loopback-портов. OTLP на хост не опубликован. Все версии закреплены в Compose/Dockerfile. Запуск, сигналы, ограничения и результаты приведены в REPORT.md. Действия при сбое описаны в runbook.md.
scripts/restart.sh перезапускает бизнес-хранилища и приложения с существующими volumes. scripts/stop-one.sh останавливает API-реплику по TARGET_INSTANCE_ID. Историю наблюдений проверяет student-tests/telemetry_persistence.py. Сценарии student-tests/observability.py воспроизводят остановку worker, блокировку БД, retry после отказа S3 и недоступность Tempo. Каждый fault-сценарий выполняется отдельно и восстанавливает остановленные сервисы.
+446
View File
@@ -0,0 +1,446 @@
{
"uid": "messenger",
"title": "Messenger",
"schemaVersion": 40,
"version": 1,
"refresh": "5s",
"time": {
"from": "now-15m",
"to": "now"
},
"tags": [
"lab6"
],
"panels": [
{
"id": 1,
"title": "API requests per second",
"type": "timeseries",
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 0
},
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "reqps"
},
"overrides": []
},
"targets": [
{
"refId": "A",
"expr": "sum by (method,route)(rate(messenger_http_requests_total[1m]))",
"legendFormat": "{{method}} {{route}}"
}
]
},
{
"id": 2,
"title": "API latency p50 / p95 / p99",
"type": "timeseries",
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 0
},
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
},
"targets": [
{
"refId": "A",
"expr": "histogram_quantile(0.5, sum by (le)(rate(messenger_http_duration_seconds_bucket[1m])))",
"legendFormat": "p50"
},
{
"refId": "B",
"expr": "histogram_quantile(0.95, sum by (le)(rate(messenger_http_duration_seconds_bucket[1m])))",
"legendFormat": "p95"
},
{
"refId": "C",
"expr": "histogram_quantile(0.99, sum by (le)(rate(messenger_http_duration_seconds_bucket[1m])))",
"legendFormat": "p99"
}
]
},
{
"id": 3,
"title": "API 5xx ratio and 5-minute SLI",
"type": "timeseries",
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 8
},
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "percentunit"
},
"overrides": []
},
"targets": [
{
"refId": "A",
"expr": "(sum(rate(messenger_http_requests_total{status_class=\"5xx\"}[1m])) or vector(0)) / clamp_min(sum(rate(messenger_http_requests_total[1m])),0.001)",
"legendFormat": "5xx ratio"
},
{
"refId": "B",
"expr": "(1 - ((sum(increase(messenger_http_requests_total{status_class=\"5xx\"}[5m])) or vector(0)) / clamp_min(sum(increase(messenger_http_requests_total[5m])),1))) and (sum(increase(messenger_http_requests_total[5m])) > 0)",
"legendFormat": "SLI 5m"
}
]
},
{
"id": 4,
"title": "Unfinished jobs and RabbitMQ ready messages",
"type": "timeseries",
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 8
},
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"targets": [
{
"refId": "A",
"expr": "max((messenger_queue_depth) and on(service_instance_id) (time() - messenger_queue_observed_at_seconds < 10))",
"legendFormat": "unfinished jobs"
},
{
"refId": "B",
"expr": "max((messenger_queue_broker_ready) and on(service_instance_id) (time() - messenger_queue_observed_at_seconds < 10))",
"legendFormat": "broker ready"
}
]
},
{
"id": 5,
"title": "PostgreSQL pool use / capacity / acquisition",
"type": "timeseries",
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 16
},
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"targets": [
{
"refId": "A",
"expr": "(messenger_db_pool_used) and on(service_instance_id) (time() - messenger_process_observed_at_seconds < 10)",
"legendFormat": "used {{service_name}} {{service_instance_id}}"
},
{
"refId": "B",
"expr": "(messenger_db_pool_capacity) and on(service_instance_id) (time() - messenger_process_observed_at_seconds < 10)",
"legendFormat": "capacity {{service_name}} {{service_instance_id}}"
},
{
"refId": "C",
"expr": "(messenger_db_pool_waiting) and on(service_instance_id) (time() - messenger_process_observed_at_seconds < 10)",
"legendFormat": "acquiring {{service_name}} {{service_instance_id}}"
}
]
},
{
"id": 6,
"title": "Oldest unfinished job age",
"type": "timeseries",
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 16
},
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
},
"targets": [
{
"refId": "A",
"expr": "max((messenger_queue_oldest_age_seconds) and on(service_instance_id) (time() - messenger_queue_observed_at_seconds < 10))",
"legendFormat": "age"
}
]
},
{
"id": 7,
"title": "Worker busy / capacity",
"type": "timeseries",
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 24
},
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"targets": [
{
"refId": "A",
"expr": "sum((messenger_worker_busy{service_name=\"messenger-worker\"}) and on(service_instance_id) (time() - messenger_process_observed_at_seconds < 10)) or vector(0)",
"legendFormat": "busy"
},
{
"refId": "B",
"expr": "sum((messenger_worker_capacity{service_name=\"messenger-worker\"}) and on(service_instance_id) (time() - messenger_process_observed_at_seconds < 10)) or vector(0)",
"legendFormat": "capacity"
}
]
},
{
"id": 8,
"title": "Worker attempts by outcome",
"type": "timeseries",
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 24
},
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "ops"
},
"overrides": []
},
"targets": [
{
"refId": "A",
"expr": "sum by (outcome)(rate(messenger_jobs_processed_total[1m]))",
"legendFormat": "{{outcome}}"
}
]
},
{
"id": 9,
"title": "Worker processing p95 and accepted-to-ready p95",
"type": "timeseries",
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 32
},
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
},
"targets": [
{
"refId": "A",
"expr": "histogram_quantile(0.95,sum by(le)(rate(messenger_job_duration_seconds_bucket[1m])))",
"legendFormat": "processing p95"
},
{
"refId": "B",
"expr": "histogram_quantile(0.95,sum by(le)(rate(messenger_attachment_time_to_ready_seconds_bucket[1m])))",
"legendFormat": "time to ready p95"
}
]
},
{
"id": 10,
"title": "Process resident memory",
"type": "timeseries",
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 32
},
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "bytes"
},
"overrides": []
},
"targets": [
{
"refId": "A",
"expr": "(messenger_process_memory_bytes) and on(service_instance_id) (time() - messenger_process_observed_at_seconds < 10)",
"legendFormat": "{{service_name}} {{service_instance_id}}"
}
]
},
{
"id": 11,
"title": "Process CPU cores",
"type": "timeseries",
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 40
},
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"targets": [
{
"refId": "A",
"expr": "(rate(messenger_process_cpu_seconds_total[1m])) and on(service_instance_id) (time() - messenger_process_observed_at_seconds < 10)",
"legendFormat": "{{service_name}} {{service_instance_id}}"
}
]
},
{
"id": 12,
"title": "Collector exporter failures / lost records",
"type": "timeseries",
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 40
},
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"targets": [
{
"refId": "A",
"expr": "{__name__=~\"otelcol_exporter_(send_failed|enqueue_failed)_(spans|log_records|metric_points).*\"}",
"legendFormat": "{{__name__}} {{exporter}}"
}
]
},
{
"id": 13,
"title": "Application logs with trace IDs",
"type": "logs",
"gridPos": {
"h": 9,
"w": 24,
"x": 0,
"y": 48
},
"datasource": {
"type": "loki",
"uid": "loki"
},
"targets": [
{
"refId": "A",
"expr": "{service_name=~\"messenger-.*\"}"
}
],
"options": {
"showTime": true,
"wrapLogMessage": true
}
},
{
"id": 14,
"title": "Recent application traces",
"type": "table",
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 57
},
"datasource": {
"type": "tempo",
"uid": "tempo"
},
"targets": [
{
"refId": "A",
"queryType": "traceql",
"query": "{resource.service.name =~ \"messenger-.*\"}",
"tableType": "traces"
}
]
}
],
"annotations": {
"list": []
},
"templating": {
"list": []
}
}
@@ -0,0 +1,10 @@
apiVersion: 1
providers:
- name: messenger
orgId: 1
folder: Messenger
type: file
disableDeletion: true
updateIntervalSeconds: 10
options:
path: /var/lib/grafana/dashboards
@@ -0,0 +1,33 @@
apiVersion: 1
datasources:
- name: Prometheus
uid: prometheus
type: prometheus
access: proxy
url: http://prometheus:9090
isDefault: true
jsonData:
timeInterval: 3s
- name: Loki
uid: loki
type: loki
access: proxy
url: http://loki:3100
jsonData:
derivedFields:
- name: TraceID
datasourceUid: tempo
matcherRegex: '"trace_id":\s*"([0-9a-f]{32})"'
url: '$${__value.raw}'
- name: Tempo
uid: tempo
type: tempo
access: proxy
url: http://tempo:3200
jsonData:
tracesToLogsV2:
datasourceUid: loki
spanStartTimeShift: '-1m'
spanEndTimeShift: '1m'
tags: [{key: service.name, value: service_name}]
filterByTraceID: true
+53
View File
@@ -0,0 +1,53 @@
#!/bin/sh
# Active readiness bridge for open-source Nginx. No Docker socket or credentials.
set -eu
probe() {
if wget -T 1 -q -O /dev/null "http://$1:8080/health/ready" 2>/dev/null; then
echo up > "/tmp/$1.health"
else
echo down > "/tmp/$1.health"
fi
}
control() { cat "/tmp/control-$1" 2>/dev/null || echo serve:boot; }
echo $$ > /tmp/healthwatch.pid
while :; do
sleep 1
probe api-a &
first=$!
probe api-b &
second=$!
wait "$first"
wait "$second"
# Read desired state AFTER probes. A generation is acknowledged only after
# the matching upstream configuration has been applied, never from a stale probe.
a_control=$(control api-a)
b_control=$(control api-b)
a_state=$(cat /tmp/api-a.health)
b_state=$(cat /tmp/api-b.health)
case "$a_control" in drain:*) a_state=down;; esac
case "$b_control" in drain:*) b_state=down;; esac
: > /tmp/upstreams.next
for service in api-a api-b; do
if [ "$service" = api-a ]; then state=$a_state; else state=$b_state; fi
suffix=''
[ "$state" = up ] || suffix=' down'
printf 'server %s:8080 resolve max_fails=1 fail_timeout=2s%s;\n' "$service" "$suffix" >> /tmp/upstreams.next
done
if ! cmp -s /tmp/upstreams.next /etc/nginx/upstreams.conf; then
cp /etc/nginx/upstreams.conf /tmp/upstreams.previous
# Rename on the same filesystem: readers never see missing/partial config.
cp /tmp/upstreams.next /etc/nginx/upstreams.next
mv -f /etc/nginx/upstreams.next /etc/nginx/upstreams.conf
if nginx -t && nginx -s reload; then
echo 'readiness upstream set updated'
else
cp /tmp/upstreams.previous /etc/nginx/upstreams.next
mv -f /etc/nginx/upstreams.next /etc/nginx/upstreams.conf
exit 1
fi
fi
printf '%s %s\n' "$a_control" "$a_state" > /tmp/applied-api-a.next
mv -f /tmp/applied-api-a.next /tmp/applied-api-a
printf '%s %s\n' "$b_control" "$b_state" > /tmp/applied-api-b.next
mv -f /tmp/applied-api-b.next /tmp/applied-api-b
done
+31
View File
@@ -0,0 +1,31 @@
auth_enabled: false
server:
http_listen_port: 3100
common:
path_prefix: /loki
replication_factor: 1
ring:
kvstore:
store: inmemory
storage:
filesystem:
chunks_directory: /loki/chunks
rules_directory: /loki/rules
schema_config:
configs:
- from: 2024-01-01
store: tsdb
object_store: filesystem
schema: v13
index:
prefix: index_
period: 24h
limits_config:
allow_structured_metadata: true
retention_period: 72h
compactor:
working_directory: /loki/compactor
retention_enabled: true
delete_request_store: filesystem
analytics:
reporting_enabled: false
+49
View File
@@ -0,0 +1,49 @@
worker_processes 1;
worker_shutdown_timeout 6s;
events { worker_connections 1024; }
http {
resolver 127.0.0.11 valid=1s ipv6=off;
resolver_timeout 1s;
log_format lab '$time_iso8601 $request_method $uri $status upstream=$upstream_addr '
'instance=$upstream_http_x_instance_id time=$request_time';
access_log /dev/stdout lab;
error_log /dev/stderr warn;
upstream api {
zone api 64k;
include /etc/nginx/upstreams.conf;
}
server {
listen 8080;
client_max_body_size 1m;
client_body_timeout 5s;
proxy_http_version 1.1;
proxy_set_header Connection close;
proxy_set_header Host $host;
proxy_set_header X-Request-Id $http_x_request_id;
proxy_connect_timeout 1s;
proxy_read_timeout 5s;
proxy_send_timeout 5s;
proxy_next_upstream error timeout http_502 http_504;
proxy_next_upstream_tries 2;
proxy_next_upstream_timeout 6s;
location ~ ^/api/v1/chats/[^/]+/attachments$ {
# Keep a proxy ceiling above the API limit so the API can return its JSON 413.
client_max_body_size 11m;
proxy_read_timeout 45s;
proxy_send_timeout 30s;
proxy_next_upstream off;
proxy_pass http://api;
}
location ~ ^/api/v1/attachments/[^/]+/content$ {
proxy_buffering off;
proxy_read_timeout 10s;
# A shared S3 outage returns 503 without marking the API itself down.
proxy_next_upstream error timeout http_502 http_504;
proxy_pass http://api;
}
location / {
proxy_pass http://api;
}
}
}
+8 -4
View File
@@ -1,7 +1,11 @@
# Конфигурация наблюдаемости — реализовать
# OpenTelemetry
Добавьте `collector.yaml` с OTLP receivers, processors и exporters выбранной версии Collector. Укажите явные bind endpoints, внутренние сети, memory limits/batch и поведение при backpressure по уровню. Сопоставьте все три pipeline с реально развёрнутыми backend.
API, publisher и worker используют SDK 1.38.0 и OTLP/gRPC на otel-collector:4317. HTTP receiver 4318 также доступен внутри сети. Ресурсы содержат service.name, service.version, service.instance.id и deployment.environment.name=lab. Используются ручные spans и собственные имена метрик, соответствие приведено в REPORT.md.
Добавьте конфигурации storage, Grafana provisioning/datasources, provisioning/dashboards и импортируемые dashboard JSON. На 4 — alert rules и runbooks. Пустой dashboard или debug exporter, который только печатает данные в stdout, не заменяют хранилища/доказательства из `contracts/observability.md`.
Collector contrib 0.123.0 применяет memory_limiter и batch ко всем трём сигналам. Метрики доступны Prometheus на порту 8889, внутренние метрики Collector — на 8888. Логи отправляются OTLP/HTTP на Loki /otlp, трассы OTLP/gRPC в Tempo. Для Loki включены structured metadata и schema v13.
Не публикуйте OTLP/metrics/административные ports в общую сеть без необходимости. В lab7 реализуйте TLS/mTLS по trust matrix; экспорт Authorization и message bodies запрещён.
Очереди SDK ограничены 256 spans и logs на процесс. Collector имеет по 128 batches для Loki и Tempo, до 512 записей в batch. Retry ограничен 10 секундами. После заполнения/исчерпания retry возможны потери телеметрии, бизнес-транзакции не ждут успешного экспорта. Контейнер Collector ограничен 192 MiB, memory_limiter — 128 MiB. Критерии проверяются сценарием student-tests/observability.py backend-outage.
Grafana, Prometheus, Loki и Tempo имеют собственные volumes и retention 72 часа для сигналов. Административные HTTP-порты опубликованы только на loopback. Collector не публикует OTLP на хост.
Первичные материалы — https://opentelemetry.io/docs/collector/configuration/ и https://grafana.com/docs/loki/latest/send-data/otel/.
+73
View File
@@ -0,0 +1,73 @@
receivers:
otlp:
protocols:
grpc:
endpoint: 0.0.0.0:4317
http:
endpoint: 0.0.0.0:4318
processors:
memory_limiter:
check_interval: 1s
limit_mib: 128
spike_limit_mib: 32
batch:
timeout: 1s
send_batch_size: 256
send_batch_max_size: 512
exporters:
prometheus:
endpoint: 0.0.0.0:8889
resource_to_telemetry_conversion:
enabled: true
otlphttp/loki:
endpoint: http://loki:3100/otlp
timeout: 2s
sending_queue:
enabled: true
num_consumers: 2
queue_size: 128
retry_on_failure:
enabled: true
initial_interval: 1s
max_interval: 3s
max_elapsed_time: 10s
otlp/tempo:
endpoint: tempo:4317
tls:
insecure: true
timeout: 2s
sending_queue:
enabled: true
num_consumers: 2
queue_size: 128
retry_on_failure:
enabled: true
initial_interval: 1s
max_interval: 3s
max_elapsed_time: 10s
extensions:
health_check:
endpoint: 0.0.0.0:13133
service:
extensions: [health_check]
telemetry:
metrics:
readers:
- pull:
exporter:
prometheus:
host: 0.0.0.0
port: 8888
pipelines:
traces:
receivers: [otlp]
processors: [memory_limiter, batch]
exporters: [otlp/tempo]
metrics:
receivers: [otlp]
processors: [memory_limiter, batch]
exporters: [prometheus]
logs:
receivers: [otlp]
processors: [memory_limiter, batch]
exporters: [otlphttp/loki]
+28
View File
@@ -0,0 +1,28 @@
groups:
- name: messenger
rules:
- alert: HighApiErrorRate
expr: sum(rate(messenger_http_requests_total{status_class="5xx"}[1m])) / clamp_min(sum(rate(messenger_http_requests_total[1m])), 0.001) > 0.01
for: 6s
labels:
severity: warning
annotations:
summary: API 5xx exceeds 1% over 1 minute
description: SLO is 99% non-5xx business requests over 5 minutes. Inspect PostgreSQL client spans and pool metrics.
runbook_url: http://localhost:13000/d/messenger/messenger?viewPanel=5
- alert: QueueLagGrowing
expr: messenger_queue_oldest_age_seconds > 15 and messenger_queue_depth > 0 and on(service_instance_id) (time() - messenger_queue_observed_at_seconds < 10)
for: 6s
labels:
severity: warning
annotations:
summary: Oldest unfinished job exceeds 15 seconds
description: Inspect worker capacity, busy gauge and consumer traces. Restore worker and verify backlog drains. See infra/runbook.md.
- alert: QueueObservationStale
expr: time() - max(messenger_queue_observed_at_seconds) > 15
for: 6s
labels:
severity: warning
annotations:
summary: Publisher backlog observations are stale
description: Check publisher and PostgreSQL availability before trusting queue gauges.
+11
View File
@@ -0,0 +1,11 @@
global:
scrape_interval: 3s
evaluation_interval: 3s
rule_files: [/etc/prometheus/alerts.yaml]
scrape_configs:
- job_name: application
static_configs:
- targets: [otel-collector:8889]
- job_name: collector
static_configs:
- targets: [otel-collector:8888]
+30
View File
@@ -0,0 +1,30 @@
# Диагностика лабораторной 6
Grafana — http://localhost:13000, dashboard Messenger. Prometheus — http://localhost:19090/alerts. Учётная запись Grafana admin, пароль в локальном secrets/grafana_password.
## Ошибки API
1. Посмотреть RPS, долю 5xx и p95/p99 за интервал алерта HighApiErrorRate.
2. В Tempo найти server span с ошибкой, раскрыть postgresql.transaction и s3 spans. HTTP server span включает шаблон маршрута и статус. SQL-параметры и содержимое объектов не экспортируются.
3. В Loki найти логи по service_name и trace_id. X-Request-Id из ответа помогает найти HTTP-событие.
4. При ошибках БД проверить здоровье postgres и число занятых соединений. getconn в psycopg2 fail-fast, waiting показывает только текущие попытки получения соединения. Exhausted и histogram acquire дополняют этот gauge.
5. Восстановить postgres. Проверить успешные запросы и исчезновение firing после выхода ошибки из минутного окна.
## Рост возраста очереди
1. Сопоставить messenger_queue_depth, messenger_queue_oldest_age_seconds, broker_ready и worker busy/capacity.
2. Backlog включает queued/processing в PostgreSQL, в том числе outbox и отложенные retry. Broker ready показывает только ожидающие доставки RabbitMQ сообщения. Processing duration измеряет одну попытку. Time to ready включает ожидание и retry.
3. Если нет свежих worker метрик, проверить worker. Gauge после остановки процесса временно сохраняется в exporter, время последнего получения данных проверять отдельно.
4. Найти trace upload по X-Trace-Id, затем producer, consumer и операции S3/DB. Попытки имеют job.attempt в traces, без job_id в metric labels.
5. Запустить worker. Убедиться, что backlog и oldest age равны нулю, алерт resolved. При QueueObservationStale проверить publisher/БД.
## Хранилище телеметрии недоступно
1. Посмотреть собственные метрики Collector, exporter send_failed и enqueue_failed, а также docker stats.
2. SDK ограничены 256 spans/logs на процесс, batch 64 и timeout 1 секунда. Collector ограничен memory_limiter 128 MiB, контейнером 192 MiB, batch максимум 512 записей и экспортными очередями по 128 batches. При заполнении/исчерпании retry телеметрия может теряться. Prometheus экспортируется pull endpoint, его остановка оставляет пробел в scrape-истории.
3. Восстановить Loki/Tempo, убедиться, что свежий trace и лог снова доступны. Подтвердить, что принятые сообщения и вложения доступны через API.
4. Разница accepted/sent и exporter failure counters помогает оценивать проблему. send_failed учитывает попытки отправки, поэтому не является точным числом уникальных потерянных spans. Без sequence IDs точные потери не заявляются.
## SLI/SLO
SLI = 1 − число 5xx / число всех бизнес-запросов за скользящие 5 минут. Health исключён при записи HTTP-метрик. SLO ≥ 99%. При отсутствии запросов окно не даёт оценки доступности. 4xx остаются в знаменателе и не считаются отказами сервера. Алерт ошибок использует более короткое окно 1 минута, порог 1%, for 6 секунд. QueueLagGrowing — возраст старейшего незавершённого задания >15 секунд, backlog >0, for 6 секунд. Это учебные пороги для воспроизводимой проверки.
+73
View File
@@ -0,0 +1,73 @@
#!/bin/sh
set -eu
umask 077
bucket=${S3_BUCKET:-messenger-attachments}
case "$bucket" in
''|*[!a-z0-9.-]*) echo 'Invalid S3 bucket name' >&2; exit 1 ;;
esac
config=$(mktemp -d)
trap 'rm -rf "$config"' EXIT
export MC_CONFIG_DIR="$config"
root_user=$(cat /run/secrets/s3_root_user)
root_password=$(cat /run/secrets/s3_root_password)
api_user=$(cat /run/secrets/s3_access_key_id)
api_password=$(cat /run/secrets/s3_secret_access_key)
endpoint=${S3_ENDPOINT:-http://s3:9000}
if [ "${1:-}" = verify ]; then
export MC_HOST_api="http://$api_user:$api_password@${endpoint#http://}"
key="attachments/probe-${config##*/}"
object="api/$bucket/$key"
trap 'mc rm "$object" >/dev/null 2>&1 || true; rm -rf "$config"' EXIT
printf '%s' 'lab6-private-bucket-probe' | mc pipe "$object" >/dev/null
test "$(mc cat "$object")" = 'lab6-private-bucket-probe'
python3 - "$endpoint/$bucket/$key" <<'PYTEST'
import sys
from urllib.error import HTTPError
from urllib.request import urlopen
try:
with urlopen(sys.argv[1], timeout=5):
raise SystemExit("FAIL: anonymous GET succeeded")
except HTTPError as exc:
if exc.code != 403:
raise
print("PASS: authenticated round trip; anonymous GET returned 403")
PYTEST
mc rm "$object" >/dev/null
exit 0
fi
attempt=0
until mc alias set storage "$endpoint" "$root_user" "$root_password" >/dev/null 2>&1 &&
mc admin info storage >/dev/null 2>&1; do
attempt=$((attempt + 1))
if [ "$attempt" -ge 30 ]; then
echo 'S3 bootstrap could not connect or authenticate' >&2
exit 1
fi
sleep 1
done
mc mb --ignore-existing "storage/$bucket"
mc anonymous set none "storage/$bucket"
cat > "$config/api-policy.json" <<POLICY
{
"Version": "2012-10-17",
"Statement": [{
"Effect": "Allow",
"Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"],
"Resource": ["arn:aws:s3:::$bucket/attachments/*"]
}, {
"Effect": "Allow",
"Action": ["s3:GetBucketLocation"],
"Resource": ["arn:aws:s3:::$bucket"]
}]
}
POLICY
mc admin policy create storage lab6-attachments "$config/api-policy.json"
mc admin user add storage "$api_user" "$api_password" >/dev/null
mc admin policy attach storage lab6-attachments --user "$api_user"
echo 'PASS: private S3 bucket and scoped API policy configured'
+22
View File
@@ -0,0 +1,22 @@
server:
http_listen_port: 3200
distributor:
receivers:
otlp:
protocols:
grpc:
endpoint: 0.0.0.0:4317
http:
endpoint: 0.0.0.0:4318
storage:
trace:
backend: local
wal:
path: /var/tempo/wal
local:
path: /var/tempo/blocks
compactor:
compaction:
block_retention: 72h
usage_report:
reporting_enabled: false
+2
View File
@@ -0,0 +1,2 @@
server api-a:8080 resolve max_fails=1 fail_timeout=2s;
server api-b:8080 resolve max_fails=1 fail_timeout=2s;
+51
View File
@@ -0,0 +1,51 @@
-- Migration 0001: initial schema for lab2.
-- Idempotent: safe to re-run on an existing database.
CREATE TABLE IF NOT EXISTS users (
id uuid PRIMARY KEY,
username text NOT NULL UNIQUE,
password_hash text NOT NULL,
display_name text NOT NULL,
created_at timestamptz NOT NULL DEFAULT now()
);
CREATE TABLE IF NOT EXISTS chats (
id uuid PRIMARY KEY,
title text NOT NULL,
created_at timestamptz NOT NULL DEFAULT now()
);
CREATE TABLE IF NOT EXISTS chat_members (
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
PRIMARY KEY (chat_id, user_id)
);
CREATE TABLE IF NOT EXISTS messages (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
sender_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
text text NOT NULL,
created_at timestamptz NOT NULL DEFAULT now()
);
-- Keyset pagination index: history by (chat_id, created_at, id) avoids
-- full table scan on every page read.
CREATE INDEX IF NOT EXISTS idx_messages_chat_created_id
ON messages (chat_id, created_at, id);
CREATE INDEX IF NOT EXISTS idx_messages_chat_id
ON messages (chat_id);
CREATE TABLE IF NOT EXISTS sessions (
token_hash text PRIMARY KEY,
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
expires_at timestamptz NOT NULL,
created_at timestamptz NOT NULL DEFAULT now()
);
CREATE INDEX IF NOT EXISTS idx_sessions_user_id
ON sessions (user_id);
CREATE INDEX IF NOT EXISTS idx_sessions_expires_at
ON sessions (expires_at);
+14
View File
@@ -0,0 +1,14 @@
-- Additive migration: compatible with the lab2 schema and existing messages.
CREATE TABLE IF NOT EXISTS message_idempotency (
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
method text NOT NULL,
path text NOT NULL,
key text NOT NULL,
request_hash text NOT NULL,
response_status integer NOT NULL,
response_body jsonb NOT NULL,
created_at timestamptz NOT NULL DEFAULT now(),
PRIMARY KEY (user_id, method, path, key)
);
-- Results are retained indefinitely in this lab (therefore at least 24 hours).
-- There is intentionally no independent cache expiry that could allow duplicates.
+37
View File
@@ -0,0 +1,37 @@
-- Additive migration. Existing text messages have no attachment links.
ALTER TABLE messages
ADD CONSTRAINT messages_id_chat_id_key UNIQUE (id, chat_id);
CREATE TABLE attachments (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
owner_id uuid NOT NULL REFERENCES users(id),
filename text NOT NULL CHECK (char_length(filename) BETWEEN 1 AND 255),
content_type text NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg')),
size_bytes bigint NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760),
sha256 text NOT NULL CHECK (sha256 ~ '^[0-9a-f]{64}$'),
object_key text NOT NULL UNIQUE CHECK (object_key LIKE 'attachments/%'),
-- pending is internal. The upload API exposes ready after committing metadata.
status text NOT NULL CHECK (status IN ('pending', 'ready', 'failed')),
error_code text,
created_at timestamptz NOT NULL DEFAULT now(),
updated_at timestamptz NOT NULL DEFAULT now(),
UNIQUE (id, chat_id),
CHECK ((status = 'failed') = (error_code IS NOT NULL))
);
CREATE INDEX attachments_pending_updated_idx
ON attachments (updated_at, id) WHERE status = 'pending';
CREATE TABLE message_attachments (
message_id uuid NOT NULL,
attachment_id uuid NOT NULL,
chat_id uuid NOT NULL,
position smallint NOT NULL CHECK (position BETWEEN 0 AND 9),
PRIMARY KEY (message_id, position),
UNIQUE (message_id, attachment_id),
FOREIGN KEY (message_id, chat_id) REFERENCES messages(id, chat_id) ON DELETE CASCADE,
FOREIGN KEY (attachment_id, chat_id) REFERENCES attachments(id, chat_id)
);
CREATE INDEX message_attachments_attachment_idx ON message_attachments (attachment_id);
+33
View File
@@ -0,0 +1,33 @@
ALTER TABLE attachments DROP CONSTRAINT attachments_status_check;
ALTER TABLE attachments ADD CHECK (status IN ('pending','queued','processing','ready','failed'));
CREATE TABLE attachment_jobs (
id uuid PRIMARY KEY,
attachment_id uuid NOT NULL REFERENCES attachments(id) ON DELETE CASCADE,
pipeline_version integer NOT NULL CHECK (pipeline_version > 0),
attempt integer NOT NULL DEFAULT 1 CHECK (attempt > 0),
created_at timestamptz NOT NULL DEFAULT now(),
UNIQUE (attachment_id, pipeline_version)
);
CREATE TABLE attachment_outbox (
job_id uuid NOT NULL REFERENCES attachment_jobs(id) ON DELETE CASCADE,
attempt integer NOT NULL CHECK (attempt > 0),
available_at timestamptz NOT NULL DEFAULT now(),
published_at timestamptz,
PRIMARY KEY (job_id, attempt)
);
CREATE INDEX attachment_outbox_due_idx ON attachment_outbox (available_at) WHERE published_at IS NULL;
CREATE TABLE attachment_variants (
attachment_id uuid NOT NULL REFERENCES attachments(id) ON DELETE CASCADE,
pipeline_version integer NOT NULL CHECK (pipeline_version > 0),
variant text NOT NULL CHECK (variant IN ('thumbnail','crop','optimized')),
object_key text NOT NULL UNIQUE,
content_type text NOT NULL CHECK (content_type IN ('image/jpeg','image/webp')),
size_bytes bigint NOT NULL CHECK (size_bytes > 0),
sha256 text NOT NULL CHECK (sha256 ~ '^[0-9a-f]{64}$'),
width integer NOT NULL CHECK (width > 0),
height integer NOT NULL CHECK (height > 0),
PRIMARY KEY (attachment_id, pipeline_version, variant)
);
+12
View File
@@ -0,0 +1,12 @@
ALTER TABLE attachment_jobs ADD COLUMN lease_until timestamptz;
ALTER TABLE attachment_jobs ADD COLUMN attempts_used integer NOT NULL DEFAULT 0;
ALTER TABLE attachment_jobs ADD COLUMN staging_cleaned_at timestamptz;
ALTER TABLE attachment_outbox ADD COLUMN objects_cleaned_at timestamptz;
CREATE INDEX attachment_jobs_lease_idx ON attachment_jobs (lease_until) WHERE lease_until IS NOT NULL;
CREATE TABLE attachment_replays (
job_id uuid NOT NULL REFERENCES attachment_jobs(id) ON DELETE CASCADE,
attempt integer NOT NULL,
operator text NOT NULL,
created_at timestamptz NOT NULL DEFAULT now(),
PRIMARY KEY (job_id, attempt)
);
+2
View File
@@ -0,0 +1,2 @@
-- Preserve causal context across outbox retries and publisher restarts.
ALTER TABLE attachment_jobs ADD COLUMN trace_context jsonb NOT NULL DEFAULT '{}'::jsonb;
+31
View File
@@ -0,0 +1,31 @@
#!/usr/bin/env python3
"""Generate local credentials once, never print them or overwrite existing files."""
from pathlib import Path
import os
import secrets
root = Path(__file__).resolve().parents[1]
folder = root / 'secrets'
folder.mkdir(mode=0o700, exist_ok=True)
password_file = folder / 'postgres_password'
def save(name, value):
path = folder / name
if path.exists():
return
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
with os.fdopen(fd, 'w') as handle:
handle.write(value + '\n')
save('postgres_password', secrets.token_hex(24))
password = password_file.read_text().strip()
save('database_url', 'postgresql://messenger:%s@postgres:5432/messenger' % password)
save('cache_url', 'redis://redis:6379/0')
save('cursor_secret', secrets.token_hex(32))
save('s3_root_user', 'lab6-admin-' + secrets.token_hex(8))
save('s3_root_password', secrets.token_hex(32))
save('s3_access_key_id', 'lab6-api-' + secrets.token_hex(8))
save('s3_secret_access_key', secrets.token_hex(24))
save('rabbitmq_password', secrets.token_hex(24))
save('grafana_password', secrets.token_hex(24))
print('PASS: local secret files ready (existing values preserved)')
+157
View File
@@ -0,0 +1,157 @@
#!/usr/bin/env python3
"""Explicit, bounded operator actions for this lab's Compose project only."""
import argparse
import json
import os
from pathlib import Path
import subprocess
import time
import uuid
ROOT = Path(__file__).resolve().parents[1]
SERVICES = ('api-a', 'api-b')
def command(args, capture=False):
result = subprocess.run(args, cwd=ROOT, check=True, text=True,
stdout=subprocess.PIPE if capture else None)
return result.stdout.strip() if capture else None
def compose(*args, capture=False):
return command(['docker', 'compose', '--env-file', '.env', *args], capture)
def inspect(container):
return json.loads(command(['docker', 'inspect', container], True))[0]
def containers(service):
return compose('ps', '-a', '-q', service, capture=True).splitlines()
def start_services(*services):
for service in services:
for container in containers(service):
command(['docker', 'start', container])
def wait_healthy(service, timeout=90):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
ids = containers(service)
if ids and all(inspect(c)['State'].get('Health', {}).get('Status') == 'healthy' for c in ids):
return
time.sleep(.5)
raise RuntimeError('%s did not become healthy' % service)
def target_service(target):
if not target:
raise RuntimeError('TARGET_INSTANCE_ID is required')
for service in SERVICES:
for container in containers(service):
data = inspect(container)
if data['Config']['Hostname'] == target:
return service, container
raise RuntimeError('Unknown API instance alias')
def set_backend(service, enabled):
# Fixed allowlist; shell receives no untrusted instance or credential input.
if service not in SERVICES:
raise ValueError('Unknown service')
token = ('serve:' if enabled else 'drain:') + uuid.uuid4().hex
path = '/tmp/control-' + service
compose('exec', '-T', 'nginx', 'sh', '-ec',
"printf '%s\\n' '" + token + "' > " + path + '.next; mv -f ' + path + '.next ' + path)
deadline = time.monotonic() + 10
expected = token + (' up' if enabled else ' down')
while time.monotonic() < deadline:
ack = compose('exec', '-T', 'nginx', 'sh', '-c',
'cat /tmp/applied-' + service + ' 2>/dev/null || true', capture=True)
if ack == expected:
return
time.sleep(.2)
raise RuntimeError('Readiness controller did not acknowledge backend state for ' + service)
def stop_one():
service, container = target_service(os.environ.get('TARGET_INSTANCE_ID'))
running = [c for name in SERVICES for c in containers(name)
if inspect(c)['State'].get('Health', {}).get('Status') == 'healthy'
and inspect(c)['State']['Running']]
if len(running) < 2:
raise RuntimeError('Restore the other replica before this experiment')
print('Stopping API replica:', service, container[:12], flush=True)
command(['docker', 'stop', '--time', '8', container])
if inspect(container)['State']['Running']:
raise RuntimeError('Target is still running')
def restart():
# Never start the one-shot migration service as part of persistence testing.
compose('restart', 'postgres', 'redis', 's3', 'rabbitmq', *SERVICES, 'nginx', 'outbox-publisher', 'worker')
for service in (*SERVICES, 'nginx'):
wait_healthy(service)
def rolling_update():
# Run additive migrations once before replacing either API version.
compose('run', '--rm', '--no-deps', 'migrate')
for service in SERVICES:
other = SERVICES[1] if service == SERVICES[0] else SERVICES[0]
wait_healthy(other)
set_backend(other, True)
before = containers(service)
set_backend(service, False)
# The old nginx worker can retain the old upstream table until it exits.
# worker_shutdown_timeout=6s bounds that drain; no new work then reaches it.
time.sleep(6.5)
compose('up', '-d', '--no-deps', '--force-recreate', '--no-build', service)
wait_healthy(service)
set_backend(service, True)
time.sleep(2) # DNS valid=1s + readiness before moving to the next replica.
after = containers(service)
if before == after:
raise RuntimeError('Replica was not replaced')
print(json.dumps({'service': service, 'before': before, 'after': after}), flush=True)
def restore():
start_services(*SERVICES)
for service in SERVICES:
wait_healthy(service)
for service in SERVICES:
set_backend(service, True)
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('action', choices=('stop-one', 'restart', 'restore', 'rolling-update', 'status'))
args = parser.parse_args()
if args.action == 'stop-one':
stop_one()
elif args.action == 'restart':
restart()
elif args.action == 'rolling-update':
rolling_update()
elif args.action == 'restore':
restore()
else:
for service in ('postgres', 'redis', *SERVICES, 'nginx'):
for container in containers(service):
data = inspect(container)
print(json.dumps({'service': service, 'id': data['Id'][:12],
'instance': data['Config']['Hostname'],
'running': data['State']['Running'],
'started_at': data['State']['StartedAt'],
'health': data['State'].get('Health', {}).get('Status'),
'image_id': data['Image'],
'cpu_limit': data['HostConfig']['NanoCpus'] / 1e9,
'memory_limit_bytes': data['HostConfig']['Memory'],
'published_ports': data['HostConfig'].get('PortBindings')}))
if __name__ == '__main__':
main()
+4
View File
@@ -0,0 +1,4 @@
#!/bin/sh
set -eu
cd "$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)"
exec python3 scripts/manage.py restart
+4
View File
@@ -0,0 +1,4 @@
#!/bin/sh
set -eu
cd "$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)"
exec python3 scripts/manage.py rolling-update
+4
View File
@@ -0,0 +1,4 @@
#!/bin/sh
set -eu
cd "$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)"
exec python3 scripts/manage.py stop-one
+27
View File
@@ -0,0 +1,27 @@
#!/usr/bin/env python3
"""Wait for the public API and local telemetry backends, with a shared deadline."""
import time
from urllib.error import URLError
from urllib.request import urlopen
pending = {
'api': 'http://localhost:18083/health/ready',
'prometheus': 'http://localhost:19090/-/ready',
'loki': 'http://localhost:13100/ready',
'tempo': 'http://localhost:13200/ready',
'grafana': 'http://localhost:13000/api/health',
}
deadline = time.monotonic() + 120
while pending and time.monotonic() < deadline:
for name, url in list(pending.items()):
try:
with urlopen(url, timeout=2) as response:
if response.status == 200:
del pending[name]
except (URLError, TimeoutError, OSError):
pass
if pending:
time.sleep(1)
if pending:
raise SystemExit('Readiness timeout: ' + ', '.join(sorted(pending)))
print('PASS: API and telemetry backends ready')
+3
View File
@@ -0,0 +1,3 @@
"""MTUSI Messenger lab1: in-memory HTTP API (standard library only)."""
__version__ = '1.0.0'
+7
View File
@@ -0,0 +1,7 @@
"""Entry point: python -m messenger."""
import sys
from .server import main
if __name__ == '__main__':
sys.exit(main())
+299
View File
@@ -0,0 +1,299 @@
"""Domain logic and routing for the lab2 API: PostgreSQL state, Bearer sessions."""
from __future__ import annotations
import base64
import datetime as dt
import json
import re
import uuid
from . import cursor as cursor_codec
from . import security
from . import validation
from .attachments import AttachmentStorage, filename_from, upload_file
from .db import Database
from .errors import ApiError, Conflict, InvalidRequest, NotFound, Unauthorized, Unavailable, UnsupportedMediaType
from .store import Store, utc_now, rfc3339
SCOPE_CHATS = 'chats'
SCOPE_MESSAGES = 'chat/{chat_id}/messages'
class Request:
"""One parsed HTTP request; the API layer never touches sockets."""
__slots__ = ('method', 'path', 'query', 'headers', 'body', 'request_id', 'client')
def __init__(self, method, path, query, headers, body, request_id, client):
self.method = method
self.path = path
self.query = query
self.headers = headers
self.body = body
self.request_id = request_id
self.client = client
class Api:
UPLOAD_PATH = re.compile(r'^/api/v1/chats/(?P<chat_id>[^/]+)/attachments$')
ROUTES = (
('POST', UPLOAD_PATH, 'upload_attachment'),
('GET', re.compile(r'^/api/v1/attachments/(?P<id>[^/]+)$'), 'get_attachment'),
('GET', re.compile(r'^/api/v1/attachments/(?P<id>[^/]+)/content$'), 'download_attachment'),
('GET', re.compile(r'^/health/live$'), 'health_live'),
('GET', re.compile(r'^/health/ready$'), 'health_ready'),
('POST', re.compile(r'^/api/v1/users$'), 'create_user'),
('GET', re.compile(r'^/api/v1/users/me$'), 'current_user'),
('GET', re.compile(r'^/api/v1/chats$'), 'list_chats'),
('POST', re.compile(r'^/api/v1/chats$'), 'create_chat'),
('GET', re.compile(r'^/api/v1/chats/(?P<chat_id>[^/]+)$'), 'get_chat'),
('GET', re.compile(r'^/api/v1/chats/(?P<chat_id>[^/]+)/messages$'), 'list_messages'),
('POST', re.compile(r'^/api/v1/chats/(?P<chat_id>[^/]+)/messages$'), 'post_message'),
('POST', re.compile(r'^/api/v1/auth/sessions$'), 'create_session'),
('DELETE', re.compile(r'^/api/v1/auth/sessions/current$'), 'revoke_session'),
)
def __init__(self, config, store: Store, database: Database):
self.config = config
self.store = store
self.database = database
self.attachments = AttachmentStorage()
def dispatch(self, request):
"""Return (status, payload). Only ApiError and payloads leave this method."""
path_matched = False
for method, pattern, handler in self.ROUTES:
match = pattern.match(request.path)
if match:
path_matched = True
if method == request.method:
return getattr(self, handler)(request, match.groupdict())
raise NotFound('Маршрут не найден' if not path_matched else 'Метод не поддерживается')
# ------------------------------------------------------------- health
def health_live(self, request, path_params):
return 200, {'status': 'ok'}
def health_ready(self, request, path_params):
try:
self.database.ping()
except Exception:
raise Unavailable('База данных недоступна')
return 200, {'status': 'ok'}
# -------------------------------------------------------------- users
def create_user(self, request, path_params):
payload = self._json_object(request)
validation.require_fields(payload, ('username', 'display_name', 'password'),
('username', 'display_name', 'password'))
username = validation.require_username(payload)
display_name = validation.require_text_field(
payload, 'display_name', 1, self.config.max_display_name_code_points)
password = validation.require_text_field(
payload, 'password', 12, 128, strip_check=False)
password_hash = security.hash_password(password)
return 201, self.store.create_user(username, display_name, password_hash)
def current_user(self, request, path_params):
return 200, self._authenticate(request)
# -------------------------------------------------------------- chats
def create_chat(self, request, path_params):
actor = self._authenticate(request)
payload = self._json_object(request)
validation.require_fields(payload, ('title', 'member_ids'), ('title', 'member_ids'))
title = validation.require_text_field(payload, 'title', 1,
self.config.max_title_code_points)
members = validation.require_member_ids(payload, 100)
if actor['id'] not in members:
raise InvalidRequest('Создатель чата должен быть указан в member_ids')
if not self.store.users_exist(members):
raise InvalidRequest('member_ids содержит неизвестных пользователей')
return 201, self.store.create_chat(title, members)
def list_chats(self, request, path_params):
actor = self._authenticate(request)
limit, after_key = self._pagination(request, SCOPE_CHATS)
items, has_more = self.store.page_chats(actor['id'], after_key, limit)
for item in items:
item['member_ids'] = self._chat_member_ids(item['id'])
return 200, {'items': items, 'next_cursor': self._next_cursor(SCOPE_CHATS, items, has_more)}
def get_chat(self, request, path_params):
actor = self._authenticate(request)
chat = self._member_chat(actor, path_params['chat_id'])
chat['member_ids'] = self._chat_member_ids(chat['id'])
return 200, chat
# ----------------------------------------------------------- messages
def post_message(self, request, path_params):
actor = self._authenticate(request)
chat = self._member_chat(actor, path_params['chat_id'])
key = validation.require_idempotency_key(request.headers.get('Idempotency-Key'))
payload = self._json_object(request)
validation.require_fields(payload, ('text',), ('text', 'attachment_ids'))
attachment_ids = validation.require_attachment_ids(payload)
text = validation.require_text_field(payload, 'text', 0 if attachment_ids else 1,
self.config.max_text_code_points,
strip_check=not attachment_ids)
return 201, self.store.add_message(chat['id'], actor['id'], text, key, attachment_ids)
def list_messages(self, request, path_params):
actor = self._authenticate(request)
chat = self._member_chat(actor, path_params['chat_id'])
scope = SCOPE_MESSAGES.format(chat_id=chat['id'])
limit, after_key = self._pagination(request, scope)
items, has_more = self.store.page_messages(chat['id'], after_key, limit)
return 200, {'items': items, 'next_cursor': self._next_cursor(scope, items, has_more)}
def upload_attachment(self, request, path_params):
actor = self._authenticate(request)
chat = self._member_chat(actor, path_params['chat_id'])
filename = filename_from(request.query)
mime = request.headers.get('Content-Type', '').split(';', 1)[0].strip().lower()
with upload_file(request.body, mime) as (file, size, digest):
aid = str(uuid.uuid4())
key = 'attachments/staging/' + aid
self.store.reserve_attachment(aid, chat['id'], actor['id'], filename,
mime, size, digest, key)
row = self.store.queue_attachment(
aid, lambda object_key: self.attachments.put(object_key, file, size, mime))
return 202, self.store.public_attachment(row)
def _visible_attachment(self, request, path_params):
actor = self._authenticate(request)
aid = validation.require_uuid_string(path_params['id'], 'attachment_id')
row = self.store.visible_attachment(aid, actor['id'])
if row is None:
raise NotFound('Вложение не найдено')
return row
def get_attachment(self, request, path_params):
row = self._visible_attachment(request, path_params)
variants = self.store.attachment_variants(row['id']) if row['status'] == 'ready' else ()
return 200, self.store.public_attachment(row, variants)
def download_attachment(self, request, path_params):
row = self._visible_attachment(request, path_params)
if row['status'] != 'ready':
raise Conflict('Вложение не готово')
variant = request.query.get('variant', ['original'])
if variant != ['original']:
row = next((item for item in self.store.attachment_variants(row['id'])
if [item['name']] == variant), None)
if row is None:
raise NotFound('Вариант вложения не найден')
return 200, self.attachments.download(row)
# ----------------------------------------------------------- sessions
def create_session(self, request, path_params):
"""Login: only Authorization: Basic username:password, no JSON body."""
authorization = request.headers.get('Authorization') or ''
if not authorization.lower().startswith('basic '):
raise Unauthorized('Требуется авторизация Basic')
try:
raw = base64.b64decode(authorization[len('basic '):].strip()).decode('utf-8')
except Exception:
raise Unauthorized('Некорректные учётные данные') from None
if ':' not in raw:
raise Unauthorized('Некорректные учётные данные')
username, password = raw.split(':', 1)
row = self.store.user_by_username(username)
if row is None or not security.verify_password(password, row['password_hash']):
raise Unauthorized('Неверное имя пользователя или пароль', www_authenticate='Basic realm="messenger", charset="UTF-8"')
token = security.generate_token()
token_hash = security.hash_token(token)
expires_at = utc_now() + dt.timedelta(seconds=self.config.session_ttl_seconds)
self.store.create_session(token_hash, row['id'], expires_at)
user = self.store._public_user({
'id': row['id'],
'username': row['username'],
'display_name': row['display_name'],
'created_at': row['created_at'],
})
return 201, {
'token': token,
'token_type': 'Bearer',
'expires_at': rfc3339(expires_at),
'user': user,
}
def revoke_session(self, request, path_params):
actor = self._authenticate(request)
token = self._bearer_token(request)
self.store.delete_session(security.hash_token(token))
return 204, None
# ------------------------------------------------------------- helpers
def _authenticate(self, request):
"""Lab2 identity: only Authorization: Bearer <token> identifies the actor.
X-User-Id is ignored: it does not change the session owner and alone
grants no access. Basic on business endpoints is 401.
"""
token = self._bearer_token(request)
if token is None:
raise Unauthorized('Требуется идентификация пользователя',
www_authenticate='Bearer realm="messenger"')
user_id = self.store.get_session(security.hash_token(token))
if user_id is None:
raise Unauthorized('Сессия недействительна или истекла',
www_authenticate='Bearer realm="messenger"')
user = self.store.user(user_id)
if user is None:
raise Unauthorized('Пользователь не найден',
www_authenticate='Bearer realm="messenger"')
return user
def _bearer_token(self, request):
authorization = request.headers.get('Authorization') or ''
if not authorization.lower().startswith('bearer '):
return None
return authorization[len('bearer '):].strip()
def _chat_member_ids(self, chat_id):
with self.database.connection() as conn:
with conn.cursor() as cur:
cur.execute(
'SELECT user_id FROM chat_members WHERE chat_id = %s', (str(chat_id),))
return [str(row['user_id']) for row in cur.fetchall()]
def _member_chat(self, actor, raw_chat_id):
"""Foreign and missing chats are indistinguishable: both are 404."""
chat_id = validation.require_uuid_string(raw_chat_id, 'chat_id')
chat = self.store.chat(chat_id)
if chat is None or not self.store.is_member(chat, actor['id']):
raise NotFound('Чат не найден')
return chat
def _json_object(self, request):
content_type = request.headers.get('Content-Type') or ''
if content_type.split(';')[0].strip().lower() != 'application/json':
raise UnsupportedMediaType('Ожидается Content-Type: application/json')
if not request.body:
raise InvalidRequest('Требуется JSON-тело запроса')
try:
payload = json.loads(request.body.decode('utf-8'))
except (UnicodeError, ValueError):
raise InvalidRequest('Тело запроса не является корректным JSON') from None
return validation.require_object(payload)
def _pagination(self, request, scope):
raw_limit = (request.query.get('limit') or [None])[0]
limit = validation.parse_limit(raw_limit, self.config.default_page_limit,
self.config.max_page_limit)
raw_cursor = validation.parse_cursor((request.query.get('cursor') or [None])[0])
if raw_cursor is None:
return limit, None
try:
stamp, identifier = cursor_codec.decode(raw_cursor, scope, self.config.cursor_secret)
except cursor_codec.InvalidCursor:
raise InvalidRequest('cursor некорректен или относится к другому ресурсу') from None
return limit, (stamp, identifier)
def _next_cursor(self, scope, items, has_more):
if not has_more or not items:
return None
last = items[-1]
return cursor_codec.encode(scope, last['created_at'], last['id'],
self.config.cursor_secret)
+147
View File
@@ -0,0 +1,147 @@
"""Bounded image validation and private S3 storage."""
from contextlib import contextmanager
import hashlib
import os
import struct
import tempfile
import threading
import unicodedata
from functools import cached_property
from . import telemetry
import boto3
from botocore.config import Config
from botocore.exceptions import BotoCoreError, ClientError
from PIL import Image
from .errors import InvalidRequest, NotFound, PayloadTooLarge, Unavailable, UnsupportedMediaType
MAX_UPLOAD_BYTES = 10 * 1024 * 1024
MAX_IMAGE_PIXELS = 40_000_000
CHUNK_SIZE = 64 * 1024
FORMATS = {'PNG': 'image/png', 'JPEG': 'image/jpeg'}
# Pillow can allocate a full decoded image. Allow only one decode per API process.
_decode_slot = threading.Lock()
_upload_slots = threading.BoundedSemaphore(10)
def filename_from(query):
names = query.get('filename', [])
if len(names) != 1:
raise InvalidRequest('Укажите одно имя файла')
name = names[0]
if (not 1 <= len(name) <= 255 or not name.strip()
or any(c in '/\\' or unicodedata.category(c) in {'Cc', 'Cs'} for c in name)):
raise InvalidRequest('Некорректное имя файла')
return name
def validate_image(file, declared_type):
try:
with _decode_slot:
file.seek(0)
with Image.open(file, formats=list(FORMATS)) as image:
if image.width * image.height > MAX_IMAGE_PIXELS:
raise PayloadTooLarge('Изображение превышает 40 млн пикселей')
if FORMATS[image.format] != declared_type:
raise UnsupportedMediaType('Тип изображения не совпадает с Content-Type')
image.verify()
file.seek(0)
with Image.open(file, formats=list(FORMATS)) as image:
image.load()
except (Image.DecompressionBombError, Image.DecompressionBombWarning):
raise PayloadTooLarge('Изображение превышает 40 млн пикселей') from None
except (OSError, SyntaxError, ValueError, OverflowError, EOFError, struct.error):
raise UnsupportedMediaType('Повреждённое или неподдерживаемое изображение') from None
@contextmanager
def upload_file(read_chunks, content_type):
if content_type not in FORMATS.values():
raise UnsupportedMediaType('Ожидается PNG или JPEG')
if not _upload_slots.acquire(blocking=False):
raise Unavailable('Слишком много одновременных загрузок')
try:
with tempfile.TemporaryFile() as file:
digest = hashlib.sha256()
size = 0
for chunk in read_chunks():
size += len(chunk)
if size > MAX_UPLOAD_BYTES:
raise PayloadTooLarge()
digest.update(chunk)
file.write(chunk)
if size == 0:
raise UnsupportedMediaType('Пустое изображение')
validate_image(file, content_type)
file.seek(0)
yield file, size, digest.hexdigest()
finally:
_upload_slots.release()
class BinaryResponse:
def __init__(self, stream, content_type, size):
self.stream = stream
self.content_type = content_type
self.size = size
def chunks(self):
try:
yield from self.stream.iter_chunks(CHUNK_SIZE)
except (BotoCoreError, OSError) as exc:
raise OSError('S3 response interrupted') from exc
def close(self):
self.stream.close()
class AttachmentStorage:
@cached_property
def client(self):
def secret(name):
with open(os.environ[name + '_FILE'], encoding='utf-8') as file:
return file.read().strip()
return boto3.session.Session().client(
's3', endpoint_url=os.environ['S3_ENDPOINT'],
region_name=os.environ.get('S3_REGION', 'us-east-1'),
aws_access_key_id=secret('S3_ACCESS_KEY_ID'),
aws_secret_access_key=secret('S3_SECRET_ACCESS_KEY'),
config=Config(connect_timeout=2, read_timeout=5, retries={'total_max_attempts': 1},
request_checksum_calculation="when_required",
s3={"addressing_style": "path"}))
@property
def bucket(self):
return os.environ['S3_BUCKET']
@telemetry.operation("s3.put")
def put(self, key, file, size, content_type):
try:
self.client.put_object(Bucket=self.bucket, Key=key, Body=file,
ContentLength=size, ContentType=content_type)
except (BotoCoreError, ClientError, OSError, KeyError):
raise Unavailable('Хранилище вложений недоступно') from None
@telemetry.operation("s3.delete")
def delete(self, key):
try:
self.client.delete_object(Bucket=self.bucket, Key=key)
except (BotoCoreError, ClientError, OSError, KeyError):
raise Unavailable('Хранилище вложений недоступно') from None
@telemetry.operation("s3.download")
def download(self, row):
try:
obj = self.client.get_object(Bucket=self.bucket, Key=row['object_key'])
except ClientError as exc:
if exc.response.get('Error', {}).get('Code') in {'NoSuchKey', '404'}:
raise NotFound('Оригинал вложения не найден') from None
raise Unavailable('Хранилище вложений недоступно') from None
except (BotoCoreError, OSError, KeyError):
raise Unavailable('Хранилище вложений недоступно') from None
if obj['ContentLength'] != row['size_bytes']:
obj['Body'].close()
raise Unavailable('Размер оригинала не совпадает с метаданными')
return BinaryResponse(obj['Body'], row['content_type'], row['size_bytes'])
+95
View File
@@ -0,0 +1,95 @@
"""Redis session cache with TTL and graceful degradation.
PostgreSQL is the source of truth for sessions. Redis is a look-aside cache:
- write: session is persisted in PostgreSQL first, then copied to Redis with TTL
- read: Redis first (fast); on miss, read PostgreSQL and backfill Redis
- when Redis is unavailable: fall back to PostgreSQL only, never treat an
anonymous request as trusted
"""
from __future__ import annotations
import logging
import redis
from .config import Config
logger = logging.getLogger('messenger.cache')
class SessionCache:
"""Look-aside cache for session tokens in Redis/Valkey."""
def __init__(self, config: Config):
self._config = config
self._client = None
self._available = False
def open(self):
if not self._config.cache_url:
self._available = False
return
try:
self._client = redis.Redis.from_url(
self._config.cache_url,
socket_connect_timeout=2,
socket_timeout=2,
socket_keepalive=True,
health_check_interval=30,
decode_responses=True,
)
self._client.ping()
self._available = True
logger.info('cache_connected', extra={'event': 'cache_connected'})
except Exception as exc: # noqa: BLE001 - Redis is optional, degrade safely
self._available = False
self._client = None
logger.warning('cache_unavailable', extra={
'event': 'cache_unavailable', 'detail': 'falling back to PostgreSQL'})
def close(self):
if self._client is not None:
try:
self._client.close()
except Exception:
pass
self._client = None
self._available = False
@property
def available(self):
return self._available and self._client is not None
def get(self, token_hash):
"""Return cached user_id or None (miss or unavailable)."""
if not self.available:
return None
try:
value = self._client.get('session:%s' % token_hash)
return value
except Exception as exc: # noqa: BLE001
logger.warning('cache_read_failed', extra={
'event': 'cache_read_failed', 'detail': 'falling back to PostgreSQL'})
self._available = False
return None
def set(self, token_hash, user_id, ttl_seconds):
"""Cache a session with TTL. Failure is non-fatal."""
if not self.available:
return
try:
self._client.setex('session:%s' % token_hash, ttl_seconds, user_id)
except Exception as exc: # noqa: BLE001
logger.warning('cache_write_failed', extra={
'event': 'cache_write_failed', 'detail': 'continuing without cache'})
self._available = False
def delete(self, token_hash):
"""Remove a session from the cache (logout). Failure is non-fatal."""
if not self.available:
return
try:
self._client.delete('session:%s' % token_hash)
except Exception as exc: # noqa: BLE001
logger.warning('cache_delete_failed', extra={
'event': 'cache_delete_failed'})
+58
View File
@@ -0,0 +1,58 @@
"""Remove expired upload reservations and their S3 objects."""
import argparse
import logging
import time
from .attachments import AttachmentStorage
from .config import Config
from .db import Database
from .errors import Unavailable
from .store import Store
from .job_lifecycle import clean_staging, clean_abandoned_outputs
GRACE_SECONDS = 15 * 60
INTERVAL_SECONDS = 60
BATCH_SIZE = 25
def clean_batch(store, storage):
count = 0
while count < BATCH_SIZE and store.clean_pending_attachment(GRACE_SECONDS, storage.delete):
count += 1
return count
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--once', action='store_true', help='Process one bounded batch and exit')
args = parser.parse_args()
logging.basicConfig(level=logging.INFO)
config = Config.from_env()
database = Database(config)
store = Store(config, database, None)
storage = AttachmentStorage()
try:
database.open()
while True:
try:
deleted = clean_batch(store, storage)
for cleanup in (clean_staging, clean_abandoned_outputs):
for _ in range(BATCH_SIZE):
if not cleanup(database, storage.delete):
break
deleted += 1
if deleted or args.once:
logging.info('Expired attachment reservations removed: %s', deleted)
except Unavailable:
logging.warning('Attachment cleanup deferred because a dependency is unavailable')
if args.once:
return 1
if args.once:
return 0
time.sleep(INTERVAL_SECONDS)
finally:
database.close()
if __name__ == '__main__':
raise SystemExit(main())
+149
View File
@@ -0,0 +1,149 @@
"""Runtime configuration. Every value is read from the environment."""
from __future__ import annotations
import os
import secrets
# Contract limits: they cannot be raised by configuration.
CONTRACT_MAX_TEXT_CODE_POINTS = 4000
CONTRACT_MAX_PAGE_LIMIT = 100
CONTRACT_SESSION_TTL_MIN = 1
CONTRACT_SESSION_TTL_MAX = 86400
class ConfigError(ValueError):
"""Raised when an environment value cannot be used."""
def _str(environ, name, default):
value = environ.get(name)
if value is None or value == '':
return default
return value
def _int(environ, name, default, minimum, maximum):
raw = environ.get(name)
if raw is None or raw == '':
return default
try:
value = int(raw)
except ValueError:
raise ConfigError('%s must be an integer, got %r' % (name, raw)) from None
if not minimum <= value <= maximum:
raise ConfigError('%s must be in %d..%d, got %d' % (name, minimum, maximum, value))
return value
def _log_level(environ, name, default):
value = _str(environ, name, default).upper()
if value not in ('DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'):
raise ConfigError('%s must be one of DEBUG, INFO, WARNING, ERROR, CRITICAL' % name)
return value
def _secret_from_file(environ, name, default=None):
"""Читает секрет из файла, путь к которому задаёт переменная окружения.
Секреты не лежат в переменных окружения напрямую: контейнер монтирует
файл (например, Docker secret), а приложение читает его содержимое.
Если файл не задан или не существует, используется значение переменной
окружения без суффикса _FILE (удобно для локального запуска без секретов).
"""
path_name = name + '_FILE'
path = environ.get(path_name)
if path:
try:
with open(path, 'r', encoding='utf-8') as handle:
return handle.read().strip()
except OSError:
pass
return environ.get(name) or default
class Config:
"""Effective configuration of one process instance."""
def __init__(self, host, port, env, log_level, cursor_secret, shutdown_grace_seconds,
max_text_code_points, max_display_name_code_points, max_title_code_points,
default_page_limit, max_page_limit, max_body_bytes,
database_url, cache_url, session_ttl_seconds, session_store,
pool_min, pool_max, connect_timeout, statement_timeout):
self.host = host
self.port = port
self.env = env
self.log_level = log_level
self.cursor_secret = cursor_secret
self.shutdown_grace_seconds = shutdown_grace_seconds
self.max_text_code_points = max_text_code_points
self.max_display_name_code_points = max_display_name_code_points
self.max_title_code_points = max_title_code_points
self.default_page_limit = default_page_limit
self.max_page_limit = max_page_limit
self.max_body_bytes = max_body_bytes
self.database_url = database_url
self.cache_url = cache_url
self.session_ttl_seconds = session_ttl_seconds
self.session_store = session_store
self.pool_min = pool_min
self.pool_max = pool_max
self.connect_timeout = connect_timeout
self.statement_timeout = statement_timeout
@classmethod
def from_env(cls, environ=None):
environ = os.environ if environ is None else environ
secret = _secret_from_file(environ, 'CURSOR_SECRET', '')
if not secret:
secret = secrets.token_hex(32)
session_store = _str(environ, 'SESSION_STORE', 'postgres')
if session_store not in ('postgres', 'redis'):
raise ConfigError('SESSION_STORE должен быть postgres или redis, got %r' % session_store)
return cls(
host=_str(environ, 'APP_HOST', '0.0.0.0'),
port=_int(environ, 'APP_PORT', 8080, 1, 65535),
env=_str(environ, 'APP_ENV', 'lab'),
log_level=_log_level(environ, 'LOG_LEVEL', 'INFO'),
cursor_secret=secret.encode('utf-8'),
shutdown_grace_seconds=_int(environ, 'SHUTDOWN_GRACE_SECONDS', 8, 1, 10),
max_text_code_points=_int(environ, 'MAX_TEXT_CODE_POINTS', CONTRACT_MAX_TEXT_CODE_POINTS,
1, CONTRACT_MAX_TEXT_CODE_POINTS),
max_display_name_code_points=_int(environ, 'MAX_DISPLAY_NAME_CODE_POINTS', 100, 1, 100),
max_title_code_points=_int(environ, 'MAX_TITLE_CODE_POINTS', 100, 1, 100),
default_page_limit=_int(environ, 'DEFAULT_PAGE_LIMIT', 50, 1, CONTRACT_MAX_PAGE_LIMIT),
max_page_limit=_int(environ, 'MAX_PAGE_LIMIT', CONTRACT_MAX_PAGE_LIMIT, 1,
CONTRACT_MAX_PAGE_LIMIT),
max_body_bytes=_int(environ, 'MAX_BODY_BYTES', 1048576, 1024, 16 * 1048576),
database_url=_secret_from_file(environ, 'DATABASE_URL', None),
cache_url=_secret_from_file(environ, 'CACHE_URL', None),
session_ttl_seconds=_int(environ, 'SESSION_TTL_SECONDS', 3600,
CONTRACT_SESSION_TTL_MIN, CONTRACT_SESSION_TTL_MAX),
session_store=session_store,
pool_min=_int(environ, 'DB_POOL_MIN', 2, 1, 50),
pool_max=_int(environ, 'DB_POOL_MAX', 10, 1, 100),
connect_timeout=_int(environ, 'DB_CONNECT_TIMEOUT', 5, 1, 60),
statement_timeout=_int(environ, 'DB_STATEMENT_TIMEOUT', 10, 1, 300),
)
def as_log_fields(self):
"""Configuration summary safe for logs (no secrets)."""
return {
'app_env': self.env,
'host': self.host,
'port': self.port,
'log_level': self.log_level,
'shutdown_grace_seconds': self.shutdown_grace_seconds,
'default_page_limit': self.default_page_limit,
'max_page_limit': self.max_page_limit,
'max_text_code_points': self.max_text_code_points,
'max_body_bytes': self.max_body_bytes,
'cursor_secret_configured': bool(os.environ.get('CURSOR_SECRET')),
'database_url_configured': bool(self.database_url),
'cache_url_configured': bool(self.cache_url),
'session_ttl_seconds': self.session_ttl_seconds,
'session_store': self.session_store,
'db_pool_min': self.pool_min,
'db_pool_max': self.pool_max,
'db_connect_timeout': self.connect_timeout,
'db_statement_timeout': self.statement_timeout,
}
+66
View File
@@ -0,0 +1,66 @@
"""Opaque keyset cursors: HMAC-protected (created_at, id) plus a resource scope."""
from __future__ import annotations
import base64
import binascii
import datetime as dt
import hashlib
import hmac
import json
import uuid
VERSION = 1
MAC_BYTES = 16
class InvalidCursor(ValueError):
"""Cursor is malformed, forged, expired or belongs to another resource."""
def _key(secret):
return hashlib.sha256(b'mtusi-messenger-cursor-v1' + secret).digest()
def encode(scope, created_at, object_id, secret):
"""Serialize a keyset position bound to one resource scope."""
payload = json.dumps(
{'v': VERSION, 'scope': scope, 'created_at': created_at, 'id': str(object_id)},
separators=(',', ':'), sort_keys=True).encode('utf-8')
mac = hmac.new(_key(secret), payload, hashlib.sha256).digest()[:MAC_BYTES]
return base64.urlsafe_b64encode(payload + mac).decode('ascii').rstrip('=')
def decode(token, scope, secret):
"""Return (created_at datetime, uuid) or raise InvalidCursor."""
if not isinstance(token, str) or not token:
raise InvalidCursor('empty cursor')
padding = '=' * (-len(token) % 4)
try:
raw = base64.urlsafe_b64decode(token + padding)
except (binascii.Error, ValueError):
raise InvalidCursor('cursor is not base64url') from None
if len(raw) <= MAC_BYTES:
raise InvalidCursor('cursor is too short')
payload, mac = raw[:-MAC_BYTES], raw[-MAC_BYTES:]
if not hmac.compare_digest(mac, hmac.new(_key(secret), payload, hashlib.sha256).digest()[:MAC_BYTES]):
raise InvalidCursor('cursor signature mismatch')
try:
data = json.loads(payload.decode('utf-8'))
except (ValueError, UnicodeError):
raise InvalidCursor('cursor payload is not JSON') from None
if not isinstance(data, dict) or data.get('v') != VERSION:
raise InvalidCursor('unsupported cursor version')
if data.get('scope') != scope:
raise InvalidCursor('cursor belongs to another resource')
created_at = data.get('created_at')
object_id = data.get('id')
if not isinstance(created_at, str) or not isinstance(object_id, str):
raise InvalidCursor('cursor payload types')
try:
stamp = dt.datetime.fromisoformat(created_at.replace('Z', '+00:00'))
identifier = uuid.UUID(object_id)
except ValueError:
raise InvalidCursor('cursor payload values') from None
if stamp.tzinfo is None:
raise InvalidCursor('cursor timestamp has no timezone')
return stamp.astimezone(dt.timezone.utc), identifier
+114
View File
@@ -0,0 +1,114 @@
"""PostgreSQL connection pool and schema migrations."""
from __future__ import annotations
import os
import time
from . import telemetry
from contextlib import contextmanager
from pathlib import Path
from psycopg2 import pool, OperationalError, InterfaceError
from psycopg2.extras import RealDictCursor
from .config import Config
from .errors import Unavailable
_ROOT = Path(__file__).resolve().parents[2]
_MIGRATIONS_DIR = _ROOT / 'migrations'
class Database:
"""Thread-safe connection pool with prepared statement timeouts.
All queries are parameterised (%s). Connections are returned to the pool
after every operation; the pool itself is closed on shutdown.
"""
def __init__(self, config: Config):
self._config = config
self._pool = None
def open(self):
if self._pool is not None:
return
try:
self._pool = pool.ThreadedConnectionPool(
minconn=self._config.pool_min,
maxconn=self._config.pool_max,
dsn=self._config.database_url,
connect_timeout=self._config.connect_timeout,
cursor_factory=RealDictCursor,
)
except Exception as exc: # psycopg2.Error or configuration error
raise Unavailable('Не удалось подключиться к PostgreSQL: %s' % exc) from None
def close(self):
if self._pool is not None:
self._pool.closeall()
self._pool = None
@contextmanager
def connection(self):
with telemetry.span('postgresql.transaction', telemetry.trace.SpanKind.CLIENT,
attributes={'db.system.name': 'postgresql'}):
with self._connection() as conn:
yield conn
@contextmanager
def _connection(self):
"""Yield a connection; rollback on exception, return to pool on exit."""
if self._pool is None:
raise Unavailable('База данных не инициализирована')
started = time.monotonic()
telemetry.pool_waiting.add(1)
try:
conn = self._pool.getconn()
except (pool.PoolError, OperationalError, InterfaceError):
telemetry.pool_exhausted.add(1)
raise Unavailable('Пул соединений PostgreSQL недоступен') from None
finally:
telemetry.pool_waiting.add(-1)
telemetry.pool_acquire.record(time.monotonic() - started)
telemetry.pool_used.add(1)
try:
with conn.cursor() as cur:
cur.execute('SET statement_timeout = %s',
(self._config.statement_timeout * 1000,))
yield conn
except Exception as exc:
try:
conn.rollback()
except Exception:
pass
if isinstance(exc, (OperationalError, InterfaceError)):
raise Unavailable('PostgreSQL недоступен') from None
raise
finally:
telemetry.pool_used.add(-1)
self._pool.putconn(conn, close=bool(conn.closed))
@contextmanager
def cursor(self):
"""Yield a cursor inside a connection; commit on success, rollback on failure."""
with self.connection() as conn:
with conn.cursor() as cur:
yield cur
conn.commit()
def ping(self):
with self.cursor() as cur:
cur.execute('SELECT 1')
cur.fetchone()
def run_migrations(self):
"""Apply versioned migrations once, under a transaction advisory lock."""
with self.cursor() as cur:
cur.execute('SELECT pg_advisory_xact_lock(3072026)')
cur.execute('CREATE TABLE IF NOT EXISTS schema_migrations '
'(name text PRIMARY KEY, applied_at timestamptz NOT NULL DEFAULT now())')
for path in sorted(_MIGRATIONS_DIR.glob('*.sql')):
cur.execute('SELECT 1 FROM schema_migrations WHERE name = %s', (path.name,))
if cur.fetchone() is not None:
continue
cur.execute(path.read_text(encoding='utf-8'))
cur.execute('INSERT INTO schema_migrations(name) VALUES (%s)', (path.name,))
+73
View File
@@ -0,0 +1,73 @@
"""API error taxonomy; every message is safe to return to a client."""
class ApiError(Exception):
"""Base class for errors that map onto the public error payload."""
status = 500
code = 'internal_error'
default_message = 'Внутренняя ошибка сервера'
def __init__(self, message=None):
self.message = message or self.default_message
super().__init__(self.message)
class InvalidRequest(ApiError):
status = 400
code = 'invalid_request'
default_message = 'Некорректный запрос'
class Unauthorized(ApiError):
status = 401
code = 'unauthorized'
default_message = 'Требуется идентификация пользователя'
www_authenticate = None
def __init__(self, message=None, www_authenticate=None):
self.message = message or self.default_message
self.www_authenticate = www_authenticate
super().__init__(self.message)
class NotFound(ApiError):
status = 404
code = 'not_found'
default_message = 'Объект не найден'
class Conflict(ApiError):
status = 409
code = 'conflict'
default_message = 'Конфликт состояния'
class PayloadTooLarge(ApiError):
status = 413
code = 'payload_too_large'
default_message = 'Тело запроса слишком велико'
class UnsupportedMediaType(ApiError):
status = 415
code = 'unsupported_media_type'
default_message = 'Поддерживается только application/json'
class RateLimited(ApiError):
status = 429
code = 'rate_limited'
default_message = 'Слишком много запросов'
class Unavailable(ApiError):
status = 503
code = 'unavailable'
default_message = 'Сервис временно недоступен'
class InternalError(ApiError):
status = 500
code = 'internal_error'
default_message = 'Внутренняя ошибка сервера'
+39
View File
@@ -0,0 +1,39 @@
"""List failed jobs and explicitly replay a failed job with retained staging."""
import argparse
import json
from uuid import UUID
from .config import Config
from .db import Database
from .errors import ApiError
from .job_lifecycle import replay
def main():
parser = argparse.ArgumentParser(description=__doc__)
actions = parser.add_subparsers(dest='action', required=True)
actions.add_parser('list')
command = actions.add_parser('replay')
command.add_argument('job_id', type=UUID)
command.add_argument('--operator', required=True)
args = parser.parse_args()
database = Database(Config.from_env())
database.open()
try:
if args.action == 'replay':
print(json.dumps({'job_id': str(args.job_id), 'attempt': replay(database, str(args.job_id), args.operator)}))
else:
with database.cursor() as cur:
cur.execute("SELECT j.id AS job_id,a.id AS attachment_id,a.error_code,j.attempt, "
"j.staging_cleaned_at IS NULL AS staging_retained "
"FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id "
"WHERE a.status='failed' ORDER BY a.updated_at DESC LIMIT 100")
print(json.dumps(cur.fetchall(), default=str))
except ApiError as exc:
parser.exit(1, str(exc) + '\n')
finally:
database.close()
if __name__ == '__main__':
main()
+140
View File
@@ -0,0 +1,140 @@
"""Transactional claims, bounded retries and explicit failed-job recovery."""
import os
import random
import datetime as dt
from . import telemetry
from .errors import Conflict, NotFound
LEASE_SECONDS = 60
STAGING_RETENTION_SECONDS = 86400
MAX_ATTEMPTS = int(os.getenv('JOB_MAX_ATTEMPTS', '5'))
if not 1 <= MAX_ATTEMPTS <= 20:
raise ValueError('JOB_MAX_ATTEMPTS must be between 1 and 20')
def _locked_job(cur, job_id):
cur.execute(
'SELECT j.*, a.status, a.updated_at, a.error_code, '
'j.lease_until > clock_timestamp() AS lease_valid FROM attachment_jobs j '
'JOIN attachments a ON a.id=j.attachment_id WHERE j.id=%s FOR UPDATE OF j,a',
(job_id,))
return cur.fetchone()
def claim(db, delivery):
with db.cursor() as cur:
row = _locked_job(cur, delivery['job_id'])
if (not row or row['status'] != 'queued' or row['attempt'] != delivery['attempt']
or str(row['attachment_id']) != delivery['attachment_id']
or row['pipeline_version'] != delivery['pipeline_version']):
return None
cur.execute('UPDATE attachment_jobs SET lease_until=clock_timestamp()+make_interval(secs => %s), '
'attempts_used=attempts_used+1 WHERE id=%s', (LEASE_SECONDS, row['id']))
cur.execute("UPDATE attachments SET status='processing', updated_at=now() "
"WHERE id=%s RETURNING *", (row['attachment_id'],))
return cur.fetchone()
def _requeue(cur, row, delay, reset=False):
attempt = row['attempt'] + 1
cur.execute('UPDATE attachment_jobs SET attempt=%s, lease_until=NULL, attempts_used=%s WHERE id=%s',
(attempt, 0 if reset else row['attempts_used'], row['id']))
cur.execute("UPDATE attachments SET status='queued', error_code=NULL, updated_at=now() WHERE id=%s",
(row['attachment_id'],))
cur.execute('INSERT INTO attachment_outbox (job_id,attempt,available_at) '
'VALUES (%s,%s,now()+make_interval(secs => %s))', (row['id'], attempt, delay))
return attempt
def _fail_or_retry(cur, row, error_code):
if error_code is None and row['attempts_used'] < MAX_ATTEMPTS:
delay = min(60, 2 ** row['attempts_used']) * random.uniform(.75, 1.25)
_requeue(cur, row, delay)
return
cur.execute('UPDATE attachment_jobs SET lease_until=NULL WHERE id=%s', (row['id'],))
cur.execute("UPDATE attachments SET status='failed', error_code=%s, updated_at=now() WHERE id=%s",
(error_code or 'retry_exhausted', row['attachment_id']))
def complete(db, delivery, result=None, error_code=None):
with db.cursor() as cur:
row = _locked_job(cur, delivery['job_id'])
if (not row or row['status'] != 'processing'
or row['attempt'] != delivery['attempt'] or not row['lease_valid']):
return False
if result is None:
_fail_or_retry(cur, row, error_code)
return True
key, variants = result
cur.executemany(
'INSERT INTO attachment_variants (attachment_id,pipeline_version,variant,object_key,'
'content_type,size_bytes,sha256,width,height) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s)',
[(row['attachment_id'], row['pipeline_version'], *item) for item in variants])
cur.execute("UPDATE attachments SET object_key=%s,status='ready',updated_at=now() WHERE id=%s",
(key, row['attachment_id']))
cur.execute('UPDATE attachment_jobs SET lease_until=NULL WHERE id=%s', (row['id'],))
elapsed = (dt.datetime.now(dt.timezone.utc) - row['created_at']).total_seconds()
telemetry.time_to_ready.record(max(0, elapsed), {'job_type': 'attachment.process.v1'})
return True
def recover_expired(db):
with db.cursor() as cur:
cur.execute(
"SELECT j.* FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id "
"WHERE a.status='processing' AND j.lease_until <= now() "
"ORDER BY j.lease_until LIMIT 10 FOR UPDATE OF j,a SKIP LOCKED")
for row in cur.fetchall():
_fail_or_retry(cur, row, None)
def replay(db, job_id, operator):
with db.cursor() as cur:
row = _locked_job(cur, job_id)
if row is None:
raise NotFound('Задача не найдена')
if row['status'] != 'failed' or row['staging_cleaned_at'] is not None:
raise Conflict('Повтор доступен только для failed с сохранённым исходником')
attempt = _requeue(cur, row, 0, reset=True)
cur.execute('INSERT INTO attachment_replays (job_id,attempt,operator) VALUES (%s,%s,%s)',
(job_id, attempt, operator))
return attempt
def clean_staging(db, remove):
with db.cursor() as cur:
cur.execute(
"SELECT j.id,j.attachment_id FROM attachment_jobs j "
"JOIN attachments a ON a.id=j.attachment_id "
"WHERE j.staging_cleaned_at IS NULL AND (a.status='ready' OR "
"(a.status='failed' AND a.updated_at < now()-make_interval(secs => %s))) "
"LIMIT 1 FOR UPDATE OF j,a SKIP LOCKED", (STAGING_RETENTION_SECONDS,))
row = cur.fetchone()
if not row:
return False
remove('attachments/staging/' + str(row['attachment_id']))
cur.execute('UPDATE attachment_jobs SET staging_cleaned_at=now() WHERE id=%s', (row['id'],))
return True
def clean_abandoned_outputs(db, remove):
with db.cursor() as cur:
cur.execute(
"SELECT j.id,j.attachment_id,j.pipeline_version,o.attempt "
"FROM attachment_outbox o JOIN attachment_jobs j ON j.id=o.job_id "
"JOIN attachments a ON a.id=j.attachment_id "
"WHERE o.objects_cleaned_at IS NULL AND o.available_at < now()-interval '15 minutes' "
"AND (o.attempt < j.attempt OR (a.status='failed' AND "
"a.updated_at < now()-interval '15 minutes')) "
"ORDER BY o.available_at LIMIT 1 FOR UPDATE OF o SKIP LOCKED")
row = cur.fetchone()
if not row:
return False
prefix = 'attachments/final/{attachment_id}/v{pipeline_version}/a{attempt}/'.format(**row)
for name in ('original', 'thumbnail.jpeg', 'crop.jpeg', 'optimized.webp'):
remove(prefix + name)
cur.execute('UPDATE attachment_outbox SET objects_cleaned_at=now() WHERE job_id=%s AND attempt=%s',
(row['id'], row['attempt']))
return True
+58
View File
@@ -0,0 +1,58 @@
"""Structured stdout logging without payload contents."""
from __future__ import annotations
import json
import logging
import os
import sys
import time
from opentelemetry import trace
from . import telemetry
_FIELDS = ('event', 'request_id', 'method', 'route', 'status', 'duration_ms', 'actor_id',
'job_id', 'attempt', 'error_code', 'client', 'outcome', 'count', 'detail', 'signal', 'version')
class JsonFormatter(logging.Formatter):
"""One JSON object per line; message text of API payloads is never logged."""
def format(self, record):
payload = {
'ts': time.strftime('%Y-%m-%dT%H:%M:%S', time.gmtime(record.created)) +
'.%03dZ' % int(record.msecs),
'level': record.levelname.lower(),
'logger': record.name,
'message': record.getMessage(),
}
context = trace.get_current_span().get_span_context()
payload['service.name'] = os.getenv('OTEL_SERVICE_NAME', 'messenger-api')
if context.is_valid:
payload['trace_id'] = format(context.trace_id, '032x')
payload['span_id'] = format(context.span_id, '016x')
for field in _FIELDS:
value = getattr(record, field, None)
if value is not None:
payload[field] = value
if record.exc_info and record.levelno >= logging.ERROR:
payload['exception'] = record.exc_info[0].__name__
return json.dumps(payload, ensure_ascii=False, sort_keys=True)
def configure(level_name):
handler = logging.StreamHandler(sys.stdout)
handler.setFormatter(JsonFormatter())
root = logging.getLogger()
for existing in list(root.handlers):
root.removeHandler(existing)
root.addHandler(handler)
root.setLevel(getattr(logging, level_name))
telemetry.configure()
for exporter in root.handlers:
if exporter is not handler:
exporter.setFormatter(JsonFormatter())
return logging.getLogger('messenger')
def access_logger():
return logging.getLogger('messenger.access')
+17
View File
@@ -0,0 +1,17 @@
"""One-shot schema migration entry point; API processes never migrate."""
from .config import Config
from .db import Database
def main():
database = Database(Config.from_env())
try:
database.open()
database.run_migrations()
print('PASS: migrations applied')
finally:
database.close()
if __name__ == '__main__':
main()
+97
View File
@@ -0,0 +1,97 @@
"""Publish committed attachment jobs using RabbitMQ publisher confirms."""
import json
import logging
import os
from pathlib import Path
import time
import pika
from . import telemetry, logging_utils
from .config import Config
from .db import Database
from .errors import Unavailable
from .store import rfc3339
from .job_lifecycle import recover_expired
QUEUE = 'attachment.process.v1'
log = logging.getLogger('messenger.publish_jobs')
def connect():
password = Path(os.environ['RABBITMQ_PASSWORD_FILE']).read_text().strip()
return pika.BlockingConnection(pika.ConnectionParameters(
host='rabbitmq', credentials=pika.PlainCredentials('lab6', password),
heartbeat=30, socket_timeout=3, stack_timeout=5, blocked_connection_timeout=5))
def publish_one(database, channel):
with database.cursor() as cur:
cur.execute(
'SELECT j.id, j.attachment_id, j.pipeline_version, j.created_at, j.trace_context, o.attempt '
'FROM attachment_outbox o JOIN attachment_jobs j ON j.id=o.job_id '
'WHERE o.published_at IS NULL AND o.available_at <= now() '
'ORDER BY o.available_at, o.job_id LIMIT 1 FOR UPDATE OF o SKIP LOCKED')
row = cur.fetchone()
if not row:
return False
job = {
'schema_version': 1, 'job_id': str(row['id']), 'type': QUEUE,
'attachment_id': str(row['attachment_id']),
'pipeline_version': row['pipeline_version'], 'attempt': row['attempt'],
'created_at': rfc3339(row['created_at']),
}
with telemetry.span('attachment.publish', telemetry.trace.SpanKind.PRODUCER,
context=telemetry.extract(row['trace_context']),
attributes={'messaging.system': 'rabbitmq',
'messaging.destination.name': QUEUE,
'job.id': job['job_id'], 'job.attempt': job['attempt']}):
channel.basic_publish(
exchange='', routing_key=QUEUE, body=json.dumps(job).encode(),
properties=pika.BasicProperties(content_type='application/json',
delivery_mode=2, message_id=job['job_id'],
headers=telemetry.carrier()), mandatory=True)
log.info('job_published', extra={'event': 'job_published',
'job_id': job['job_id'], 'attempt': job['attempt']})
cur.execute('UPDATE attachment_outbox SET published_at=now() WHERE job_id=%s AND attempt=%s',
(row['id'], row['attempt']))
return True
def main():
logging_utils.configure("INFO")
database = Database(Config.from_env())
database.open()
try:
while True:
connection = None
try:
connection = connect()
channel = connection.channel()
channel.queue_declare(queue=QUEUE, durable=True)
channel.confirm_delivery()
next_recovery = 0
while True:
if time.monotonic() >= next_recovery:
recover_expired(database)
telemetry.observe_queue(database, channel)
next_recovery = time.monotonic() + 1
if not publish_one(database, channel):
connection.sleep(.25)
else:
connection.process_data_events()
except (pika.exceptions.AMQPError, OSError, Unavailable) as exc:
log.warning('Publisher will retry: %s', type(exc).__name__)
time.sleep(2)
finally:
if connection and connection.is_open:
try:
connection.close()
except pika.exceptions.AMQPError:
pass
finally:
database.close()
if __name__ == '__main__':
main()
+99
View File
@@ -0,0 +1,99 @@
"""Password hashing and session tokens.
Passwords are hashed with a library password KDF: scrypt from the standard
library when available, otherwise bcrypt with a SHA-256 pre-hash so that
passwords longer than bcrypt's 72-byte limit are still supported.
Session tokens are opaque random values (>= 256 bits of entropy); only their
SHA-256 is stored in the database.
"""
from __future__ import annotations
import base64
import hashlib
import hmac
import secrets
import bcrypt
try: # scrypt is part of hashlib on OpenSSL-enabled builds
hashlib.scrypt(b'x', salt=b's', n=16, r=8, p=1, dklen=16)
_HAS_SCRYPT = True
except Exception:
_HAS_SCRYPT = False
if not _HAS_SCRYPT:
import bcrypt # type: ignore
SCRYPT_N = 16384
SCRYPT_R = 8
SCRYPT_P = 1
SCRYPT_DKLEN = 32
SALT_BYTES = 16
def hash_password(password: str) -> str:
"""Hash a UTF-8 password with a library password KDF.
Returns a single storable string:
scrypt:$n:$r:$p:$salt_b64:$hash_b64
or
bcrypt:$salt_b64:$hash_b64
"""
raw = password.encode('utf-8')
if _HAS_SCRYPT:
salt = secrets.token_bytes(SALT_BYTES)
dk = hashlib.scrypt(
raw,
salt=salt,
n=SCRYPT_N,
r=SCRYPT_R,
p=SCRYPT_P,
dklen=SCRYPT_DKLEN,
)
return 'scrypt:%d:%d:%d:%s:%s' % (
SCRYPT_N, SCRYPT_R, SCRYPT_P,
base64.b64encode(salt).decode('ascii'),
base64.b64encode(dk).decode('ascii'),
)
# bcrypt limits input to 72 bytes; pre-hash with SHA-256 so long passwords
# (up to 128 code points per contract) are fully covered.
digest = hashlib.sha256(raw).digest()
hashed = bcrypt.hashpw(digest, bcrypt.gensalt(rounds=12))
return 'bcrypt:%s' % hashed.decode('ascii')
def verify_password(password: str, password_hash: str) -> bool:
"""Verify a password against a KDF hash string. Constant-time compare."""
try:
raw = password.encode('utf-8')
if password_hash.startswith('scrypt:'):
parts = password_hash.split(':')
if len(parts) != 6 or parts[0] != 'scrypt':
return False
_, n, r, p, salt_b64, hash_b64 = parts
salt = base64.b64decode(salt_b64)
expected = base64.b64decode(hash_b64)
dk = hashlib.scrypt(
raw,
salt=salt,
n=int(n), r=int(r), p=int(p),
dklen=len(expected),
)
return hmac.compare_digest(dk, expected)
if password_hash.startswith('bcrypt:'):
stored = password_hash[len('bcrypt:'):].encode('ascii')
digest = hashlib.sha256(raw).digest()
return bcrypt.checkpw(digest, stored)
return False
except Exception:
return False
def generate_token() -> str:
"""Opaque random bearer token, at least 256 bits of entropy."""
return secrets.token_urlsafe(32)
def hash_token(token: str) -> str:
"""SHA-256 of a bearer token, stored in the database instead of the token itself."""
return hashlib.sha256(token.encode('utf-8')).hexdigest()
+327
View File
@@ -0,0 +1,327 @@
"""HTTP/1.1 front end: request ids, structured logs, graceful SIGTERM shutdown."""
from __future__ import annotations
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
import json
import os
import platform
import re
import signal
import sys
import threading
import time
import uuid
from urllib.parse import parse_qs, urlsplit
from . import __version__, logging_utils, telemetry
from .api import Api, Request
from .attachments import BinaryResponse, CHUNK_SIZE, MAX_UPLOAD_BYTES
from .cache import SessionCache
from .config import Config, ConfigError
from .db import Database
from .errors import ApiError, InvalidRequest, PayloadTooLarge, Unavailable
from .store import Store
MAX_REQUEST_ID_LENGTH = 128
MAX_LOGGED_LINE = 200
def sanitize_request_id(raw):
"""Echo a safe client id, otherwise generate one. Never trust raw input."""
if raw and len(raw) <= MAX_REQUEST_ID_LENGTH and all(0x21 <= ord(ch) <= 0x7E for ch in raw):
return raw
return uuid.uuid4().hex
def route_template(path):
for _method, pattern, _handler in Api.ROUTES:
if pattern.match(path):
return re.sub(r'\(\?P<([^>]+)>\[\^/\]\+\)', r'{\1}', pattern.pattern).strip('^$')
return 'unmatched'
def encode_payload(payload):
return json.dumps(payload, ensure_ascii=False, separators=(',', ':')).encode('utf-8')
class ApiServer(ThreadingHTTPServer):
"""Threaded server that tracks in-flight requests and stops accepting on demand."""
daemon_threads = True
allow_reuse_address = True
request_queue_size = 128
def __init__(self, address, api, config, logger, database):
self.api = api
self.config = config
self.logger = logger
self.database = database
self.instance_id = os.environ.get("HOSTNAME") or uuid.uuid4().hex
self.shutting_down = False
self._active = 0
self._idle = threading.Condition()
super().__init__(address, ApiHandler)
def enter(self):
with self._idle:
self._active += 1
def leave(self):
with self._idle:
self._active -= 1
if self._active == 0:
self._idle.notify_all()
def wait_idle(self, timeout):
deadline = time.monotonic() + timeout
with self._idle:
while self._active > 0:
remaining = deadline - time.monotonic()
if remaining <= 0:
return False
self._idle.wait(remaining)
return True
class ApiHandler(BaseHTTPRequestHandler):
protocol_version = 'HTTP/1.1'
server_version = 'mtusi-messenger/' + __version__
sys_version = ''
timeout = 30
# --------------------------------------------------------- entry points
def do_GET(self):
self._handle()
do_HEAD = do_POST = do_PUT = do_PATCH = do_DELETE = do_OPTIONS = do_GET
def log_message(self, fmt, *args):
"""Base-class chatter stays debug-only; access logs are structured."""
self.server.logger.debug('http_note', extra={
'event': 'http_note', 'detail': (fmt % args)[:MAX_LOGGED_LINE]})
# ------------------------------------------------------------- pipeline
def _handle(self):
route = route_template(urlsplit(self.path).path)
with telemetry.span(self.command + ' ' + route, telemetry.trace.SpanKind.SERVER,
context=telemetry.extract({k.lower(): v for k, v in self.headers.items()}),
attributes={'http.request.method': self.command, 'http.route': route}):
self._handle_request()
def _handle_request(self):
started = time.monotonic()
server = self.server
request_id = sanitize_request_id(self.headers.get('X-Request-Id'))
head_only = self.command == 'HEAD'
path = urlsplit(self.path).path
status, error_code = 500, None
self._last_exc = None
server.enter()
try:
try:
if server.shutting_down:
raise Unavailable('Сервер завершает работу')
if self.command == 'POST' and Api.UPLOAD_PATH.fullmatch(path):
# Rejected uploads may leave unread bytes in the connection.
self.close_connection = True
body = self._upload_chunks
else:
body = self._read_body()
query = parse_qs(urlsplit(self.path).query, keep_blank_values=True)
request = Request('GET' if head_only else self.command, path, query, self.headers,
body, request_id, self.client_address[0])
status, payload = server.api.dispatch(request)
encoded = payload if isinstance(payload, BinaryResponse) else encode_payload(payload)
except ApiError as exc:
self._last_exc = exc
status, error_code = exc.status, exc.code
encoded = self._error_payload(exc, request_id)
except Exception: # last resort: log privately, answer generically
server.logger.exception('unhandled_error', extra={
'event': 'unhandled_error', 'request_id': request_id,
'method': self.command, 'route': route_template(path)})
status, error_code = 500, 'internal_error'
encoded = self._error_payload(ApiError(), request_id)
try:
self._send(status, encoded, request_id, head_only)
finally:
if isinstance(encoded, BinaryResponse):
encoded.close()
finally:
duration = time.monotonic() - started
current = telemetry.trace.get_current_span()
current.set_attribute('http.response.status_code', status)
if status >= 500:
current.set_status(telemetry.trace.Status(telemetry.trace.StatusCode.ERROR))
if not path.startswith('/health/'):
labels = {'method': self.command, 'route': route_template(path),
'status_class': str(status // 100) + 'xx'}
telemetry.http_requests.add(1, labels)
telemetry.http_duration.record(duration, labels)
server.logger.info('http_request', extra={
'event': 'http_request', 'request_id': request_id, 'method': self.command,
'route': route_template(path), 'status': status,
'duration_ms': round(1000 * (time.monotonic() - started), 3),
'error_code': error_code, 'client': self.client_address[0]})
server.leave()
def _read_body(self):
if self.headers.get('Transfer-Encoding'):
self.close_connection = True
raise InvalidRequest('Transfer-Encoding не поддерживается')
raw_length = self.headers.get('Content-Length')
if raw_length is None:
return b''
try:
length = int(raw_length.strip())
except ValueError:
raise InvalidRequest('Некорректный Content-Length') from None
if length < 0:
raise InvalidRequest('Некорректный Content-Length')
if length > self.server.config.max_body_bytes:
self.close_connection = True
raise PayloadTooLarge('Тело запроса больше %d байт' % self.server.config.max_body_bytes)
if length == 0:
return b''
return self.rfile.read(length)
def _upload_chunks(self):
if self.headers.get('Transfer-Encoding'):
raise InvalidRequest('Требуется Content-Length')
lengths = self.headers.get_all('Content-Length') or []
try:
length = int(lengths[0]) if len(lengths) == 1 else -1
except ValueError:
length = -1
if length < 0:
raise InvalidRequest('Некорректный Content-Length')
if length > MAX_UPLOAD_BYTES:
raise PayloadTooLarge()
deadline = time.monotonic() + 30
while length:
remaining = deadline - time.monotonic()
if remaining <= 0:
raise InvalidRequest('Истекло время загрузки')
self.connection.settimeout(min(5, remaining))
try:
chunk = self.rfile.read1(min(CHUNK_SIZE, length))
except OSError:
raise InvalidRequest('Загрузка прервана') from None
finally:
self.connection.settimeout(self.timeout)
if not chunk:
raise InvalidRequest('Загрузка прервана')
length -= len(chunk)
yield chunk
@staticmethod
def _error_payload(exc, request_id):
return encode_payload({'error': {
'code': exc.code, 'message': exc.message, 'request_id': request_id}})
def _send(self, status, encoded, request_id, head_only):
try:
self.send_response(status)
binary = isinstance(encoded, BinaryResponse)
self.send_header('Content-Type', encoded.content_type if binary else 'application/json; charset=utf-8')
self.send_header('X-Request-Id', request_id)
ctx = telemetry.trace.get_current_span().get_span_context()
if ctx.is_valid:
self.send_header('X-Trace-Id', format(ctx.trace_id, '032x'))
self.send_header('X-Instance-Id', self.server.instance_id)
self.send_header('Cache-Control', 'no-store')
if status == 204:
self.send_header('Content-Length', '0')
if status == 503:
self.send_header('Retry-After', '1')
self.end_headers()
return
self.send_header('Content-Length', str(encoded.size if binary else len(encoded)))
if status == 503:
self.send_header('Retry-After', '1')
if status == 401 and self._last_exc is not None:
www_auth = getattr(self._last_exc, 'www_authenticate', None)
if www_auth:
self.send_header('WWW-Authenticate', www_auth)
if self.close_connection:
self.send_header('Connection', 'close')
self.end_headers()
if not head_only and encoded:
if binary:
for chunk in encoded.chunks():
self.wfile.write(chunk)
else:
self.wfile.write(encoded)
except OSError:
self.close_connection = True
def build_server(config, logger, api, database):
"""Create the listening server; raises OSError when the port is taken."""
return ApiServer((config.host, config.port), api, config, logger, database)
def run(config, logger, database, cache):
"""Serve until SIGTERM/SIGINT, then drain in-flight work within the grace period."""
api = Api(config, Store(config, database, cache), database)
try:
server = build_server(config, logger, api, database)
except OSError as exc:
logger.error('listen_failed', extra={'event': 'listen_failed', 'detail': str(exc)})
return 1
stop = threading.Event()
def handle_signal(signum, _frame):
if stop.is_set():
return
stop.set()
server.shutting_down = True
logger.warning('shutdown_signal', extra={
'event': 'shutdown_signal', 'signal': signum,
'detail': 'stop accepting new requests'})
threading.Thread(target=server.shutdown, name='shutdown', daemon=True).start()
for name in ('SIGTERM', 'SIGINT'):
if hasattr(signal, name):
signal.signal(getattr(signal, name), handle_signal)
logger.info('server_started', extra=dict(
{'event': 'server_started', 'version': __version__,
'detail': 'pid %d, python %s' % (os.getpid(), platform.python_version())},
**config.as_log_fields()))
started = time.monotonic()
try:
server.serve_forever(poll_interval=0.2)
finally:
server.server_close()
drained = server.wait_idle(config.shutdown_grace_seconds)
database.close()
logger.info('server_stopped', extra={
'event': 'server_stopped',
'outcome': 'graceful' if drained else 'grace_period_exceeded',
'duration_ms': round(1000 * (time.monotonic() - started), 3)})
return 0
def main(argv=None):
try:
config = Config.from_env()
except ConfigError as exc:
print('configuration error: %s' % exc, file=sys.stderr)
return 2
logger = logging_utils.configure(config.log_level)
database = Database(config)
cache = SessionCache(config)
try:
database.open()
cache.open()
except Exception as exc: # noqa: BLE001 - startup must fail loud and clear
logger.error('database_start_failed', extra={
'event': 'database_start_failed', 'detail': str(exc)})
database.close()
cache.close()
return 1
try:
return run(config, logger, database, cache)
finally:
cache.close()
+392
View File
@@ -0,0 +1,392 @@
"""PostgreSQL-backed state store. All writes go through parameterised queries
inside explicit transactions; reads use keyset pagination over indexed columns."""
from __future__ import annotations
from . import telemetry
import datetime as dt
import hashlib
import json
from psycopg2.extras import Json
from psycopg2.errors import UniqueViolation, ForeignKeyViolation
import uuid
from . import cursor as cursor_codec
from . import security
from .cache import SessionCache
from .config import Config
from .db import Database
from .errors import Conflict, InvalidRequest, NotFound
def utc_now():
return dt.datetime.now(dt.timezone.utc)
def rfc3339(stamp):
"""RFC 3339 UTC with fixed microsecond precision and the Z suffix."""
return stamp.astimezone(dt.timezone.utc).isoformat(timespec='microseconds').replace('+00:00', 'Z')
class Store:
"""PostgreSQL state: users, chats, membership, messages, sessions."""
def __init__(self, config: Config, database: Database, cache: SessionCache):
self._config = config
self._db = database
self._cache = cache
# ---------------------------------------------------------------- reads
def stats(self):
with self._db.cursor() as cur:
cur.execute('SELECT (SELECT count(*) FROM users) AS users,'
' (SELECT count(*) FROM chats) AS chats,'
' (SELECT count(*) FROM messages) AS messages')
row = cur.fetchone()
return {
'users': row['users'],
'chats': row['chats'],
'messages': row['messages'],
}
def user(self, user_id):
with self._db.cursor() as cur:
cur.execute(
'SELECT id, username, display_name, created_at FROM users WHERE id = %s',
(str(user_id),))
row = cur.fetchone()
return self._public_user(row) if row else None
def user_by_username(self, username):
"""Return user row including password_hash, or None."""
with self._db.cursor() as cur:
cur.execute(
'SELECT id, username, password_hash, display_name, created_at FROM users '
'WHERE username = %s', (username,))
return cur.fetchone()
def users_exist(self, identifiers):
if not identifiers:
return True
with self._db.cursor() as cur:
cur.execute(
'SELECT count(*) AS c FROM users WHERE id = ANY(%s::uuid[])',
([str(i) for i in identifiers],))
row = cur.fetchone()
return row['c'] == len(identifiers)
# --------------------------------------------------------------- writes
def create_user(self, username, display_name, password_hash):
with self._db.cursor() as cur:
try:
cur.execute(
'INSERT INTO users (id, username, password_hash, display_name, created_at) '
'VALUES (%s, %s, %s, %s, %s) '
'RETURNING id, username, display_name, created_at',
(str(uuid.uuid4()), username, password_hash, display_name, utc_now()))
except UniqueViolation:
# Unique violation on username -> conflict, rolled back by cursor context.
raise Conflict('username уже занят') from None
row = cur.fetchone()
return self._public_user(row)
def create_chat(self, title, member_ids):
"""Atomically create a chat and its membership. Rollback on any failure."""
identifier = str(uuid.uuid4())
stamp = utc_now()
with self._db.cursor() as cur:
try:
cur.execute(
'INSERT INTO chats (id, title, created_at) VALUES (%s, %s, %s)',
(identifier, title, stamp))
for member_id in member_ids:
cur.execute(
'INSERT INTO chat_members (chat_id, user_id) VALUES (%s, %s)',
(identifier, str(member_id)))
except ForeignKeyViolation:
raise InvalidRequest('member_ids содержит неизвестных пользователей') from None
return {
'id': identifier,
'title': title,
'member_ids': list(member_ids),
'created_at': rfc3339(stamp),
}
def reserve_attachment(self, aid, chat_id, owner_id, filename, mime, size, digest, key):
with self._db.cursor() as cur:
cur.execute(
"INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, "
"size_bytes, sha256, object_key, status) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,'pending')",
(aid, chat_id, owner_id, filename, mime, size, digest, key))
def queue_attachment(self, aid, put_object):
"""Keep a row lock from before PUT until staging metadata, job and outbox commit."""
with self._db.cursor() as cur:
cur.execute("SELECT object_key FROM attachments WHERE id=%s "
"AND status='pending' FOR UPDATE", (aid,))
pending = cur.fetchone()
if pending is None:
raise Conflict('Загрузка вложения не может быть завершена')
put_object(pending['object_key'])
job_id = str(uuid.uuid4())
cur.execute('INSERT INTO attachment_jobs (id, attachment_id, pipeline_version, trace_context, created_at) '
'VALUES (%s,%s,1,%s::jsonb,clock_timestamp())', (job_id, aid, json.dumps(telemetry.carrier())))
cur.execute('INSERT INTO attachment_outbox (job_id, attempt) VALUES (%s,1)', (job_id,))
cur.execute("UPDATE attachments SET status='queued', updated_at=now() "
"WHERE id=%s RETURNING *", (aid,))
return cur.fetchone()
def clean_pending_attachment(self, grace_seconds, delete_object):
"""Remove one expired reservation and its object, or leave it for retry."""
with self._db.cursor() as cur:
cur.execute(
"SELECT id, object_key FROM attachments WHERE status='pending' "
"AND updated_at < now() - make_interval(secs => %s) "
"ORDER BY updated_at, id LIMIT 1 FOR UPDATE SKIP LOCKED",
(grace_seconds,))
row = cur.fetchone()
if row is None:
return False
delete_object(row['object_key'])
cur.execute('DELETE FROM attachments WHERE id=%s', (row['id'],))
return True
def visible_attachment(self, aid, user_id):
with self._db.cursor() as cur:
cur.execute(
"SELECT a.* FROM attachments a JOIN chat_members m ON m.chat_id=a.chat_id "
"WHERE a.id=%s AND m.user_id=%s AND a.status<>'pending'", (aid, user_id))
return cur.fetchone()
def attachment_variants(self, aid):
with self._db.cursor() as cur:
cur.execute(
'SELECT DISTINCT ON (variant) variant AS name, object_key, content_type, '
'size_bytes, sha256, width, height FROM attachment_variants '
'WHERE attachment_id=%s ORDER BY variant, pipeline_version DESC', (aid,))
return cur.fetchall()
@staticmethod
def public_attachment(row, variants=()):
return {
'id': str(row['id']), 'chat_id': str(row['chat_id']), 'owner_id': str(row['owner_id']),
'filename': row['filename'], 'content_type': row['content_type'],
'size_bytes': row['size_bytes'], 'sha256': row['sha256'], 'status': row['status'],
'error_code': row['error_code'],
'variants': [{k: v for k, v in item.items() if k != 'object_key'} for item in variants],
'created_at': rfc3339(row['created_at']), 'updated_at': rfc3339(row['updated_at']),
}
def add_message(self, chat_id, sender_id, text, key, attachment_ids=()):
"""One transaction owns both the message and its replayable result.
The advisory lock serializes only the same user/method/path/key across
processes. Hash collisions serialize extra requests, never alias keys:
the complete key is still checked against the primary key in the table.
A crash before COMMIT rolls back the message, links and replay record. A lost reply after COMMIT
is recovered by replay. Transient failures are never recorded.
"""
path = '/api/v1/chats/%s/messages' % chat_id
body = {'text': text}
# Keep the lab3 digest for text-only requests, including explicit [].
if attachment_ids:
body['attachment_ids'] = list(attachment_ids)
body_hash = hashlib.sha256(json.dumps(body, ensure_ascii=False,
sort_keys=True).encode('utf-8')).hexdigest()
scope = json.dumps([str(sender_id), 'POST', path, key], separators=(',', ':'))
lock_id = int.from_bytes(hashlib.sha256(scope.encode()).digest()[:8], 'big', signed=True)
identifier = str(uuid.uuid4())
stamp = utc_now()
with self._db.cursor() as cur:
cur.execute('SELECT pg_advisory_xact_lock(%s)', (lock_id,))
cur.execute('SELECT request_hash, response_body FROM message_idempotency '
'WHERE user_id=%s AND method=%s AND path=%s AND key=%s',
(str(sender_id), 'POST', path, key))
replay = cur.fetchone()
if replay is not None:
if replay['request_hash'] != body_hash:
raise Conflict('Idempotency-Key уже использован с другим телом запроса')
return {'attachment_ids': [], **replay['response_body']}
if attachment_ids:
cur.execute(
'SELECT id, status FROM attachments WHERE chat_id=%s AND id=ANY(%s::uuid[]) FOR SHARE',
(str(chat_id), list(attachment_ids)))
attachments = cur.fetchall()
if len(attachments) != len(attachment_ids):
raise NotFound('Вложение не найдено')
if any(row['status'] not in ('ready', 'queued', 'processing') for row in attachments):
raise Conflict('Вложение не готово')
cur.execute(
'INSERT INTO messages (id, chat_id, sender_id, text, created_at) '
'VALUES (%s, %s, %s, %s, %s) '
'RETURNING id, chat_id, sender_id, text, created_at',
(identifier, str(chat_id), str(sender_id), text, stamp))
result = self._public_message(cur.fetchone())
if attachment_ids:
cur.executemany(
'INSERT INTO message_attachments (message_id, attachment_id, chat_id, position) '
'VALUES (%s,%s,%s,%s)',
[(identifier, aid, str(chat_id), pos) for pos, aid in enumerate(attachment_ids)])
result['attachment_ids'] = list(attachment_ids)
cur.execute('INSERT INTO message_idempotency '
'(user_id, method, path, key, request_hash, response_status, response_body) '
'VALUES (%s,%s,%s,%s,%s,201,%s)',
(str(sender_id), 'POST', path, key, body_hash, Json(result)))
return result
# --------------------------------------------------------------- pages
def page_messages(self, chat_id, after_key, limit):
"""Return (public messages, has_more) for a keyset window."""
with self._db.cursor() as cur:
if after_key is None:
cur.execute(
'SELECT id, chat_id, sender_id, text, created_at FROM messages '
'WHERE chat_id = %s '
'ORDER BY created_at, id '
'LIMIT %s', (str(chat_id), limit + 1))
else:
stamp, identifier = after_key
cur.execute(
'SELECT id, chat_id, sender_id, text, created_at FROM messages '
'WHERE chat_id = %s '
'AND (created_at, id) > (%s, %s) '
'ORDER BY created_at, id '
'LIMIT %s',
(str(chat_id), stamp, str(identifier), limit + 1))
rows = cur.fetchall()
items = [self._public_message(row) for row in rows[:limit]]
if items:
by_id = {item['id']: item for item in items}
cur.execute(
'SELECT message_id, attachment_id FROM message_attachments '
'WHERE message_id=ANY(%s::uuid[]) ORDER BY message_id, position',
(list(by_id),))
for row in cur.fetchall():
by_id[str(row['message_id'])]['attachment_ids'].append(str(row['attachment_id']))
has_more = len(rows) > limit
return items, has_more
def page_chats(self, user_id, after_key, limit):
"""Return (chats visible to the user, has_more) ordered by (created_at, id)."""
with self._db.cursor() as cur:
if after_key is None:
cur.execute(
'SELECT c.id, c.title, c.created_at FROM chats c '
'JOIN chat_members cm ON cm.chat_id = c.id '
'WHERE cm.user_id = %s '
'ORDER BY c.created_at, c.id '
'LIMIT %s', (str(user_id), limit + 1))
else:
stamp, identifier = after_key
cur.execute(
'SELECT c.id, c.title, c.created_at FROM chats c '
'JOIN chat_members cm ON cm.chat_id = c.id '
'WHERE cm.user_id = %s '
'AND (c.created_at, c.id) > (%s, %s) '
'ORDER BY c.created_at, c.id '
'LIMIT %s',
(str(user_id), stamp, str(identifier), limit + 1))
rows = cur.fetchall()
has_more = len(rows) > limit
items = [self._public_chat(dict(row)) for row in rows[:limit]]
return items, has_more
# ----------------------------------------------------------- projection
def chat(self, chat_id):
with self._db.cursor() as cur:
cur.execute(
'SELECT id, title, created_at FROM chats WHERE id = %s', (str(chat_id),))
row = cur.fetchone()
if row is None:
return None
chat = self._public_chat(dict(row))
cur.execute(
'SELECT user_id FROM chat_members WHERE chat_id = %s', (str(chat_id),))
chat['member_ids'] = [str(r['user_id']) for r in cur.fetchall()]
return chat
@staticmethod
def is_member(chat, user_id):
return str(user_id) in chat['member_ids']
@staticmethod
def _public_user(row):
return {
'id': str(row['id']),
'username': row['username'],
'display_name': row['display_name'],
'created_at': rfc3339(row['created_at']),
}
@staticmethod
def _public_chat(row):
return {
'id': str(row['id']),
'title': row['title'],
'member_ids': [], # filled by API when needed
'created_at': rfc3339(row['created_at']),
}
@staticmethod
def _public_message(row):
return {
'id': str(row['id']),
'chat_id': str(row['chat_id']),
'sender_id': str(row['sender_id']),
'text': row['text'],
'attachment_ids': [],
'created_at': rfc3339(row['created_at']),
}
# --------------------------------------------------------------- sessions
def create_session(self, token_hash, user_id, expires_at):
with self._db.cursor() as cur:
cur.execute(
'INSERT INTO sessions (token_hash, user_id, expires_at, created_at) '
'VALUES (%s, %s, %s, %s)',
(token_hash, str(user_id), expires_at, utc_now()))
ttl = max(1, int((expires_at - utc_now()).total_seconds()))
self._cache.set(token_hash, str(user_id), ttl)
def get_session(self, token_hash):
"""Return user_id for a valid non-expired session, or None.
The cache is consulted first, but every cache hit is re-validated
against PostgreSQL so that a revoked or expired session is never
trusted from a stale Redis key. On miss, read PostgreSQL and
backfill the cache. Expired sessions found during lookup are deleted.
"""
cached = self._cache.get(token_hash)
if cached is not None:
with self._db.cursor() as cur:
cur.execute(
'SELECT 1 FROM sessions WHERE token_hash = %s AND expires_at > %s',
(token_hash, utc_now()))
if cur.fetchone() is None:
self._cache.delete(token_hash)
return None
return cached
with self._db.cursor() as cur:
cur.execute(
'SELECT user_id, expires_at FROM sessions WHERE token_hash = %s',
(token_hash,))
row = cur.fetchone()
if row is None:
return None
if row['expires_at'] <= utc_now():
cur.execute('DELETE FROM sessions WHERE token_hash = %s', (token_hash,))
return None
user_id = str(row['user_id'])
ttl = max(1, int((row['expires_at'] - utc_now()).total_seconds()))
self._cache.set(token_hash, user_id, ttl)
return user_id
def delete_session(self, token_hash):
self._cache.delete(token_hash)
with self._db.cursor() as cur:
cur.execute('DELETE FROM sessions WHERE token_hash = %s', (token_hash,))
def cleanup_expired_sessions(self):
with self._db.cursor() as cur:
cur.execute('DELETE FROM sessions WHERE expires_at <= %s', (utc_now(),))
+153
View File
@@ -0,0 +1,153 @@
"""Bounded OTLP export and low-cardinality application signals."""
from contextlib import contextmanager
from functools import wraps
import atexit
import copy
import logging
import os
from pathlib import Path
import resource
import time
from opentelemetry import metrics, trace
from opentelemetry.propagate import extract, inject
from opentelemetry.sdk.resources import Resource
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor
from opentelemetry.sdk.metrics import MeterProvider, Histogram
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
from opentelemetry.sdk.metrics.view import View, ExplicitBucketHistogramAggregation
from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import OTLPMetricExporter
from opentelemetry.exporter.otlp.proto.grpc._log_exporter import OTLPLogExporter
tracer = trace.get_tracer('messenger', '1.0')
meter = metrics.get_meter('messenger', '1.0')
http_requests = meter.create_counter('messenger.http.requests', unit='{request}')
http_duration = meter.create_histogram('messenger.http.duration', unit='s')
jobs = meter.create_counter('messenger.jobs.processed', unit='{job}')
job_duration = meter.create_histogram('messenger.job.duration', unit='s')
time_to_ready = meter.create_histogram('messenger.attachment.time_to_ready', unit='s')
pool_used = meter.create_up_down_counter('messenger.db.pool.used', unit='{connection}')
pool_waiting = meter.create_up_down_counter('messenger.db.pool.waiting', unit='{request}')
pool_exhausted = meter.create_counter('messenger.db.pool.exhausted', unit='{event}')
pool_acquire = meter.create_histogram('messenger.db.pool.acquire', unit='s')
worker_busy = meter.create_up_down_counter('messenger.worker.busy', unit='{worker}')
_state = {'depth': 0, 'age': 0, 'broker_ready': 0, 'updated': 0}
_configured = False
def configure():
global _configured
if _configured or not os.getenv('OTEL_EXPORTER_OTLP_ENDPOINT'):
return
if os.getenv('OTEL_SDK_DISABLED', '').lower() == 'true':
return
_configured = True
res = Resource.create({
'service.name': os.getenv('OTEL_SERVICE_NAME', 'messenger-api'),
'service.version': os.getenv('APP_REVISION', 'local'),
'service.instance.id': os.getenv('HOSTNAME', 'local'),
'deployment.environment.name': 'lab',
})
endpoint = os.environ['OTEL_EXPORTER_OTLP_ENDPOINT']
options = {'endpoint': endpoint, 'insecure': True, 'timeout': 1,
'channel_options': (('grpc.initial_reconnect_backoff_ms', 500),
('grpc.min_reconnect_backoff_ms', 500),
('grpc.max_reconnect_backoff_ms', 2000),
('grpc.dns_min_time_between_resolutions_ms', 1000))}
traces = TracerProvider(resource=res)
traces.add_span_processor(BatchSpanProcessor(
OTLPSpanExporter(**options), max_queue_size=256, max_export_batch_size=64,
schedule_delay_millis=500, export_timeout_millis=1500))
trace.set_tracer_provider(traces)
reader = PeriodicExportingMetricReader(
OTLPMetricExporter(**options), export_interval_millis=3000, export_timeout_millis=2000)
metric_provider = MeterProvider(resource=res, metric_readers=[reader], views=[View(
instrument_type=Histogram,
aggregation=ExplicitBucketHistogramAggregation(
boundaries=(.005, .01, .025, .05, .1, .25, .5, 1, 2, 5, 10, 30, 60, 120)))])
metrics.set_meter_provider(metric_provider)
logs = LoggerProvider(resource=res)
logs.add_log_record_processor(BatchLogRecordProcessor(
OTLPLogExporter(**options), max_queue_size=256, max_export_batch_size=64,
schedule_delay_millis=500, export_timeout_millis=1500))
handler = LoggingHandler(level=logging.INFO, logger_provider=logs)
def safe_record(record):
if not record.name.startswith('messenger'):
return False
safe = copy.copy(record)
# OTLP's logging bridge otherwise exports full exception text/stack traces.
if safe.exc_info:
safe.error_type = safe.exc_info[0].__name__
safe.exc_info = None
safe.exc_text = None
return safe
handler.addFilter(safe_record)
logging.getLogger().addHandler(handler)
atexit.register(logs.shutdown)
# SDK providers install their own shutdown hooks. All exporters have finite timeouts.
meter.create_observable_gauge('messenger.process.observed_at', unit='s', callbacks=[
lambda _: [metrics.Observation(time.time())]])
meter.create_observable_gauge('messenger.process.memory', unit='By', callbacks=[
lambda _: [metrics.Observation(int(Path('/proc/self/statm').read_text().split()[1]) *
os.sysconf('SC_PAGE_SIZE'))]])
meter.create_observable_counter('messenger.process.cpu', unit='s', callbacks=[
lambda _: [metrics.Observation(resource.getrusage(resource.RUSAGE_SELF).ru_utime +
resource.getrusage(resource.RUSAGE_SELF).ru_stime)]])
meter.create_observable_gauge('messenger.worker.capacity', unit='{worker}', callbacks=[
lambda _: [metrics.Observation(1 if os.getenv('OTEL_SERVICE_NAME') == 'messenger-worker' else 0)]])
meter.create_observable_gauge('messenger.db.pool.capacity', unit='{connection}', callbacks=[
lambda _: [metrics.Observation(int(os.getenv('DB_POOL_MAX', '10')))]])
if os.getenv('OTEL_SERVICE_NAME') == 'messenger-publisher':
for name, key, unit in (
('messenger.queue.depth', 'depth', '{job}'),
('messenger.queue.oldest_age', 'age', 's'),
('messenger.queue.broker_ready', 'broker_ready', '{message}'),
('messenger.queue.observed_at', 'updated', 's'),
):
meter.create_observable_gauge(name, unit=unit, callbacks=[
lambda _, k=key: [metrics.Observation(_state[k], {'queue': 'attachment.process.v1'})]])
@contextmanager
def span(name, kind=trace.SpanKind.INTERNAL, context=None, attributes=None):
with tracer.start_as_current_span(name, kind=kind, context=context,
attributes=attributes, record_exception=False,
set_status_on_exception=False) as current:
try:
yield current
except BaseException as exc:
current.set_status(trace.Status(trace.StatusCode.ERROR, type(exc).__name__))
current.set_attribute('error.type', type(exc).__name__)
raise
def operation(name):
def decorate(function):
@wraps(function)
def wrapped(*args, **kwargs):
with span(name, trace.SpanKind.CLIENT):
return function(*args, **kwargs)
return wrapped
return decorate
def carrier():
value = {}
inject(value)
return value
def observe_queue(database, channel):
"""Include outbox and active leases in backlog, using durable acceptance time."""
with database.cursor() as cur:
cur.execute("SELECT count(*) AS depth, COALESCE(EXTRACT(EPOCH FROM "
"clock_timestamp()-min(j.created_at)),0) AS age FROM attachment_jobs j "
"JOIN attachments a ON a.id=j.attachment_id WHERE a.status IN ('queued','processing')")
row = cur.fetchone()
broker = channel.queue_declare(queue='attachment.process.v1', durable=True, passive=True)
_state.update(depth=int(row['depth']), age=float(row['age']),
broker_ready=broker.method.message_count, updated=time.time())
+142
View File
@@ -0,0 +1,142 @@
"""Strict request payload validation; any deviation is 400 invalid_request."""
from __future__ import annotations
import re
import uuid
from .errors import InvalidRequest
USERNAME_PATTERN = re.compile(r'[a-z0-9_]{3,32}')
DIGITS_PATTERN = re.compile(r'[0-9]+')
def safe_label(value):
"""Render untrusted text for an error message without breaking its encoding.
Field names come straight from the client JSON, so they may contain the same
lone surrogates that the payload itself is rejected for.
"""
text = value if isinstance(value, str) else repr(value)
return text.encode('utf-8', 'replace').decode('utf-8')
def require_encodable(value, name):
"""Reject text that cannot survive a UTF-8 round trip.
`json.loads` happily produces lone surrogates from `\ud800` escapes. Such a
string cannot be UTF-8 encoded, so it must be rejected before it is stored;
otherwise the write is committed and the response encoding fails afterwards.
"""
try:
value.encode('utf-8')
except UnicodeEncodeError:
raise InvalidRequest(
'Поле %s содержит недопустимые символы Unicode' % name) from None
return value
def require_object(payload):
if not isinstance(payload, dict):
raise InvalidRequest('Тело запроса должно быть JSON-объектом')
return payload
def require_fields(payload, required, allowed):
"""Reject missing required fields and any unknown field."""
missing = [name for name in required if name not in payload]
if missing:
raise InvalidRequest('Отсутствуют обязательные поля: ' + ', '.join(sorted(missing)))
extra = [name for name in payload if name not in allowed]
if extra:
raise InvalidRequest('Неизвестные поля: ' + ', '.join(sorted(safe_label(n) for n in extra)))
return payload
def require_text_field(payload, name, minimum, maximum, strip_check=True):
value = payload.get(name)
if not isinstance(value, str):
raise InvalidRequest('Поле %s должно быть строкой' % name)
require_encodable(value, name)
if strip_check and not value.strip():
raise InvalidRequest('Поле %s не может быть пустым или состоять из пробелов' % name)
if len(value) < minimum:
raise InvalidRequest('Поле %s короче %d символов' % (name, minimum))
if len(value) > maximum:
raise InvalidRequest('Поле %s длиннее %d символов' % (name, maximum))
return value
def require_username(payload):
username = payload.get('username')
# fullmatch, not match: a trailing "$" would also accept a final newline.
if not isinstance(username, str) or USERNAME_PATTERN.fullmatch(username) is None:
raise InvalidRequest('username должен соответствовать ^[a-z0-9_]{3,32}$')
return require_encodable(username, 'username')
def require_uuid_string(value, name):
if not isinstance(value, str):
raise InvalidRequest('Поле %s должно быть строкой UUID' % name)
try:
return str(uuid.UUID(value))
except ValueError:
raise InvalidRequest('Поле %s не является корректным UUID' % name) from None
def require_attachment_ids(payload):
value = payload.get('attachment_ids', [])
if not isinstance(value, list) or len(value) > 10:
raise InvalidRequest('attachment_ids должен содержать от 0 до 10 UUID')
identifiers = [require_uuid_string(item, 'attachment_ids') for item in value]
if len(set(identifiers)) != len(identifiers):
raise InvalidRequest('attachment_ids содержит повторяющиеся UUID')
return identifiers
def require_member_ids(payload, maximum, minimum=1):
value = payload.get('member_ids')
if not isinstance(value, list):
raise InvalidRequest('member_ids должен быть массивом UUID')
if not minimum <= len(value) <= maximum:
raise InvalidRequest('member_ids должен содержать от %d до %d участников' % (minimum, maximum))
identifiers = [require_uuid_string(item, 'member_ids') for item in value]
if len(set(identifiers)) != len(identifiers):
raise InvalidRequest('member_ids содержит повторяющихся участников')
return identifiers
def parse_limit(raw, default, maximum):
"""Parse a limit query value strictly: absent means default, anything else must be a plain integer.
An empty value is a client error rather than an absent parameter, and the
digit run is length-bounded before int() because Python refuses to convert
strings longer than its integer-string limit, which would surface as a 500.
"""
if raw is None:
return default
if not DIGITS_PATTERN.fullmatch(raw):
raise InvalidRequest('limit должен быть целым числом от 1 до %d' % maximum)
# int() refuses absurdly long digit strings (ValueError -> 500). Comparing the
# significant digits against the maximum first rejects them without converting.
significant = raw.lstrip('0') or '0'
if len(significant) > len(str(maximum)):
raise InvalidRequest('limit должен быть в диапазоне 1..%d' % maximum)
value = int(significant)
if not 1 <= value <= maximum:
raise InvalidRequest('limit должен быть в диапазоне 1..%d' % maximum)
return value
def parse_cursor(raw):
if raw is None:
return None
if not isinstance(raw, str) or not raw:
raise InvalidRequest('cursor должен быть непустой строкой')
return raw
def require_idempotency_key(value):
"""Only the contract's bounded ASCII token is accepted."""
if not isinstance(value, str) or re.fullmatch(r'[A-Za-z0-9._:-]{1,128}', value) is None:
raise InvalidRequest('Требуется корректный Idempotency-Key (1–128 ASCII символов)')
return value
+170
View File
@@ -0,0 +1,170 @@
"""Process attachment jobs and acknowledge only committed results."""
import hashlib
from io import BytesIO
import json
import logging
import signal
import tempfile
import time
from uuid import UUID
from PIL import Image, ImageOps
import pika
from . import telemetry, logging_utils
from .attachments import AttachmentStorage, MAX_UPLOAD_BYTES, validate_image
from .config import Config
from .db import Database
from .errors import NotFound, PayloadTooLarge, Unavailable, UnsupportedMediaType
from .job_lifecycle import claim, complete
from .publish_jobs import QUEUE, connect
log = logging.getLogger('messenger.worker')
def fitted(image, bound):
width, height = image.size
longest = max(width, height)
if longest <= bound:
return image.copy()
return image.resize((max(1, width * bound // longest),
max(1, height * bound // longest)), Image.Resampling.LANCZOS)
def save_variants(source, storage, prefix):
with Image.open(source, formats=['PNG', 'JPEG']) as original:
with ImageOps.exif_transpose(original) as corrected:
with corrected.convert('RGBA') as rgba:
rgb = Image.new('RGB', rgba.size, 'white')
rgb.paste(rgba, mask=rgba.getchannel('A'))
with rgb:
specs = (
('thumbnail', 'jpeg', 85, lambda: fitted(rgb, 256)),
('crop', 'jpeg', 85, lambda: ImageOps.fit(rgb, (128, 128), Image.Resampling.LANCZOS)),
('optimized', 'webp', 82, lambda: fitted(rgb, 1280)),
)
results = []
for name, fmt, quality, render in specs:
with render() as image, BytesIO() as file:
image.save(file, format=fmt.upper(), quality=quality)
size = file.tell()
digest = hashlib.sha256(file.getbuffer()).hexdigest()
file.seek(0)
key = f'{prefix}/{name}.{fmt}'
mime = f'image/{fmt}'
storage.put(key, file, size, mime)
results.append((name, key, mime, size, digest, image.width, image.height))
return results
def materialize(attachment, storage, version, attempt):
prefix = f"attachments/final/{attachment['id']}/v{version}/a{attempt}"
key = prefix + '/original'
stream = storage.download(attachment)
try:
with tempfile.TemporaryFile() as source:
digest, size = hashlib.sha256(), 0
for chunk in stream.chunks():
size += len(chunk)
if size > MAX_UPLOAD_BYTES:
raise PayloadTooLarge()
digest.update(chunk)
source.write(chunk)
if size != attachment['size_bytes'] or digest.hexdigest() != attachment['sha256']:
raise UnsupportedMediaType('Контрольная сумма оригинала не совпадает')
validate_image(source, attachment['content_type'])
source.seek(0)
variants = save_variants(source, storage, prefix)
source.seek(0)
storage.put(key, source, size, attachment['content_type'])
finally:
stream.close()
return key, variants
def process(database, storage, job):
attachment = claim(database, job)
if attachment is None:
log.info('job_skipped', extra={'event': 'job_skipped', 'job_id': job['job_id']})
return
started = time.monotonic()
outcome = 'error'
telemetry.worker_busy.add(1)
try:
try:
result = materialize(attachment, storage, job['pipeline_version'], job['attempt'])
except (PayloadTooLarge, UnsupportedMediaType, NotFound):
complete(database, job, error_code='invalid_image')
except (Unavailable, OSError):
complete(database, job)
else:
outcome = 'success' if complete(database, job, result=result) else 'stale'
finally:
telemetry.worker_busy.add(-1)
labels = {'job_type': QUEUE, 'outcome': outcome}
telemetry.jobs.add(1, labels)
telemetry.job_duration.record(time.monotonic() - started, labels)
if outcome == 'error':
telemetry.trace.get_current_span().set_status(
telemetry.trace.Status(telemetry.trace.StatusCode.ERROR))
log.info('job_processed', extra={'event': 'job_processed', 'outcome': outcome,
'job_id': job['job_id'], 'attempt': job['attempt']})
def parse_job(body):
job = json.loads(body)
if not isinstance(job, dict) or job.get('type') != QUEUE or job.get('schema_version') != 1:
raise ValueError('Unknown job format')
for field in ('job_id', 'attachment_id'):
UUID(job[field])
for field in ('pipeline_version', 'attempt'):
if type(job[field]) is not int or job[field] < 1:
raise ValueError('Invalid job number')
return job
def main():
logging_utils.configure("INFO")
database = Database(Config.from_env())
database.open()
storage = AttachmentStorage()
connection = None
def timeout(_signal, _frame):
raise TimeoutError('Attachment processing exceeded 20 seconds')
signal.signal(signal.SIGALRM, timeout)
def consume(channel, method, _properties, body):
try:
job = parse_job(body)
except (ValueError, KeyError, TypeError, AttributeError):
channel.basic_reject(method.delivery_tag, requeue=False)
return
with telemetry.span('attachment.consume', telemetry.trace.SpanKind.CONSUMER,
context=telemetry.extract(_properties.headers or {}),
attributes={'messaging.system': 'rabbitmq',
'messaging.destination.name': QUEUE,
'job.id': job['job_id'], 'job.attempt': job['attempt']}):
signal.alarm(20)
try:
process(database, storage, job)
finally:
signal.alarm(0)
channel.basic_ack(method.delivery_tag)
try:
connection = connect()
channel = connection.channel()
channel.queue_declare(queue=QUEUE, durable=True)
channel.basic_qos(prefetch_count=1)
channel.basic_consume(queue=QUEUE, on_message_callback=consume)
channel.start_consuming()
finally:
if connection and connection.is_open:
connection.close()
database.close()
if __name__ == '__main__':
main()
+106
View File
@@ -0,0 +1,106 @@
"""Run cleanup scenarios against a disposable PostgreSQL database."""
import os
import sys
from pathlib import Path
from uuid import uuid4
import psycopg2
from psycopg2 import sql
from psycopg2.extensions import make_dsn
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'src'))
from messenger.config import Config
from messenger.db import Database
from messenger.errors import Unavailable
from messenger.store import Store
GRACE_SECONDS = 900
def unavailable(_key):
raise Unavailable('synthetic S3 failure')
def expect_unavailable(operation):
try:
operation()
except Unavailable:
return
raise AssertionError('Expected Unavailable')
def exercise(database):
store = Store(None, database, None)
user, chat = str(uuid4()), str(uuid4())
with database.cursor() as cur:
cur.execute("INSERT INTO users VALUES (%s,'cleanup','unused','Cleanup')", (user,))
cur.execute("INSERT INTO chats VALUES (%s,'Cleanup')", (chat,))
def reserve(expired=False):
aid = str(uuid4())
key = 'attachments/' + aid
store.reserve_attachment(aid, chat, user, 'x.png', 'image/png', 1, 'a' * 64, key)
if expired:
with database.cursor() as cur:
cur.execute("UPDATE attachments SET updated_at=now()-interval '16 minutes' WHERE id=%s", (aid,))
return aid, key
objects = set()
fresh, fresh_key = reserve()
objects.add(fresh_key)
ready, ready_key = reserve(expired=True)
store.finish_attachment(ready, objects.add)
with database.cursor() as cur:
cur.execute("UPDATE attachments SET updated_at=now()-interval '16 minutes' WHERE id=%s", (ready,))
active, active_key = reserve(expired=True)
def active_put(key):
objects.add(key)
assert not store.clean_pending_attachment(GRACE_SECONDS, objects.discard)
store.finish_attachment(active, active_put)
failed, failed_key = reserve(expired=True)
def put_then_fail(key):
objects.add(key)
# Inject a failure after the object exists and before the ready update.
raise Unavailable('synthetic failure after PUT')
expect_unavailable(lambda: store.finish_attachment(failed, put_then_fail))
assert failed_key in objects
expect_unavailable(lambda: store.clean_pending_attachment(GRACE_SECONDS, unavailable))
with database.cursor() as cur:
cur.execute('SELECT status FROM attachments WHERE id=%s', (failed,))
assert cur.fetchone()['status'] == 'pending'
assert store.clean_pending_attachment(GRACE_SECONDS, objects.discard)
assert objects == {fresh_key, ready_key, active_key}
reserve(expired=True) # Reservation whose PUT never created an object.
assert store.clean_pending_attachment(GRACE_SECONDS, objects.discard)
assert not store.clean_pending_attachment(GRACE_SECONDS, objects.discard)
with database.cursor() as cur:
cur.execute('SELECT id::text AS id, status FROM attachments')
assert {row['id']: row['status'] for row in cur.fetchall()} == {
fresh: 'pending', ready: 'ready', active: 'ready',
}
print('PASS: PostgreSQL row lock protects active PUT; fresh and ready attachments preserved')
print('PASS: interrupted PUT and failure after PUT cleaned; S3 failure retries safely')
def main():
dsn = os.environ['LAB4_TEST_DATABASE_URL']
schema = 'cleanup_' + uuid4().hex
with psycopg2.connect(dsn) as conn, conn.cursor() as cur:
cur.execute(sql.SQL('CREATE SCHEMA {}').format(sql.Identifier(schema)))
config = Config.from_env({'DATABASE_URL': make_dsn(dsn, options='-csearch_path=' + schema)})
database = Database(config)
try:
database.open()
database.run_migrations()
exercise(database)
finally:
database.close()
with psycopg2.connect(dsn) as conn, conn.cursor() as cur:
cur.execute(sql.SQL('DROP SCHEMA {} CASCADE').format(sql.Identifier(schema)))
if __name__ == '__main__':
main()
+68
View File
@@ -0,0 +1,68 @@
"""Check upload errors through Nginx and exercise an actual S3 outage."""
from io import BytesIO
from pathlib import Path
import subprocess
import sys
import time
from PIL import Image
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client, Failure, arguments, check, run
def main():
args = arguments(__doc__)
client = Client(args.base_url, args.ca_file, timeout=45)
alice, _, eve, chat = client.bootstrap()
data = (ROOT / 'tests/fixtures/sample.png').read_bytes()
path = f"/api/v1/chats/{chat['id']}/attachments"
good = path + '?filename=sample.png'
item, _ = client.attachment(alice, chat)
content = f"/api/v1/attachments/{item['id']}/content"
def upload(url, payload, expected, mime='image/png', actor=alice):
return client.call('POST', url, expected, actor=actor, raw=payload,
headers={'Content-Type': mime})
huge = BytesIO()
Image.new('1', (8001, 5000)).save(huge, 'PNG')
for payload, expected, mime in [
(b'x' * (10 * 1024 * 1024 + 1), 413, 'image/png'),
(huge.getvalue(), 413, 'image/png'),
(data[:30], 415, 'image/png'),
(data, 415, 'image/jpeg'),
]:
upload(good, payload, expected, mime)
upload(path + '?filename=a%2Fb.png', data, 400)
upload(good, data, 404, actor=eve)
client.call('GET', content, 404, actor=eve)
print('PASS: size, pixel count, corruption, MIME, filename and chat ACL over Nginx')
def s3_command(action):
subprocess.run(['docker', 'compose', action, 's3'], cwd=ROOT, check=True, timeout=30)
try:
s3_command('stop')
upload(good, data, 503)
client.call('GET', content, 503, actor=alice)
client.message(alice, chat, 'Текст при недоступном S3')
finally:
s3_command('start')
started = time.monotonic()
for _ in range(40):
try:
downloaded = client.call('GET', content, actor=alice)
break
except Failure:
time.sleep(.25)
else:
raise Failure('S3 download did not recover after restart')
check(downloaded['bytes'] == data, 'original changed after S3 restart')
print(f'PASS: S3 outage returned 503, text remained available, original recovered in {time.monotonic()-started:.2f}s')
if __name__ == '__main__':
run(main)
+54
View File
@@ -0,0 +1,54 @@
"""Stop the actual upload replica and check original bytes after failover and restart."""
from hashlib import sha256
import os
from pathlib import Path
import subprocess
import sys
import time
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client, arguments, check, run
def main():
args = arguments(__doc__)
client = Client(args.base_url, args.ca_file, timeout=55)
alice, bob, _, chat = client.bootstrap()
path = f"/api/v1/chats/{chat['id']}/attachments?filename=sample.png"
original = (ROOT / 'tests/fixtures/sample.png').read_bytes()
reply = client.call('POST', path, 201, actor=alice, raw=original,
headers={'Content-Type': 'image/png'}, schema='Attachment')
aid = reply['json']['id']
upload_instance = reply['headers']['X-Instance-Id']
digest = sha256(original).hexdigest()
content = '/api/v1/attachments/' + aid + '/content'
text = client.message(alice, chat, '', attachments=[aid])['json']
def verify():
received = client.call('GET', content, actor=bob)
check(received['bytes'] == original and sha256(received['bytes']).hexdigest() == digest,
'original changed')
history = client.call('GET', f"/api/v1/chats/{chat['id']}/messages",
actor=bob, schema='MessagePage')['json']
check(text in history['items'], 'message attachment link lost')
return received['headers']['X-Instance-Id']
start = time.monotonic()
try:
subprocess.run(['scripts/stop-one.sh'], cwd=ROOT, check=True, timeout=30,
env={**os.environ, 'TARGET_INSTANCE_ID': upload_instance})
instance = verify()
check(instance != upload_instance, 'download was served by the stopped uploader')
print(f'PASS: {upload_instance} -> {instance}, SHA-256 {digest}, {time.monotonic()-start:.2f}s')
finally:
subprocess.run([sys.executable, 'scripts/manage.py', 'restore'], cwd=ROOT, check=True, timeout=100)
start = time.monotonic()
subprocess.run(['scripts/restart.sh'], cwd=ROOT, check=True, timeout=100)
instance = verify()
print(f'PASS: after full restart via {instance}, SHA-256 {digest}, {time.monotonic()-start:.2f}s')
if __name__ == '__main__':
run(main)
+141
View File
@@ -0,0 +1,141 @@
from datetime import datetime, timezone
from hashlib import sha256
from io import BytesIO
import logging
import os
import http.client
import socket
from pathlib import Path
import threading
from urllib.error import HTTPError
from urllib.request import Request, urlopen
from uuid import uuid4
import json
import sys
from botocore.response import StreamingBody
from PIL import Image
root = Path(__file__).resolve().parents[1]
os.environ['S3_BUCKET'] = 'test'
sys.path.insert(0, str(root / 'src'))
from messenger.api import Api
from messenger.config import Config
from messenger.security import hash_token
from messenger.server import build_server
from messenger.store import Store
chat, member, outsider = [str(uuid4()) for _ in range(3)]
class Metadata:
rows = {}
def get_session(self, token):
return {hash_token('member'): member, hash_token('outsider'): outsider}.get(token)
def user(self, uid):
return {'id': uid}
def chat(self, cid):
return {'id': cid} if cid == chat else None
def is_member(self, _chat, uid):
return uid == member
def reserve_attachment(self, aid, cid, uid, filename, mime, size, digest, key):
now = datetime.now(timezone.utc)
self.rows[aid] = dict(id=aid, chat_id=cid, owner_id=uid, filename=filename,
content_type=mime, size_bytes=size, sha256=digest,
object_key=key, status='pending', error_code=None,
created_at=now, updated_at=now)
def finish_attachment(self, aid, put_object):
put_object(self.rows[aid]['object_key'])
self.rows[aid]['status'] = 'ready'
return self.rows[aid]
def visible_attachment(self, aid, uid):
row = self.rows.get(aid)
if uid == member and row and row['status'] == 'ready':
return row
return None
public_attachment = staticmethod(Store.public_attachment)
class S3:
def __init__(self):
self.objects = {}
self.fail = False
def put_object(self, **kw):
if self.fail:
from botocore.exceptions import EndpointConnectionError
raise EndpointConnectionError(endpoint_url='http://s3.invalid')
data = kw['Body'].read()
assert len(data) == kw['ContentLength']
self.objects[kw['Key']] = data
def get_object(self, **kw):
data = self.objects[kw['Key']]
return {'ContentLength': len(data), 'Body': StreamingBody(BytesIO(data), len(data))}
config = Config.from_env({'APP_HOST':'127.0.0.1','APP_PORT':'8080'})
config.port = 0
api = Api(config, Metadata(), None)
s3 = S3()
api.attachments.client = s3
server = build_server(config, logging.getLogger('lab4-check'), api, None)
thread = threading.Thread(target=server.serve_forever, daemon=True)
thread.start()
base = 'http://127.0.0.1:' + str(server.server_address[1])
upload = f'/api/v1/chats/{chat}/attachments?filename=x.png'
def call(path, method='GET', data=None, mime='image/png', token='member'):
headers = {'Authorization': 'Bearer ' + token, 'Content-Type': mime}
try:
response = urlopen(Request(base + path, data=data, headers=headers, method=method), timeout=12)
except HTTPError as exc:
response = exc
with response:
return response.code, response.headers, response.read()
try:
for image_format, mime in [('PNG', 'image/png'), ('JPEG', 'image/jpeg')]:
buf = BytesIO()
Image.new('RGB', (10, 10), 'green').save(buf, format=image_format)
data = buf.getvalue()
status, _, raw = call(upload, 'POST', data, mime)
assert status == 201, (status, raw)
item = json.loads(raw)
assert item['status'] == 'ready' and item['sha256'] == sha256(data).hexdigest()
assert item['size_bytes'] == len(data) and 'object_key' not in item
path = '/api/v1/attachments/' + item['id']
assert json.loads(call(path)[2]) == item
status, headers, actual = call(path + '/content')
assert (status, headers['Content-Type'], actual) == (200, mime, data)
assert call(path, token='outsider')[0] == 404
assert call(path + '/content', token='outsider')[0] == 404
assert call(path + '/content?variant=thumbnail')[0] == 404
png = (root / 'tests/fixtures/sample.png').read_bytes()
assert call(upload, 'POST', png)[0] == 201
assert call(upload, 'POST', png, 'image/jpeg')[0] == 415
assert call(upload, 'POST', png[:30])[0] == 415
assert call(upload.replace('x.png','a%2Fb.png'), 'POST', png)[0] == 400
before = len(api.store.rows)
for length, partial, expected in [(10 * 1024 * 1024 + 1, b'', 413), (64, b'abc', 400)]:
conn = http.client.HTTPConnection(*server.server_address, timeout=5)
try:
conn.request('POST', upload, body=partial, headers={
'Authorization': 'Bearer member', 'Content-Type': 'image/png',
'Content-Length': str(length)})
if partial:
conn.sock.shutdown(socket.SHUT_WR)
with conn.getresponse() as response:
assert response.status == expected, response.status
response.read()
finally:
conn.close()
assert len(api.store.rows) == before
buf = BytesIO()
Image.new('1', (8001, 5000)).save(buf, 'PNG')
assert call(upload, 'POST', buf.getvalue())[0] == 413
s3.fail = True
assert call(upload, 'POST', png)[0] == 503
assert list(api.store.rows.values())[-1]['status'] == 'pending'
print('PASS: local HTTP PNG/JPEG round trip, checksum, metadata, ACL and missing variant')
print('PASS: malformed image, mismatched MIME, unsafe filename, byte/pixel limits and S3 error mapping')
print('NOTE: metadata and S3 are test substitutes; real Compose integration not exercised')
finally:
server.shutdown()
server.server_close()
thread.join()
+98
View File
@@ -0,0 +1,98 @@
"""Exercise broker persistence and kill a worker after S3 writes, before SQL commit/ACK."""
from pathlib import Path
import subprocess
import sys
import time
from uuid import uuid4
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client, arguments, check, run
def cmd(*args, capture=False):
return subprocess.run(args, cwd=ROOT, text=True, check=True, timeout=100,
stdout=subprocess.PIPE if capture else None).stdout
def compose(*args, capture=False):
return cmd('docker', 'compose', *args, capture=capture)
def wait_for(probe, timeout, label):
until = time.monotonic() + timeout
while time.monotonic() < until:
if probe():
return
time.sleep(.25)
raise RuntimeError(label)
def confirmed(aid):
count = compose('exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger',
'-At', '-c', f"SELECT count(*) FROM attachment_outbox o JOIN attachment_jobs j "
f"ON j.id=o.job_id WHERE j.attachment_id='{aid}' AND o.published_at IS NOT NULL",
capture=True)
return int(count.strip()) > 0
def main():
args = arguments(__doc__)
client = Client(args.base_url, args.ca_file)
marker = 'AFTER_S3_BEFORE_COMMIT'
fault_worker = 'lab5-fault-' + uuid4().hex[:10]
compose('stop', 'worker')
try:
alice, bob, _, chat = client.bootstrap()
attachment, original = client.attachment(alice, chat)
client.message(alice, chat, '', attachments=[attachment['id']])
path = '/api/v1/attachments/' + attachment['id']
client.call('GET', path + '/content', 409, actor=bob)
wait_for(lambda: confirmed(attachment['id']), 30, 'no publisher confirm')
compose('restart', 'rabbitmq', 'api-a', 'api-b')
compose('up', '-d', '--no-build', '--no-deps', '--wait', 'api-a', 'api-b')
queued = client.call('GET', path, actor=bob, schema='Attachment')['json']
check(queued['status'] == 'queued', 'job lost during restart')
compose('start', 'worker')
client.wait_ready(bob, attachment)
check(client.call('GET', path + '/content', actor=bob)['bytes'] == original, 'original lost')
print('PASS: worker stopped; after API and RabbitMQ restart the same attachment became ready', flush=True)
compose('stop', 'worker')
attachment, original = client.attachment(alice, chat)
path = '/api/v1/attachments/' + attachment['id']
code = """
import sys,time
from messenger import worker
commit = worker.complete
def pause(db, job, result=None, error_code=None):
if job['attachment_id'] == sys.argv[1] and result is not None:
print('AFTER_S3_BEFORE_COMMIT', flush=True)
time.sleep(120)
return commit(db, job, result, error_code)
worker.complete = pause
worker.main()
"""
compose('run', '-d', '--no-deps', '--name', fault_worker, 'worker',
'python', '-c', code, attachment['id'])
wait_for(lambda: marker in cmd('docker', 'logs', fault_worker, capture=True),
40, 'worker did not reach fault point')
cmd('docker', 'kill', fault_worker)
compose('start', 'worker')
start = time.monotonic()
def is_ready():
state = client.call('GET', path, actor=bob, schema='Attachment')['json']
check(state['status'] != 'failed', 'processing failed after crash')
return state if state['status'] == 'ready' else None
wait_for(is_ready, 90, 'expired processing lease did not recover')
ready = is_ready()
check(len(ready['variants']) == 3, 'expected exactly one set of variants')
check(client.call('GET', path + '/content', actor=bob)['bytes'] == original, 'original changed')
print(f'PASS: worker killed after S3 write before COMMIT/ACK; recovered in {time.monotonic()-start:.2f}s')
finally:
subprocess.run(['docker', 'rm', '-f', fault_worker], cwd=ROOT, capture_output=True, timeout=15)
compose('start', 'worker')
if __name__ == '__main__':
run(main)
+24
View File
@@ -0,0 +1,24 @@
"""Regression: HTTP histogram includes response transmission."""
from email.message import Message
import logging
from types import SimpleNamespace
import time
from unittest.mock import Mock, patch
from messenger.server import ApiHandler
from messenger import telemetry
handler = object.__new__(ApiHandler)
handler.command = 'GET'
handler.path = '/api/v1/users/me'
handler.headers = Message()
handler.client_address = ('127.0.0.1', 1)
handler.server = SimpleNamespace(
shutting_down=False, config=SimpleNamespace(), logger=logging.getLogger('test'),
api=SimpleNamespace(dispatch=lambda _: (200, {})), enter=Mock(), leave=Mock())
handler._read_body = lambda: b''
handler._send = lambda *_: time.sleep(.08)
with patch.object(telemetry.http_duration, 'record') as record:
handler._handle_request()
assert record.call_args.args[0] >= .075, 'histogram omits response transmission'
handler.server.leave.assert_called_once()
print('PASS: HTTP histogram includes response transmission')
+89
View File
@@ -0,0 +1,89 @@
"""Exercise job fencing, failed replay and cleanup on the stage-5 stand."""
import subprocess
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client, arguments, check, run
def main():
args = arguments(__doc__)
client = Client(args.base_url, args.ca_file)
alice, bob, _, chat = client.bootstrap()
attachment, original = client.attachment(alice, chat)
code = """
import sys
from messenger.attachments import AttachmentStorage
from messenger.config import Config
from messenger.db import Database
from messenger.errors import Conflict
import messenger.job_lifecycle as lifecycle
from messenger.worker import materialize, process
db = Database(Config.from_env())
db.open()
s3 = AttachmentStorage()
aid = sys.argv[1]
def delivery():
with db.cursor() as cur:
cur.execute('SELECT id AS job_id,attachment_id,pipeline_version,attempt FROM attachment_jobs WHERE attachment_id=%s', (aid,))
return cur.fetchone()
try:
first = delivery()
row = lifecycle.claim(db, first)
assert row['status'] == 'processing' and lifecycle.claim(db, first) is None
stale_result = materialize(row, s3, first['pipeline_version'], first['attempt'])
with db.cursor() as cur:
cur.execute("UPDATE attachment_jobs SET lease_until=now()-interval '1 second' WHERE id=%s", (first['job_id'],))
lifecycle.recover_expired(db)
second = delivery()
assert second['attempt'] == first['attempt'] + 1
assert not lifecycle.complete(db, first, result=stale_result)
assert lifecycle.claim(db, second)
lifecycle.MAX_ATTEMPTS = 2
assert lifecycle.complete(db, second)
with db.cursor() as cur:
cur.execute('SELECT status,error_code FROM attachments WHERE id=%s', (aid,))
assert cur.fetchone() == {'status':'failed','error_code':'retry_exhausted'}
assert lifecycle.replay(db, first['job_id'], 'stage5-test') == 3
third = delivery()
process(db, s3, third)
process(db, s3, third)
try:
lifecycle.replay(db, first['job_id'], 'stage5-test')
except Conflict:
pass
else:
raise AssertionError('ready job was replayed')
with db.cursor() as cur:
cur.execute('SELECT count(*) AS n FROM attachment_variants WHERE attachment_id=%s', (aid,))
assert cur.fetchone()['n'] == 3
cur.execute('SELECT count(*) AS n FROM attachment_replays WHERE job_id=%s', (first['job_id'],))
assert cur.fetchone()['n'] == 1
cur.execute("UPDATE attachment_outbox SET available_at=now()-interval '16 minutes' WHERE job_id=%s AND attempt=1",
(first['job_id'],))
# Check that the cleaner can remove an obsolete attempt even after replay succeeded.
removed = []
while lifecycle.clean_abandoned_outputs(db, lambda key: (s3.delete(key), removed.append(key))):
pass
assert stale_result[0] in removed, removed
while lifecycle.clean_staging(db, s3.delete):
pass
with db.cursor() as cur:
cur.execute('SELECT staging_cleaned_at,lease_until FROM attachment_jobs WHERE id=%s', (first['job_id'],))
row = cur.fetchone()
assert row['staging_cleaned_at'] is not None and row['lease_until'] is None
print('PASS: expired claim fenced, retry exhausted, audited replay, stale objects and staging cleaned')
finally:
db.close()
"""
subprocess.run(['docker', 'compose', 'exec', '-T', 'api-a', 'python', '-c',
code, attachment['id']], cwd=ROOT, check=True, timeout=45)
path = f"/api/v1/attachments/{attachment['id']}/content"
check(client.call('GET', path, actor=bob)['bytes'] == original, 'cleanup damaged original')
print('PASS: ready original survives cleanup and replay requests')
if __name__ == '__main__':
run(main)
+53
View File
@@ -0,0 +1,53 @@
"""Exercise attachment links and message replay over the running lab4 API."""
import sys
import uuid
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
from client import Client, arguments, check, run
def main():
args = arguments('Message attachment links and replay')
client = Client(args.base_url, args.ca_file)
alice, bob, eve, chat = client.bootstrap()
first, _ = client.attachment(alice, chat)
second, _ = client.attachment(bob, chat)
ids = [second['id'], first['id']]
key = uuid.uuid4().hex
sent = client.message(bob, chat, '', key=key, attachments=ids)['json']
replay = client.message(bob, chat, '', key=key, attachments=ids)['json']
check(sent == replay and sent['attachment_ids'] == ids, 'ordered attachment replay mismatch')
path = f"/api/v1/chats/{chat['id']}/messages"
client.call('POST', path, 409, actor=bob, body={'text': '', 'attachment_ids': ids[::-1]},
headers={'Idempotency-Key': key})
for invalid in [[ids[0]] * 2, [str(uuid.uuid4()) for _ in range(11)], ['bad UUID'], None]:
client.call('POST', path, 400, actor=bob, body={'text': '', 'attachment_ids': invalid},
headers={'Idempotency-Key': uuid.uuid4().hex})
client.call('POST', path, 400, actor=bob, body={'text': ''},
headers={'Idempotency-Key': uuid.uuid4().hex})
client.call('POST', path, 404, actor=bob,
body={'text': '', 'attachment_ids': [str(uuid.uuid4())]},
headers={'Idempotency-Key': uuid.uuid4().hex})
other = client.call('POST', '/api/v1/chats', 201, actor=alice,
body={'title': 'Другой чат', 'member_ids': [alice['id'], eve['id']]},
schema='Chat')['json']
foreign, _ = client.attachment(alice, other)
client.call('POST', path, 404, actor=alice,
body={'text': '', 'attachment_ids': [foreign['id']]},
headers={'Idempotency-Key': uuid.uuid4().hex})
plain_key = uuid.uuid4().hex
plain = client.message(alice, chat, 'Обычный текст', key=plain_key)['json']
check(client.message(alice, chat, 'Обычный текст', key=plain_key, attachments=[])['json'] == plain,
'omitted and empty attachment_ids must replay identically')
page = client.call('GET', path, actor=alice, schema='MessagePage')['json']
check([item for item in page['items'] if item['id'] == sent['id']] == [sent],
'ordered attachment IDs missing or duplicated in history')
print('PASS: ordered attachments, replay, invalid input, cross-chat rejection and history')
if __name__ == '__main__':
run(main)
+297
View File
@@ -0,0 +1,297 @@
"""Live lab6 checks. Fault modes restore every stopped service in finally."""
import argparse
import base64
from concurrent.futures import ThreadPoolExecutor
import json
import re
from pathlib import Path
import subprocess
import sys
import time
import uuid
from urllib.error import HTTPError, URLError
from urllib.parse import urlencode
from urllib.request import Request, urlopen
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client, check
PROM = 'http://localhost:19090'
LOKI = 'http://localhost:13100'
TEMPO = 'http://localhost:13200'
BASE = 'http://localhost:18083'
def get(url, headers=None):
with urlopen(Request(url, headers=headers or {}), timeout=8) as response:
return json.load(response)
def prom(query):
return get(PROM + '/api/v1/query?' + urlencode({'query': query}))['data']['result']
def eventually(function, timeout=75):
deadline = time.monotonic() + timeout
last = None
while time.monotonic() < deadline:
try:
value = function()
if value:
return value
except OSError as exc:
last = type(exc).__name__
time.sleep(2)
raise AssertionError('condition timeout: ' + str(last or function.__name__))
def compose(*args):
subprocess.run(['docker', 'compose', '--env-file', '.env', *args], cwd=ROOT,
check=True, timeout=100, stdout=subprocess.DEVNULL)
def alert_firing(name):
alerts = get(PROM + '/api/v1/alerts')['data']['alerts']
return any(a['labels']['alertname'] == name and a['state'] == 'firing' for a in alerts)
def trace_spans(trace_id):
data = get(TEMPO + '/api/traces/' + trace_id, {'Accept': 'application/json'})
spans = []
def visit(item):
if isinstance(item, dict):
if 'spanId' in item and 'name' in item:
spans.append(item)
for child in item.values():
visit(child)
elif isinstance(item, list):
for child in item:
visit(child)
visit(data)
return spans
def upload(client, actor, chat):
original = (ROOT / 'tests/fixtures/sample.png').read_bytes()
incoming = uuid.uuid4().hex
reply = client.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=sample.png',
202, actor=actor, raw=original,
headers={'Content-Type': 'image/png',
'traceparent': '00-' + incoming + '-1234567890abcdef-01',
'tracestate': 'lab=lab6'}, schema='Attachment')
check(reply['headers']['X-Trace-Id'] == incoming, 'upload context lost')
return reply['json'], reply['headers']['X-Trace-Id']
def linked_trace(trace_id, attempts=1):
def ready():
spans = trace_spans(trace_id)
names = [s['name'] for s in spans]
if (any(n.startswith('POST /api/v1/chats/') for n in names)
and names.count('attachment.publish') >= attempts
and names.count('attachment.consume') >= attempts
and 's3.put' in names and 'postgresql.transaction' in names):
return spans
spans = eventually(ready)
ids = {s['spanId'] for s in spans}
for item in spans:
if item['name'] in ('attachment.publish', 'attachment.consume'):
check(item.get('parentSpanId') in ids, 'producer/consumer parent missing')
check(item.get('traceState') == 'lab=lab6', 'AMQP tracestate lost')
return spans
def correlated_logs(trace_id):
query = '{service_name=~"messenger-.*"} |= "' + trace_id + '"'
def available():
response = get(LOKI + '/loki/api/v1/query_range?' + urlencode({
'query': query, 'since': '1h', 'limit': 1000}))
rows = response['data']['result']
names = {r['stream'].get('service_name') for r in rows}
return rows if {'messenger-api', 'messenger-worker', 'messenger-publisher'} <= names else None
return eventually(available)
def smoke(client, alice, bob, chat):
att, tid = upload(client, alice, chat)
client.wait_ready(bob, att)
spans = linked_trace(tid)
logs = correlated_logs(tid)
required = ['messenger_http_requests_total', 'messenger_http_duration_seconds_bucket',
'messenger_queue_depth', 'messenger_queue_oldest_age_seconds',
'messenger_db_pool_used', 'messenger_worker_busy',
'messenger_job_duration_seconds_bucket', 'messenger_attachment_time_to_ready_seconds_bucket',
'messenger_process_memory_bytes', 'messenger_process_cpu_seconds_total']
for name in required:
eventually(lambda: prom(name))
# Validate propagation from an incoming W3C header, including header case normalization.
incoming_id = '1234567890abcdef1234567890abcdef'
reply = client.call('GET', '/api/v1/users/me', actor=alice,
headers={'traceparent': '00-' + incoming_id + '-1234567890abcdef-01',
'tracestate': 'lab=propagation'})
check(reply['headers']['X-Trace-Id'] == incoming_id, 'incoming trace context lost')
# Synthetic message and token must never appear in persisted telemetry.
canary = 'PRIVATE_BODY_' + tid
client.message(alice, chat, canary)
time.sleep(5)
for secret in (canary, alice['token']):
query = '{service_name=~"messenger-.*"} |= "' + secret + '"'
found = get(LOKI + '/loki/api/v1/query_range?' + urlencode({
'query': query, 'since': '1h', 'limit': 10}))['data']['result']
check(not found, 'private content found in telemetry')
password = (ROOT / 'secrets/grafana_password').read_text().strip()
auth = 'Basic ' + base64.b64encode(('admin:' + password).encode()).decode()
dashboard = get('http://localhost:13000/api/dashboards/uid/messenger', {'Authorization': auth})
check(len(dashboard['dashboard']['panels']) >= 12, 'dashboard not provisioned')
for panel in dashboard['dashboard']['panels']:
for target in panel.get('targets', []):
if 'expr' in target:
prom(target['expr'])
search = urlencode({'q': '{resource.service.name =~ "messenger-.*"}',
'start': int(time.time() - 900), 'end': int(time.time()), 'limit': 20})
traces = get('http://localhost:13000/api/datasources/proxy/uid/tempo/api/search?' + search,
{'Authorization': auth})
check(traces.get('traces'), 'Tempo search through Grafana failed')
return {'trace_id': tid, 'span_names': [s['name'] for s in spans],
'log_services': sorted({r['stream']['service_name'] for r in logs}),
'metric_names_checked': required, 'dashboard_panels': len(dashboard['dashboard']['panels']),
'dashboard_queries_valid': True, 'incoming_w3c_context': True, 'body_and_token_absent_from_logs': True}
def worker_stop(client, alice, bob, chat):
compose('stop', 'worker')
try:
att, tid = upload(client, alice, chat)
started = time.monotonic()
eventually(lambda: alert_firing('QueueLagGrowing'))
observation = {'seconds_to_firing': round(time.monotonic() - started, 2),
'oldest_job_age': prom('messenger_queue_oldest_age_seconds'),
'backlog': prom('messenger_queue_depth'),
'alerts_firing': get(PROM + '/api/v1/alerts')['data']['alerts']}
finally:
compose('start', 'worker')
client.wait_ready(bob, att)
eventually(lambda: not alert_firing('QueueLagGrowing'))
spans = linked_trace(tid)
observation.update(resolved=True, trace_id=tid, span_count=len(spans),
alerts_after=get(PROM + '/api/v1/alerts')['data']['alerts'])
return observation
def db_outage(client, alice):
# A held table lock makes actual PostgreSQL operations time out. Readiness SELECT 1 stays healthy.
locker = subprocess.Popen(['docker', 'compose', 'exec', '-T', 'postgres', 'psql',
'-U', 'messenger', '-d', 'messenger', '-v', 'ON_ERROR_STOP=1', '-c',
'BEGIN; LOCK TABLE users IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep(35); COMMIT;'],
cwd=ROOT, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
failures = []
try:
time.sleep(2)
def request(_):
reply = client.call('GET', '/api/v1/users/me', 503, actor=alice)
return reply['headers']['X-Trace-Id']
with ThreadPoolExecutor(max_workers=2) as executor:
failures = list(executor.map(request, range(8)))
eventually(lambda: alert_firing('HighApiErrorRate'))
error_spans = eventually(lambda: [s for s in trace_spans(failures[0])
if s['name'] == 'postgresql.transaction' and
s.get('status', {}).get('code') in (2, 'STATUS_CODE_ERROR')])
observation = {'firing': True, 'failed_requests': len(failures),
'trace_id': failures[0], 'db_error_spans': len(error_spans),
'alerts_firing': get(PROM + '/api/v1/alerts')['data']['alerts'],
'error_ratio': prom('sum(rate(messenger_http_requests_total{status_class="5xx"}[1m])) / sum(rate(messenger_http_requests_total[1m]))')}
finally:
locker.wait(timeout=50)
eventually(lambda: client.call('GET', '/api/v1/users/me', actor=alice, schema='User')['json'])
eventually(lambda: not alert_firing('HighApiErrorRate'), timeout=100)
observation['resolved'] = True
return observation
def retry(client, alice, bob, chat):
compose('stop', 'worker')
try:
att, tid = upload(client, alice, chat)
compose('stop', 's3')
# Compose start also starts dependencies in this version. Keep S3 stopped here.
subprocess.run(['docker', 'start', 'mtusi-lab6-worker-1'], check=True,
stdout=subprocess.DEVNULL)
def retried():
spans = trace_spans(tid)
return any(s['name'] == 'attachment.consume' and
s.get('status', {}).get('code') in (2, 'STATUS_CODE_ERROR') for s in spans)
eventually(retried, timeout=45)
finally:
compose('start', 's3', 'worker')
client.wait_ready(bob, att)
spans = linked_trace(tid, attempts=2)
return {'trace_id': tid, 'consumer_attempts': sum(s['name'] == 'attachment.consume' for s in spans),
'span_names': [s['name'] for s in spans], 'ready_after_retry': True}
def backend_outage(client, alice, bob, chat):
before = prom('sum(otelcol_exporter_send_failed_spans_total)')
tracked = ['mtusi-lab6-' + name + '-1' for name in
('api-a', 'api-b', 'worker', 'outbox-publisher', 'otel-collector')]
before_containers = json.loads(subprocess.check_output(['docker', 'inspect', *tracked], text=True))
outage_start = str(int(time.time()))
compose('stop', 'tempo')
count = 0
samples = []
try:
started = time.monotonic()
while time.monotonic() - started < 40:
client.message(alice, chat, 'telemetry-outage')
count += 1
if count % 15 == 0:
raw = subprocess.check_output(['docker', 'stats', '--no-stream', '--format',
'{{json .}}', *tracked], cwd=ROOT, text=True)
samples.extend(json.loads(line) for line in raw.splitlines())
time.sleep(.15)
att, _ = upload(client, alice, chat)
client.wait_ready(bob, att)
failures = prom('sum(otelcol_exporter_send_failed_spans_total)')
before_count = float(before[0]['value'][1]) if before else 0
check(failures and float(failures[0]['value'][1]) > before_count, 'no new exporter failures')
finally:
compose('start', 'tempo')
collector_logs = subprocess.check_output(['docker', 'compose', 'logs', '--since', outage_start,
'otel-collector'], cwd=ROOT, text=True)
dropped = sum(int(value) for value in re.findall(r'"dropped_items"\s*:\s*(\d+)', collector_logs))
after_containers = json.loads(subprocess.check_output(['docker', 'inspect', *tracked], text=True))
for before_item, after_item in zip(before_containers, after_containers):
check(after_item['State']['Running'] and not after_item['State']['OOMKilled'], 'container died')
check(before_item['RestartCount'] == after_item['RestartCount'], 'container restarted')
restored_att, restored_trace = upload(client, alice, chat)
client.wait_ready(bob, restored_att)
linked_trace(restored_trace)
correlated_logs(restored_trace)
return {'successful_messages': count, 'attachment_ready_during_outage': True,
'collector_reported_dropped_spans': dropped, 'export_failures_before': before, 'export_failures_after': failures,
'application_and_collector_stats': samples, 'no_oom_or_restarts': True, 'telemetry_restored_trace': restored_trace}
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('mode', choices=['smoke', 'worker-stop', 'db-outage', 'retry', 'backend-outage'])
args = parser.parse_args()
client = Client(BASE, None, timeout=20)
alice, bob, _, chat = client.bootstrap()
if args.mode == 'smoke':
result = smoke(client, alice, bob, chat)
elif args.mode == 'worker-stop':
result = worker_stop(client, alice, bob, chat)
elif args.mode == 'db-outage':
result = db_outage(client, alice)
elif args.mode == 'retry':
result = retry(client, alice, bob, chat)
else:
result = backend_outage(client, alice, bob, chat)
path = ROOT / 'evidence' / ('observability-' + args.mode + '.json')
path.write_text(json.dumps(result, ensure_ascii=False, indent=2) + '\n')
print('PASS', args.mode, path.name)
if __name__ == '__main__':
main()
+27
View File
@@ -0,0 +1,27 @@
#!/bin/sh
# Run against a lab5 stage-3 stand without a worker.
set -eu
cd "$(dirname "$0")/.."
base_url=${BASE_URL:-http://localhost:18082}
sql() {
docker compose exec -T postgres psql -U messenger -d messenger -At -v ON_ERROR_STOP=1 -c "$1"
}
docker compose stop outbox-publisher
trap 'docker compose start outbox-publisher >/dev/null' EXIT
upload=$(PYTHONDONTWRITEBYTECODE=1 python3 student-tests/queued_upload.py --base-url "$base_url")
printf '%s\n' "$upload"
aid=$(printf '%s\n' "$upload" | awk '/Attachment ID:/ {print $3}')
query="SELECT count(*) FROM attachment_outbox o JOIN attachment_jobs j ON j.id=o.job_id WHERE j.attachment_id='$aid' AND o.published_at IS"
test "$(sql "$query NULL")" = 1
echo 'PASS: upload committed while publisher was stopped'
docker compose start outbox-publisher
trap - EXIT
for i in $(seq 1 40); do
if test "$(sql "$query NOT NULL")" = 1; then
echo 'PASS: same attachment job confirmed after publisher startup'
exit 0
fi
sleep 1
done
echo 'FAIL: outbox row not confirmed' >&2
exit 1
+27
View File
@@ -0,0 +1,27 @@
"""Verify durable handoff before a publisher or worker is enabled."""
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
from client import Client, arguments, check, run
def main():
args = arguments(__doc__)
client = Client(args.base_url, args.ca_file)
alice, bob, eve, chat = client.bootstrap()
attachment, _ = client.attachment(alice, chat)
path = '/api/v1/attachments/' + attachment['id']
check(attachment['status'] == 'queued', 'expected queued with worker disabled')
metadata = client.call('GET', path, actor=bob, schema='Attachment')['json']
check(metadata == attachment, 'queued metadata differs')
client.call('GET', path, 404, actor=eve)
client.call('GET', path + '/content', 409, actor=bob)
sent = client.message(alice, chat, '', attachments=[attachment['id']])['json']
check(sent['attachment_ids'] == [attachment['id']], 'queued attachment not linked')
print('PASS: 202 queued, metadata ACL, 409 before ready, and queued attachment linked')
print('Attachment ID:', attachment['id'])
if __name__ == '__main__':
run(main)
+72
View File
@@ -0,0 +1,72 @@
"""Run via docker compose exec -T api-a python - with this file on stdin."""
from hashlib import sha256
from io import BytesIO
from uuid import uuid4
from botocore.exceptions import ClientError
from PIL import Image
from messenger.attachments import AttachmentStorage
from messenger.clean_attachments import GRACE_SECONDS, clean_batch
from messenger.config import Config
from messenger.db import Database
from messenger.errors import Unavailable
from messenger.store import Store
def main():
config = Config.from_env()
db = Database(config)
db.open()
store = Store(config, db, None)
s3 = AttachmentStorage()
user, chat, aid = [str(uuid4()) for _ in range(3)]
key = 'attachments/' + aid
payload = BytesIO()
Image.new('RGB', (2, 2), 'green').save(payload, format='PNG')
data = payload.getvalue()
try:
with db.cursor() as cur:
cur.execute('INSERT INTO users (id,username,password_hash,display_name) VALUES (%s,%s,%s,%s)',
(user, 'gc_' + user[:8], 'unused', 'Cleanup check'))
cur.execute("INSERT INTO chats (id,title) VALUES (%s,'Cleanup check')", (chat,))
store.reserve_attachment(aid, chat, user, 'probe.png', 'image/png',
len(data), sha256(data).hexdigest(), key)
def put_then_fail(object_key):
s3.put(object_key, BytesIO(data), len(data), 'image/png')
raise Unavailable('synthetic failure after successful PUT')
try:
store.finish_attachment(aid, put_then_fail)
except Unavailable as exc:
assert str(exc) == 'synthetic failure after successful PUT'
else:
raise AssertionError('fault did not run')
obj = s3.client.get_object(Bucket=s3.bucket, Key=key)
with obj['Body'] as body:
assert body.read() == data
with db.cursor() as cur:
cur.execute('SELECT status FROM attachments WHERE id=%s', (aid,))
assert cur.fetchone()['status'] == 'pending'
cur.execute("UPDATE attachments SET updated_at=now()-make_interval(secs => %s) WHERE id=%s",
(GRACE_SECONDS + 1, aid))
clean_batch(store, s3)
with db.cursor() as cur:
cur.execute('SELECT 1 FROM attachments WHERE id=%s', (aid,))
assert cur.fetchone() is None
try:
obj = s3.client.get_object(Bucket=s3.bucket, Key=key)
except ClientError as exc:
assert exc.response['ResponseMetadata']['HTTPStatusCode'] == 404, exc
else:
obj['Body'].close()
raise AssertionError('orphan object still exists')
print('PASS: actual MinIO object existed after failed commit and was removed by cleanup')
finally:
db.close()
if __name__ == '__main__':
main()
+20
View File
@@ -0,0 +1,20 @@
import sys,json,time,urllib.parse,subprocess
from pathlib import Path
sys.path.insert(0,'student-tests');import observability as o
trace_id=json.loads(Path('evidence/observability-smoke.json').read_text())['trace_id']
before=o.trace_spans(trace_id)
o.correlated_logs(trace_id)
stamp=time.time()-10
query=o.PROM+'/api/v1/query?'+urllib.parse.urlencode({'query':'sum(messenger_http_requests_total)','time':stamp})
old=o.get(query)['data']['result']
assert old
for service in ('otel-collector','prometheus','loki','tempo','grafana'):
o.compose('restart',service)
subprocess.run([sys.executable, 'scripts/wait_ready.py'], check=True)
o.eventually(lambda:o.trace_spans(trace_id),timeout=100)
o.correlated_logs(trace_id)
after=o.get(query)['data']['result']
assert after==old,'historical metric changed after restart'
assert {s['spanId'] for s in before}=={s['spanId'] for s in o.trace_spans(trace_id)},'trace history changed'
Path('evidence/telemetry-persistence.json').write_text(json.dumps({'trace_id':trace_id,'trace_spans_preserved':len(before),'api_worker_publisher_logs_preserved':True,'historical_metric_preserved':True,'restarted':['otel-collector','prometheus','loki','tempo','grafana']},indent=2)+'\n')
print('PASS metrics/logs/traces history survives backend restart')
+104
View File
@@ -0,0 +1,104 @@
"""Measure sampled cgroup memory during ten simultaneous HTTP uploads."""
from concurrent.futures import ThreadPoolExecutor
import hashlib
from io import BytesIO
import json
import math
from pathlib import Path
import random
import subprocess
import sys
from threading import Barrier
import time
from PIL import Image
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client, Failure, arguments, check, run
COUNT = 10
SAMPLER = """
import select, sys, time
from pathlib import Path
usage = Path('/sys/fs/cgroup/memory.current')
limit = Path('/sys/fs/cgroup/memory.max')
cpu = Path('/sys/fs/cgroup/cpu.stat')
print(limit.read_text().strip(), flush=True)
while True:
stats = dict(line.split() for line in cpu.read_text().splitlines())
print(usage.read_text().strip(), stats['usage_usec'], flush=True)
if select.select([sys.stdin], [], [], .05)[0]:
break
"""
def main():
args = arguments(__doc__)
client = Client(args.base_url, args.ca_file, timeout=60)
alice, _, _, chat = client.bootstrap()
width = 1700
source = Image.frombytes('RGB', (width, width), random.Random(2026).randbytes(width * width * 3))
buffer = BytesIO()
source.save(buffer, format='PNG', compress_level=0)
payload = buffer.getvalue()
digest = hashlib.sha256(payload).hexdigest()
check(len(payload) <= 10 * 1024 * 1024, 'fixture exceeds upload size limit')
path = f"/api/v1/chats/{chat['id']}/attachments?filename=load.png"
barrier = Barrier(COUNT + 1)
samplers = {}
def upload():
local = Client(args.base_url, args.ca_file, timeout=60)
barrier.wait(timeout=30)
start = time.monotonic()
try:
reply = local.call('POST', path, 201, actor=alice, raw=payload,
headers={'Content-Type': 'image/png'}, schema='Attachment')
check(reply['json']['sha256'] == digest, 'upload hash mismatch')
return {'instance': reply['headers']['X-Instance-Id'], 'id': reply['json']['id'],
'seconds': time.monotonic() - start}
except Failure as exc:
return {'error': str(exc), 'seconds': time.monotonic() - start}
try:
for service in ('api-a', 'api-b'):
proc = subprocess.Popen(
['docker', 'compose', 'exec', '-T', service, 'python', '-u', '-c', SAMPLER],
cwd=ROOT, stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True)
samplers[service] = proc
with ThreadPoolExecutor(max_workers=COUNT) as pool:
futures = [pool.submit(upload) for _ in range(COUNT)]
limits = {service: int(proc.stdout.readline()) for service, proc in samplers.items()}
initial = {service: tuple(map(int, proc.stdout.readline().split()))
for service, proc in samplers.items()}
start = time.monotonic()
barrier.wait(timeout=30)
requests = [future.result() for future in futures]
elapsed = time.monotonic() - start
finally:
for proc in samplers.values():
if proc.poll() is None:
proc.stdin.close()
resources = {}
for service, proc in samplers.items():
samples = [initial[service]] + [tuple(map(int, line.split())) for line in proc.stdout]
check(proc.wait(timeout=10) == 0 and len(samples) > 1, 'memory sampler failed')
resources[service] = {
'baseline_bytes': samples[0][0], 'peak_sample_bytes': max(row[0] for row in samples),
'limit_bytes': limits[service], 'samples': len(samples),
'cpu_seconds': (samples[-1][1] - samples[0][1]) / 1e6,
}
successes = [row for row in requests if 'id' in row]
durations = sorted(row['seconds'] for row in successes)
percentiles = {f'p{p}': durations[math.ceil(p / 100 * len(durations)) - 1]
for p in (50, 95, 99)} if durations else {}
print(json.dumps({'concurrency': COUNT, 'payload_bytes': len(payload), 'sha256': digest,
'elapsed_seconds': elapsed, 'successful_rps': len(successes) / elapsed,
'latency_seconds': percentiles, 'requests': requests,
'memory_sample_interval_seconds': .05, 'resources': resources}, indent=2))
check(len({row['id'] for row in successes}) == COUNT, 'not all uploads succeeded uniquely')
if __name__ == '__main__':
run(main)
+62
View File
@@ -0,0 +1,62 @@
"""Verify worker effects through HTTP and MinIO with a direct job handoff."""
import hashlib
import subprocess
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client, arguments, check, run
from image_probe import jpeg_size
def main():
args = arguments(__doc__)
client = Client(args.base_url, args.ca_file)
alice, bob, _, chat = client.bootstrap()
attachment, original = client.attachment(alice, chat)
code = """
from messenger.attachments import AttachmentStorage
from messenger.config import Config
from messenger.db import Database
from messenger.worker import process
import sys
db = Database(Config.from_env())
db.open()
try:
with db.cursor() as cur:
cur.execute('SELECT id AS job_id, attachment_id, pipeline_version, attempt '
'FROM attachment_jobs WHERE attachment_id=%s', (sys.argv[1],))
job = cur.fetchone()
storage = AttachmentStorage()
process(db, storage, job)
process(db, storage, job)
finally:
db.close()
"""
subprocess.run(['docker', 'compose', 'exec', '-T', 'api-a', 'python', '-c',
code, attachment['id']], cwd=ROOT, check=True, timeout=30)
path = '/api/v1/attachments/' + attachment['id']
ready = client.call('GET', path, actor=bob, schema='Attachment')['json']
check(ready['status'] == 'ready' and len(ready['variants']) == 3, 'expected three variants')
check(client.call('GET', path + '/content', actor=bob)['bytes'] == original, 'original changed')
expected = {'thumbnail': (256, 160, 'image/jpeg'), 'crop': (128, 128, 'image/jpeg'),
'optimized': (320, 200, 'image/webp')}
for item in ready['variants']:
width, height, mime = expected[item['name']]
reply = client.call('GET', path + '/content?variant=' + item['name'], actor=bob)
data = reply['bytes']
check((item['width'], item['height'], item['content_type']) == (width, height, mime),
'incorrect variant metadata')
check(reply['headers'].get_content_type() == mime and len(data) == item['size_bytes']
and hashlib.sha256(data).hexdigest() == item['sha256'], 'incorrect variant payload')
if mime == 'image/jpeg':
check(jpeg_size(data) == (width, height), 'invalid JPEG dimensions')
else:
check(data[:4] == b'RIFF' and data[8:12] == b'WEBP', 'invalid WebP signature')
print('PASS: original unchanged, thumbnail 256x160, crop 128x128, WebP 320x200')
print('PASS: processing the same committed job twice keeps one set of variants')
if __name__ == '__main__':
run(main)