79 changed files with 10345 additions and 24 deletions
+6
View File
@@ -17,4 +17,10 @@ backups
__pycache__
node_modules
evidence
student-tests
tests
research
REPORT.md
SCHEMA.md
requirements-dev.txt
.DS_Store
+9
View File
@@ -28,3 +28,12 @@ OTEL_SERVICE_NAME=messenger-api
OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318
OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab
# Local ports and telemetry mode.
PUBLIC_PORT=8086
GRAFANA_PORT=3006
PROMETHEUS_PORT=9096
LOKI_PORT=3106
TEMPO_PORT=3206
TELEMETRY_ENABLED=1
OTEL_SAMPLE_RATIO=1
+1
View File
@@ -0,0 +1 @@
3.14.7
+29
View File
@@ -0,0 +1,29 @@
FROM python:3.14.7-slim-bookworm@sha256:82bc3c539b8813ada9d68c63b40158fa002f7f33de9bf3312a3dfdc0620dff56 AS builder
WORKDIR /app
RUN python -m pip install --no-cache-dir uv==0.12.13
ENV UV_PYTHON_DOWNLOADS=never
COPY pyproject.toml uv.lock .python-version ./
RUN --mount=type=cache,target=/root/.cache/uv uv sync --frozen --no-dev --no-install-project --compile-bytecode
COPY src ./src
RUN --mount=type=cache,target=/root/.cache/uv uv sync --frozen --no-dev --no-editable --compile-bytecode
FROM python:3.14.7-slim-bookworm@sha256:82bc3c539b8813ada9d68c63b40158fa002f7f33de9bf3312a3dfdc0620dff56
RUN groupadd --gid 10001 app && useradd --uid 10001 --gid app --no-create-home app
WORKDIR /app
COPY --from=builder /app/.venv /app/.venv
COPY migrations /app/migrations
ARG VCS_REF=uncommitted
LABEL org.opencontainers.image.revision=$VCS_REF
ENV PATH="/app/.venv/bin:$PATH" \
PYTHONUNBUFFERED=1 \
PYTHONDONTWRITEBYTECODE=1 \
APP_HOST=0.0.0.0 \
APP_PORT=8080 \
APP_ENV=lab \
APP_LOG_LEVEL=INFO
USER 10001:10001
EXPOSE 8080
STOPSIGNAL SIGTERM
HEALTHCHECK --interval=2s --timeout=2s --start-period=5s --retries=3 \
CMD ["python", "-c", "import os, urllib.request; urllib.request.urlopen('http://127.0.0.1:' + os.environ.get('APP_PORT', '8080') + '/health/ready', timeout=1).close()"]
CMD ["python", "-m", "messenger"]
+19 -1
View File
@@ -1,5 +1,5 @@
PYTHON ?= python3
BASE_URL ?= http://localhost:8080
BASE_URL ?= http://127.0.0.1:8086
CA_FILE ?=
ACTION_SCRIPT ?=
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
@@ -28,3 +28,21 @@ test-persistence:
test-failover:
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
.PHONY: bootstrap up down unit lint verify-cluster verify-pipeline
bootstrap:
$(PYTHON) scripts/bootstrap.py
up:
docker compose --env-file .env up -d --build --wait --wait-timeout 180
down:
docker compose --env-file .env stop --timeout 7
unit:
uv run pytest --tb=short
lint:
uv run ruff check .
uv run ruff format --check .
verify-cluster:
uv run python scripts/verify_cluster.py --base-url "$(BASE_URL)"
verify-pipeline:
uv run python scripts/verify_pipeline.py --base-url "$(BASE_URL)"
+86 -23
View File
@@ -1,37 +1,100 @@
# Отчёт — лабораторная 6
# ЛР6. Наблюдаемость мессенджера
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
**Горбанюк, БВТ2301.** Целевая оценка 5. Трек 1 «Цена наблюдаемости».
Использованы материалы задания, документация [OpenTelemetry](https://opentelemetry.io/docs/collector/configuration/), [Loki OTLP](https://grafana.com/docs/loki/latest/send-data/otel/) и [Prometheus](https://prometheus.io/docs/practices/histograms/). За основу взято собственное решение ЛР5. При подготовке реализации, проверок и отчёта использовалась помощь ИИ.
## Запуск из чистого clone
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
Нужны Docker с Compose и Python 3.14 для локальных проверок. Runtime приложения — Python 3.14.7, зависимости в `uv.lock`, OpenTelemetry Python 1.45.2. Стенд включает Collector Contrib 0.147.0, Prometheus 3.9.1, Loki 3.6.3, Tempo 2.10.0 и Grafana 12.3.3. Образы и лимиты указаны в `compose.yaml`. Docker сам запускает миграции и создаёт приватный bucket.
## Архитектура и решения
```sh
make bootstrap
make up
curl -fsS http://127.0.0.1:8086/health/ready
make test
uv sync --locked
make check
make validate PYTHON=.venv/bin/python
make lint
```
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
Grafana доступна на `http://127.0.0.1:3006`; пользователь `admin`, пароль в локальном `secrets/grafana_password`. Панель **MTUSI Messenger** и три источника появляются при старте. Прямые порты Prometheus 9096, Loki 3106 и Tempo 3206 привязаны к loopback для локальных проверок. `make down` останавливает стенд с сохранением данных. `docker compose down -v` удаляет учебные данные этого проекта.
## Доказательства критериев
Для набора собственных тестов и диагностики с инъекцией одноразового сбоя:
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
```sh
docker compose -f compose.yaml -f compose.test.yaml up -d --build --wait
make unit
make test-persistence ACTION_SCRIPT=scripts/restart.sh
make test-failover ACTION_SCRIPT=scripts/stop-one.sh
uv run python scripts/verify_pipeline.py --output evidence/pipeline-repeat.json
uv run python scripts/observe.py --incidents --retry --output evidence/observe-repeat.json
uv run python scripts/telemetry_cost.py --output evidence/cost-repeat.json
```
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
## Архитектура
## Эксперименты и отказы
Две реплики FastAPI за HAProxy используют общие PostgreSQL, Valkey и S3. Из ЛР5 сохранены загрузка со статусом 202 после приёма файла и фиксации job/outbox, relay, отдельный durable Valkey Streams и обработка в дочернем процессе worker. PostgreSQL хранит статус и версии, S3 — байты. Повторная доставка ограничена состоянием задачи и транзакционной проверкой fence.
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
API, relay и worker отправляют OTLP/HTTP в Collector. Prometheus опрашивает экспортёр метрик и сам Collector. Loki хранит структурные логи, Tempo хранит трассы. Grafana загружает источники и панель из Git. У хранилищ отдельные volumes, задано удержание примерно 72 часа; для Prometheus также действует предел размера 256 MB.
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
`traceparent` и `tracestate` добавляются к job той же транзакцией, которая фиксирует её приём. Relay создаёт producer span под контекстом из БД и вкладывает свой контекст в envelope; worker создаёт consumer span. Повторная публикация использует исходный сохранённый контекст. Для обработки видны дочерние операции PostgreSQL и S3. Супервизор агрегирует исходы и задержки процессов, включая таймауты, в стабильную серию worker.
## Ограничения и следующие шаги
В метриках HTTP используется шаблон маршрута и класс статуса; health исключён. В логах остаются созданные сервисом request ID, trace ID, span ID и в событиях обработки job ID для поиска. Инструментация не читает заголовки авторизации, содержимое запросов, SQL, имена файлов или ключи объектов. Исключения в span записывают только тип. `service.instance.id` отличает процессы, `service.version` берётся из установленного приложения, окружение `lab`. Нестандартные имена описаны ниже.
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
| Сигнал | Prometheus | Единицы и агрегация |
| --- | --- | --- |
| Запросы и ошибки | `messenger_http_requests_total` | rate по всем API; ошибки — 5xx / бизнес-запросы |
| Длительность HTTP | `messenger_http_duration_seconds_bucket` | p50/p95/p99 через `histogram_quantile` после суммы rate buckets по `le` |
| Задачи и время обработки | `messenger_jobs_processed_total`, `messenger_jobs_duration_seconds_bucket` | исходы и гистограмма в секундах |
| Backlog и возраст | `messenger_queue_depth`, `messenger_queue_oldest_seconds` | число и возраст незавершённых job; снимок PostgreSQL каждые 2 с |
| Пул БД и worker | `messenger_db_pool_used`, `messenger_db_pool_waiting`, `messenger_worker_busy`, `messenger_worker_capacity` | используемые и ожидающие соединения при обращении к пулу, занятые слоты worker |
| Время до ready | `messenger_ready_duration_seconds_bucket` | от времени создания записи в транзакции приёма до фиксации результата; секунды |
| CPU и память | `messenger_process_cpu_seconds_total`, `messenger_process_rss_bytes` | CPU процесса с завершёнными дочерними процессами, RSS процесса; для полного дерева дополнительно Docker stats |
## Диагностика
Учебные цели для окна 5 минут: 99% бизнес-запросов без 5xx, 95% принятых и завершённых изображений готовы за 30 секунд. Для второго SLI доля берётся из buckets, а незавершённые и окончательно failed контролируются отдельно через backlog и исходы задач. Пороги демонстрационных алертов рассчитаны на короткий стенд. HTTP alert требует >5% 5xx в окне 20 секунд с выдержкой 6 секунд. Queue alert требует незавершённые job и возраст >5 секунд с выдержкой 6 секунд. Дополнительный DB alert использует рост ошибок пула в окне 20 секунд.
1. При `MessengerQueueLag` проверить число и возраст незавершённых job, слоты worker, затем потребление и логи worker. Из лога загрузки перейти к trace ID и искать `jobs.publish` и `jobs.consume`. Рост очереди при исчезнувших consume и стабильном HTTP указывает на остановку обработки.
2. При HTTP 5xx проверить `MessengerDatabaseErrors`, взять request ID из ответа и найти лог в Loki. В Tempo сравнить длительность HTTP span и `db.read`. Узкий участок виден внутри БД; для сравнения обычный запрос укладывался в миллисекунды.
3. Если данные исчезли из Tempo, сверить accepted/sent/failed у Collector и проверить readiness API. Отдельная отправка из SDK и очередь Collector ограничивают влияние на мессенджер. В SDK до 512 span и 512 log records на процесс; у Collector memory limiter 128 MiB, cgroup 256 MiB и 16 batches на экспортёр, таймаут попытки 2 секунды, окно повторов 5 секунд. RSS процесса отличается от управляемой памяти Go.
| Воздействие | Наблюдение |
| --- | --- |
| Остановка worker и 3 загрузки | Алерт firing при глубине 3 и возрасте 16.9 с. После запуска 3/3 ready, очередь 0 и alert resolved. |
| Ограниченная блокировка таблицы sessions | 12 запросов 503, максимум около 1.01 с. Длительность локализуется в `db.read`; оба алерта firing, затем resolved. |
| Остановка Tempo | 15 записей сообщений успешны, 21 span учтён счётчиком ошибок экспорта. Снимок RSS Collector около 197 MiB. API сохранил готовность. |
Доказательства запросов к реальным хранилищам и статусов алертов сохранены в [observability.json](evidence/observability.json). [Трасса повтора](evidence/trace-retry.json) показывает две попытки worker под одним trace ID после одноразового сбоя. В [наследуемой матрице pipeline](evidence/pipeline.json) прошли 10 сценариев ЛР5, 11 подтверждённых загрузок сохранены, устаревших перезаписей нет. При отказе метрики и логи остаются доступны; старые трассы, отброшенные после исчерпания повторов, не воспроизводятся автоматически.
## Трек 1. Цена наблюдаемости
Гипотеза: пакетный экспорт сохраняет сопоставимую задержку под выбранной нагрузкой; уменьшение head sampling экономит объём трасс и оставляет метрики и логи. На одном Docker Desktop ARM64, VM 18 vCPU и 7.75 GiB, сравнивались четыре режима. API — 2 × 1 CPU/512 MiB, worker — 2 CPU/1536 MiB, Collector — 0.5 CPU/256 MiB. Один и тот же образ и лимиты. В каждом запуске отдельный чат из 25 сообщений, затем 3 с прогрева и 10 с измерения с фиксированной подачей 18 запросов/с. Запросы: страницы истории до 20 сообщений, около 20% новых сообщений, одна загрузка маленького PNG каждые 45 операций. Два повтора с обратным порядком режимов. Отброшены только результаты прогрева; его записи остаются. Сохраняются четыре снимка CPU/RSS на режим.
Медианы повторов, 180 из 180 успешных операций в каждом из восьми запусков:
| Режим | p50, мс | p95, мс | p99, мс | Принято span | Байт трасс в Tempo | CPU приложения, % одного ядра | Память приложения, MiB |
| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: |
| off | 5.15 | 8.28 | 22.17 | 0 | 0 | 41.74 | 308.82 |
| 100% | 5.31 | 8.46 | 17.11 | 997 | 142 250 | 79.81 | 402.25 |
| 50% | 5.41 | 10.08 | 21.53 | 522 | 80 782 | 76.21 | 391.50 |
| 25% | 5.03 | 9.49 | 27.17 | 238 | 41 374 | 76.44 | 365.21 |
CPU/RAM приложения — сумма API, relay и worker в снимке Docker stats. В [телеметрии эксперимента](evidence/telemetry-cost.json) также есть снимки четырёх компонентов наблюдаемости и фактическое время каждого прогона. Измерено число записей логов и точек метрик, объём байтов — protobuf трасс после ограничений на входе Tempo. Во включённых режимах за прогон принято около 188 логов; меняется sampling трасс, логи и метрики сохраняются. Разность accepted/sent в несколько span объяснима границами scrape и остатком batch. Счётчик ошибок экспорта в этих восьми прогонах не рос; эксперимент с остановкой Tempo отдельно показывает реальный сбой доставки. Измерение не видит потерю записи до приёма Collector и не устанавливает отсутствие потерь во всех условиях.
Режим 50% уменьшил объём трасс примерно на 43% относительно 100% при сохранении метрик и логов. По CPU основной рост связан с включением всей телеметрии, переключение sampling само по себе не убирает стоимость метрик и логов. По двум коротким повторам нельзя установить выигрыш в задержке: p95 и особенно p99 колеблются. Head sampling выбирает trace до ответа, поэтому пропуск редкого медленного или ошибочного запроса так же вероятен, как пропуск обычного. При 50% вероятность пропуска одного уникального инцидента примерно 1/2, при 25% — 3/4. Для разборов можно временно использовать 100%, для наблюдения при данной нагрузке проверен 50%. Исходный режим защиты — 100%, чтобы учебная трасса воспроизводилась детерминированно.
## Проверки и границы
- Публичный smoke, OpenAPI и 84 собственных сценария, включая ACL, сохранение и восстановление pipeline. [Чистый clone](evidence/clean-clone.txt) поднят отдельным Compose project с новыми volumes и локальными credentials.
- Метрики и provisioning подтверждены запросами к данным, `scripts/observe.py` проверяет связь Loki/Tempo и восстанавливает затронутые сервисы.
- Compose развёрнут на одном хосте; потеря VM или volume не покрывается. Отказ backend телеметрии допускает потерю сигналов. Публичная сеть, внутренние TLS и управление сервисными секретами рассматриваются в ЛР7.
## Вопросы к защите
- **Чем отличаются метрика, лог и трасса?** Метрика даёт агрегированный симптом, лог — конкретное событие, трасса — путь и время операции между процессами.
- **Как считается p95 нескольких реплик?** Сначала суммируются скорости bucket counters по границе, потом находится квантиль общей гистограммы.
- **Как связаны HTTP и worker?** Контекст хранится в job, relay передаёт его в envelope, worker продолжает trace. Request ID называет HTTP-попытку, trace ID объединяет операции.
- **Что при отказе хранилища наблюдаемости?** SDK и Collector буферизуют в пределах лимитов. По таймауту экспорт допускает потери, сообщения и задания продолжают жить в прикладных хранилищах.
+21
View File
@@ -0,0 +1,21 @@
# Only DB/cache test ports; API replicas always stay behind the LB.
services:
postgres:
ports: ["127.0.0.1:${TEST_PG_PORT:-55467}:5432"]
valkey:
ports: ["127.0.0.1:${TEST_CACHE_PORT:-56386}:6379"]
s3:
ports: ["127.0.0.1:${TEST_S3_PORT:-59006}:9000"]
bucket-init:
environment:
CREATE_TEST_BUCKET: '1'
broker:
ports: ["127.0.0.1:${TEST_BROKER_PORT:-56387}:6379"]
worker:
environment:
ENABLE_FAULT_INJECTION: '1'
JOB_TIMEOUT_SECONDS: ${TEST_JOB_TIMEOUT_SECONDS:-5}
relay:
environment:
ENABLE_FAULT_INJECTION: '1'
JOB_TIMEOUT_SECONDS: ${TEST_JOB_TIMEOUT_SECONDS:-5}
+323
View File
@@ -0,0 +1,323 @@
name: mtusi-lab6
x-api: &api
image: ${API_IMAGE:-mtusi-messenger:lab6}
build:
context: .
args:
VCS_REF: ${VCS_REF:-uncommitted}
user: "10001:10001"
read_only: true
tmpfs: ["/tmp:rw,noexec,nosuid,size=32m"]
cap_drop: [ALL]
security_opt: [no-new-privileges:true]
pids_limit: 96
cpus: ${API_CPU:-1}
mem_limit: ${API_MEMORY:-512m}
memswap_limit: ${API_MEMORY:-512m}
stop_grace_period: 7s
secrets:
- database_url
- cache_url
- source: s3_api_access_key_id
target: s3_access_key_id
- source: s3_api_secret_access_key
target: s3_secret_access_key
environment:
APP_HOST: 0.0.0.0
APP_PORT: 8080
BROKER_URL_FILE: /run/secrets/broker_url
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-20}
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
APP_ENV: lab6
TELEMETRY_ENABLED: ${TELEMETRY_ENABLED:-1}
OTEL_SAMPLE_RATIO: ${OTEL_SAMPLE_RATIO:-1}
OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
DATABASE_READ_AUTOCOMMIT: ${DATABASE_READ_AUTOCOMMIT:-1}
DATABASE_POOL_SIZE: ${DATABASE_POOL_SIZE:-8}
DATABASE_URL_FILE: /run/secrets/database_url
CACHE_URL_FILE: /run/secrets/cache_url
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000}
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
S3_REGION: ${S3_REGION:-us-east-1}
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
networks: [backend]
services:
postgres:
image: postgres:18.3
environment:
POSTGRES_USER: messenger
POSTGRES_DB: messenger
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
secrets: [postgres_password]
volumes: [postgres_data:/var/lib/postgresql]
command: [postgres, -c, fsync=on, -c, synchronous_commit=on, -c, full_page_writes=on,
-c, shared_preload_libraries=pg_stat_statements]
healthcheck:
test: [CMD-SHELL, "pg_isready -U messenger -d messenger"]
interval: 2s
timeout: 2s
retries: 30
cpus: ${POSTGRES_CPU:-1}
mem_limit: 768m
memswap_limit: 768m
networks: [backend]
valkey:
image: valkey/valkey:8.1.3-alpine
command: [valkey-server, --save, "", --appendonly, "no", --maxmemory, 64mb,
--maxmemory-policy, allkeys-lru]
healthcheck:
test: [CMD, valkey-cli, ping]
interval: 2s
timeout: 1s
retries: 20
cpus: 0.5
mem_limit: 128m
memswap_limit: 128m
networks: [backend]
migrate:
<<: *api
command: [python, -m, messenger.migrate]
secrets: [database_url]
healthcheck: {disable: true}
depends_on:
postgres: {condition: service_healthy}
api:
<<: *api
deploy:
replicas: ${API_REPLICAS:-2}
depends_on:
migrate: {condition: service_completed_successfully}
valkey: {condition: service_healthy}
bucket-init: {condition: service_completed_successfully}
lb:
image: haproxy:3.2.1-alpine
volumes: [./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro]
ports: ["127.0.0.1:${PUBLIC_PORT:-8086}:8080"]
read_only: true
tmpfs: ["/tmp:rw,noexec,nosuid,size=4m"]
cap_drop: [ALL]
security_opt: [no-new-privileges:true]
cpus: 0.5
mem_limit: 128m
memswap_limit: 128m
healthcheck:
test: [CMD, wget, -q, -O, /dev/null, http://127.0.0.1:8080/health/ready]
interval: 2s
timeout: 2s
retries: 30
depends_on:
api: {condition: service_healthy}
networks: [backend]
broker:
image: valkey/valkey:8.1.3-alpine
command: [valkey-server, --appendonly, "yes", --appendfsync, always, --save, "60 1",
--maxmemory, 192mb, --maxmemory-policy, noeviction]
volumes: [broker_data:/data]
cpus: 0.5
mem_limit: 384m
memswap_limit: 384m
healthcheck:
test: [CMD, valkey-cli, ping]
interval: 2s
timeout: 1s
retries: 30
networks: [backend]
relay:
<<: *api
command: [python, -m, messenger.relay]
restart: unless-stopped
healthcheck: {disable: true}
secrets: [database_url, cache_url, broker_url]
depends_on:
migrate: {condition: service_completed_successfully}
broker: {condition: service_healthy}
worker:
<<: *api
command: [python, -m, messenger.worker]
restart: unless-stopped
cpus: 2
mem_limit: 1536m
memswap_limit: 1536m
tmpfs: ["/tmp:rw,noexec,nosuid,size=64m"]
healthcheck: {disable: true}
secrets:
- database_url
- cache_url
- broker_url
- source: s3_worker_access_key_id
target: s3_access_key_id
- source: s3_worker_secret_access_key
target: s3_secret_access_key
depends_on:
migrate: {condition: service_completed_successfully}
broker: {condition: service_healthy}
bucket-init: {condition: service_completed_successfully}
s3:
image: mtusi-lab6-s3:2025-10-15
build: {context: ., dockerfile: infra/s3.Dockerfile, target: server}
command: [server, /data, --address, ':9000', --console-address, '127.0.0.1:9001']
secrets: [s3_root_user, s3_root_password]
volumes: [s3_data:/data]
read_only: true
tmpfs: ["/tmp:rw,noexec,nosuid,size=32m"]
cap_drop: [ALL]
cpus: 1
mem_limit: 768m
memswap_limit: 768m
healthcheck:
test: [CMD, wget, -q, -O, /dev/null, 'http://127.0.0.1:9000/minio/health/live']
interval: 2s
timeout: 1s
retries: 30
networks: [backend]
bucket-init:
image: mtusi-lab6-mc:2025-08-13
build: {context: ., dockerfile: infra/s3.Dockerfile, target: client}
command: [/bootstrap-s3.sh]
volumes:
- ./infra/bootstrap-s3.sh:/bootstrap-s3.sh:ro
- ./infra/api-policy.json:/policies/api-policy.json:ro
- ./infra/reconciler-policy.json:/policies/reconciler-policy.json:ro
- ./infra/worker-policy.json:/policies/worker-policy.json:ro
- ./infra/test-policy.json:/policies/test-policy.json:ro
secrets: [s3_root_user, s3_root_password, s3_api_access_key_id, s3_api_secret_access_key,
s3_reconciler_access_key_id, s3_reconciler_secret_access_key,
s3_test_access_key_id, s3_test_secret_access_key,
s3_worker_access_key_id, s3_worker_secret_access_key]
environment:
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
read_only: true
tmpfs: ["/tmp:rw,noexec,nosuid,size=16m"]
cpus: 0.5
mem_limit: 128m
networks: [backend]
depends_on:
s3: {condition: service_healthy}
reconciler:
<<: *api
profiles: [tools]
command: [python, -m, messenger.cleanup]
healthcheck: {disable: true}
secrets:
- database_url
- cache_url
- source: s3_reconciler_access_key_id
target: s3_access_key_id
- source: s3_reconciler_secret_access_key
target: s3_secret_access_key
depends_on:
migrate: {condition: service_completed_successfully}
bucket-init: {condition: service_completed_successfully}
otel-collector:
image: otel/opentelemetry-collector-contrib:0.147.0
command: [--config=/etc/otelcol/config.yaml]
volumes: [./infra/otel/collector.yaml:/etc/otelcol/config.yaml:ro]
mem_limit: 256m
cpus: 0.5
read_only: true
cap_drop: [ALL]
security_opt: [no-new-privileges:true]
networks: [backend]
prometheus:
image: prom/prometheus:v3.9.1
command: [--config.file=/etc/prometheus/prometheus.yaml, --storage.tsdb.path=/prometheus, --storage.tsdb.retention.time=3d, --storage.tsdb.retention.size=256MB]
volumes:
- ./infra/otel/prometheus.yaml:/etc/prometheus/prometheus.yaml:ro
- ./infra/otel/alerts.yaml:/etc/prometheus/alerts.yaml:ro
- prometheus_data:/prometheus
ports: ["127.0.0.1:${PROMETHEUS_PORT:-9096}:9090"]
mem_limit: 384m
cpus: 0.5
networks: [backend]
loki:
image: grafana/loki:3.6.3
command: [-config.file=/etc/loki/local.yaml]
volumes:
- ./infra/otel/loki.yaml:/etc/loki/local.yaml:ro
- loki_data:/loki
ports: ["127.0.0.1:${LOKI_PORT:-3106}:3100"]
mem_limit: 512m
cpus: 0.5
networks: [backend]
tempo:
image: grafana/tempo:2.10.0
command: [-config.file=/etc/tempo/local.yaml]
volumes:
- ./infra/otel/tempo.yaml:/etc/tempo/local.yaml:ro
- tempo_data:/var/tempo
ports: ["127.0.0.1:${TEMPO_PORT:-3206}:3200"]
mem_limit: 512m
cpus: 0.5
networks: [backend]
grafana:
image: grafana/grafana:12.3.3
environment:
GF_SECURITY_ADMIN_PASSWORD__FILE: /run/secrets/grafana_password
GF_USERS_ALLOW_SIGN_UP: "false"
GF_ANALYTICS_REPORTING_ENABLED: "false"
GF_ANALYTICS_CHECK_FOR_UPDATES: "false"
GF_ANALYTICS_CHECK_FOR_PLUGIN_UPDATES: "false"
secrets: [grafana_password]
volumes:
- grafana_data:/var/lib/grafana
- ./infra/otel/grafana/provisioning:/etc/grafana/provisioning:ro
- ./infra/otel/grafana/dashboards:/var/lib/grafana/dashboards:ro
ports: ["127.0.0.1:${GRAFANA_PORT:-3006}:3000"]
mem_limit: 384m
cpus: 0.5
networks: [backend]
networks:
backend: {}
volumes:
prometheus_data:
loki_data:
tempo_data:
grafana_data:
broker_data:
s3_data:
postgres_data:
secrets:
grafana_password:
file: ./secrets/grafana_password
postgres_password:
file: ./secrets/postgres_password
database_url:
file: ./secrets/database_url
cache_url:
file: ./secrets/cache_url
s3_root_user:
file: ./secrets/s3_root_user
s3_root_password:
file: ./secrets/s3_root_password
s3_api_access_key_id:
file: ./secrets/s3_api_access_key_id
s3_api_secret_access_key:
file: ./secrets/s3_api_secret_access_key
s3_reconciler_access_key_id:
file: ./secrets/s3_reconciler_access_key_id
s3_reconciler_secret_access_key:
file: ./secrets/s3_reconciler_secret_access_key
s3_test_access_key_id:
file: ./secrets/s3_test_access_key_id
s3_test_secret_access_key:
file: ./secrets/s3_test_secret_access_key
broker_url:
file: ./secrets/broker_url
s3_worker_access_key_id:
file: ./secrets/s3_worker_access_key_id
s3_worker_secret_access_key:
file: ./secrets/s3_worker_secret_access_key
+4
View File
@@ -0,0 +1,4 @@
python3 tests/smoke.py --base-url "http://127.0.0.1:18086"
PASS: lab6 public API scenario; infrastructure and grading evidence remain separate.
python3 scripts/check_contract.py
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
+4
View File
@@ -0,0 +1,4 @@
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://127.0.0.1:8086"
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
{"observed_instances_before": 2, "successful_writes": 128, "failed_attempts": 3, "first_recovery_seconds": 4.763}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
+57
View File
@@ -0,0 +1,57 @@
{
"correlation": {
"request_id": "26db18d7-d97b-488a-b9e5-2e3a3f48fb03",
"trace_id": "d72b60b8bdea49a5cff62c0abd43f1e0",
"span_names": [
"POST /api/v1/chats/{chat_id}/attachments",
"db.read",
"db.transaction",
"jobs.consume",
"jobs.publish",
"s3.download",
"s3.put"
],
"worker_parent": "jobs.publish",
"api_and_worker_loki": true
},
"worker_stop": {
"depth_at_firing": 3.0,
"oldest_seconds_at_firing": 16.917512,
"state": [
"firing",
"resolved"
],
"uploads_ready": 3
},
"database_lock": {
"statuses": [
503,
503,
503,
503,
503,
503,
503,
503,
503,
503,
503,
503
],
"max_latency_ms": 1007.0772920007585,
"trace_id": "d3e8b679ae370a4de65821ead5d74acc",
"state": [
"firing",
"resolved"
],
"spans": [
"GET /api/v1/users/me",
"db.read"
]
},
"tempo_stop": {
"successful_messages": 15,
"failed_export_spans": 21.0,
"collector_rss_bytes": 206360576.0
}
}
+3
View File
@@ -0,0 +1,3 @@
python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://127.0.0.1:8086"
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
File diff suppressed because it is too large Load Diff
+862
View File
@@ -0,0 +1,862 @@
{
"seconds": 10,
"offered_requests_per_s": 18,
"runs": [
{
"mode": "off",
"sample_ratio": 0.0,
"load": {
"requests": 180,
"elapsed_s": 9.952042167002219,
"latency_ms": {
"p50": 4.991917001461843,
"p95": 7.5844580023840535,
"p99": 16.816375002235873
}
},
"container_samples": [
[
{
"container": "mtusi-lab6-api-1",
"cpu_percent": 2.45,
"memory": "91.67MiB"
},
{
"container": "mtusi-lab6-api-2",
"cpu_percent": 2.93,
"memory": "91.85MiB"
},
{
"container": "mtusi-lab6-loki-1",
"cpu_percent": 0.46,
"memory": "107.5MiB"
},
{
"container": "mtusi-lab6-otel-collector-1",
"cpu_percent": 0.2,
"memory": "58.3MiB"
},
{
"container": "mtusi-lab6-prometheus-1",
"cpu_percent": 0.09,
"memory": "40.25MiB"
},
{
"container": "mtusi-lab6-relay-1",
"cpu_percent": 0.38,
"memory": "46.33MiB"
},
{
"container": "mtusi-lab6-tempo-1",
"cpu_percent": 0.13,
"memory": "88.37MiB"
},
{
"container": "mtusi-lab6-worker-1",
"cpu_percent": 60.82,
"memory": "75.3MiB"
}
],
[
{
"container": "mtusi-lab6-api-1",
"cpu_percent": 15.95,
"memory": "91.73MiB"
},
{
"container": "mtusi-lab6-api-2",
"cpu_percent": 3.69,
"memory": "93.02MiB"
},
{
"container": "mtusi-lab6-loki-1",
"cpu_percent": 0.45,
"memory": "107MiB"
},
{
"container": "mtusi-lab6-otel-collector-1",
"cpu_percent": 0.19,
"memory": "58.55MiB"
},
{
"container": "mtusi-lab6-prometheus-1",
"cpu_percent": 0.09,
"memory": "40.26MiB"
},
{
"container": "mtusi-lab6-relay-1",
"cpu_percent": 0.59,
"memory": "46.33MiB"
},
{
"container": "mtusi-lab6-tempo-1",
"cpu_percent": 0.14,
"memory": "88.38MiB"
},
{
"container": "mtusi-lab6-worker-1",
"cpu_percent": 0.88,
"memory": "78.04MiB"
}
]
],
"ingest": {
"trace_ingest_proto_bytes": 0.0,
"accepted_spans": 0.0,
"sent_spans": 0.0,
"failed_spans": 0.0,
"accepted_logs": 0.0,
"accepted_metric_points": 0.0
}
},
{
"mode": "full",
"sample_ratio": 1.0,
"load": {
"requests": 180,
"elapsed_s": 9.950246666998282,
"latency_ms": {
"p50": 5.275124996842351,
"p95": 8.54145799894468,
"p99": 16.04162500007078
}
},
"container_samples": [
[
{
"container": "mtusi-lab6-api-1",
"cpu_percent": 15.27,
"memory": "94.31MiB"
},
{
"container": "mtusi-lab6-api-2",
"cpu_percent": 15.43,
"memory": "96.71MiB"
},
{
"container": "mtusi-lab6-loki-1",
"cpu_percent": 0.55,
"memory": "107MiB"
},
{
"container": "mtusi-lab6-otel-collector-1",
"cpu_percent": 0.45,
"memory": "59.33MiB"
},
{
"container": "mtusi-lab6-prometheus-1",
"cpu_percent": 0.01,
"memory": "40.25MiB"
},
{
"container": "mtusi-lab6-relay-1",
"cpu_percent": 0.52,
"memory": "56.45MiB"
},
{
"container": "mtusi-lab6-tempo-1",
"cpu_percent": 0.29,
"memory": "94.84MiB"
},
{
"container": "mtusi-lab6-worker-1",
"cpu_percent": 46.73,
"memory": "155.4MiB"
}
],
[
{
"container": "mtusi-lab6-api-1",
"cpu_percent": 15.41,
"memory": "93.86MiB"
},
{
"container": "mtusi-lab6-api-2",
"cpu_percent": 7.99,
"memory": "106.5MiB"
},
{
"container": "mtusi-lab6-loki-1",
"cpu_percent": 0.62,
"memory": "107.3MiB"
},
{
"container": "mtusi-lab6-otel-collector-1",
"cpu_percent": 0.57,
"memory": "59.59MiB"
},
{
"container": "mtusi-lab6-prometheus-1",
"cpu_percent": 0.0,
"memory": "40.26MiB"
},
{
"container": "mtusi-lab6-relay-1",
"cpu_percent": 0.85,
"memory": "56.21MiB"
},
{
"container": "mtusi-lab6-tempo-1",
"cpu_percent": 0.29,
"memory": "94.63MiB"
},
{
"container": "mtusi-lab6-worker-1",
"cpu_percent": 53.06,
"memory": "160.2MiB"
}
]
],
"ingest": {
"trace_ingest_proto_bytes": 142249.0,
"accepted_spans": 998.0,
"sent_spans": 997.0,
"failed_spans": 0.0,
"accepted_logs": 188.0,
"accepted_metric_points": 271.0
}
},
{
"mode": "half",
"sample_ratio": 0.5,
"load": {
"requests": 180,
"elapsed_s": 9.95444554200003,
"latency_ms": {
"p50": 5.507500001840526,
"p95": 11.544834000233095,
"p99": 24.414750001596985
}
},
"container_samples": [
[
{
"container": "mtusi-lab6-api-1",
"cpu_percent": 15.49,
"memory": "96.14MiB"
},
{
"container": "mtusi-lab6-api-2",
"cpu_percent": 16.11,
"memory": "95.01MiB"
},
{
"container": "mtusi-lab6-loki-1",
"cpu_percent": 0.52,
"memory": "113.9MiB"
},
{
"container": "mtusi-lab6-otel-collector-1",
"cpu_percent": 0.52,
"memory": "58.38MiB"
},
{
"container": "mtusi-lab6-prometheus-1",
"cpu_percent": 0.05,
"memory": "40.26MiB"
},
{
"container": "mtusi-lab6-relay-1",
"cpu_percent": 0.42,
"memory": "54.59MiB"
},
{
"container": "mtusi-lab6-tempo-1",
"cpu_percent": 0.3,
"memory": "107MiB"
},
{
"container": "mtusi-lab6-worker-1",
"cpu_percent": 45.37,
"memory": "132.9MiB"
}
],
[
{
"container": "mtusi-lab6-api-1",
"cpu_percent": 14.36,
"memory": "108.8MiB"
},
{
"container": "mtusi-lab6-api-2",
"cpu_percent": 6.47,
"memory": "104.6MiB"
},
{
"container": "mtusi-lab6-loki-1",
"cpu_percent": 0.68,
"memory": "113.9MiB"
},
{
"container": "mtusi-lab6-otel-collector-1",
"cpu_percent": 0.62,
"memory": "58.88MiB"
},
{
"container": "mtusi-lab6-prometheus-1",
"cpu_percent": 0.1,
"memory": "40.26MiB"
},
{
"container": "mtusi-lab6-relay-1",
"cpu_percent": 0.83,
"memory": "54.34MiB"
},
{
"container": "mtusi-lab6-tempo-1",
"cpu_percent": 0.32,
"memory": "109MiB"
},
{
"container": "mtusi-lab6-worker-1",
"cpu_percent": 53.37,
"memory": "157.9MiB"
}
]
],
"ingest": {
"trace_ingest_proto_bytes": 80752.0,
"accepted_spans": 522.0,
"sent_spans": 523.0,
"failed_spans": 0.0,
"accepted_logs": 188.0,
"accepted_metric_points": 287.0
}
},
{
"mode": "quarter",
"sample_ratio": 0.25,
"load": {
"requests": 180,
"elapsed_s": 9.952545791999,
"latency_ms": {
"p50": 5.109958998218644,
"p95": 8.02891599960276,
"p99": 18.22395799899823
}
},
"container_samples": [
[
{
"container": "mtusi-lab6-api-1",
"cpu_percent": 14.66,
"memory": "94.55MiB"
},
{
"container": "mtusi-lab6-api-2",
"cpu_percent": 15.32,
"memory": "94.49MiB"
},
{
"container": "mtusi-lab6-loki-1",
"cpu_percent": 0.89,
"memory": "114.4MiB"
},
{
"container": "mtusi-lab6-otel-collector-1",
"cpu_percent": 0.51,
"memory": "59.14MiB"
},
{
"container": "mtusi-lab6-prometheus-1",
"cpu_percent": 0.16,
"memory": "40.51MiB"
},
{
"container": "mtusi-lab6-relay-1",
"cpu_percent": 0.49,
"memory": "56.43MiB"
},
{
"container": "mtusi-lab6-tempo-1",
"cpu_percent": 0.64,
"memory": "115.5MiB"
},
{
"container": "mtusi-lab6-worker-1",
"cpu_percent": 46.14,
"memory": "158.4MiB"
}
],
[
{
"container": "mtusi-lab6-api-1",
"cpu_percent": 16.03,
"memory": "94.77MiB"
},
{
"container": "mtusi-lab6-api-2",
"cpu_percent": 3.98,
"memory": "95.34MiB"
},
{
"container": "mtusi-lab6-loki-1",
"cpu_percent": 0.69,
"memory": "114.9MiB"
},
{
"container": "mtusi-lab6-otel-collector-1",
"cpu_percent": 0.52,
"memory": "59.16MiB"
},
{
"container": "mtusi-lab6-prometheus-1",
"cpu_percent": 0.14,
"memory": "40.26MiB"
},
{
"container": "mtusi-lab6-relay-1",
"cpu_percent": 0.77,
"memory": "56.18MiB"
},
{
"container": "mtusi-lab6-tempo-1",
"cpu_percent": 0.33,
"memory": "117MiB"
},
{
"container": "mtusi-lab6-worker-1",
"cpu_percent": 55.48,
"memory": "157.7MiB"
}
]
],
"ingest": {
"trace_ingest_proto_bytes": 41309.0,
"accepted_spans": 238.0,
"sent_spans": 237.0,
"failed_spans": 0.0,
"accepted_logs": 188.0,
"accepted_metric_points": 273.0
}
},
{
"mode": "quarter",
"sample_ratio": 0.25,
"load": {
"requests": 180,
"elapsed_s": 9.9528304580017,
"latency_ms": {
"p50": 4.951999999320833,
"p95": 10.958250000840053,
"p99": 36.11366699988139
}
},
"container_samples": [
[
{
"container": "mtusi-lab6-api-1",
"cpu_percent": 17.98,
"memory": "94.93MiB"
},
{
"container": "mtusi-lab6-api-2",
"cpu_percent": 3.8,
"memory": "94.62MiB"
},
{
"container": "mtusi-lab6-loki-1",
"cpu_percent": 0.78,
"memory": "114.9MiB"
},
{
"container": "mtusi-lab6-otel-collector-1",
"cpu_percent": 0.53,
"memory": "59.66MiB"
},
{
"container": "mtusi-lab6-prometheus-1",
"cpu_percent": 0.31,
"memory": "40.51MiB"
},
{
"container": "mtusi-lab6-relay-1",
"cpu_percent": 0.98,
"memory": "54.59MiB"
},
{
"container": "mtusi-lab6-tempo-1",
"cpu_percent": 0.32,
"memory": "117.3MiB"
},
{
"container": "mtusi-lab6-worker-1",
"cpu_percent": 65.06,
"memory": "79.88MiB"
}
],
[
{
"container": "mtusi-lab6-api-1",
"cpu_percent": 3.32,
"memory": "95.67MiB"
},
{
"container": "mtusi-lab6-api-2",
"cpu_percent": 11.75,
"memory": "95.2MiB"
},
{
"container": "mtusi-lab6-loki-1",
"cpu_percent": 0.66,
"memory": "113.9MiB"
},
{
"container": "mtusi-lab6-otel-collector-1",
"cpu_percent": 0.5,
"memory": "59.87MiB"
},
{
"container": "mtusi-lab6-prometheus-1",
"cpu_percent": 0.0,
"memory": "40.27MiB"
},
{
"container": "mtusi-lab6-relay-1",
"cpu_percent": 0.58,
"memory": "54.6MiB"
},
{
"container": "mtusi-lab6-tempo-1",
"cpu_percent": 0.26,
"memory": "117.8MiB"
},
{
"container": "mtusi-lab6-worker-1",
"cpu_percent": 58.04,
"memory": "81.07MiB"
}
]
],
"ingest": {
"trace_ingest_proto_bytes": 41438.0,
"accepted_spans": 238.0,
"sent_spans": 235.0,
"failed_spans": 0.0,
"accepted_logs": 188.0,
"accepted_metric_points": 287.0
}
},
{
"mode": "half",
"sample_ratio": 0.5,
"load": {
"requests": 180,
"elapsed_s": 9.95298329200159,
"latency_ms": {
"p50": 5.302292000123998,
"p95": 8.62283300011768,
"p99": 18.653041999641573
}
},
"container_samples": [
[
{
"container": "mtusi-lab6-api-1",
"cpu_percent": 2.95,
"memory": "96.92MiB"
},
{
"container": "mtusi-lab6-api-2",
"cpu_percent": 3.27,
"memory": "95.07MiB"
},
{
"container": "mtusi-lab6-loki-1",
"cpu_percent": 0.61,
"memory": "114.5MiB"
},
{
"container": "mtusi-lab6-otel-collector-1",
"cpu_percent": 0.47,
"memory": "59.41MiB"
},
{
"container": "mtusi-lab6-prometheus-1",
"cpu_percent": 0.05,
"memory": "40.52MiB"
},
{
"container": "mtusi-lab6-relay-1",
"cpu_percent": 0.49,
"memory": "54.61MiB"
},
{
"container": "mtusi-lab6-tempo-1",
"cpu_percent": 0.27,
"memory": "118.1MiB"
},
{
"container": "mtusi-lab6-worker-1",
"cpu_percent": 15.31,
"memory": "78.76MiB"
}
],
[
{
"container": "mtusi-lab6-api-1",
"cpu_percent": 15.06,
"memory": "96.86MiB"
},
{
"container": "mtusi-lab6-api-2",
"cpu_percent": 15.71,
"memory": "95.6MiB"
},
{
"container": "mtusi-lab6-loki-1",
"cpu_percent": 0.57,
"memory": "114.7MiB"
},
{
"container": "mtusi-lab6-otel-collector-1",
"cpu_percent": 0.56,
"memory": "59.4MiB"
},
{
"container": "mtusi-lab6-prometheus-1",
"cpu_percent": 0.06,
"memory": "40.27MiB"
},
{
"container": "mtusi-lab6-relay-1",
"cpu_percent": 0.65,
"memory": "54.61MiB"
},
{
"container": "mtusi-lab6-tempo-1",
"cpu_percent": 0.3,
"memory": "117.4MiB"
},
{
"container": "mtusi-lab6-worker-1",
"cpu_percent": 49.13,
"memory": "157.3MiB"
}
]
],
"ingest": {
"trace_ingest_proto_bytes": 80811.0,
"accepted_spans": 521.0,
"sent_spans": 520.0,
"failed_spans": 0.0,
"accepted_logs": 188.0,
"accepted_metric_points": 273.0
}
},
{
"mode": "full",
"sample_ratio": 1.0,
"load": {
"requests": 180,
"elapsed_s": 9.95264620799935,
"latency_ms": {
"p50": 5.342791999282781,
"p95": 8.375000001251465,
"p99": 18.184750002546934
}
},
"container_samples": [
[
{
"container": "mtusi-lab6-api-1",
"cpu_percent": 16.07,
"memory": "94.52MiB"
},
{
"container": "mtusi-lab6-api-2",
"cpu_percent": 15.33,
"memory": "94.31MiB"
},
{
"container": "mtusi-lab6-loki-1",
"cpu_percent": 0.52,
"memory": "114.2MiB"
},
{
"container": "mtusi-lab6-otel-collector-1",
"cpu_percent": 0.69,
"memory": "58.92MiB"
},
{
"container": "mtusi-lab6-prometheus-1",
"cpu_percent": 0.09,
"memory": "40.27MiB"
},
{
"container": "mtusi-lab6-relay-1",
"cpu_percent": 0.86,
"memory": "56.7MiB"
},
{
"container": "mtusi-lab6-tempo-1",
"cpu_percent": 0.37,
"memory": "117.5MiB"
},
{
"container": "mtusi-lab6-worker-1",
"cpu_percent": 49.41,
"memory": "156.1MiB"
}
],
[
{
"container": "mtusi-lab6-api-1",
"cpu_percent": 4.12,
"memory": "93.79MiB"
},
{
"container": "mtusi-lab6-api-2",
"cpu_percent": 15.92,
"memory": "94.05MiB"
},
{
"container": "mtusi-lab6-loki-1",
"cpu_percent": 0.87,
"memory": "114.7MiB"
},
{
"container": "mtusi-lab6-otel-collector-1",
"cpu_percent": 0.52,
"memory": "59.92MiB"
},
{
"container": "mtusi-lab6-prometheus-1",
"cpu_percent": 0.01,
"memory": "40.27MiB"
},
{
"container": "mtusi-lab6-relay-1",
"cpu_percent": 0.77,
"memory": "56.45MiB"
},
{
"container": "mtusi-lab6-tempo-1",
"cpu_percent": 0.31,
"memory": "118.2MiB"
},
{
"container": "mtusi-lab6-worker-1",
"cpu_percent": 61.5,
"memory": "153.1MiB"
}
]
],
"ingest": {
"trace_ingest_proto_bytes": 142250.0,
"accepted_spans": 996.0,
"sent_spans": 998.0,
"failed_spans": 0.0,
"accepted_logs": 188.0,
"accepted_metric_points": 271.0
}
},
{
"mode": "off",
"sample_ratio": 0.0,
"load": {
"requests": 180,
"elapsed_s": 9.951078292000602,
"latency_ms": {
"p50": 5.305624999891734,
"p95": 8.97895800153492,
"p99": 27.521125000930624
}
},
"container_samples": [
[
{
"container": "mtusi-lab6-api-1",
"cpu_percent": 2.74,
"memory": "93.45MiB"
},
{
"container": "mtusi-lab6-api-2",
"cpu_percent": 2.7,
"memory": "91.64MiB"
},
{
"container": "mtusi-lab6-loki-1",
"cpu_percent": 0.41,
"memory": "114.9MiB"
},
{
"container": "mtusi-lab6-otel-collector-1",
"cpu_percent": 0.17,
"memory": "58.39MiB"
},
{
"container": "mtusi-lab6-prometheus-1",
"cpu_percent": 0.13,
"memory": "40.27MiB"
},
{
"container": "mtusi-lab6-relay-1",
"cpu_percent": 0.42,
"memory": "46.32MiB"
},
{
"container": "mtusi-lab6-tempo-1",
"cpu_percent": 0.13,
"memory": "116MiB"
},
{
"container": "mtusi-lab6-worker-1",
"cpu_percent": 56.51,
"memory": "77.11MiB"
}
],
[
{
"container": "mtusi-lab6-api-1",
"cpu_percent": 3.66,
"memory": "93.92MiB"
},
{
"container": "mtusi-lab6-api-2",
"cpu_percent": 3.46,
"memory": "92.96MiB"
},
{
"container": "mtusi-lab6-loki-1",
"cpu_percent": 0.47,
"memory": "114.2MiB"
},
{
"container": "mtusi-lab6-otel-collector-1",
"cpu_percent": 0.22,
"memory": "58.39MiB"
},
{
"container": "mtusi-lab6-prometheus-1",
"cpu_percent": 0.08,
"memory": "40.52MiB"
},
{
"container": "mtusi-lab6-relay-1",
"cpu_percent": 0.34,
"memory": "46.07MiB"
},
{
"container": "mtusi-lab6-tempo-1",
"cpu_percent": 0.18,
"memory": "116MiB"
},
{
"container": "mtusi-lab6-worker-1",
"cpu_percent": 0.22,
"memory": "76.44MiB"
}
]
],
"ingest": {
"trace_ingest_proto_bytes": 0.0,
"accepted_spans": 0.0,
"sent_spans": 0.0,
"failed_spans": 0.0,
"accepted_logs": 0.0,
"accepted_metric_points": 0.0
}
}
]
}
+18
View File
@@ -0,0 +1,18 @@
{
"correlation": {
"request_id": "bda1ff17-5d60-4d88-a0c1-2eaf450d5769",
"trace_id": "fc3682b58c4df5515088f7fef7f0d559",
"span_names": [
"POST /api/v1/chats/{chat_id}/attachments",
"db.read",
"db.transaction",
"jobs.consume",
"jobs.publish",
"s3.download",
"s3.put"
],
"worker_parent": "jobs.publish",
"worker_attempts": 2,
"api_and_worker_loki": true
}
}
+1
View File
@@ -0,0 +1 @@
======================== 84 passed, 1 warning in 13.36s ========================
+33
View File
@@ -0,0 +1,33 @@
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": [
"s3:ListBucket",
"s3:GetBucketLocation"
],
"Resource": [
"arn:aws:s3:::BUCKET"
]
},
{
"Effect": "Allow",
"Action": [
"s3:PutObject"
],
"Resource": [
"arn:aws:s3:::BUCKET/staging/*"
]
},
{
"Effect": "Allow",
"Action": [
"s3:GetObject"
],
"Resource": [
"arn:aws:s3:::BUCKET/objects/*"
]
}
]
}
+31
View File
@@ -0,0 +1,31 @@
#!/bin/sh
set -eu
# Never print mc arguments/errors: they can include access key identifiers.
exec 3>&1
exec > /tmp/bootstrap.log 2>&1
trap 'printf "%s\n" "S3 bootstrap failed; credentials suppressed" >&3' EXIT
bucket="$S3_BUCKET"
case "$bucket" in ''|*[!a-z0-9.-]*) exit 2;; esac
mc --config-dir /tmp/mc alias set -- local http://s3:9000 "$(cat /run/secrets/s3_root_user)" "$(cat /run/secrets/s3_root_password)"
mc --config-dir /tmp/mc mb --ignore-existing "local/$bucket"
mc --config-dir /tmp/mc anonymous set none "local/$bucket"
for role in api worker reconciler test; do
target_bucket="$bucket"
policy_source=/policies/api-policy.json
if [ "$role" = reconciler ]; then policy_source=/policies/reconciler-policy.json; fi
if [ "$role" = worker ]; then policy_source=/policies/worker-policy.json; fi
if [ "$role" = test ]; then
if [ "${CREATE_TEST_BUCKET:-0}" != 1 ]; then continue; fi
policy_source=/policies/test-policy.json
target_bucket="$bucket-tests"
mc --config-dir /tmp/mc mb --ignore-existing "local/$target_bucket"
mc --config-dir /tmp/mc anonymous set none "local/$target_bucket"
fi
sed "s/BUCKET/$target_bucket/g" "$policy_source" > "/tmp/$role-policy.json"
mc --config-dir /tmp/mc admin policy create local "messenger-$role" "/tmp/$role-policy.json"
key="$(cat /run/secrets/s3_${role}_access_key_id)"
mc --config-dir /tmp/mc admin user add -- local "$key" "$(cat /run/secrets/s3_${role}_secret_access_key)"
mc --config-dir /tmp/mc admin policy attach local "messenger-$role" --user "$key"
done
trap - EXIT
printf '%s\n' 'PASS: private bucket and scoped identities ready' >&3
+44
View File
@@ -0,0 +1,44 @@
global
log stdout format raw local0
maxconn 2048
defaults
log global
mode http
log-format "lb status=%ST backend=%b server=%s duration_ms=%Tt"
timeout connect 500ms
timeout check 500ms
timeout client 20s
timeout server 30s
timeout http-request 3s
retries 0
resolvers docker
nameserver embedded 127.0.0.11:53
accepted_payload_size 8192
resolve_retries 3
timeout resolve 1s
timeout retry 1s
hold valid 1s
hold obsolete 0s
hold nx 1s
hold refused 1s
hold timeout 1s
hold other 1s
frontend public
bind :8080
default_backend messenger
backend messenger
balance roundrobin
option httpchk GET /health/ready
http-check expect status 200
default-server inter 500ms fastinter 250ms downinter 500ms fall 2 rise 1
server-template app 1-8 api:8080 check resolvers docker resolve-prefer ipv4 init-addr none
listen metrics
bind :8404
stats enable
stats uri /stats
stats refresh 1s
+27
View File
@@ -0,0 +1,27 @@
groups:
- name: messenger
rules:
- alert: MessengerHttpErrors
expr: |
sum(rate(messenger_http_requests_total{route!~"/health/.*",status_class="5xx"}[20s]))
/ clamp_min(sum(rate(messenger_http_requests_total{route!~"/health/.*"}[20s])), 0.01)
> 0.05
for: 6s
labels: {severity: warning}
annotations:
summary: More than 5% of business requests have failed for six seconds
runbook: "REPORT.md#диагностика"
- alert: MessengerQueueLag
expr: max(messenger_queue_oldest_seconds) > 5 and max(messenger_queue_depth) > 0
for: 6s
labels: {severity: warning}
annotations:
summary: The oldest unfinished job is more than five seconds old
runbook: "REPORT.md#диагностика"
- alert: MessengerDatabaseErrors
expr: sum(increase(messenger_db_errors_total[20s])) > 0
for: 4s
labels: {severity: warning}
annotations:
summary: PostgreSQL requests are failing
runbook: "REPORT.md#диагностика"
+53
View File
@@ -0,0 +1,53 @@
receivers:
otlp:
protocols:
http:
endpoint: 0.0.0.0:4318
processors:
memory_limiter:
check_interval: 1s
limit_mib: 128
spike_limit_mib: 32
batch:
timeout: 500ms
send_batch_size: 128
send_batch_max_size: 256
exporters:
prometheus:
endpoint: 0.0.0.0:8889
metric_expiration: 15s
otlp_http/tempo:
endpoint: http://tempo:4318
timeout: 2s
sending_queue: {queue_size: 16, num_consumers: 1}
retry_on_failure: {initial_interval: 500ms, max_interval: 2s, max_elapsed_time: 5s}
otlp_http/loki:
endpoint: http://loki:3100/otlp
timeout: 2s
sending_queue: {queue_size: 16, num_consumers: 1}
retry_on_failure: {initial_interval: 500ms, max_interval: 2s, max_elapsed_time: 5s}
service:
telemetry:
metrics:
readers:
- pull:
exporter:
prometheus:
host: 0.0.0.0
port: 8888
pipelines:
metrics:
receivers: [otlp]
processors: [memory_limiter, batch]
exporters: [prometheus]
logs:
receivers: [otlp]
processors: [memory_limiter, batch]
exporters: [otlp_http/loki]
traces:
receivers: [otlp]
processors: [memory_limiter, batch]
exporters: [otlp_http/tempo]
@@ -0,0 +1,453 @@
{
"uid": "messenger",
"title": "MTUSI Messenger",
"schemaVersion": 41,
"version": 1,
"editable": false,
"timezone": "browser",
"refresh": "5s",
"time": {
"from": "now-15m",
"to": "now"
},
"panels": [
{
"id": 1,
"type": "timeseries",
"title": "API / запросов в секунду",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 0,
"w": 12,
"h": 8
},
"fieldConfig": {
"defaults": {
"unit": "reqps"
},
"overrides": []
},
"targets": [
{
"refId": "A",
"expr": "sum(rate(messenger_http_requests_total{route!~\"/health/.*\"}[30s]))",
"legendFormat": ""
}
]
},
{
"id": 2,
"type": "timeseries",
"title": "API / доля 5xx",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 0,
"w": 12,
"h": 8
},
"fieldConfig": {
"defaults": {
"unit": "percentunit"
},
"overrides": []
},
"targets": [
{
"refId": "A",
"expr": "sum(rate(messenger_http_requests_total{status_class=\"5xx\",route!~\"/health/.*\"}[30s])) / clamp_min(sum(rate(messenger_http_requests_total{route!~\"/health/.*\"}[30s])),0.001)",
"legendFormat": ""
}
]
},
{
"id": 3,
"type": "timeseries",
"title": "API / p50, p95, p99",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 8,
"w": 12,
"h": 8
},
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
},
"targets": [
{
"refId": "A",
"expr": "histogram_quantile(0.5, sum by (le) (rate(messenger_http_duration_seconds_bucket{route!~\"/health/.*\"}[30s])))",
"legendFormat": "p50"
},
{
"refId": "B",
"expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_http_duration_seconds_bucket{route!~\"/health/.*\"}[30s])))",
"legendFormat": "p95"
},
{
"refId": "C",
"expr": "histogram_quantile(0.99, sum by (le) (rate(messenger_http_duration_seconds_bucket{route!~\"/health/.*\"}[30s])))",
"legendFormat": "p99"
}
]
},
{
"id": 4,
"type": "timeseries",
"title": "Worker / исходы в секунду",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 8,
"w": 12,
"h": 8
},
"fieldConfig": {
"defaults": {
"unit": "ops"
},
"overrides": []
},
"targets": [
{
"refId": "A",
"expr": "sum by (outcome) (rate(messenger_jobs_processed_total[30s]))",
"legendFormat": "{{outcome}}"
}
]
},
{
"id": 5,
"type": "timeseries",
"title": "Worker / p95 по исходу",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 16,
"w": 12,
"h": 8
},
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
},
"targets": [
{
"refId": "A",
"expr": "histogram_quantile(0.95, sum by (le,outcome) (rate(messenger_jobs_duration_seconds_bucket[30s])))",
"legendFormat": ""
}
]
},
{
"id": 6,
"type": "timeseries",
"title": "Upload → ready / p95",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 16,
"w": 12,
"h": 8
},
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
},
"targets": [
{
"refId": "A",
"expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_ready_duration_seconds_bucket[30s])))",
"legendFormat": ""
}
]
},
{
"id": 7,
"type": "timeseries",
"title": "Backlog / заданий",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 24,
"w": 12,
"h": 8
},
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"targets": [
{
"refId": "A",
"expr": "max(messenger_queue_depth)",
"legendFormat": ""
}
]
},
{
"id": 8,
"type": "timeseries",
"title": "Возраст старейшего задания",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 24,
"w": 12,
"h": 8
},
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
},
"targets": [
{
"refId": "A",
"expr": "max(messenger_queue_oldest_seconds)",
"legendFormat": ""
}
]
},
{
"id": 9,
"type": "timeseries",
"title": "PostgreSQL / занято и ожидает",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 32,
"w": 12,
"h": 8
},
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"targets": [
{
"refId": "A",
"expr": "messenger_db_pool_used",
"legendFormat": ""
},
{
"refId": "B",
"expr": "messenger_db_pool_waiting",
"legendFormat": ""
}
]
},
{
"id": 10,
"type": "timeseries",
"title": "Worker / занято и доступно",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 32,
"w": 12,
"h": 8
},
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"targets": [
{
"refId": "A",
"expr": "sum(messenger_worker_busy)",
"legendFormat": ""
},
{
"refId": "B",
"expr": "sum(messenger_worker_capacity)",
"legendFormat": ""
}
]
},
{
"id": 11,
"type": "timeseries",
"title": "CPU процессов",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 40,
"w": 12,
"h": 8
},
"fieldConfig": {
"defaults": {
"unit": "cores"
},
"overrides": []
},
"targets": [
{
"refId": "A",
"expr": "rate(messenger_process_cpu_seconds_total[30s])",
"legendFormat": ""
}
]
},
{
"id": 12,
"type": "timeseries",
"title": "RSS процессов",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 40,
"w": 12,
"h": 8
},
"fieldConfig": {
"defaults": {
"unit": "bytes"
},
"overrides": []
},
"targets": [
{
"refId": "A",
"expr": "messenger_process_rss_bytes",
"legendFormat": ""
}
]
},
{
"id": 13,
"type": "timeseries",
"title": "Потери экспорта Collector",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 48,
"w": 12,
"h": 8
},
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"targets": [
{
"refId": "A",
"expr": "sum by (exporter) (increase(otelcol_exporter_send_failed_spans_total[30s]))",
"legendFormat": ""
},
{
"refId": "B",
"expr": "sum by (exporter) (increase(otelcol_exporter_send_failed_log_records_total[30s]))",
"legendFormat": ""
}
]
},
{
"id": 14,
"type": "timeseries",
"title": "Алерты",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 48,
"w": 12,
"h": 8
},
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"targets": [
{
"refId": "A",
"expr": "ALERTS{alertname=~\"Messenger.*\"}",
"legendFormat": ""
}
]
},
{
"id": 30,
"type": "logs",
"title": "API, relay и worker / связанные логи",
"datasource": {
"type": "loki",
"uid": "loki"
},
"gridPos": {
"x": 0,
"y": 56,
"w": 24,
"h": 10
},
"targets": [
{
"refId": "A",
"expr": "{service_name=~\"messenger-(api|relay|worker)\"}"
}
]
}
]
}
@@ -0,0 +1,7 @@
apiVersion: 1
providers:
- name: messenger
folder: MTUSI
type: file
allowUiUpdates: false
options: {path: /var/lib/grafana/dashboards}
@@ -0,0 +1,33 @@
apiVersion: 1
datasources:
- name: Prometheus
type: prometheus
uid: prometheus
url: http://prometheus:9090
isDefault: true
editable: false
- name: Loki
type: loki
uid: loki
url: http://loki:3100
editable: false
jsonData:
derivedFields:
- name: Trace
matcherType: label
matcherRegex: trace_id
datasourceUid: tempo
url: '$${__value.raw}'
- name: Tempo
type: tempo
uid: tempo
url: http://tempo:3200
editable: false
jsonData:
tracesToLogsV2:
datasourceUid: loki
spanStartTimeShift: '-2m'
spanEndTimeShift: '2m'
filterByTraceID: true
tags:
- {key: service.name, value: service_name}
+36
View File
@@ -0,0 +1,36 @@
auth_enabled: false
server:
http_listen_port: 3100
common:
instance_addr: 127.0.0.1
path_prefix: /loki
storage:
filesystem:
chunks_directory: /loki/chunks
rules_directory: /loki/rules
replication_factor: 1
ring:
kvstore:
store: inmemory
schema_config:
configs:
- from: 2024-01-01
store: tsdb
object_store: filesystem
schema: v13
index: {prefix: index_, period: 24h}
compactor:
working_directory: /loki/compactor
retention_enabled: true
delete_request_store: filesystem
limits_config:
retention_period: 72h
allow_structured_metadata: true
otlp_config:
resource_attributes:
ignore_defaults: true
attributes_config:
- action: index_label
attributes: [service.name, deployment.environment.name]
analytics:
reporting_enabled: false
+13
View File
@@ -0,0 +1,13 @@
global:
scrape_interval: 2s
evaluation_interval: 2s
rule_files:
- /etc/prometheus/alerts.yaml
scrape_configs:
- job_name: messenger-otel
honor_labels: true
static_configs: [{targets: ["otel-collector:8889"]}]
- job_name: collector
static_configs: [{targets: ["otel-collector:8888"]}]
+18
View File
@@ -0,0 +1,18 @@
server:
http_listen_port: 3200
distributor:
receivers:
otlp:
protocols:
http:
endpoint: 0.0.0.0:4318
storage:
trace:
backend: local
wal: {path: /var/tempo/wal}
local: {path: /var/tempo/blocks}
compactor:
compaction:
block_retention: 72h
usage_report:
reporting_enabled: false
+26
View File
@@ -0,0 +1,26 @@
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": [
"s3:ListBucket",
"s3:GetBucketLocation"
],
"Resource": [
"arn:aws:s3:::BUCKET"
]
},
{
"Effect": "Allow",
"Action": [
"s3:GetObject",
"s3:DeleteObject"
],
"Resource": [
"arn:aws:s3:::BUCKET/staging/*",
"arn:aws:s3:::BUCKET/objects/*"
]
}
]
}
+18
View File
@@ -0,0 +1,18 @@
# Official security-fixed source release; no dependency on retired image tags.
FROM golang:1.26.7-alpine3.23@sha256:b17af760035fc2f338eed92d448a6c67f2d45438844fc6c60678fa5f99e44b57 AS builder
ENV CGO_ENABLED=0 GOTOOLCHAIN=local GOMAXPROCS=4 GOFLAGS=-p=4
RUN go install -ldflags='-s -w' github.com/minio/minio@RELEASE.2025-10-15T17-29-55Z
RUN go install -ldflags='-s -w' github.com/minio/mc@RELEASE.2025-08-13T08-35-41Z
FROM alpine:3.23@sha256:85fe1e81d6758c208f3e1eed4338a1997e19d4be002d4dd32d3100c9a8c010a0 AS server
COPY --from=builder /go/bin/minio /usr/local/bin/minio
COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
RUN addgroup -g 10001 app && adduser -D -u 10001 -G app app && mkdir /data && chown app:app /data
COPY --chmod=755 infra/start-s3.sh /usr/local/bin/start-s3.sh
USER 10001:10001
ENTRYPOINT ["/usr/local/bin/start-s3.sh"]
FROM alpine:3.23@sha256:85fe1e81d6758c208f3e1eed4338a1997e19d4be002d4dd32d3100c9a8c010a0 AS client
COPY --from=builder /go/bin/mc /usr/local/bin/mc
COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
ENTRYPOINT ["/bin/sh"]
+5
View File
@@ -0,0 +1,5 @@
#!/bin/sh
set -eu
export MINIO_ROOT_USER="$(cat /run/secrets/s3_root_user)"
export MINIO_ROOT_PASSWORD="$(cat /run/secrets/s3_root_password)"
exec minio "$@"
+27
View File
@@ -0,0 +1,27 @@
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": [
"s3:ListBucket",
"s3:GetBucketLocation"
],
"Resource": [
"arn:aws:s3:::BUCKET"
]
},
{
"Effect": "Allow",
"Action": [
"s3:GetObject",
"s3:PutObject",
"s3:DeleteObject"
],
"Resource": [
"arn:aws:s3:::BUCKET/staging/*",
"arn:aws:s3:::BUCKET/objects/*"
]
}
]
}
+34
View File
@@ -0,0 +1,34 @@
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": [
"s3:ListBucket",
"s3:GetBucketLocation"
],
"Resource": [
"arn:aws:s3:::BUCKET"
]
},
{
"Effect": "Allow",
"Action": [
"s3:GetObject"
],
"Resource": [
"arn:aws:s3:::BUCKET/staging/*",
"arn:aws:s3:::BUCKET/objects/*"
]
},
{
"Effect": "Allow",
"Action": [
"s3:PutObject"
],
"Resource": [
"arn:aws:s3:::BUCKET/objects/*"
]
}
]
}
+37
View File
@@ -0,0 +1,37 @@
CREATE TABLE users (
id uuid PRIMARY KEY,
username varchar(32) COLLATE "C" NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'),
display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100),
password_hash text NOT NULL,
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
);
CREATE TABLE chats (
id uuid PRIMARY KEY,
title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100),
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
);
CREATE TABLE chat_members (
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
user_id uuid NOT NULL REFERENCES users(id),
position smallint NOT NULL CHECK (position BETWEEN 0 AND 99),
PRIMARY KEY (chat_id,user_id),
UNIQUE (chat_id,position)
);
CREATE INDEX chat_members_by_user ON chat_members (user_id,chat_id);
CREATE TABLE messages (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats(id),
sender_id uuid NOT NULL REFERENCES users(id),
text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000),
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
FOREIGN KEY (chat_id,sender_id) REFERENCES chat_members(chat_id,user_id)
);
CREATE INDEX messages_history ON messages (chat_id,created_at,id);
CREATE TABLE sessions (
token_hash bytea PRIMARY KEY CHECK (octet_length(token_hash)=32),
user_id uuid NOT NULL REFERENCES users(id),
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
expires_at timestamptz NOT NULL,
CHECK (expires_at > created_at)
);
CREATE INDEX sessions_expiry ON sessions (expires_at);
+14
View File
@@ -0,0 +1,14 @@
CREATE TABLE message_requests (
user_id uuid NOT NULL REFERENCES users(id),
http_method text NOT NULL DEFAULT 'POST' CHECK (http_method='POST'),
chat_id uuid NOT NULL REFERENCES chats(id),
idempotency_key varchar(128) COLLATE "C" NOT NULL
CHECK (idempotency_key ~ '^[A-Za-z0-9._:-]{1,128}$'),
payload_hash bytea NOT NULL CHECK (octet_length(payload_hash)=32),
response_message_id uuid NOT NULL REFERENCES messages(id),
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
expires_at timestamptz NOT NULL,
PRIMARY KEY (user_id,http_method,chat_id,idempotency_key),
CHECK (expires_at > created_at)
);
CREATE INDEX message_requests_expiry ON message_requests (expires_at);
+30
View File
@@ -0,0 +1,30 @@
CREATE TABLE attachments (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats(id),
owner_id uuid NOT NULL REFERENCES users(id),
filename text NOT NULL CHECK (char_length(filename) BETWEEN 1 AND 255),
content_type text NOT NULL CHECK (content_type IN ('image/png','image/jpeg')),
size_bytes bigint NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760),
sha256 text NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'),
object_key text NOT NULL UNIQUE,
storage_state text NOT NULL CHECK (storage_state IN ('pending','ready','missing')),
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
updated_at timestamptz NOT NULL DEFAULT clock_timestamp(),
UNIQUE (id,chat_id),
FOREIGN KEY (chat_id,owner_id) REFERENCES chat_members(chat_id,user_id)
);
CREATE INDEX attachments_reconcile ON attachments (updated_at,id);
ALTER TABLE messages DROP CONSTRAINT messages_text_check;
ALTER TABLE messages ADD CHECK (char_length(text) BETWEEN 0 AND 4000);
ALTER TABLE messages ADD UNIQUE (id,chat_id);
CREATE TABLE message_attachments (
message_id uuid NOT NULL,
chat_id uuid NOT NULL,
attachment_id uuid NOT NULL,
position smallint NOT NULL CHECK (position BETWEEN 0 AND 9),
PRIMARY KEY (message_id,attachment_id),
UNIQUE (message_id,position),
FOREIGN KEY (message_id,chat_id) REFERENCES messages(id,chat_id),
FOREIGN KEY (attachment_id,chat_id) REFERENCES attachments(id,chat_id)
);
CREATE INDEX message_attachments_by_attachment ON message_attachments (attachment_id);
+46
View File
@@ -0,0 +1,46 @@
-- Preserve earlier migration checksums; transition lab4 storage to versioned jobs.
ALTER TABLE attachments DROP CONSTRAINT attachments_storage_state_check;
ALTER TABLE attachments ADD CHECK (storage_state IN ('queued','processing','ready','failed')) NOT VALID;
UPDATE attachments SET storage_state='queued' WHERE storage_state IN ('pending','missing');
ALTER TABLE attachments ADD COLUMN pipeline_version integer NOT NULL DEFAULT 1 CHECK (pipeline_version > 0);
ALTER TABLE attachments ADD COLUMN variants jsonb NOT NULL DEFAULT '[]' CHECK (jsonb_typeof(variants)='array');
ALTER TABLE attachments ADD COLUMN error_code text;
CREATE TABLE attachment_jobs (
job_id uuid PRIMARY KEY,
attachment_id uuid NOT NULL REFERENCES attachments(id) ON DELETE CASCADE,
pipeline_version integer NOT NULL CHECK (pipeline_version > 0),
source_key text NOT NULL,
state text NOT NULL DEFAULT 'queued' CHECK (state IN ('queued','processing','ready','failed','superseded')),
attempt integer NOT NULL DEFAULT 0 CHECK (attempt >= 0),
fence uuid,
lease_until timestamptz,
next_attempt_at timestamptz NOT NULL DEFAULT clock_timestamp(),
error_code text,
source_expired boolean NOT NULL DEFAULT false,
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
updated_at timestamptz NOT NULL DEFAULT clock_timestamp(),
UNIQUE (attachment_id,pipeline_version)
);
CREATE INDEX jobs_recovery ON attachment_jobs(state,lease_until,next_attempt_at);
CREATE TABLE job_outbox (
job_id uuid PRIMARY KEY REFERENCES attachment_jobs(job_id) ON DELETE CASCADE,
due_at timestamptz NOT NULL DEFAULT clock_timestamp(),
published_at timestamptz,
stream_id text
);
CREATE INDEX outbox_due ON job_outbox(due_at) WHERE published_at IS NULL;
CREATE TABLE pipeline_audit (
sequence bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
job_id uuid NOT NULL REFERENCES attachment_jobs(job_id) ON DELETE CASCADE,
event text NOT NULL,
details jsonb NOT NULL DEFAULT '{}',
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
);
-- Only an operator with DB access can configure faults; production ignores this table.
CREATE TABLE pipeline_faults (
job_id uuid REFERENCES attachment_jobs(job_id) ON DELETE CASCADE,
phase text NOT NULL,
action text NOT NULL CHECK (action IN ('crash','hang','transient','permanent')),
remaining integer NOT NULL DEFAULT 1 CHECK (remaining >= 0),
PRIMARY KEY(job_id,phase)
);
+8
View File
@@ -0,0 +1,8 @@
-- A paused API must not acknowledge an object that GC has already retired.
CREATE TABLE upload_reservations (
id uuid PRIMARY KEY,
object_key text NOT NULL UNIQUE,
state text NOT NULL DEFAULT 'uploading' CHECK (state IN ('uploading','accepted','abandoned')),
expires_at timestamptz NOT NULL DEFAULT clock_timestamp()+interval '120 seconds',
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
);
+5
View File
@@ -0,0 +1,5 @@
-- Optional W3C context follows the durable job, including outbox retries.
ALTER TABLE attachment_jobs ADD COLUMN traceparent text CHECK (
traceparent ~ '^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$'
);
ALTER TABLE attachment_jobs ADD COLUMN tracestate text CHECK (length(tracestate) <= 512);
+48
View File
@@ -0,0 +1,48 @@
[project]
name = "mtusi-messenger"
version = "0.6.0"
description = "MTUSI lab 6: messenger observability"
requires-python = ">=3.14,<3.15"
dependencies = [
"opentelemetry-sdk==1.45.2",
"opentelemetry-exporter-otlp-proto-http==1.45.2",
"argon2-cffi==25.1.0",
"boto3==1.43.109",
"fastapi==0.143.3",
"pillow==13.0.0",
"psycopg[binary,pool]==3.3.6",
"pydantic==2.14.1",
"redis==8.1.0",
"uvicorn==0.54.0",
]
[build-system]
requires = ["hatchling==1.27.0"]
build-backend = "hatchling.build"
[tool.hatch.build.targets.wheel]
packages = ["src/messenger"]
[dependency-groups]
dev = [
"httpx==0.28.1",
"openapi-spec-validator==0.7.2",
"pytest==9.1.1",
"ruff==0.17.1",
]
research = ["matplotlib==3.11.2"]
[tool.pytest.ini_options]
testpaths = ["student-tests"]
addopts = "-ra --tb=short"
[tool.ruff]
extend-exclude = ["tests", "scripts/check_contract.py", "scripts/validate_openapi.py"]
target-version = "py314"
line-length = 100
[tool.ruff.lint]
select = ["E", "F", "I", "B", "UP"]
[tool.ruff.per-file-target-version]
"scripts/*.py" = "py39"
+50
View File
@@ -0,0 +1,50 @@
#!/usr/bin/env python3
"""Create local synthetic credentials once, without printing their values."""
import os
import secrets
from pathlib import Path
root = Path(__file__).resolve().parents[1]
directory = root / "secrets"
directory.mkdir(mode=0o700, exist_ok=True)
password_file = directory / "postgres_password"
if not password_file.exists():
password_file.write_text(secrets.token_urlsafe(32) + "\n")
password = password_file.read_text().strip()
for name, value in {
"database_url": f"postgresql://messenger:{password}@postgres:5432/messenger",
"grafana_password": secrets.token_urlsafe(24),
"cache_url": "redis://valkey:6379/0",
"broker_url": "redis://broker:6379/0",
}.items():
path = directory / name
if not path.exists():
path.write_text(value + "\n")
# Compose bind-mount secrets must be readable by the non-root container process.
# The parent directory remains 0700 on the host.
for role in ("root", "api", "worker", "reconciler", "test"):
names = (
("s3_root_user", "s3_root_password")
if role == "root"
else ("s3_" + role + "_access_key_id", "s3_" + role + "_secret_access_key")
)
for index, name in enumerate(names):
path = directory / name
if not path.exists():
path.write_text(
(secrets.token_hex(10) if index == 0 else secrets.token_urlsafe(32)) + "\n"
)
for name in (
"postgres_password",
"grafana_password",
"database_url",
"cache_url",
"broker_url",
*[p.name for p in directory.glob("s3_*")],
):
os.chmod(directory / name, 0o644)
env = root / ".env"
if not env.exists():
env.write_text((root / ".env.example").read_text())
print("Local secret files and environment are ready; existing values preserved.")
+82
View File
@@ -0,0 +1,82 @@
"""Read-only discovery and scoped Compose operations for the local lab5 stack."""
import json
import subprocess
import time
import urllib.error
import urllib.request
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
def compose(*args, env=None):
return subprocess.run(
["docker", "compose", "--env-file", ".env", *args],
cwd=ROOT,
env=env,
check=True,
capture_output=True,
text=True,
timeout=90,
).stdout
def inspect(container):
template = (
'{"id":{{json .Id}},"started_at":{{json .State.StartedAt}},'
'"pid":{{.State.Pid}},"running":{{json .State.Running}},'
'"image":{{json .Image}},"memory":{{.HostConfig.Memory}},'
'"nano_cpus":{{.HostConfig.NanoCpus}},'
'"published_ports":{{json .HostConfig.PortBindings}}}'
)
r = subprocess.run(
["docker", "inspect", "--format", template, container],
capture_output=True,
text=True,
check=True,
timeout=5,
)
return json.loads(r.stdout)
def instance(container):
code = (
"import urllib.request; r=urllib.request.urlopen("
"'http://127.0.0.1:8080/health/live',timeout=1); print(r.headers['X-Instance-Id'])"
)
r = subprocess.run(
["docker", "exec", container, "python", "-c", code],
capture_output=True,
text=True,
check=True,
timeout=3,
)
value = inspect(container)
value["instance"] = r.stdout.strip()
return value
def instances():
ids = compose("ps", "-q", "api").split()
with ThreadPoolExecutor(max_workers=8) as pool:
return list(pool.map(instance, ids))
def wait_lb(base="http://127.0.0.1:8086", count=2, timeout=45):
deadline, seen = time.monotonic() + timeout, set()
while time.monotonic() < deadline:
try:
current = {value["instance"] for value in instances()}
if len(current) != count:
time.sleep(0.2)
continue
with urllib.request.urlopen(base + "/health/ready", timeout=2) as response:
seen.add(response.headers["X-Instance-Id"])
if current <= seen:
return sorted(current)
except (OSError, urllib.error.URLError, subprocess.SubprocessError):
pass
time.sleep(0.1)
raise RuntimeError("Expected healthy API replicas were not observed through LB")
+279
View File
@@ -0,0 +1,279 @@
#!/usr/bin/env python3
"""Collect sanitized evidence from the local lab6 stand."""
import argparse
import json
import subprocess
import sys
import time
from pathlib import Path
from urllib.error import HTTPError, URLError
from urllib.parse import urlencode
from urllib.request import urlopen
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client # noqa: E402
def compose(*args, **kwargs):
return subprocess.run(["docker", "compose", *args], cwd=ROOT, check=True, **kwargs)
def api_json(port, path, **params):
url = f"http://127.0.0.1:{port}{path}"
if params:
url += "?" + urlencode(params)
with urlopen(url, timeout=5) as response:
return json.load(response)
def prom(query):
data = api_json(9096, "/api/v1/query", query=query)["data"]["result"]
return float(data[0]["value"][1]) if data else 0
def until(check, timeout=45):
end = time.monotonic() + timeout
while time.monotonic() < end:
try:
result = check()
if result:
return result
except (HTTPError, URLError):
pass
time.sleep(0.5)
raise RuntimeError(f"Timed out waiting for {check.__name__}")
def log_for(request_id):
data = api_json(
3106,
"/loki/api/v1/query_range",
query='{service_name="messenger-api"} | request_id="' + request_id + '"',
start=str(time.time_ns() - 180_000_000_000),
limit=5,
)["data"]["result"]
for stream in data:
for value in stream["values"]:
return value[2] if len(value) > 2 else stream["stream"]
return None
def span_names(trace_id):
data = api_json(3206, "/api/traces/" + trace_id)
return [
(span["name"], span.get("kind"), span["spanId"], span.get("parentSpanId"))
for batch in data.get("batches", data.get("resourceSpans", []))
for group in batch.get("scopeSpans", batch.get("instrumentationLibrarySpans", []))
for span in group["spans"]
]
def alerts():
return {
a["labels"]["alertname"]: a["state"]
for a in api_json(9096, "/api/v1/alerts")["data"]["alerts"]
}
def correlation(client, actor, chat, retry=False):
if retry:
compose("stop", "worker", stdout=subprocess.DEVNULL)
try:
attachment = client.call(
"POST",
f"/api/v1/chats/{chat['id']}/attachments?filename=observability.png",
202,
actor=actor,
raw=(ROOT / "tests/fixtures/sample.png").read_bytes(),
headers={"Content-Type": "image/png"},
schema="Attachment",
)
if retry:
job_attachment = attachment["json"]["id"]
compose(
"exec",
"-T",
"postgres",
"psql",
"-U",
"messenger",
"-Xq",
"-v",
"ON_ERROR_STOP=1",
"-v",
"attachment=" + job_attachment,
input=(
"INSERT INTO pipeline_faults(job_id,phase,action,remaining) "
"SELECT job_id,'before_effect','transient',1 FROM attachment_jobs "
"WHERE attachment_id=:'attachment';\n"
),
text=True,
stdout=subprocess.DEVNULL,
)
finally:
if retry:
compose("start", "worker", stdout=subprocess.DEVNULL)
client.wait_ready(actor, attachment["json"])
request_id = attachment["headers"]["X-Request-Id"]
log = until(lambda: log_for(request_id))
trace_id = log["trace_id"]
expected = {"jobs.publish", "jobs.consume", "s3.put", "s3.download", "db.transaction"}
def connected():
spans = span_names(trace_id)
names = {row[0] for row in spans}
consumers = sum(row[0] == "jobs.consume" for row in spans)
return spans if expected <= names and consumers >= (2 if retry else 1) else None
spans = until(connected)
by_id = {row[2]: row for row in spans}
consumers = [row for row in spans if row[0] == "jobs.consume"]
assert all(by_id[row[3]][0] == "jobs.publish" for row in consumers)
logs = until(
lambda: api_json(
3106,
"/loki/api/v1/query_range",
query='{service_name="messenger-worker"} | trace_id="' + trace_id + '"',
start=str(time.time_ns() - 180_000_000_000),
limit=5,
)["data"]["result"]
)
assert any(value[1] == "job_processed" for row in logs for value in row["values"])
return {
"request_id": request_id,
"trace_id": trace_id,
"span_names": sorted({r[0] for r in spans}),
"worker_parent": by_id[consumers[0][3]][0],
"worker_attempts": len(consumers),
"api_and_worker_loki": True,
}
def worker_incident(client, actor, chat):
compose("stop", "worker", stdout=subprocess.DEVNULL)
try:
uploads = [client.attachment(actor, chat)[0] for _ in range(3)]
until(lambda: alerts().get("MessengerQueueLag") == "firing")
record = {
"depth_at_firing": prom("max(messenger_queue_depth)"),
"oldest_seconds_at_firing": prom("max(messenger_queue_oldest_seconds)"),
}
assert record["depth_at_firing"] >= 3
finally:
compose("start", "worker", stdout=subprocess.DEVNULL)
for attachment in uploads:
client.wait_ready(actor, attachment)
until(lambda: not prom("max(messenger_queue_depth)") and "MessengerQueueLag" not in alerts())
return {**record, "state": ["firing", "resolved"], "uploads_ready": len(uploads)}
def database_incident(client, actor):
# Independent, bounded transaction. Lock sessions only; readiness can still use the schema.
command = [
"docker",
"compose",
"exec",
"-T",
"postgres",
"psql",
"-U",
"messenger",
"-d",
"messenger",
"-XqAt",
"-v",
"ON_ERROR_STOP=1",
]
start = time.monotonic()
lock = subprocess.Popen(
command, cwd=ROOT, stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True
)
lock.stdin.write(
"BEGIN;\nLOCK TABLE sessions IN ACCESS EXCLUSIVE MODE;\n"
"\\echo locked\nSELECT pg_sleep(42);\nROLLBACK;\n"
)
lock.stdin.close()
try:
assert lock.stdout.readline().strip() == "locked"
attempts = []
while time.monotonic() - start < 34:
attempt = client.call("GET", "/api/v1/users/me", [200, 503], actor=actor, schema="User")
attempts.append(attempt)
states = alerts()
if all(
states.get(a) == "firing"
for a in ("MessengerHttpErrors", "MessengerDatabaseErrors")
):
break
assert any(item["status"] == 503 for item in attempts)
active = alerts()
assert active.get("MessengerDatabaseErrors") == "firing", active
assert active.get("MessengerHttpErrors") == "firing", active
finally:
lock.wait(timeout=46)
lock.stdout.close()
client.call("GET", "/api/v1/users/me", actor=actor, schema="User")
# Keep successful traffic to publish fresh data while the alert window slides.
def resolved():
client.call("GET", "/api/v1/users/me", actor=actor, schema="User")
return not ({"MessengerDatabaseErrors", "MessengerHttpErrors"} & alerts().keys())
until(resolved)
failure = next(item for item in attempts if item["status"] == 503)
failure_log = until(lambda: log_for(failure["headers"]["X-Request-Id"]))
names = span_names(failure_log["trace_id"])
assert "db.read" in {row[0] for row in names}
return {
"statuses": [item["status"] for item in attempts],
"max_latency_ms": max(item["elapsed"] for item in attempts) * 1000,
"trace_id": failure_log["trace_id"],
"state": ["firing", "resolved"],
"spans": sorted({row[0] for row in names}),
}
def backend_incident(client, actor, chat):
lost = "sum(otelcol_exporter_send_failed_spans_total)"
baseline = prom(lost)
compose("stop", "tempo", stdout=subprocess.DEVNULL)
try:
for _ in range(15):
client.message(actor, chat, "synthetic telemetry outage probe")
time.sleep(0.5)
delta = until(lambda: value if (value := prom(lost) - baseline) > 0 else None)
memory = prom("max(otelcol_process_memory_rss_bytes)")
assert memory > 0
finally:
compose("start", "tempo", stdout=subprocess.DEVNULL)
client.call("GET", "/health/ready", schema="Health")
return {"successful_messages": 15, "failed_export_spans": delta, "collector_rss_bytes": memory}
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--base-url", default="http://127.0.0.1:8086")
parser.add_argument("--output", type=Path, required=True)
parser.add_argument("--incidents", action="store_true")
parser.add_argument("--retry", action="store_true")
args = parser.parse_args()
if args.output.exists():
parser.error("choose a new evidence file")
client = Client(args.base_url)
alice, _, _, chat = client.bootstrap()
result = {"correlation": correlation(client, alice, chat, retry=args.retry)}
print("PASS: same Tempo trace and Loki logs for upload and worker", flush=True)
if args.incidents:
result["worker_stop"] = worker_incident(client, alice, chat)
print("PASS: queue alert firing and resolved", flush=True)
result["database_lock"] = database_incident(client, alice)
print("PASS: database and HTTP alerts firing and resolved", flush=True)
result["tempo_stop"] = backend_incident(client, alice, chat)
print("PASS: messages persisted during Tempo outage", flush=True)
args.output.write_text(json.dumps(result, indent=2) + "\n")
if __name__ == "__main__":
main()
+33
View File
@@ -0,0 +1,33 @@
"""Local test operator configuration. Credentials stay in ignored files/in memory."""
import os
from pathlib import Path
from psycopg.conninfo import make_conninfo
from messenger.settings import Settings
ROOT = Path(__file__).resolve().parents[1]
def settings(role="worker"):
prefix = "s3_root" if role == "root" else "s3_" + role
access, secret = (
("user", "password") if role == "root" else ("access_key_id", "secret_access_key")
)
return Settings(
make_conninfo(
host="127.0.0.1",
port=int(os.getenv("TEST_PG_PORT", "55467")),
user="messenger",
dbname="messenger",
password=(ROOT / "secrets/postgres_password").read_text().strip(),
),
"redis://127.0.0.1:" + os.getenv("TEST_CACHE_PORT", "56386") + "/0",
s3_endpoint="http://127.0.0.1:" + os.getenv("TEST_S3_PORT", "59006"),
s3_access_key=(ROOT / ("secrets/" + prefix + "_" + access)).read_text().strip(),
s3_secret_key=(ROOT / ("secrets/" + prefix + "_" + secret)).read_text().strip(),
broker_url="redis://127.0.0.1:" + os.getenv("TEST_BROKER_PORT", "56387") + "/0",
job_timeout=int(os.getenv("TEST_JOB_TIMEOUT_SECONDS", "5")),
fault_injection=True,
)
+5
View File
@@ -0,0 +1,5 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
# All running application/state services; no migration, down -v, flush or seed.
docker compose --env-file .env restart --timeout 7 api postgres valkey s3 broker relay worker lb
+49
View File
@@ -0,0 +1,49 @@
#!/usr/bin/env python3
"""Sequential process replacement; at least one existing API stays running."""
import argparse
import json
import subprocess
from cluster import compose, instances, wait_lb
def replace_all(base="http://127.0.0.1:8084"):
original = instances()
if len(original) < 2:
raise RuntimeError("Rolling replacement requires at least two healthy replicas")
wait_lb(base, len(original))
replacements = []
for current in original:
subprocess.run(
["docker", "stop", "--time", "5", current["id"]],
check=True,
capture_output=True,
timeout=8,
)
subprocess.run(["docker", "rm", current["id"]], check=True, capture_output=True)
compose(
"up", "-d", "--no-deps", "--no-recreate", "--scale", "api=" + str(len(original)), "api"
)
aliases = wait_lb(base, len(original))
if current["instance"] in aliases:
raise RuntimeError("Old API process is still serving after replacement")
replacements.append(
{
"old_container": current["id"],
"old_instance": current["instance"],
"live_instances": aliases,
}
)
return replacements
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--base-url", default="http://127.0.0.1:8084")
args = parser.parse_args()
print(json.dumps(replace_all(args.base_url)))
if __name__ == "__main__":
main()
+48
View File
@@ -0,0 +1,48 @@
#!/usr/bin/env python3
"""Map a process alias to ONE container belonging to this Compose service."""
import json
import os
import re
import subprocess
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
def alias(container):
code = (
"import urllib.request; r=urllib.request.urlopen("
"'http://127.0.0.1:8080/health/live',timeout=1); print(r.headers['X-Instance-Id'])"
)
r = subprocess.run(
["docker", "exec", container, "python", "-c", code],
cwd=ROOT,
capture_output=True,
text=True,
timeout=3,
check=True,
)
return container, r.stdout.strip()
def main():
target = os.environ.get("TARGET_INSTANCE_ID", "")
if not re.fullmatch(r"[a-f0-9]{32}", target):
raise SystemExit("TARGET_INSTANCE_ID must name a live API process")
ids = subprocess.check_output(
["docker", "compose", "ps", "-q", "api"], cwd=ROOT, text=True
).split()
with ThreadPoolExecutor(max_workers=8) as pool:
matches = [cid for cid, identity in pool.map(alias, ids) if identity == target]
if len(matches) != 1:
raise SystemExit("Expected exactly one matching API in this project")
subprocess.run(
["docker", "stop", "--time", "5", matches[0]], check=True, capture_output=True, timeout=7
)
print(json.dumps({"stopped_container": matches[0], "instance": target}))
if __name__ == "__main__":
main()
+4
View File
@@ -0,0 +1,4 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
exec python3 scripts/stop-one.py
+184
View File
@@ -0,0 +1,184 @@
#!/usr/bin/env python3
"""Fixed offered-load comparison. Stops only named services in the current lab6 project."""
import argparse
import concurrent.futures
import json
import math
import os
import re
import subprocess
import time
from pathlib import Path
from urllib.request import urlopen
from observe import Client, compose, prom, until
MODES = (("off", "0", "0"), ("full", "1", "1"), ("half", "1", "0.5"), ("quarter", "1", "0.25"))
COUNTERS = {
"accepted_spans": "sum(otelcol_receiver_accepted_spans_total)",
"sent_spans": "sum(otelcol_exporter_sent_spans_total)",
"failed_spans": "sum(otelcol_exporter_send_failed_spans_total)",
"accepted_logs": "sum(otelcol_receiver_accepted_log_records_total)",
"accepted_metric_points": "sum(otelcol_receiver_accepted_metric_points_total)",
}
def configure(enabled, ratio):
compose(
"up",
"-d",
"--no-build",
"--no-deps",
"--force-recreate",
"api",
"worker",
"relay",
env={**os.environ, "TELEMETRY_ENABLED": enabled, "OTEL_SAMPLE_RATIO": ratio},
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
def ready():
try:
with urlopen("http://127.0.0.1:8086/health/ready", timeout=2) as response:
return response.status == 200
except OSError:
return False
until(ready)
def snapshot():
with urlopen("http://127.0.0.1:3206/metrics", timeout=4) as response:
metrics = response.read().decode()
received_bytes = sum(
float(value)
for value in re.findall(
r"^tempo_distributor_bytes_received_total\{[^}]+\} (\S+)", metrics, re.M
)
)
return {
"trace_ingest_proto_bytes": received_bytes,
**{name: prom(expression) for name, expression in COUNTERS.items()},
}
def workload(client, actor, chat, seconds, rate):
def operation(n):
if n % 45 == 0:
started = time.perf_counter()
client.attachment(actor, chat)
return (time.perf_counter() - started) * 1000
if n % 5 == 0:
response = client.message(actor, chat, "synthetic cost experiment")
else:
response = client.call(
"GET",
f"/api/v1/chats/{chat['id']}/messages?limit=20",
actor=actor,
schema="MessagePage",
)
return response["elapsed"] * 1000
start = time.perf_counter()
with concurrent.futures.ThreadPoolExecutor(max_workers=12) as pool:
futures = []
for number in range(seconds * rate):
delay = start + number / rate - time.perf_counter()
if delay > 0:
time.sleep(delay)
futures.append(pool.submit(operation, number))
observations = sorted(future.result() for future in futures)
elapsed = time.perf_counter() - start
return {
"requests": len(observations),
"elapsed_s": elapsed,
"latency_ms": {
f"p{int(q * 100)}": observations[math.ceil(q * len(observations)) - 1]
for q in (0.50, 0.95, 0.99)
},
}
def stats():
services = ("api", "worker", "relay", "otel-collector", "prometheus", "loki", "tempo")
ids = compose("ps", "-q", *services, capture_output=True, text=True).stdout.split()
raw = subprocess.check_output(
["docker", "stats", "--no-stream", "--format", "{{json .}}", *ids], text=True
)
result = []
for line in raw.splitlines():
row = json.loads(line)
result.append(
{
"container": row["Name"],
"cpu_percent": float(row["CPUPerc"].rstrip("%")),
"memory": row["MemUsage"].split(" / ")[0],
}
)
return result
def run_one(mode, seconds, rate):
label, enabled, ratio = mode
configure(enabled, ratio)
client = Client("http://127.0.0.1:8086")
actor, _, _, chat = client.bootstrap()
for _ in range(25):
client.message(actor, chat, "seed")
workload(client, actor, chat, 3, rate)
time.sleep(3)
initial = snapshot()
with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool:
run = pool.submit(workload, client, actor, chat, seconds, rate)
samples = []
for _ in range(2):
time.sleep(2)
samples.append(stats())
receipt = run.result()
# Collector emits batches and Prometheus scrapes every two seconds.
time.sleep(4)
final = snapshot()
return {
"mode": label,
"sample_ratio": float(ratio),
"load": receipt,
"container_samples": samples,
"ingest": {k: final[k] - v for k, v in initial.items()},
}
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--seconds", type=int, default=10)
parser.add_argument("--rate", type=int, default=18)
parser.add_argument("--repeats", type=int, default=2)
parser.add_argument("--output", type=Path, required=True)
args = parser.parse_args()
if args.output.exists() or args.seconds < 8 or args.rate < 1 or args.repeats < 1:
parser.error("use a new output and a positive load for at least 8 s")
results = []
try:
for repeat in range(args.repeats):
for mode in MODES if repeat % 2 == 0 else MODES[::-1]:
row = run_one(mode, args.seconds, args.rate)
print(row["mode"], row["load"]["latency_ms"], row["ingest"], flush=True)
results.append(row)
args.output.write_text(
json.dumps(
{
"seconds": args.seconds,
"offered_requests_per_s": args.rate,
"runs": results,
},
indent=2,
)
+ "\n"
)
finally:
configure("1", "1")
if __name__ == "__main__":
main()
+172
View File
@@ -0,0 +1,172 @@
#!/usr/bin/env python3
"""Two-phase real-age GC check: seed, run other checks, then verify after >=300 seconds."""
import argparse
import hashlib
import io
import json
import sys
import time
from datetime import UTC, datetime
from pathlib import Path
from uuid import uuid4
from cluster import ROOT, compose
from pipeline_lab import settings
from messenger.cleanup import collect
from messenger.pipeline import Pipeline, intent
from messenger.repository import PostgresRepository
from messenger.storage import ObjectStore
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, check # noqa: E402
def main():
parser = argparse.ArgumentParser(__doc__)
parser.add_argument("action", choices=("seed", "verify"))
parser.add_argument("--seed", type=Path, required=True)
parser.add_argument("--output", type=Path)
parser.add_argument("--base-url", default="http://127.0.0.1:8086")
args = parser.parse_args()
config = settings("reconciler")
db = PostgresRepository(config.database_url, pool_size=3)
db.open()
store = ObjectStore(config)
try:
if args.action == "seed":
check(not args.seed.exists(), "Seed already exists")
compose("stop", "--timeout", "2", "worker", "relay")
try:
c = Client(args.base_url)
alice, _, _, room = c.bootstrap()
active, data = c.attachment(alice, room)
failed, _ = c.attachment(alice, room)
orphan = str(uuid4())
Pipeline(db, config).reserve(orphan, "staging/" + orphan)
uploader = ObjectStore(settings("api"))
try:
uploader.put(
"staging/" + orphan,
io.BytesIO(data),
len(data),
"image/png",
hashlib.sha256(data).hexdigest(),
)
finally:
uploader.close()
with db.connection(write=True) as conn:
j = conn.execute(
"SELECT * FROM attachment_jobs WHERE attachment_id=%s", (active["id"],)
).fetchone()
conn.execute(
"UPDATE attachment_jobs SET "
"next_attempt_at=clock_timestamp()+interval '1 hour' "
"WHERE job_id=%s",
(j["job_id"],),
)
conn.execute(
"UPDATE job_outbox SET due_at=clock_timestamp()+interval '1 "
"hour' WHERE job_id=%s",
(j["job_id"],),
)
f = conn.execute(
"SELECT job_id FROM attachment_jobs WHERE attachment_id=%s", (failed["id"],)
).fetchone()
conn.execute(
"INSERT INTO pipeline_faults(job_id,phase,action) VALUES "
"(%s,'before_effect','permanent')",
(f["job_id"],),
)
seed = {
"created_at": datetime.now(UTC).isoformat(),
"active_id": active["id"],
"failed_id": failed["id"],
"orphan_id": orphan,
}
args.seed.write_text(json.dumps(seed, indent=2) + "\n")
print(
"PASS: active job, failed candidate and unaccepted orphan seeded; no "
"credentials saved"
)
finally:
compose("start", "worker", "relay")
else:
check(args.output is not None and not args.output.exists(), "Choose a new output file")
seed = json.loads(args.seed.read_text())
# Real S3 timestamps and failed retention; no modified clocks.
while True:
with db.connection() as conn:
f = conn.execute(
"SELECT * FROM attachment_jobs WHERE attachment_id=%s", (seed["failed_id"],)
).fetchone()
check(f["state"] == "failed", "Failed seed did not reach terminal state")
age = (datetime.now(UTC) - f["updated_at"]).total_seconds()
if age >= 301:
break
check(age >= 0, "Clock mismatch")
time.sleep(min(1, 301 - age))
with db.connection() as conn:
ready = conn.execute(
"SELECT * FROM attachments WHERE storage_state='ready' LIMIT 1"
).fetchone()
check(ready is not None, "Need a published attachment to protect")
keys = [ready["object_key"], *[v["object_key"] for v in ready["variants"]]]
before = {key: store.head(key)["ETag"] for key in keys}
preview = collect(db, store, 300, False)
result = collect(db, store, 300, True)
check(
store.head("staging/" + seed["active_id"]) is not None, "Active staging was deleted"
)
check(store.head("staging/" + seed["orphan_id"]) is None, "Orphan not deleted")
check(
store.head("staging/" + seed["failed_id"]) is None,
"Expired failed source not deleted",
)
check(
all(store.head(k)["ETag"] == tag for k, tag in before.items()),
"Published result changed",
)
with db.connection(write=True) as conn:
expired = conn.execute(
"SELECT source_expired FROM attachment_jobs WHERE attachment_id=%s",
(seed["failed_id"],),
).fetchone()
check(expired["source_expired"], "Expired source not registered")
j = conn.execute(
"UPDATE attachment_jobs SET next_attempt_at=clock_timestamp() WHERE "
"attachment_id=%s RETURNING job_id",
(seed["active_id"],),
).fetchone()
intent(conn, j["job_id"])
deadline = time.monotonic() + 60
while time.monotonic() < deadline:
if db.attachment(seed["active_id"])["storage_state"] == "ready":
break
time.sleep(0.2)
check(
db.attachment(seed["active_id"])["storage_state"] == "ready",
"Protected work failed to complete",
)
proof = {
"retention_seconds": 300,
"failed_real_age_seconds": age,
"seed": seed,
"dry_run": preview,
"apply": result,
"active_protected_then_ready": True,
"orphan_deleted": True,
"failed_source_expired": True,
"published_objects_unchanged": len(keys),
}
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps(proof, indent=2) + "\n")
print("PASS: real-age cleanup preserves active work and published results")
finally:
store.close()
db.close()
if __name__ == "__main__":
main()
+253
View File
@@ -0,0 +1,253 @@
#!/usr/bin/env python3
"""Real LB requests during each replica failure and sequential replacement."""
import argparse
import json
import os
import subprocess
import sys
import threading
import time
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
from uuid import uuid4
from cluster import ROOT, compose, inspect, instance, instances, wait_lb
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, Failure, check # noqa: E402
from resilience import history # noqa: E402
def percentile(values, fraction):
values = sorted(values)
return values[min(int((len(values) - 1) * fraction), len(values) - 1)] if values else None
class Traffic:
def __init__(self, base, actor, room, workers=6):
self.base, self.actor, self.room = base, actor, room
self.prefix = uuid4().hex
self.stop = threading.Event()
self.lock = threading.Lock()
self.events = []
self.executor = ThreadPoolExecutor(max_workers=workers)
self.workers = workers
def worker(self, worker):
client, iteration = Client(self.base, timeout=3), 0
while not self.stop.is_set():
started = time.monotonic()
write = iteration % 10 == 9
try:
if write:
key = f"{self.prefix}:{worker}:{iteration}"
response = client.message(self.actor, self.room, key, key=key)
else:
response = client.call(
"GET", "/api/v1/users/me", actor=self.actor, schema="User"
)
event = {
"at": time.monotonic(),
"ok": True,
"instance": response["headers"]["X-Instance-Id"],
"latency_ms": (time.monotonic() - started) * 1000,
"message": response["json"] if write else None,
}
iteration += 1
except Failure:
event = {
"at": time.monotonic(),
"ok": False,
"latency_ms": (time.monotonic() - started) * 1000,
}
with self.lock:
self.events.append(event)
def __enter__(self):
self.started = time.monotonic()
self.futures = [self.executor.submit(self.worker, i) for i in range(self.workers)]
return self
def __exit__(self, *args):
self.stop.set()
self.executor.shutdown(wait=True)
for future in self.futures:
future.result()
self.finished = time.monotonic()
def report(self, action_start, action_end):
events = sorted(self.events, key=lambda row: row["at"])
ok = [row for row in events if row["ok"]]
after = [row for row in ok if row["at"] >= action_end]
check(bool(after), "No successful request after action")
elapsed = self.finished - self.started
latency = [row["latency_ms"] for row in ok]
return {
"duration_seconds": elapsed,
"workers": self.workers,
"requests": len(events),
"successes": len(ok),
"errors": len(events) - len(ok),
"successful_rps": len(ok) / elapsed,
"error_fraction": (len(events) - len(ok)) / len(events),
"first_success_after_action_seconds": after[0]["at"] - action_start,
"action_seconds": action_end - action_start,
"p50_ms": percentile(latency, 0.5),
"p95_ms": percentile(latency, 0.95),
"p99_ms": percentile(latency, 0.99),
"instances": sorted({row["instance"] for row in ok}),
"tail_instances": sorted({row["instance"] for row in after[-10:]}),
"confirmed_writes": sum(row["message"] is not None for row in ok),
}
def verify_messages(self, client, actor, room):
actual = history(client, actor, room)
by_id = {value["id"]: value for value in actual}
check(len(by_id) == len(actual), "History has duplicate IDs")
texts = [row["text"] for row in actual if row["text"].startswith(self.prefix)]
check(len(texts) == len(set(texts)), "An uncertain retry created a duplicate effect")
for event in self.events:
if event["ok"] and event["message"]:
check(
by_id.get(event["message"]["id"]) == event["message"],
"A confirmed message was lost or changed",
)
def public_test(target, base):
result = subprocess.run(
[
"make",
target,
"ACTION_SCRIPT=scripts/"
+ ("restart.sh" if target == "test-persistence" else "stop-one.sh"),
"BASE_URL=" + base,
],
cwd=ROOT,
check=True,
capture_output=True,
text=True,
timeout=150,
)
return "\n".join(line.rstrip() for line in result.stdout.splitlines()) + "\n"
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--base-url", default="http://127.0.0.1:8086")
parser.add_argument(
"--output", type=Path, default=ROOT / f"evidence/cluster-{time.time_ns()}.json"
)
args = parser.parse_args()
if args.output.exists():
raise SystemExit("Use a new evidence filename")
os.environ["NO_PROXY"] = os.environ["no_proxy"] = "127.0.0.1,localhost"
baseline = instances()
check(len(baseline) == 2, "This verification expects the default two API replicas")
check(not any(row["published_ports"] for row in baseline), "API must not publish host ports")
wait_lb(args.base_url, 2)
result = {"initial_instances": baseline, "failures": []}
client = Client(args.base_url, timeout=4)
alice, bob, eve, room = client.bootstrap()
key = uuid4().hex
def same_request(_):
response = Client(args.base_url).message(alice, room, "concurrent replay", key=key)
return response["json"], response["headers"]["X-Instance-Id"]
with ThreadPoolExecutor(max_workers=10) as pool:
responses = list(pool.map(same_request, range(30)))
check(
len({row[0]["id"] for row in responses}) == 1, "Cross-replica replay duplicated a message"
)
check(len({row[1] for row in responses}) == 2, "Concurrent replay did not cross real replicas")
result["concurrent_replays"] = 30
result["concurrent_effects"] = 1
try:
for selected in baseline:
selected = instance(selected["id"])
with Traffic(args.base_url, alice, room) as traffic:
time.sleep(1)
action_start = time.monotonic()
env = {**os.environ, "TARGET_INSTANCE_ID": selected["instance"]}
subprocess.run(
[str(ROOT / "scripts/stop-one.sh")],
cwd=ROOT,
env=env,
check=True,
capture_output=True,
timeout=10,
)
action_end = time.monotonic()
check(not inspect(selected["id"])["running"], "Chosen process was not stopped")
time.sleep(3)
report = traffic.report(action_start, action_end)
check(report["first_success_after_action_seconds"] <= 10, "Failover took too long")
check(selected["instance"] not in report["tail_instances"], "Stopped API still serves")
report["stopped"] = selected
traffic.verify_messages(client, bob, room)
result["failures"].append(report)
subprocess.run(["docker", "start", selected["id"]], check=True, capture_output=True)
wait_lb(args.base_url, 2)
print("Verified failure and restoration of one API", flush=True)
with Traffic(args.base_url, alice, room) as traffic:
time.sleep(1)
action_start = time.monotonic()
replaced = subprocess.run(
[
sys.executable,
str(ROOT / "scripts/rolling-update.py"),
"--base-url",
args.base_url,
],
cwd=ROOT,
check=True,
capture_output=True,
text=True,
timeout=120,
)
action_end = time.monotonic()
time.sleep(3)
traffic.verify_messages(client, bob, room)
result["rolling"] = traffic.report(action_start, action_end)
result["rolling"]["replacements"] = json.loads(replaced.stdout)
result["rolling"]["confirmed_messages_preserved"] = True
print("Verified sequential replacement under traffic", flush=True)
(ROOT / "evidence/failover.txt").write_text(public_test("test-failover", args.base_url))
for value in compose("ps", "-aq", "api").split():
if not inspect(value)["running"]:
subprocess.run(["docker", "start", value], check=True, capture_output=True)
wait_lb(args.base_url, 2)
before = {
service: inspect(compose("ps", "-q", service).strip())
for service in ("postgres", "valkey", "s3", "broker", "lb")
}
(ROOT / "evidence/persistence.txt").write_text(
public_test("test-persistence", args.base_url)
)
after = {service: inspect(compose("ps", "-q", service).strip()) for service in before}
check(
all(before[s]["started_at"] != after[s]["started_at"] for s in before),
"Persistence hook did not restart all state/LB services",
)
result["persistence"] = {"before": before, "after": after, "same_token_public_test": True}
result["final_instances"] = instances()
logs = compose("logs", "--no-color", "api", "lb")
check(
not any(
a["token"] in logs or a["registration"]["password"] in logs
for a in (alice, bob, eve)
),
"Credential appeared in logs",
)
args.output.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n")
print("PASS: real replica failures, replay, rolling replacement and persistence")
finally:
for value in compose("ps", "-aq", "api").split():
if not inspect(value)["running"]:
subprocess.run(["docker", "start", value], check=True, capture_output=True)
if __name__ == "__main__":
main()
+513
View File
@@ -0,0 +1,513 @@
#!/usr/bin/env python3
"""Real-process fault matrix for the versioned pipeline; test Compose override required."""
import argparse
import hashlib
import io
import json
import subprocess
import sys
import time
from pathlib import Path
from uuid import uuid4
from botocore.exceptions import ClientError
from cluster import ROOT, compose, inspect, instances, wait_lb
from PIL import Image
from pipeline_lab import settings
from messenger.broker import STREAM, Broker, Job
from messenger.pipeline import Pipeline
from messenger.repository import PostgresRepository, stamp
from messenger.storage import ObjectStore
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, check # noqa: E402
def until(fn, timeout=60):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
result = fn()
if result:
return result
time.sleep(0.2)
raise AssertionError("Timed out waiting for a durable pipeline condition")
def main():
parser = argparse.ArgumentParser(__doc__)
parser.add_argument("--base-url", default="http://127.0.0.1:8086")
parser.add_argument("--output", type=Path, default=ROOT / "evidence/pipeline.json")
args = parser.parse_args()
if args.output.exists():
parser.error("Output already exists; choose a new evidence file")
config = settings()
db = PostgresRepository(config.database_url, pool_size=4)
db.open()
broker, store = Broker(config), ObjectStore(config)
c = Client(args.base_url)
alice, bob, eve, room = c.bootstrap()
records, accepted = [], []
data = (ROOT / "tests/fixtures/sample.png").read_bytes()
def query(sql, params=()):
with db.connection() as conn:
return conn.execute(sql, params).fetchall()
def job(a, version=1):
return query(
"SELECT * FROM attachment_jobs WHERE attachment_id=%s AND pipeline_version=%s",
(a["id"], version),
)[0]
def audit(j):
return query(
"SELECT event,details,created_at FROM pipeline_audit WHERE job_id=%s ORDER BY sequence",
(j["job_id"],),
)
def injected(j, phase):
return any(
e["event"] == "fault_injected" and e["details"]["phase"] == phase for e in audit(j)
)
def fault(j, phase, action, count=1):
with db.connection(write=True) as conn:
conn.execute(
"INSERT INTO pipeline_faults(job_id,phase,action,remaining) VALUES (%s,%s,%s,%s)",
(j["job_id"], phase, action, count),
)
def upload():
a, original = c.attachment(alice, room)
check(original == data, "Unexpected fixture")
accepted.append(a["id"])
return a
def ready(a):
value = c.wait_ready(bob, a)
path = "/api/v1/attachments/" + a["id"]
check(c.call("GET", path + "/content", actor=bob)["bytes"] == data, "Original changed")
for v in value["variants"]:
content = c.call("GET", path + "/content?variant=" + v["name"], actor=bob)["bytes"]
check(hashlib.sha256(content).hexdigest() == v["sha256"], "Variant checksum mismatch")
with Image.open(io.BytesIO(content)) as im:
check(im.size == (v["width"], v["height"]), "Variant dimensions mismatch")
check(len(value["variants"]) == 3, "Missing variants")
check(
len(query("SELECT 1 FROM message_attachments WHERE attachment_id=%s", (a["id"],))) <= 1,
"Unexpected duplicate link",
)
return value
def saved(name, a, start, **details):
j = job(a, details.get("version", 1))
current = query(
"SELECT storage_state,pipeline_version FROM attachments WHERE id=%s", (a["id"],)
)[0]
keys = [
o["Key"]
for o in store.client.list_objects_v2(
Bucket=store.bucket,
Prefix="objects/" + a["id"] + "/v" + str(current["pipeline_version"]) + "/",
).get("Contents", [])
]
check(len(keys) == 4, "Repeated deliveries multiplied the published variant set")
record = {
"scenario": name,
"attachment_id": a["id"],
"job_id": str(j["job_id"]),
"elapsed_seconds": time.monotonic() - start,
"final_state": current["storage_state"],
"attempts": j["attempt"],
"final_version": current["pipeline_version"],
"published_objects": len(keys),
"audit": audit(j),
**details,
}
records.append(record)
print("PASS: " + name, flush=True)
def envelope(j, attempt=None):
return Job(
schema_version=1,
type="attachment.process.v1",
job_id=str(j["job_id"]),
attachment_id=str(j["attachment_id"]),
pipeline_version=j["pipeline_version"],
attempt=attempt or max(1, j["attempt"]),
created_at=stamp(j["created_at"]),
)
try:
# Actual COMMIT/publish gap: no relay exists while the acknowledged upload is stored.
compose("stop", "--timeout", "2", "worker", "relay")
start = time.monotonic()
a = upload()
j = job(a)
check(
query("SELECT published_at FROM job_outbox WHERE job_id=%s", (j["job_id"],))[0][
"published_at"
]
is None,
"Gap was not established",
)
message = c.message(alice, room, "", attachments=[a["id"]])["json"]
c.call("GET", "/api/v1/attachments/" + a["id"] + "/content", 409, actor=bob)
before = instances()
compose("restart", "--timeout", "3", "api")
wait_lb(args.base_url)
after = instances()
check(
{r["instance"] for r in before}.isdisjoint({r["instance"] for r in after}),
"API did not restart",
)
compose("start", "relay")
until(
lambda: query("SELECT published_at FROM job_outbox WHERE job_id=%s", (j["job_id"],))[0][
"published_at"
]
)
entry_ids = [
id
for id, fields in broker.client.xrange(STREAM)
if json.loads(fields["job"])["job_id"] == str(j["job_id"])
]
check(bool(entry_ids), "Broker has no accepted work")
broker_id = compose("ps", "-q", "broker").strip()
broker_before = inspect(broker_id)
subprocess.run(
["docker", "kill", "--signal", "KILL", broker_id], check=True, capture_output=True
)
subprocess.run(["docker", "start", broker_id], check=True, capture_output=True)
def broker_restored():
try:
return broker.client.ping()
except Exception:
return False
until(broker_restored)
check(
set(entry_ids) <= {id for id, _ in broker.client.xrange(STREAM)},
"Durable stream lost confirmed work",
)
compose("start", "worker")
ready(a)
history = c.call(
"GET", "/api/v1/chats/" + room["id"] + "/messages", actor=bob, schema="MessagePage"
)["json"]
check(message in history["items"], "Queued attachment message lost")
saved(
"commit_publish_gap_api_restart_broker_sigkill",
a,
start,
api_before=before,
api_after=after,
broker_before=broker_before,
broker_after=inspect(broker_id),
broker_entry_ids=entry_ids,
message_id=message["id"],
)
# A publisher can lose its COMMIT after XADD; same logical ID is sent twice.
compose("stop", "--timeout", "2", "worker", "relay")
start = time.monotonic()
a = upload()
j = job(a)
fault(j, "after_publish", "crash")
compose("start", "relay")
until(lambda: injected(j, "after_publish"))
until(
lambda: (
sum(
json.loads(f["job"])["job_id"] == str(j["job_id"])
for _, f in broker.client.xrange(STREAM)
)
>= 2
)
)
compose("start", "worker")
ready(a)
until(lambda j=j: any(e["event"] == "duplicate_ready" for e in audit(j)))
saved("publish_before_outbox_commit_duplicate", a, start)
for phase in ("before_effect", "after_effect", "after_commit"):
compose("stop", "--timeout", "2", "worker")
start = time.monotonic()
a = upload()
j = job(a)
fault(j, phase, "crash")
compose("start", "worker")
ready(a)
until(lambda j=j, phase=phase: injected(j, phase))
if phase == "after_commit":
until(lambda j=j: any(e["event"] == "duplicate_ready" for e in audit(j)))
check(job(a)["attempt"] == 1, "Committed job was processed twice")
else:
check(job(a)["attempt"] == 2, "Expired lease did not recover exactly once")
saved("worker_crash_" + phase, a, start)
for action, count, expected in (
("transient", 2, "ready"),
("hang", 1, "ready"),
("permanent", 1, "failed"),
("transient", 5, "failed"),
):
compose("stop", "--timeout", "2", "worker")
start = time.monotonic()
a = upload()
j = job(a)
fault(j, "before_effect", action, count)
compose("start", "worker")
path = "/api/v1/attachments/" + a["id"]
if expected == "failed":
until(
lambda path=path: (
c.call("GET", path, actor=bob, schema="Attachment")["json"]["status"]
== "failed"
)
)
terminal = job(a)
check(terminal["attempt"] == count, "Wrong retry budget")
c.call("GET", path + "/content", 409, actor=bob)
c.call(
"POST",
"/api/v1/chats/" + room["id"] + "/messages",
409,
actor=alice,
body={"text": "", "attachment_ids": [a["id"]]},
headers={"Idempotency-Key": uuid4().hex},
)
# Explicit CLI replay, same logical job; no public administrator endpoint.
replay = json.loads(
compose(
"exec",
"-T",
"worker",
"python",
"-m",
"messenger.operator",
"replay",
a["id"],
)
)
check(replay["job_id"] == str(j["job_id"]), "Replay changed logical job")
else:
terminal = None
ready(a)
events = audit(j)
delays = [e["details"]["delay_seconds"] for e in events if e["event"] == "queued"]
check(all(0.25 <= delay <= 30 for delay in delays), "Invalid bounded jitter")
rejected = subprocess.run(
[
"docker",
"compose",
"--env-file",
".env",
"exec",
"-T",
"worker",
"python",
"-m",
"messenger.operator",
"replay",
a["id"],
],
cwd=ROOT,
capture_output=True,
)
check(rejected.returncode != 0, "Ready replay was accepted")
saved(
action + "_" + str(count) + ("_failed_then_explicit_replay" if terminal else ""),
a,
start,
terminal_before_replay={k: terminal[k] for k in ("state", "attempt", "error_code")}
if terminal
else None,
backoff_delays=delays,
)
# Reorder and stale v1 delivered after v2; exercise the DB publication fence directly too.
compose("stop", "--timeout", "2", "worker", "relay")
start = time.monotonic()
a = upload()
old = job(a)
old_envelope = envelope(old, 1)
outcome, old_claim = Pipeline(db, config).claim(old_envelope)
check(outcome == "claimed", "Cannot establish old writer")
Pipeline(db, config).replay(a["id"], new_version=True)
new = job(a, 2)
# Publish v2 first, then v1, plus duplicates through both worker slots.
for value in (envelope(new, 1), old_envelope, envelope(new, 1), old_envelope):
broker.publish(value)
compose("start", "worker", "relay")
current = ready(a)
row = db.attachment(a["id"])
check(
not Pipeline(db, config).publish(old_claim, "objects/old-writer", []),
"Old writer overwrote v2",
)
for _ in range(20):
broker.publish(old_envelope)
broker.publish(envelope(new, 1))
until(lambda: sum(e["event"] == "stale_delivery" for e in audit(old)) >= 21)
until(lambda: sum(e["event"] == "duplicate_ready" for e in audit(new)) >= 20)
check(
db.attachment(a["id"])["object_key"] == row["object_key"],
"Stale delivery changed current pointer",
)
check(
c.call("GET", "/api/v1/attachments/" + a["id"], actor=bob, schema="Attachment")["json"]
== current,
"Old version mutated ready metadata",
)
saved(
"reorder_stale_v1_after_v2_and_publication_fence",
a,
start,
version=2,
old_audit=audit(old),
duplicates=44,
)
# ACL remains enforced through both API replicas; S3 is private and role-scoped.
a = {"id": accepted[0]}
for suffix in ("", "/content", "/content?variant=crop"):
c.call("GET", "/api/v1/attachments/" + a["id"] + suffix, 404, actor=eve)
# Direct bucket access and write/delete privilege checks use no public credentials.
import boto3
from botocore import UNSIGNED
from botocore.config import Config
first = db.attachment(accepted[0])
unsigned = boto3.client(
"s3", endpoint_url=config.s3_endpoint, config=Config(signature_version=UNSIGNED)
)
try:
unsigned.get_object(Bucket=store.bucket, Key=first["object_key"])
raise AssertionError("Anonymous S3 read succeeded")
except ClientError as exc:
check(
exc.response["ResponseMetadata"]["HTTPStatusCode"] == 403, "Wrong anonymous status"
)
finally:
unsigned.close()
denied = []
for role, action in (
("api", "put_object"),
("api", "delete_object"),
("worker", "delete_object"),
("reconciler", "put_object"),
):
scoped = ObjectStore(settings(role))
try:
kw = {"Bucket": store.bucket, "Key": "objects/" + uuid4().hex + "/forbidden"}
if action == "put_object":
kw["Body"] = b"forbidden"
try:
getattr(scoped.client, action)(**kw)
raise AssertionError("IAM action unexpectedly allowed")
except ClientError as exc:
check(
exc.response["ResponseMetadata"]["HTTPStatusCode"] == 403,
"Wrong IAM status",
)
denied.append(role + ":" + action)
finally:
scoped.close()
# Stop the actual uploader, then read original and thumbnail through another replica.
response = c.call(
"POST",
"/api/v1/chats/" + room["id"] + "/attachments?filename=cross.png",
202,
actor=alice,
raw=data,
headers={"Content-Type": "image/png"},
schema="Attachment",
)
cross = response["json"]
accepted.append(cross["id"])
ready(cross)
uploader = response["headers"]["X-Instance-Id"]
process = next(v for v in instances() if v["instance"] == uploader)
subprocess.run(
["docker", "stop", "--time", "3", process["id"]], check=True, capture_output=True
)
try:
wait_lb(args.base_url, count=1)
download = c.call("GET", "/api/v1/attachments/" + cross["id"] + "/content", actor=bob)
check(
download["bytes"] == data and download["headers"]["X-Instance-Id"] != uploader,
"Other replica could not serve the original",
)
c.call(
"GET",
"/api/v1/attachments/" + cross["id"] + "/content?variant=thumbnail",
actor=bob,
)
cross_proof = {
"uploader": uploader,
"stopped": inspect(process["id"]),
"reader": download["headers"]["X-Instance-Id"],
}
finally:
subprocess.run(["docker", "start", process["id"]], check=True, capture_output=True)
wait_lb(args.base_url)
logs = compose("logs", "--no-color", "api", "worker", "relay", "lb")
check(
not any(
actor["token"] in logs or actor["registration"]["password"] in logs
for actor in (alice, bob, eve)
),
"Credential in logs",
)
states = query(
"SELECT id,storage_state FROM attachments WHERE id=ANY(%s::uuid[])", (accepted,)
)
check(
len(states) == len(accepted) and all(s["storage_state"] == "ready" for s in states),
"Confirmed work lost",
)
stats = compose(
"exec",
"-T",
"worker",
"python",
"-c",
"import json;from pathlib import Path;print(json.dumps({"
"n:Path('/sys/fs/cgroup/'+n).read_text().strip() "
"for n in ('memory.current','memory.peak','memory.events','cpu.stat')}))",
)
result = {
"anonymous_s3_status": 403,
"denied_iam_actions": denied,
"cross_replica": cross_proof,
"track": 2,
"delivery": "at-least-once",
"effect_key": ["attachment_id", "pipeline_version", "variant"],
"worker_concurrency": config.worker_concurrency,
"prefetch_per_slot": 1,
"job_timeout_seconds": config.job_timeout,
"lease_seconds": config.lease_seconds,
"max_attempts": config.job_max_attempts,
"scenarios": records,
"accepted": len(accepted),
"lost_confirmed": 0,
"stale_overwrites": 0,
"worker_cgroup": json.loads(stats),
"logs_credentials_absent": True,
}
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps(result, ensure_ascii=False, indent=2, default=str) + "\n")
print("PASS: all acknowledged work ready; no stale overwrites", flush=True)
finally:
compose("start", "worker", "relay", "broker", "api")
broker.close()
store.close()
db.close()
if __name__ == "__main__":
main()
+1
View File
@@ -0,0 +1 @@
"""One-process implementation of the lab 1 messenger contract."""
+27
View File
@@ -0,0 +1,27 @@
import logging
import os
import uvicorn
from messenger import telemetry
def main():
level = os.getenv("APP_LOG_LEVEL", "INFO").upper()
logging.basicConfig(level=level, format="%(message)s")
with telemetry.active("api"):
uvicorn.run(
"messenger.app:create_app",
factory=True,
host=os.getenv("APP_HOST", "0.0.0.0"),
port=int(os.getenv("APP_PORT", "8080")),
workers=1,
access_log=False,
log_level=level.lower(),
timeout_keep_alive=2,
timeout_graceful_shutdown=5,
)
if __name__ == "__main__":
main()
+350
View File
@@ -0,0 +1,350 @@
"""HTTP adapter. The instructor's OpenAPI, not FastAPI defaults, defines errors."""
import json
import logging
import os
import re
from contextlib import asynccontextmanager
from time import perf_counter
from typing import Annotated
from urllib.parse import quote
from uuid import uuid4
from fastapi import Depends, FastAPI, Request
from fastapi.exceptions import RequestValidationError
from fastapi.responses import JSONResponse, Response, StreamingResponse
from opentelemetry.context import Context
from opentelemetry.trace import INVALID_SPAN, SpanKind
from starlette.exceptions import HTTPException
from starlette.types import ASGIApp, Message, Receive, Scope, Send
from messenger import telemetry
from messenger.attachments import Attachments
from messenger.auth import AuthService, Identity, basic_credentials, bearer_digest, unauthorized
from messenger.cache import SessionCache
from messenger.domain import DomainError, User, invalid, not_found
from messenger.repository import PostgresRepository
from messenger.schemas import (
AttachmentView,
ChatPage,
ChatView,
CreateChat,
CreateMessage,
CreateUser,
MessagePage,
MessageView,
SessionView,
UserView,
canonical_uuid,
)
from messenger.service import Messenger
from messenger.settings import Settings
from messenger.storage import ObjectStore
logger = logging.getLogger("messenger")
class RequestContext:
def __init__(self, app: ASGIApp, instance_id: str):
self.app = app
self.instance_id = instance_id
async def __call__(self, scope: Scope, receive: Receive, send: Send):
if scope["type"] != "http":
return await self.app(scope, receive, send)
if scope.get("path") in {"/health/live", "/health/ready"}:
await self.request(scope, receive, send, INVALID_SPAN)
return
with telemetry.span("http.request", context=Context(), kind=SpanKind.SERVER) as span:
await self.request(scope, receive, send, span)
async def request(self, scope, receive, send, span):
request_id = str(uuid4())
scope.setdefault("state", {})["request_id"] = request_id
start, status, response_started = perf_counter(), 500, False
async def with_id(message: Message):
nonlocal status, response_started
if message["type"] == "http.response.start":
response_started = True
status = message["status"]
message["headers"] = [
pair
for pair in message.get("headers", [])
if pair[0].lower() not in (b"x-request-id", b"x-instance-id")
] + [
(b"x-request-id", request_id.encode()),
(b"x-instance-id", self.instance_id.encode()),
]
await send(message)
try:
await self.app(scope, receive, with_id)
except Exception as exc:
# Do not let Uvicorn print exception values that could contain user data.
if response_started:
raise
logger.error(json.dumps({"event": "internal_error", "type": type(exc).__name__}))
response = error_response(
Request(scope), 500, "internal_error", "Внутренняя ошибка сервера"
)
await response(scope, receive, with_id)
finally:
route = getattr(scope.get("route"), "path", "unmatched")
if route not in {"/health/live", "/health/ready"}:
telemetry.http(
span, scope["method"], route, status, perf_counter() - start, request_id
)
def error_response(
request: Request, status: int, code: str, message: str, headers: dict | None = None
) -> JSONResponse:
identity = getattr(request.state, "request_id", str(uuid4()))
return JSONResponse(
status_code=status,
content={"error": {"code": code, "message": message, "request_id": identity}},
headers={"X-Request-Id": identity, **(headers or {})},
)
def create_app(settings: Settings | None = None) -> FastAPI:
settings = settings or Settings.from_env()
repository = PostgresRepository(
settings.database_url,
settings.pool_size,
settings.idempotency_ttl,
settings.read_autocommit,
)
cache = SessionCache(settings.cache_url)
store = ObjectStore(settings)
authentication = AuthService(repository, cache, settings.session_ttl)
@asynccontextmanager
async def lifespan(app: FastAPI):
repository.open()
app.state.ready = True
logger.info(json.dumps({"event": "started", "environment": os.getenv("APP_ENV", "lab")}))
try:
yield
finally:
app.state.ready = False
repository.close()
cache.close()
store.close()
logger.info(json.dumps({"event": "stopped"}))
app = FastAPI(lifespan=lifespan, docs_url=None, redoc_url=None, openapi_url=None)
app.state.instance_id = uuid4().hex
app.add_middleware(RequestContext, instance_id=app.state.instance_id)
app.state.messenger = Messenger(repository)
app.state.attachments = Attachments(repository, app.state.messenger, store, settings)
app.state.repository = repository
app.state.authentication = authentication
app.state.cache = cache
app.state.ready = False
@app.exception_handler(DomainError)
async def domain_error(request: Request, exc: DomainError):
return error_response(request, exc.status, exc.code, exc.message, exc.headers)
@app.exception_handler(RequestValidationError)
async def validation_error(request: Request, exc: RequestValidationError):
return error_response(request, 400, "invalid_request", "Некорректные поля или JSON")
@app.exception_handler(HTTPException)
async def http_error(request: Request, exc: HTTPException):
code = "not_found" if exc.status_code == 404 else "invalid_request"
return error_response(request, exc.status_code, code, "Некорректный HTTP-запрос")
@app.exception_handler(Exception)
async def unexpected_error(request: Request, exc: Exception):
logger.error(json.dumps({"event": "internal_error", "type": type(exc).__name__}))
return error_response(request, 500, "internal_error", "Внутренняя ошибка сервера")
def service(request: Request) -> Messenger:
return request.app.state.messenger
def token_digest(request: Request) -> bytes:
if len(request.headers.getlist("authorization")) != 1:
raise unauthorized()
return bearer_digest(request.headers.get("authorization"))
def identity(
digest: Annotated[bytes, Depends(token_digest)],
) -> Identity:
return authentication.identify(digest)
def actor(current: Annotated[Identity, Depends(identity)]) -> User:
return current.user
async def credentials(request: Request) -> tuple[str, str]:
if len(request.headers.getlist("authorization")) != 1:
raise unauthorized("Basic")
result = basic_credentials(request.headers.get("authorization"))
if await request.body():
raise invalid("Вход не принимает тело запроса")
return result
def resource_id(chat_id: str) -> str:
try:
return canonical_uuid(chat_id)
except ValueError:
raise invalid("Некорректный UUID чата") from None
def pagination(request: Request) -> tuple[int, str | None]:
for key in ("limit", "cursor"):
if len(request.query_params.getlist(key)) > 1:
raise invalid("Параметр пагинации указан несколько раз")
raw = request.query_params.get("limit", "50")
try:
if not re.fullmatch(r"[0-9]+", raw):
raise ValueError
limit = int(raw)
if not 1 <= limit <= 100:
raise ValueError
except ValueError:
raise invalid("limit должен быть целым от 1 до 100") from None
return limit, request.query_params.get("cursor")
def json_input(request: Request):
media = request.headers.get("content-type", "").split(";", 1)[0].strip().lower()
if media != "application/json":
raise DomainError(415, "unsupported_media_type", "Требуется application/json")
def message_key(request: Request) -> str:
values = request.headers.getlist("idempotency-key")
if len(values) != 1 or not re.fullmatch(r"[A-Za-z0-9._:-]{1,128}", values[0]):
raise invalid("Требуется корректный Idempotency-Key")
return values[0]
CurrentUser = Annotated[User, Depends(actor)]
Service = Annotated[Messenger, Depends(service)]
ChatId = Annotated[str, Depends(resource_id)]
Pagination = Annotated[tuple[int, str | None], Depends(pagination)]
@app.get("/health/live")
def live():
return {"status": "ok"}
@app.get("/health/ready")
def ready(request: Request):
if not request.app.state.ready:
raise DomainError(503, "unavailable", "Приложение не готово")
repository.ready()
store.ready()
return {"status": "ok"}
@app.post(
"/api/v1/users",
status_code=201,
response_model=UserView,
dependencies=[Depends(json_input)],
)
def register(body: CreateUser):
return authentication.register(body.username, body.display_name, body.password)
@app.post("/api/v1/auth/sessions", status_code=201, response_model=SessionView)
def login(value: Annotated[tuple[str, str], Depends(credentials)]):
return authentication.login(*value)
@app.delete("/api/v1/auth/sessions/current", status_code=204)
def logout(current: Annotated[Identity, Depends(identity)]):
authentication.logout(current)
return Response(status_code=204)
@app.get("/api/v1/users/me", response_model=UserView)
def me(current: CurrentUser):
return current
@app.post(
"/api/v1/chats",
status_code=201,
response_model=ChatView,
dependencies=[Depends(json_input)],
)
def create_chat(body: CreateChat, current: CurrentUser, messenger: Service):
return messenger.create_chat(current, body.title, body.member_ids)
@app.get("/api/v1/chats", response_model=ChatPage)
def chats(current: CurrentUser, messenger: Service, page: Pagination):
return messenger.chats(current, *page)
@app.get("/api/v1/chats/{chat_id}", response_model=ChatView)
def chat(current: CurrentUser, chat: ChatId, messenger: Service):
return messenger.chat(current, chat)
@app.post(
"/api/v1/chats/{chat_id}/messages",
status_code=201,
response_model=MessageView,
dependencies=[Depends(json_input)],
)
def send_message(
body: CreateMessage,
current: CurrentUser,
chat: ChatId,
messenger: Service,
key: Annotated[str, Depends(message_key)],
):
return messenger.send(current, chat, body.text, key, tuple(body.attachment_ids))
@app.get("/api/v1/chats/{chat_id}/messages", response_model=MessagePage)
def messages(current: CurrentUser, chat: ChatId, messenger: Service, page: Pagination):
return messenger.messages(current, chat, *page)
def attachment_id(attachment_id: str) -> str:
try:
return canonical_uuid(attachment_id)
except ValueError:
raise invalid("Некорректный UUID вложения") from None
AttachmentId = Annotated[str, Depends(attachment_id)]
@app.post("/api/v1/chats/{chat_id}/attachments", status_code=202, response_model=AttachmentView)
async def upload(request: Request, current: CurrentUser, chat: ChatId):
return await app.state.attachments.upload(request, current, chat)
@app.get("/api/v1/attachments/{attachment_id}", response_model=AttachmentView)
def attachment(current: CurrentUser, id: AttachmentId):
return app.state.attachments.metadata(current, id)
@app.get("/api/v1/attachments/{attachment_id}/content")
def content(request: Request, current: CurrentUser, id: AttachmentId):
row = app.state.attachments.record(current, id)
if row["storage_state"] != "ready":
raise DomainError(409, "conflict", "Вложение пока недоступно")
values = request.query_params.getlist("variant")
if (
len(values) > 1
or values
and values[0] not in ("original", "thumbnail", "crop", "optimized")
):
raise invalid("Некорректный вариант вложения")
if values and values[0] != "original":
row = next((v for v in row["variants"] if v["name"] == values[0]), None)
if row is None:
raise not_found()
row = {**row, "filename": values[0]}
body = store.download(row)
def chunks():
try:
while data := body.read(65536):
yield data
finally:
body.close()
return StreamingResponse(
chunks(),
media_type=row["content_type"],
headers={
"Content-Length": str(row["size_bytes"]),
"Content-Disposition": "attachment; filename*=UTF-8''"
+ quote(row["filename"], safe=""),
"Cache-Control": "private, no-store",
"X-Content-Type-Options": "nosniff",
},
)
return app
+127
View File
@@ -0,0 +1,127 @@
"""Bounded upload: verified staging and transactional job intent precede HTTP 202."""
import asyncio
import hashlib
import tempfile
import unicodedata
import warnings
from anyio import to_thread
from PIL import Image, UnidentifiedImageError
from starlette.requests import ClientDisconnect
from messenger.domain import DomainError, invalid, new_id, not_found
from messenger.pipeline import Pipeline
from messenger.repository import attachment_value
Image.MAX_IMAGE_PIXELS = 40_000_000
warnings.filterwarnings("error", category=Image.DecompressionBombWarning)
def filename(value):
if (
not value
or not value.strip()
or len(value) > 255
or any(c in "/\\" or unicodedata.category(c).startswith("C") for c in value)
):
raise invalid("Некорректное имя файла")
return value
def image_type(file, maximum):
try:
file.seek(0)
with Image.open(file) as image:
if image.width * image.height > maximum:
raise DomainError(413, "payload_too_large", "Слишком много пикселей")
if image.format not in ("PNG", "JPEG"):
raise ValueError
media = "image/png" if image.format == "PNG" else "image/jpeg"
image.verify()
file.seek(0)
with Image.open(file) as image:
image.load() # Reject truncated pixel streams, not just an intact header.
file.seek(0)
return media
except Image.DecompressionBombError, Image.DecompressionBombWarning:
raise DomainError(413, "payload_too_large", "Слишком много пикселей") from None
except UnidentifiedImageError, OSError, ValueError, SyntaxError:
raise DomainError(415, "unsupported_media_type", "Повреждённое изображение") from None
class Attachments:
def __init__(self, repository, messenger, store, settings):
self.repository, self.messenger, self.store = repository, messenger, store
self.settings = settings
self.slots = asyncio.Semaphore(settings.upload_slots)
async def upload(self, request, actor, chat_id):
await to_thread.run_sync(self.messenger.chat, actor, chat_id)
values = request.query_params.getlist("filename")
if len(values) != 1:
raise invalid("Требуется одно имя файла")
name = filename(values[0])
media = request.headers.get("content-type", "").split(";", 1)[0].strip().lower()
if media not in ("image/png", "image/jpeg"):
raise DomainError(415, "unsupported_media_type", "Требуется PNG или JPEG")
length = request.headers.get("content-length")
if length is not None:
if not length.isascii() or not length.isdecimal():
raise invalid("Некорректный Content-Length")
if int(length) > self.settings.max_upload_bytes:
raise DomainError(413, "payload_too_large", "Файл больше 10 MiB")
try:
await asyncio.wait_for(self.slots.acquire(), timeout=5)
except TimeoutError:
raise DomainError(429, "rate_limited", "Слишком много загрузок") from None
try:
with tempfile.TemporaryFile() as file:
size, digest = 0, hashlib.sha256()
try:
async with asyncio.timeout(15):
async for chunk in request.stream():
size += len(chunk)
if size > self.settings.max_upload_bytes:
raise DomainError(413, "payload_too_large", "Файл больше 10 MiB")
digest.update(chunk)
await to_thread.run_sync(file.write, chunk)
except TimeoutError:
raise DomainError(400, "invalid_request", "Время загрузки истекло") from None
except ClientDisconnect:
raise invalid("Загрузка прервана") from None
actual = await to_thread.run_sync(image_type, file, self.settings.max_image_pixels)
if actual != media:
raise DomainError(415, "unsupported_media_type", "MIME не совпадает с файлом")
id = new_id()
key = "staging/" + id
await to_thread.run_sync(Pipeline(self.repository, self.settings).reserve, id, key)
await to_thread.run_sync(
self.store.put, key, file, size, actual, digest.hexdigest()
)
# A failure before this COMMIT leaves only an unaccepted staging orphan.
# The same transaction creates attachment, job and outbox intent.
row = await to_thread.run_sync(
Pipeline(self.repository, self.settings).accept,
id,
chat_id,
actor.id,
name,
actual,
size,
digest.hexdigest(),
key,
)
return attachment_value(row)
finally:
self.slots.release()
def record(self, actor, id):
row = self.repository.attachment(id)
if row is None:
raise not_found()
self.messenger.chat(actor, str(row["chat_id"]))
return row
def metadata(self, actor, id):
return attachment_value(self.record(actor, id))
+127
View File
@@ -0,0 +1,127 @@
"""Argon2id passwords and opaque sessions. Cache hits never authorize a request."""
import base64
import binascii
import hashlib
import re
import secrets
from contextlib import contextmanager
from dataclasses import dataclass
from threading import BoundedSemaphore
from argon2 import PasswordHasher
from argon2.exceptions import InvalidHashError, VerificationError
from argon2.profiles import RFC_9106_LOW_MEMORY
from messenger.cache import SessionCache
from messenger.domain import DomainError, User
from messenger.repository import PostgresRepository
def unauthorized(scheme: str = "Bearer") -> DomainError:
challenge = scheme + ' realm="messenger"'
if scheme == "Basic":
challenge += ', charset="UTF-8"'
return DomainError(
401,
"unauthorized",
"Требуется действительная идентификация",
{"WWW-Authenticate": challenge},
)
def basic_credentials(header: str | None) -> tuple[str, str]:
try:
if header is None or len(header) > 2048:
raise ValueError
scheme, encoded = header.split(" ", 1)
if scheme.lower() != "basic":
raise ValueError
raw = base64.b64decode(encoded, validate=True).decode("utf-8")
username, password = raw.split(":", 1)
if not re.fullmatch(r"[a-z0-9_]{3,32}", username):
raise ValueError
if not 12 <= len(password) <= 128 or not password.strip():
raise ValueError
return username, password
except ValueError, UnicodeError, binascii.Error:
raise unauthorized("Basic") from None
def bearer_digest(header: str | None) -> bytes:
if header:
parts = header.split(" ")
if len(parts) == 2 and parts[0].lower() == "bearer":
if re.fullmatch(r"[A-Za-z0-9_-]{43}", parts[1]):
return hashlib.sha256(parts[1].encode()).digest()
raise unauthorized()
@dataclass(frozen=True)
class Identity:
user: User
token_hash: bytes
@dataclass(frozen=True)
class Session:
token: str
token_type: str
expires_at: str
user: User
class AuthService:
def __init__(self, repository: PostgresRepository, cache: SessionCache, ttl: int):
self.repository, self.cache, self.ttl = repository, cache, ttl
self.hasher = PasswordHasher.from_parameters(RFC_9106_LOW_MEMORY)
self.kdf_slots = BoundedSemaphore(2)
self.dummy_hash = self.hasher.hash(secrets.token_urlsafe(32))
@contextmanager
def kdf(self):
if not self.kdf_slots.acquire(timeout=0.5):
raise DomainError(503, "unavailable", "Обработка пароля временно занята")
try:
yield
finally:
self.kdf_slots.release()
def register(self, username: str, display_name: str, password: str) -> User:
with self.kdf():
encoded = self.hasher.hash(password)
return self.repository.add_user(username, display_name, encoded)
def login(self, username: str, password: str) -> Session:
credentials = self.repository.credentials(username)
encoded = credentials[1] if credentials else self.dummy_hash
try:
with self.kdf():
self.hasher.verify(encoded, password)
except VerificationError, InvalidHashError:
raise unauthorized("Basic") from None
if credentials is None:
raise unauthorized("Basic")
token = secrets.token_urlsafe(32)
digest = hashlib.sha256(token.encode()).digest()
user = credentials[0]
expires = self.repository.create_session(digest, user.id, self.ttl)
return Session(token, "Bearer", expires, user)
def identify(self, digest: bytes) -> Identity:
session = self.repository.active_session(digest)
if session is None:
raise unauthorized()
user_id = str(session["user_id"])
user = self.cache.get(digest, user_id)
if user is None:
user = self.repository.get_user(user_id)
if user is None:
raise unauthorized()
ttl = int((session["expires_at"] - session["checked_at"]).total_seconds())
self.cache.put(digest, user, ttl)
return Identity(user, digest)
def logout(self, identity: Identity):
self.repository.revoke_session(identity.token_hash)
self.cache.delete(identity.token_hash)
+96
View File
@@ -0,0 +1,96 @@
"""Dedicated durable Valkey Stream, one consumer group; never evict/trim pending work."""
import re
from datetime import datetime
from typing import Literal
from uuid import UUID
import redis
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
STREAM = "attachment-jobs"
GROUP = "image-workers-v1"
DLQ = "invalid-jobs"
class Job(BaseModel):
model_config = ConfigDict(extra="forbid", strict=True)
schema_version: Literal[1]
job_id: str
type: Literal["attachment.process.v1"]
attachment_id: str
pipeline_version: int = Field(ge=1)
attempt: int = Field(ge=1)
created_at: str
traceparent: str | None = Field(
default=None, pattern=r"^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$"
)
tracestate: str | None = Field(default=None, max_length=512)
@model_validator(mode="before")
@classmethod
def strict_carrier(cls, value):
if isinstance(value, dict):
if type(value.get("schema_version")) is not int:
raise ValueError("Integer schema version required")
if any(k in value and value[k] is None for k in ("traceparent", "tracestate")):
raise ValueError("Optional carrier fields must be strings when provided")
return value
@field_validator("job_id", "attachment_id")
@classmethod
def uuid(cls, value):
if str(UUID(value)) != value:
raise ValueError("Canonical UUID required")
return value
@field_validator("created_at")
@classmethod
def timestamp(cls, value):
if not re.fullmatch(r"\d{4}-\d\d-\d\dT.+(?:Z|[+-]\d\d:\d\d)", value):
raise ValueError("RFC3339 required")
datetime.fromisoformat(value.replace("Z", "+00:00"))
return value
class Broker:
def __init__(self, settings):
self.client = redis.Redis.from_url(
settings.broker_url,
decode_responses=True,
socket_timeout=2,
socket_connect_timeout=1,
retry_on_timeout=False,
)
def group(self):
try:
self.client.xgroup_create(STREAM, GROUP, id="0", mkstream=True)
except redis.ResponseError as exc:
if "BUSYGROUP" not in str(exc):
raise
def publish(self, job):
# XADD reply is the publisher boundary; appendfsync=always precedes that reply.
return self.client.xadd(STREAM, {"job": job.model_dump_json(exclude_none=True)})
def ack(self, id):
# A single consumer group only. Atomically remove completed entries, never MAXLEN.
self.client.eval(
"redis.call('XACK',KEYS[1],ARGV[1],ARGV[2]); return redis.call('XDEL',KEYS[1],ARGV[2])",
1,
STREAM,
GROUP,
id,
)
def quarantine(self, id, reason):
# Never copy arbitrary attacker-supplied fields into DLQ/logs.
with self.client.pipeline(transaction=True) as tx:
tx.xadd(DLQ, {"stream_id": id, "reason": reason}, maxlen=1000)
tx.xack(STREAM, GROUP, id)
tx.xdel(STREAM, id)
tx.execute()
def close(self):
self.client.close()
+57
View File
@@ -0,0 +1,57 @@
"""Disposable session-user cache; PostgreSQL always decides token validity."""
import json
import logging
from dataclasses import asdict
from redis import Redis
from redis.backoff import NoBackoff
from redis.exceptions import RedisError
from redis.retry import Retry
from messenger.domain import User
logger = logging.getLogger("messenger")
class SessionCache:
def __init__(self, url: str):
self.client = Redis.from_url(
url,
socket_connect_timeout=0.2,
socket_timeout=0.2,
max_connections=8,
decode_responses=True,
retry_on_timeout=False,
retry=Retry(NoBackoff(), 0),
)
@staticmethod
def key(digest: bytes) -> str:
return "session-user:" + digest.hex()
def get(self, digest: bytes, user_id: str) -> User | None:
try:
raw = self.client.get(self.key(digest))
if raw:
value = User(**json.loads(raw))
if value.id == user_id:
return value
except RedisError, ValueError, TypeError:
logger.warning('{"event":"session_cache_unavailable","operation":"get"}')
return None
def put(self, digest: bytes, user: User, ttl: int):
try:
self.client.set(self.key(digest), json.dumps(asdict(user)), ex=max(1, ttl))
except RedisError:
logger.warning('{"event":"session_cache_unavailable","operation":"set"}')
def delete(self, digest: bytes):
try:
self.client.delete(self.key(digest))
except RedisError:
logger.warning('{"event":"session_cache_unavailable","operation":"delete"}')
def close(self):
self.client.close()
+122
View File
@@ -0,0 +1,122 @@
"""Dry-run by default. Retention GC protects all active jobs and every published object."""
import argparse
import json
from datetime import UTC, datetime
from uuid import UUID
from messenger.pipeline import event
from messenger.repository import PostgresRepository
from messenger.settings import Settings
from messenger.storage import ObjectStore
def collect(db, store, retention, apply=False):
if retention < 300:
raise ValueError("Retention must be at least 300 seconds")
counts = {"scanned": 0, "eligible": 0, "deleted": 0, "protected": 0}
# Serialize collectors; API handoff is bounded < retention and all owned keys have UUID prefix.
with db.connection() as lock:
if not lock.execute("SELECT pg_try_advisory_lock(2301005) AS ok").fetchone()["ok"]:
return {**counts, "busy": True}
try:
for page in store.client.get_paginator("list_objects_v2").paginate(Bucket=store.bucket):
for obj in page.get("Contents", []):
counts["scanned"] += 1
key = obj["Key"]
parts = key.split("/")
try:
if (
parts[0] not in ("staging", "objects")
or str(UUID(parts[1])) != parts[1]
):
continue
except ValueError, IndexError:
continue
if (datetime.now(UTC) - obj["LastModified"]).total_seconds() < retention:
continue
with db.connection(write=True) as conn:
reservation = conn.execute(
"SELECT * FROM upload_reservations WHERE id=%s FOR UPDATE", (parts[1],)
).fetchone()
if (
reservation
and reservation["state"] == "uploading"
and reservation["expires_at"] > datetime.now(UTC)
):
counts["protected"] += 1
continue
# This row lock serializes replay/new-version with deletion eligibility.
a = conn.execute(
"SELECT * FROM attachments WHERE id=%s FOR UPDATE", (parts[1],)
).fetchone()
active = conn.execute(
"SELECT 1 FROM attachment_jobs WHERE attachment_id=%s AND "
"state IN ('queued','processing') LIMIT 1",
(parts[1],),
).fetchone()
published = (
a
and a["storage_state"] == "ready"
and (
key == a["object_key"]
or any(v["object_key"] == key for v in a["variants"])
)
)
recent_failed = conn.execute(
"SELECT 1 FROM attachment_jobs WHERE attachment_id=%s AND "
"state='failed' "
"AND updated_at>clock_timestamp()-(%s*interval '1 second') LIMIT 1",
(parts[1], retention),
).fetchone()
if active or published or recent_failed:
counts["protected"] += 1
continue
head = store.head(key)
if (
not head
or head["ETag"] != obj["ETag"]
or head["ContentLength"] != obj["Size"]
or (datetime.now(UTC) - head["LastModified"]).total_seconds()
< retention
):
continue
counts["eligible"] += 1
if apply:
if reservation and reservation["state"] == "uploading":
conn.execute(
"UPDATE upload_reservations SET state='abandoned' WHERE id=%s",
(parts[1],),
)
store.client.delete_object(Bucket=store.bucket, Key=key)
counts["deleted"] += 1
rows = conn.execute(
"UPDATE attachment_jobs SET source_expired=true WHERE "
"source_key=%s "
"AND state='failed' RETURNING job_id",
(key,),
).fetchall()
for row in rows:
event(conn, row["job_id"], "source_expired")
finally:
lock.execute("SELECT pg_advisory_unlock(2301005)")
return counts
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--apply", action="store_true")
args = parser.parse_args()
settings = Settings.from_env()
db = PostgresRepository(settings.database_url, pool_size=2)
db.open()
store = ObjectStore(settings)
try:
print(json.dumps(collect(db, store, settings.staging_retention, args.apply)))
finally:
store.close()
db.close()
if __name__ == "__main__":
main()
+41
View File
@@ -0,0 +1,41 @@
"""Versioned keyset position, bound to a list and its resource, never an offset."""
import base64
import binascii
import json
from datetime import UTC, datetime
from uuid import UUID
from messenger.domain import HistoryKey, invalid
def encode_cursor(scope: str, position: HistoryKey) -> str:
value = {"v": 1, "scope": scope, "created_at": position[0], "id": position[1]}
return (
base64.urlsafe_b64encode(json.dumps(value, separators=(",", ":")).encode())
.decode()
.rstrip("=")
)
def decode_cursor(value: str | None, scope: str) -> HistoryKey | None:
if value is None:
return None
try:
if not value or len(value) > 1024:
raise ValueError
raw = base64.b64decode(value + "=" * (-len(value) % 4), altchars=b"-_", validate=True)
data = json.loads(raw)
if not isinstance(data, dict) or set(data) != {"v", "scope", "created_at", "id"}:
raise ValueError
if type(data["v"]) is not int or data["v"] != 1 or data["scope"] != scope:
raise ValueError
if not isinstance(data["created_at"], str) or not data["created_at"].endswith("Z"):
raise ValueError
stamp = datetime.fromisoformat(data["created_at"])
if stamp.tzinfo != UTC or not isinstance(data["id"], str):
raise ValueError
identity = str(UUID(data["id"]))
return stamp.isoformat(timespec="microseconds").replace("+00:00", "Z"), identity
except ValueError, TypeError, KeyError, UnicodeError, binascii.Error:
raise invalid("Некорректный cursor или другая область пагинации") from None
+79
View File
@@ -0,0 +1,79 @@
"""Immutable domain values; no dependency on HTTP or a storage technology."""
from dataclasses import dataclass
from datetime import UTC, datetime
from uuid import uuid4
def utc_now() -> str:
return datetime.now(UTC).isoformat(timespec="microseconds").replace("+00:00", "Z")
def new_id() -> str:
return str(uuid4())
class DomainError(Exception):
def __init__(self, status: int, code: str, message: str, headers: dict | None = None):
super().__init__(message)
self.status = status
self.code = code
self.message = message
self.headers = headers or {}
def invalid(message: str = "Некорректный запрос") -> DomainError:
return DomainError(400, "invalid_request", message)
def not_found() -> DomainError:
return DomainError(404, "not_found", "Объект не найден")
@dataclass(frozen=True, slots=True)
class User:
id: str
username: str
display_name: str
created_at: str
@dataclass(frozen=True, slots=True)
class Chat:
id: str
title: str
member_ids: tuple[str, ...]
created_at: str
@dataclass(frozen=True, slots=True)
class Message:
id: str
chat_id: str
sender_id: str
text: str
created_at: str
attachment_ids: tuple[str, ...] = ()
@dataclass(frozen=True, slots=True)
class Attachment:
id: str
chat_id: str
owner_id: str
filename: str
content_type: str
size_bytes: int
sha256: str
created_at: str
updated_at: str
status: str = "ready"
variants: tuple = ()
error_code: str | None = None
type HistoryKey = tuple[str, str]
def history_key(item: Chat | Message) -> HistoryKey:
return item.created_at, item.id
+35
View File
@@ -0,0 +1,35 @@
"""Operator-only, single-use fault barriers. Disabled in the normal Compose stack."""
import os
import time
from messenger.domain import DomainError
from messenger.pipeline import event
class PermanentFailure(Exception):
pass
def checkpoint(db, settings, job, phase):
if not settings.fault_injection:
return
with db.connection(write=True) as conn:
row = conn.execute(
"UPDATE pipeline_faults SET remaining=remaining-1 WHERE job_id=%s AND phase=%s "
"AND remaining>0 RETURNING action",
(job, phase),
).fetchone()
if row:
event(conn, job, "fault_injected", phase=phase, action=row["action"], pid=os.getpid())
if not row:
return
match row["action"]:
case "crash":
os._exit(87)
case "hang":
time.sleep(settings.job_timeout * 3)
case "transient":
raise DomainError(503, "unavailable", "Synthetic transient failure")
case "permanent":
raise PermanentFailure
+51
View File
@@ -0,0 +1,51 @@
"""Pure, versioned image transforms. No network, filenames, or EXIF in output."""
import io
from PIL import Image, ImageOps
def fit(image, maximum):
scale = max(image.size)
if scale <= maximum:
return image.copy()
size = tuple(max(1, n * maximum // scale) for n in image.size)
return image.resize(size, Image.Resampling.LANCZOS)
def variants(source, version):
if version not in (1, 2):
raise ValueError("Unsupported pipeline version")
with Image.open(source) as raw:
oriented = ImageOps.exif_transpose(raw)
rgba = oriented.convert("RGBA")
image = Image.new("RGB", rgba.size, "white")
image.paste(rgba, mask=rgba.getchannel("A"))
rgba.close()
oriented.close()
try:
for name, output, format, options in (
("thumbnail", fit(image, 256), "JPEG", {"quality": 85, "subsampling": 0}),
(
"crop",
ImageOps.fit(image, (128, 128), method=Image.Resampling.LANCZOS),
"JPEG",
{"quality": 85, "subsampling": 0},
),
(
"optimized",
fit(image, 1280),
"WEBP",
{"quality": 82 if version == 1 else 76, "method": 4},
),
):
with output, io.BytesIO() as buffer:
output.save(buffer, format=format, exif=b"", **options)
yield (
name,
buffer.getvalue(),
output.size,
"image/webp" if format == "WEBP" else "image/jpeg",
)
finally:
image.close()
+46
View File
@@ -0,0 +1,46 @@
"""Transactional, checksummed migrations, serialized across concurrent starts."""
import hashlib
import os
from pathlib import Path
import psycopg
from messenger.settings import secret
def migrate(conninfo: str, directory: Path):
with psycopg.connect(conninfo, connect_timeout=5) as conn:
conn.execute("SELECT pg_advisory_xact_lock(2301002)")
conn.execute(
"CREATE TABLE IF NOT EXISTS schema_migrations ("
"version text PRIMARY KEY, checksum text NOT NULL, "
"applied_at timestamptz NOT NULL DEFAULT clock_timestamp())"
)
for path in sorted(directory.glob("*.sql")):
content = path.read_text()
checksum = hashlib.sha256(content.encode()).hexdigest()
current = conn.execute(
"SELECT checksum FROM schema_migrations WHERE version=%s", (path.stem,)
).fetchone()
if current:
if current[0] != checksum:
raise RuntimeError("Applied migration checksum changed: " + path.stem)
continue
conn.execute(content)
conn.execute(
"INSERT INTO schema_migrations (version,checksum) VALUES (%s,%s)",
(path.stem, checksum),
)
print("Applied migration " + path.stem)
def main():
migrate(
secret("DATABASE_URL_FILE", "/run/secrets/database_url"),
Path(os.getenv("MIGRATIONS_DIR", "/app/migrations")),
)
if __name__ == "__main__":
main()
+60
View File
@@ -0,0 +1,60 @@
"""Explicit local operator actions, never exposed as a public HTTP endpoint."""
import argparse
import json
from messenger.pipeline import Pipeline
from messenger.repository import PostgresRepository
from messenger.settings import Settings
from messenger.storage import ObjectStore
def main():
parser = argparse.ArgumentParser()
parser.add_argument("action", choices=("failed", "replay", "new-version"))
parser.add_argument("attachment_id", nargs="?")
args = parser.parse_args()
settings = Settings.from_env()
db = PostgresRepository(settings.database_url, pool_size=2)
db.open()
try:
if args.action == "failed":
with db.connection() as conn:
rows = conn.execute(
"SELECT job_id,attachment_id,pipeline_version,attempt,error_code,"
"source_expired FROM attachment_jobs WHERE state='failed' "
"ORDER BY updated_at LIMIT 100"
).fetchall()
print(json.dumps(rows, default=str))
else:
if not args.attachment_id:
parser.error("attachment_id is required")
with db.connection() as conn:
row = conn.execute(
"SELECT j.source_key,a.object_key,a.storage_state FROM attachment_jobs j "
"JOIN attachments a ON a.id=j.attachment_id AND "
"a.pipeline_version=j.pipeline_version "
"WHERE a.id=%s",
(args.attachment_id,),
).fetchone()
if not row:
parser.error("Unknown attachment")
store = ObjectStore(settings)
try:
key = row["object_key"] if row["storage_state"] == "ready" else row["source_key"]
if not store.head(key):
parser.error("Source is no longer retained")
finally:
store.close()
job = Pipeline(db, settings).replay(
args.attachment_id, new_version=args.action == "new-version"
)
print(json.dumps({"job_id": job, "action": args.action}))
except ValueError as exc:
parser.error(str(exc))
finally:
db.close()
if __name__ == "__main__":
main()
+266
View File
@@ -0,0 +1,266 @@
"""Job state machine. PostgreSQL fences publication; the stream is at least once."""
import os
import random
from datetime import UTC, datetime, timedelta
from psycopg.types.json import Jsonb
from messenger import telemetry
from messenger.domain import DomainError, new_id
def event(conn, job_id, name, **details):
conn.execute(
"INSERT INTO pipeline_audit(job_id,event,details) VALUES (%s,%s,%s)",
(job_id, name, Jsonb(details)),
)
def intent(conn, job_id, when=None):
conn.execute(
"INSERT INTO job_outbox(job_id,due_at) VALUES (%s,coalesce(%s,clock_timestamp())) "
"ON CONFLICT(job_id) DO UPDATE SET due_at=EXCLUDED.due_at,published_at=NULL,stream_id=NULL",
(job_id, when),
)
class Pipeline:
def __init__(self, repository, settings):
self.db, self.settings = repository, settings
def reserve(self, id, key):
with self.db.connection(write=True) as conn:
conn.execute("INSERT INTO upload_reservations(id,object_key) VALUES (%s,%s)", (id, key))
def accept(self, id, chat, owner, name, media, size, digest, key):
with self.db.connection(write=True) as conn:
reservation = conn.execute(
"SELECT * FROM upload_reservations WHERE id=%s FOR UPDATE", (id,)
).fetchone()
if (
not reservation
or reservation["state"] != "uploading"
or reservation["object_key"] != key
or reservation["expires_at"] <= datetime.now(UTC)
):
raise DomainError(503, "unavailable", "Срок приёма файла истёк")
conn.execute("UPDATE upload_reservations SET state='accepted' WHERE id=%s", (id,))
row = conn.execute(
"INSERT INTO attachments "
"(id,chat_id,owner_id,filename,content_type,size_bytes,sha256,"
"object_key,storage_state) "
"VALUES (%s,%s,%s,%s,%s,%s,%s,%s,'queued') RETURNING *",
(id, chat, owner, name, media, size, digest, key),
).fetchone()
job = new_id()
trace = telemetry.carrier()
conn.execute(
"INSERT INTO attachment_jobs(job_id,attachment_id,pipeline_version,source_key,"
"traceparent,tracestate) VALUES (%s,%s,1,%s,%s,%s)",
(job, id, key, trace.get("traceparent"), trace.get("tracestate")),
)
intent(conn, job)
event(conn, job, "accepted")
return row
def claim(self, envelope):
with self.db.connection(write=True) as conn:
# All writers lock attachment before job, including publication and operator replay.
a = conn.execute(
"SELECT * FROM attachments WHERE id=%s FOR UPDATE", (envelope.attachment_id,)
).fetchone()
if not a:
return "unknown", None
j = conn.execute(
"SELECT * FROM attachment_jobs WHERE job_id=%s FOR NO KEY UPDATE",
(envelope.job_id,),
).fetchone()
if (
not j
or j["attachment_id"] != a["id"]
or j["pipeline_version"] != envelope.pipeline_version
):
return "unknown", None
if a["pipeline_version"] != j["pipeline_version"] or j["state"] == "superseded":
event(conn, j["job_id"], "stale_delivery")
return "stale", None
if j["state"] == "ready":
event(conn, j["job_id"], "duplicate_ready")
return "ready", a
if j["state"] == "failed":
return "failed", None
now = datetime.now(UTC)
if j["state"] == "processing" or j["next_attempt_at"] > now:
return "busy", None
if envelope.attempt != j["attempt"] + 1:
# An older envelope cannot steal a scheduled retry or an explicit replay.
return "obsolete_attempt", None
fence = new_id()
conn.execute(
"UPDATE attachment_jobs SET state='processing',attempt=attempt+1,fence=%s,"
"lease_until=clock_timestamp()+(%s*interval '1 "
"second'),updated_at=clock_timestamp() "
"WHERE job_id=%s",
(fence, self.settings.lease_seconds, j["job_id"]),
)
conn.execute(
"UPDATE attachments SET storage_state='processing',"
"updated_at=clock_timestamp() WHERE id=%s",
(a["id"],),
)
event(
conn, j["job_id"], "claimed", attempt=j["attempt"] + 1, fence=fence, pid=os.getpid()
)
return "claimed", {
**a,
"job_id": str(j["job_id"]),
"source_key": j["source_key"],
"fence": fence,
"attempt": j["attempt"] + 1,
"accepted_at": j["created_at"],
}
def publish(self, row, original_key, variants):
with self.db.connection(write=True) as conn:
a = conn.execute(
"SELECT * FROM attachments WHERE id=%s FOR UPDATE", (row["id"],)
).fetchone()
j = conn.execute(
"SELECT * FROM attachment_jobs WHERE job_id=%s FOR NO KEY UPDATE", (row["job_id"],)
).fetchone()
if not self.owns(a, j, row):
event(conn, row["job_id"], "publication_fenced")
return False
conn.execute(
"UPDATE attachments SET storage_state='ready',object_key=%s,variants=%s,"
"error_code=NULL,updated_at=clock_timestamp() WHERE id=%s",
(original_key, Jsonb(variants), row["id"]),
)
conn.execute(
"UPDATE attachment_jobs SET state='ready',lease_until=NULL,error_code=NULL,"
"updated_at=clock_timestamp() WHERE job_id=%s",
(row["job_id"],),
)
event(conn, row["job_id"], "result_committed", version=row["pipeline_version"])
return True
@staticmethod
def owns(a, j, row):
return (
a
and j
and a["pipeline_version"] == row["pipeline_version"]
and j["state"] == "processing"
and str(j["fence"]) == row["fence"]
and j["lease_until"] > datetime.now(UTC)
)
def failure(self, row, code, permanent=False):
with self.db.connection(write=True) as conn:
a = conn.execute(
"SELECT * FROM attachments WHERE id=%s FOR UPDATE", (row["id"],)
).fetchone()
j = conn.execute(
"SELECT * FROM attachment_jobs WHERE job_id=%s FOR NO KEY UPDATE", (row["job_id"],)
).fetchone()
if self.owns(a, j, row):
return self._retry(conn, j, code, permanent)
return "fenced"
def _retry(self, conn, j, code, permanent=False):
failed = permanent or j["attempt"] >= self.settings.job_max_attempts
state = "failed" if failed else "queued"
if failed and not permanent:
code = "retry_exhausted"
# Full jitter; durable absolute time survives worker/relay restarts.
delay = random.uniform(0.25, min(30, 2 ** min(j["attempt"], 5)))
due = datetime.now(UTC) + timedelta(seconds=delay)
conn.execute(
"UPDATE attachment_jobs SET state=%s,error_code=%s,next_attempt_at=%s,lease_until=NULL,"
"updated_at=clock_timestamp() WHERE job_id=%s",
(state, code, due, j["job_id"]),
)
conn.execute(
"UPDATE attachments SET storage_state=%s,error_code=%s,updated_at=clock_timestamp() "
"WHERE id=%s",
(state, code if failed else None, j["attachment_id"]),
)
event(conn, j["job_id"], state, reason=code, attempt=j["attempt"], delay_seconds=delay)
if not failed:
intent(conn, j["job_id"], due)
return "failed" if failed else "retry"
def recover(self):
# Bounded SKIP LOCKED batch; multiple relays. No transaction held during Pillow.
with self.db.connection(write=True) as conn:
rows = conn.execute(
"SELECT a.id FROM attachments a JOIN attachment_jobs j ON j.attachment_id=a.id "
"AND j.pipeline_version=a.pipeline_version WHERE j.state='processing' "
"AND j.lease_until<clock_timestamp() ORDER BY a.id LIMIT 32 "
"FOR UPDATE OF a SKIP LOCKED"
).fetchall()
for row in rows:
j = conn.execute(
"SELECT j.* FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id "
"AND a.pipeline_version=j.pipeline_version WHERE a.id=%s FOR NO KEY "
"UPDATE OF j",
(row["id"],),
).fetchone()
if j["state"] == "processing" and j["lease_until"] < datetime.now(UTC):
event(conn, j["job_id"], "lease_expired")
self._retry(conn, j, "worker_timeout")
def replay(self, attachment, *, new_version=False):
with self.db.connection(write=True) as conn:
a = conn.execute(
"SELECT * FROM attachments WHERE id=%s FOR UPDATE", (attachment,)
).fetchone()
if not a:
raise ValueError("Unknown attachment")
j = conn.execute(
"SELECT * FROM attachment_jobs WHERE attachment_id=%s "
"AND pipeline_version=%s FOR UPDATE",
(attachment, a["pipeline_version"]),
).fetchone()
if new_version:
if a["pipeline_version"] != 1:
raise ValueError("Only pipeline v1 -> v2 is implemented")
conn.execute(
"UPDATE attachment_jobs SET state='superseded' WHERE job_id=%s "
"AND state IN ('queued','processing','failed')",
(j["job_id"],),
)
key = a["object_key"] if a["storage_state"] == "ready" else j["source_key"]
job = new_id()
conn.execute(
"INSERT INTO attachment_jobs(job_id,attachment_id,pipeline_version,source_key) "
"VALUES (%s,%s,2,%s)",
(job, attachment, key),
)
conn.execute(
"UPDATE attachments SET "
"pipeline_version=2,storage_state='queued',variants='[]',"
"error_code=NULL,updated_at=clock_timestamp() WHERE id=%s",
(attachment,),
)
event(conn, job, "new_version", previous_job=str(j["job_id"]))
else:
if j["state"] != "failed" or j["source_expired"]:
raise ValueError("Replay requires failed job with retained source")
job = str(j["job_id"])
conn.execute(
"UPDATE attachment_jobs SET "
"state='queued',attempt=0,fence=NULL,error_code=NULL,"
"next_attempt_at=clock_timestamp(),updated_at=clock_timestamp() WHERE "
"job_id=%s",
(job,),
)
conn.execute(
"UPDATE attachments SET storage_state='queued',error_code=NULL,"
"updated_at=clock_timestamp() WHERE id=%s",
(attachment,),
)
event(conn, job, "operator_replay")
intent(conn, job)
return job
+106
View File
@@ -0,0 +1,106 @@
"""Transactional outbox publisher and expired-lease watchdog, independently restartable."""
import signal
import time
from pathlib import Path
import redis
from opentelemetry.trace import SpanKind
from messenger import telemetry
from messenger.broker import Broker, Job
from messenger.domain import DomainError
from messenger.faults import checkpoint
from messenger.pipeline import Pipeline, event
from messenger.repository import PostgresRepository, stamp
from messenger.settings import Settings
def publish_one(db, broker, settings):
with db.connection(write=True) as conn:
row = conn.execute(
"SELECT o.job_id,j.attachment_id,j.pipeline_version,j.attempt,j.created_at,"
"j.traceparent,j.tracestate "
"FROM job_outbox o JOIN attachment_jobs j USING(job_id) "
"WHERE o.published_at IS NULL AND o.due_at<=clock_timestamp() AND j.state='queued' "
"ORDER BY o.due_at LIMIT 1 FOR UPDATE OF o SKIP LOCKED"
).fetchone()
if not row:
return False
with telemetry.span("jobs.publish", context=telemetry.context(row), kind=SpanKind.PRODUCER):
job = Job(
schema_version=1,
type=telemetry.JOB_TYPE,
job_id=str(row["job_id"]),
attachment_id=str(row["attachment_id"]),
pipeline_version=row["pipeline_version"],
attempt=row["attempt"] + 1,
created_at=stamp(row["created_at"]),
**telemetry.carrier(),
)
stream_id = broker.publish(job)
telemetry.event("job_published", job_id=job.job_id, attempt=job.attempt)
# Commit-on-return may be lost here; repeat sends the same logical job_id.
checkpoint(db, settings, row["job_id"], "after_publish")
conn.execute(
"UPDATE job_outbox SET published_at=clock_timestamp(),stream_id=%s WHERE job_id=%s",
(stream_id, row["job_id"]),
)
event(conn, row["job_id"], "published", stream_id=stream_id, attempt=job.attempt)
return True
def sample_backlog(db):
with db.connection() as conn:
row = conn.execute(
"SELECT count(*) AS depth, "
"coalesce(extract(epoch FROM clock_timestamp()-min(created_at)),0) AS age "
"FROM attachment_jobs WHERE state IN ('queued','processing')"
).fetchone()
telemetry.gauge("messenger.queue.depth", row["depth"], queue="attachment-jobs")
telemetry.gauge(
"messenger.queue.oldest", max(0, float(row["age"])), unit="s", queue="attachment-jobs"
)
def main():
with telemetry.active("relay"):
run()
def run():
settings = Settings.from_env()
db = PostgresRepository(settings.database_url, pool_size=4)
db.open()
broker = Broker(settings)
pipeline = Pipeline(db, settings)
running = True
next_sample = 0.0
def stop(*_):
nonlocal running
running = False
signal.signal(signal.SIGTERM, stop)
signal.signal(signal.SIGINT, stop)
try:
while running:
try:
if telemetry.runtime is not None and time.monotonic() >= next_sample:
sample_backlog(db)
next_sample = time.monotonic() + 2
pipeline.recover()
sent = publish_one(db, broker, settings)
Path("/tmp/relay-heartbeat").touch()
if not sent:
time.sleep(0.2)
except redis.RedisError, DomainError:
# Keep the transaction uncommitted and retry. No credentials/SQL in logs.
time.sleep(0.5)
finally:
broker.close()
db.close()
if __name__ == "__main__":
main()
+351
View File
@@ -0,0 +1,351 @@
"""PostgreSQL adapter: bounded connections, parameterized SQL, commit before return."""
import hashlib
import json
from contextlib import contextmanager, nullcontext
from datetime import datetime
from typing import Protocol
import psycopg
from psycopg import errors
from psycopg.rows import dict_row
from psycopg_pool import ConnectionPool, PoolTimeout, TooManyRequests
from messenger import telemetry
from messenger.domain import (
Attachment,
Chat,
DomainError,
HistoryKey,
Message,
User,
invalid,
new_id,
)
def stamp(value: datetime) -> str:
return value.isoformat(timespec="microseconds").replace("+00:00", "Z")
def user_value(row) -> User:
return User(str(row["id"]), row["username"], row["display_name"], stamp(row["created_at"]))
def chat_value(row) -> Chat:
return Chat(
str(row["id"]),
row["title"],
tuple(str(x) for x in row["member_ids"]),
stamp(row["created_at"]),
)
def message_value(row) -> Message:
return Message(
str(row["id"]),
str(row["chat_id"]),
str(row["sender_id"]),
row["text"],
stamp(row["created_at"]),
tuple(str(x) for x in row.get("attachment_ids", ())),
)
def attachment_value(row) -> Attachment:
return Attachment(
*(str(row[k]) for k in ("id", "chat_id", "owner_id")),
row["filename"],
row["content_type"],
row["size_bytes"],
row["sha256"],
stamp(row["created_at"]),
stamp(row["updated_at"]),
row["storage_state"],
tuple(
{k: v for k, v in item.items() if k != "object_key"} for item in row.get("variants", [])
),
row.get("error_code"),
)
class Repository(Protocol):
def get_user(self, user_id: str) -> User | None: ...
def add_chat(self, title: str, member_ids: tuple[str, ...]) -> Chat: ...
def get_chat(self, chat_id: str) -> Chat | None: ...
def page_chats(self, user_id: str, after: HistoryKey | None, limit: int) -> list[Chat]: ...
def add_message(self, chat_id: str, sender_id: str, text: str) -> Message: ...
def idempotent_message(
self,
chat_id: str,
sender_id: str,
text: str,
key: str,
attachment_ids: tuple[str, ...] = (),
) -> Message: ...
def page_messages(
self, chat_id: str, after: HistoryKey | None, limit: int
) -> list[Message]: ...
CHAT_COLUMNS = """c.id, c.title, c.created_at,
ARRAY(SELECT user_id FROM chat_members WHERE chat_id=c.id ORDER BY position) AS member_ids"""
MESSAGE_COLUMNS = """m.*, ARRAY(SELECT attachment_id FROM message_attachments
WHERE message_id=m.id ORDER BY position) AS attachment_ids"""
class PostgresRepository:
def __init__(
self,
conninfo: str,
pool_size: int = 8,
idempotency_ttl: int = 86400,
read_autocommit: bool = True,
):
self.read_autocommit = read_autocommit
self.idempotency_ttl = idempotency_ttl
self.pool = ConnectionPool(
conninfo,
min_size=1,
max_size=pool_size,
timeout=1,
max_waiting=32,
open=False,
check=ConnectionPool.check_connection,
kwargs={
"row_factory": dict_row,
"autocommit": True,
"connect_timeout": 2,
"options": "-c statement_timeout=2500 -c lock_timeout=1000 "
"-c idle_in_transaction_session_timeout=5000 -c timezone=UTC",
},
)
def open(self):
self.pool.open(wait=True, timeout=15)
def close(self):
self.pool.close(timeout=5)
@contextmanager
def connection(self, *, write: bool = False):
with telemetry.span(
"db.transaction" if write else "db.read", **{"db.system.name": "postgresql"}
):
with self._connection(write=write) as conn:
yield conn
@contextmanager
def _connection(self, *, write):
try:
with self.pool.connection() as conn:
transaction = (
conn.transaction() if write or not self.read_autocommit else nullcontext()
)
with transaction:
telemetry.pool(self.pool)
yield conn
except (
psycopg.OperationalError,
psycopg.InterfaceError,
PoolTimeout,
TooManyRequests,
errors.QueryCanceled,
errors.LockNotAvailable,
):
telemetry.pool(self.pool)
telemetry.record("messenger.db.errors", 1)
raise DomainError(503, "unavailable", "Хранилище временно недоступно") from None
def ready(self):
with self.connection() as conn:
row = conn.execute(
"SELECT count(*) AS n FROM schema_migrations "
"WHERE version IN "
"('001_initial','002_idempotency','003_attachments',"
"'004_pipeline','005_upload_handoff','006_trace_context')"
).fetchone()
if row["n"] != 6:
raise DomainError(503, "unavailable", "Схема ещё не готова")
def add_user(self, username: str, display_name: str, password_hash: str) -> User:
try:
with self.connection(write=True) as conn:
row = conn.execute(
"INSERT INTO users (id,username,display_name,password_hash) "
"VALUES (%s,%s,%s,%s) RETURNING *",
(new_id(), username, display_name, password_hash),
).fetchone()
return user_value(row)
except errors.UniqueViolation:
raise DomainError(409, "conflict", "Имя пользователя уже занято") from None
def credentials(self, username: str):
with self.connection() as conn:
row = conn.execute("SELECT * FROM users WHERE username=%s", (username,)).fetchone()
return (user_value(row), row["password_hash"]) if row else None
def get_user(self, user_id: str) -> User | None:
with self.connection() as conn:
row = conn.execute("SELECT * FROM users WHERE id=%s", (user_id,)).fetchone()
return user_value(row) if row else None
def add_chat(self, title: str, member_ids: tuple[str, ...]) -> Chat:
try:
with self.connection(write=True) as conn:
row = conn.execute(
"INSERT INTO chats (id,title) VALUES (%s,%s) RETURNING *", (new_id(), title)
).fetchone()
with conn.cursor() as cur:
cur.executemany(
"INSERT INTO chat_members (chat_id,user_id,position) VALUES (%s,%s,%s)",
[(row["id"], member, i) for i, member in enumerate(member_ids)],
)
return Chat(str(row["id"]), title, member_ids, stamp(row["created_at"]))
except errors.ForeignKeyViolation:
raise invalid("Указан неизвестный участник") from None
def get_chat(self, chat_id: str) -> Chat | None:
with self.connection() as conn:
row = conn.execute(
"SELECT " + CHAT_COLUMNS + " FROM chats c WHERE c.id=%s", (chat_id,)
).fetchone()
return chat_value(row) if row else None
def page_chats(self, user_id: str, after: HistoryKey | None, limit: int) -> list[Chat]:
condition = " AND (c.created_at,c.id) > (%s::timestamptz,%s::uuid)" if after else ""
params = (user_id, *after, limit) if after else (user_id, limit)
with self.connection() as conn:
rows = conn.execute(
"SELECT " + CHAT_COLUMNS + " FROM chats c WHERE EXISTS "
"(SELECT 1 FROM chat_members m WHERE m.chat_id=c.id AND m.user_id=%s)"
+ condition
+ " ORDER BY c.created_at,c.id LIMIT %s",
params,
).fetchall()
return [chat_value(row) for row in rows]
def add_message(self, chat_id: str, sender_id: str, text: str) -> Message:
with self.connection(write=True) as conn:
row = conn.execute(
"INSERT INTO messages (id,chat_id,sender_id,text) VALUES (%s,%s,%s,%s) RETURNING *",
(new_id(), chat_id, sender_id, text),
).fetchone()
return message_value(row)
def page_messages(self, chat_id: str, after: HistoryKey | None, limit: int) -> list[Message]:
condition = " AND (m.created_at,m.id) > (%s::timestamptz,%s::uuid)" if after else ""
params = (chat_id, *after, limit) if after else (chat_id, limit)
with self.connection() as conn:
rows = conn.execute(
"SELECT "
+ MESSAGE_COLUMNS
+ " FROM messages m WHERE m.chat_id=%s"
+ condition
+ " ORDER BY m.created_at,m.id LIMIT %s",
params,
).fetchall()
return [message_value(row) for row in rows]
def idempotent_message(
self,
chat_id: str,
sender_id: str,
text: str,
key: str,
attachment_ids: tuple[str, ...] = (),
) -> Message:
scope = json.dumps([sender_id, "POST", chat_id, key], separators=(",", ":"))
lock = int.from_bytes(hashlib.sha256(scope.encode()).digest()[:8], "big", signed=True)
payload = json.dumps(
{"text": text, "attachment_ids": list(attachment_ids)},
sort_keys=True,
separators=(",", ":"),
ensure_ascii=False,
)
digest = hashlib.sha256(payload.encode()).digest()
with self.connection(write=True) as conn:
conn.execute("SELECT pg_advisory_xact_lock(%s)", (lock,))
previous = conn.execute(
"SELECT r.payload_hash," + MESSAGE_COLUMNS + " FROM message_requests r "
"JOIN messages m ON m.id=r.response_message_id "
"WHERE r.user_id=%s AND r.http_method='POST' AND r.chat_id=%s "
"AND r.idempotency_key=%s AND r.expires_at > clock_timestamp()",
(sender_id, chat_id, key),
).fetchone()
if previous:
if previous["payload_hash"] != digest:
raise DomainError(409, "conflict", "Ключ уже использован с другим сообщением")
return message_value(previous)
conn.execute(
"DELETE FROM message_requests WHERE user_id=%s AND http_method='POST' "
"AND chat_id=%s AND idempotency_key=%s",
(sender_id, chat_id, key),
)
if attachment_ids:
rows = conn.execute(
"SELECT id,storage_state FROM attachments WHERE chat_id=%s "
"AND id=ANY(%s::uuid[]) ORDER BY id FOR UPDATE",
(chat_id, list(attachment_ids)),
).fetchall()
if len(rows) != len(attachment_ids):
raise DomainError(404, "not_found", "Объект не найден")
if any(row["storage_state"] == "failed" for row in rows):
raise DomainError(409, "conflict", "Вложение пока недоступно")
message = message_value(
conn.execute(
"INSERT INTO messages (id,chat_id,sender_id,text) "
"VALUES (%s,%s,%s,%s) RETURNING *",
(new_id(), chat_id, sender_id, text),
).fetchone()
)
for position, attachment in enumerate(attachment_ids):
conn.execute(
"INSERT INTO message_attachments VALUES (%s,%s,%s,%s)",
(message.id, chat_id, attachment, position),
)
message = Message(
message.id,
message.chat_id,
message.sender_id,
message.text,
message.created_at,
attachment_ids,
)
self._save_request(conn, sender_id, chat_id, key, digest, message.id)
return message
def _save_request(self, conn, user_id, chat_id, key, digest, message_id):
conn.execute(
"INSERT INTO message_requests "
"(user_id,chat_id,idempotency_key,payload_hash,response_message_id,expires_at) "
"VALUES (%s,%s,%s,%s,%s,clock_timestamp()+make_interval(secs => %s))",
(user_id, chat_id, key, digest, message_id, self.idempotency_ttl),
)
def create_session(self, token_hash: bytes, user_id: str, ttl: int) -> str:
with self.connection(write=True) as conn:
row = conn.execute(
"INSERT INTO sessions (token_hash,user_id,expires_at) "
"VALUES (%s,%s,clock_timestamp() + make_interval(secs => %s)) RETURNING expires_at",
(token_hash, user_id, ttl),
).fetchone()
return stamp(row["expires_at"])
def active_session(self, token_hash: bytes):
with self.connection() as conn:
return conn.execute(
"SELECT user_id,expires_at,clock_timestamp() AS checked_at FROM sessions "
"WHERE token_hash=%s AND expires_at > clock_timestamp()",
(token_hash,),
).fetchone()
def revoke_session(self, token_hash: bytes):
with self.connection(write=True) as conn:
conn.execute("DELETE FROM sessions WHERE token_hash=%s", (token_hash,))
def attachment(self, id):
with self.connection() as conn:
return conn.execute("SELECT * FROM attachments WHERE id=%s", (id,)).fetchone()
+120
View File
@@ -0,0 +1,120 @@
"""Strict lab 5 wire types; password never belongs to a response User."""
import re
from typing import Annotated
from uuid import UUID
from pydantic import AfterValidator, BaseModel, ConfigDict, Field, field_validator, model_validator
def canonical_uuid(value: str) -> str:
if not re.fullmatch(r"[0-9a-fA-F]{8}(?:-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}", value):
raise ValueError("Ожидается UUID")
return str(UUID(value))
def nonblank(value: str) -> str:
if not value.strip() or any(0xD800 <= ord(char) <= 0xDFFF for char in value):
raise ValueError("Ожидается непустая строка Unicode")
return value
type Identifier = Annotated[str, AfterValidator(canonical_uuid)]
type ShortText = Annotated[str, Field(min_length=1, max_length=100), AfterValidator(nonblank)]
class Input(BaseModel):
model_config = ConfigDict(strict=True, extra="forbid")
class CreateUser(Input):
username: str = Field(min_length=3, max_length=32)
display_name: ShortText
password: Annotated[str, Field(min_length=12, max_length=128), AfterValidator(nonblank)]
@field_validator("username")
@classmethod
def valid_username(cls, value: str) -> str:
if not re.fullmatch(r"[a-z0-9_]{3,32}", value):
raise ValueError("Недопустимое имя пользователя")
return value
class CreateChat(Input):
title: ShortText
member_ids: list[Identifier] = Field(min_length=1, max_length=100)
class CreateMessage(Input):
text: str = Field(max_length=4000)
attachment_ids: list[Identifier] = Field(default_factory=list, max_length=10)
@model_validator(mode="after")
def valid_content(self):
if (
not self.text.strip()
and not self.attachment_ids
or any(0xD800 <= ord(c) <= 0xDFFF for c in self.text)
or len(set(self.attachment_ids)) != len(self.attachment_ids)
):
raise ValueError("Некорректное содержимое сообщения")
return self
class View(BaseModel):
model_config = ConfigDict(from_attributes=True, extra="forbid")
class UserView(View):
id: str
username: str
display_name: str
created_at: str
class SessionView(View):
token: str
token_type: str
expires_at: str
user: UserView
class ChatView(View):
id: str
title: str
member_ids: list[str]
created_at: str
class MessageView(View):
attachment_ids: list[str]
id: str
chat_id: str
sender_id: str
text: str
created_at: str
class ChatPage(View):
items: list[ChatView]
next_cursor: str | None
class MessagePage(View):
items: list[MessageView]
next_cursor: str | None
class AttachmentView(View):
id: str
chat_id: str
owner_id: str
filename: str
content_type: str
size_bytes: int
sha256: str
status: str
variants: list
error_code: str | None
created_at: str
updated_at: str
+52
View File
@@ -0,0 +1,52 @@
"""Use cases shared by HTTP and tests. Authorization precedes resource access."""
from dataclasses import dataclass
from messenger.cursor import decode_cursor, encode_cursor
from messenger.domain import Chat, Message, User, history_key, invalid, not_found
from messenger.repository import Repository
@dataclass(frozen=True)
class Page[T]:
items: list[T]
next_cursor: str | None
class Messenger:
def __init__(self, repository: Repository):
self.repository = repository
def create_chat(self, actor: User, title: str, member_ids: list[str]) -> Chat:
if len(member_ids) != len(set(member_ids)) or actor.id not in member_ids:
raise invalid("Участники не должны повторяться; вызывающий должен входить в чат")
return self.repository.add_chat(title, tuple(member_ids))
def chat(self, actor: User, chat_id: str) -> Chat:
chat = self.repository.get_chat(chat_id)
if chat is None or actor.id not in chat.member_ids:
raise not_found()
return chat
def chats(self, actor: User, limit: int, cursor: str | None) -> Page[Chat]:
scope = "chats:" + actor.id
after = decode_cursor(cursor, scope)
return self._page(self.repository.page_chats(actor.id, after, limit + 1), limit, scope)
def send(
self, actor: User, chat_id: str, text: str, key: str, attachments: tuple[str, ...] = ()
) -> Message:
self.chat(actor, chat_id)
return self.repository.idempotent_message(chat_id, actor.id, text, key, attachments)
def messages(self, actor: User, chat_id: str, limit: int, cursor: str | None) -> Page[Message]:
self.chat(actor, chat_id)
scope = "messages:" + chat_id
after = decode_cursor(cursor, scope)
return self._page(self.repository.page_messages(chat_id, after, limit + 1), limit, scope)
@staticmethod
def _page[T: Chat | Message](items: list[T], limit: int, scope: str) -> Page[T]:
visible = items[:limit]
cursor = encode_cursor(scope, history_key(visible[-1])) if len(items) > limit else None
return Page(visible, cursor)
+85
View File
@@ -0,0 +1,85 @@
"""Explicit configuration; credentials are read from untracked secret files."""
import os
from dataclasses import dataclass, field
from pathlib import Path
def secret(name: str, default: str, optional=False) -> str:
path = Path(os.getenv(name, default))
return path.read_text().strip() if path.exists() or not optional else ""
@dataclass(frozen=True)
class Settings:
database_url: str = field(repr=False)
cache_url: str = field(repr=False)
session_ttl: int = 3600
pool_size: int = 8
idempotency_ttl: int = 86400
read_autocommit: bool = True
s3_endpoint: str = "http://127.0.0.1:59006"
s3_region: str = "us-east-1"
s3_bucket: str = "messenger-attachments"
s3_access_key: str = field(default="", repr=False)
s3_secret_key: str = field(default="", repr=False)
max_upload_bytes: int = 10485760
max_image_pixels: int = 40000000
upload_slots: int = 1
broker_url: str = field(default="redis://broker:6379/0", repr=False)
worker_concurrency: int = 2
job_max_attempts: int = 5
job_timeout: int = 20
staging_retention: int = 86400
fault_injection: bool = False
@property
def lease_seconds(self):
return self.job_timeout + 3
@classmethod
def from_env(cls):
ttl = int(os.getenv("SESSION_TTL_SECONDS", "3600"))
size = int(os.getenv("DATABASE_POOL_SIZE", "8"))
idempotency_ttl = int(os.getenv("IDEMPOTENCY_TTL_SECONDS", "86400"))
if idempotency_ttl < 86400:
raise ValueError("Idempotency retention must be at least 24 hours")
if not 1 <= ttl <= 2_592_000 or not 1 <= size <= 32:
raise ValueError("Invalid session lifetime or connection pool size")
maximum = int(os.getenv("MAX_UPLOAD_BYTES", "10485760"))
pixels = int(os.getenv("MAX_IMAGE_PIXELS", "40000000"))
if not 1 <= maximum <= 10485760 or not 1 <= pixels <= 40000000:
raise ValueError("Invalid upload limits")
concurrency = int(os.getenv("WORKER_CONCURRENCY", "2"))
attempts = int(os.getenv("JOB_MAX_ATTEMPTS", "5"))
timeout = int(os.getenv("JOB_TIMEOUT_SECONDS", "20"))
retention = int(os.getenv("STAGING_RETENTION_SECONDS", "86400"))
if not 1 <= concurrency <= 4 or not 1 <= attempts <= 10 or not 2 <= timeout <= 120:
raise ValueError("Invalid worker limits")
if retention < 300:
raise ValueError("Staging retention must be at least 300 seconds")
return cls(
secret("DATABASE_URL_FILE", "/run/secrets/database_url"),
secret("CACHE_URL_FILE", "/run/secrets/cache_url"),
ttl,
size,
idempotency_ttl,
os.getenv("DATABASE_READ_AUTOCOMMIT", "1") == "1",
os.getenv("S3_ENDPOINT", "http://s3:9000"),
os.getenv("S3_REGION", "us-east-1"),
os.getenv("S3_BUCKET", "messenger-attachments"),
secret("S3_ACCESS_KEY_ID_FILE", "/run/secrets/s3_access_key_id", optional=True),
secret("S3_SECRET_ACCESS_KEY_FILE", "/run/secrets/s3_secret_access_key", optional=True),
maximum,
pixels,
broker_url=secret("BROKER_URL_FILE", "/run/secrets/broker_url")
if os.path.exists(os.getenv("BROKER_URL_FILE", "/run/secrets/broker_url"))
else "redis://broker:6379/0",
worker_concurrency=concurrency,
job_max_attempts=attempts,
job_timeout=timeout,
staging_retention=retention,
fault_injection=os.getenv("ENABLE_FAULT_INJECTION", "0") == "1",
)
+82
View File
@@ -0,0 +1,82 @@
"""S3 adapter: bounded sockets, least-privilege credentials, no automatic PUT retries."""
import boto3
from botocore.config import Config
from botocore.exceptions import BotoCoreError, ClientError
from messenger import telemetry
from messenger.domain import DomainError
def unavailable():
return DomainError(503, "unavailable", "Файловое хранилище временно недоступно")
class ObjectStore:
def __init__(self, settings):
self.bucket = settings.s3_bucket
self.client = boto3.client(
"s3",
endpoint_url=settings.s3_endpoint,
region_name=settings.s3_region,
aws_access_key_id=settings.s3_access_key,
aws_secret_access_key=settings.s3_secret_key,
config=Config(
connect_timeout=0.5,
read_timeout=2,
max_pool_connections=8,
retries={"total_max_attempts": 1},
s3={"addressing_style": "path"},
request_checksum_calculation="when_required",
response_checksum_validation="when_required",
),
)
def ready(self):
try:
self.client.head_bucket(Bucket=self.bucket)
except BotoCoreError, ClientError:
raise unavailable() from None
def put(self, key, file, size, media, digest):
with telemetry.span("s3.put"):
try:
self.client.put_object(
Bucket=self.bucket,
Key=key,
Body=file,
ContentLength=size,
ContentType=media,
Metadata={"sha256": digest},
)
except BotoCoreError, ClientError:
raise unavailable() from None
def head(self, key):
with telemetry.span("s3.head"):
try:
return self.client.head_object(Bucket=self.bucket, Key=key)
except ClientError as exc:
if str(exc.response.get("Error", {}).get("Code")) in (
"404",
"NoSuchKey",
"NotFound",
):
return None
raise unavailable() from None
except BotoCoreError:
raise unavailable() from None
def download(self, row):
with telemetry.span("s3.download"):
try:
response = self.client.get_object(Bucket=self.bucket, Key=row["object_key"])
if response["ContentLength"] != row["size_bytes"]:
response["Body"].close()
raise unavailable()
return response["Body"]
except BotoCoreError, ClientError:
raise unavailable() from None
def close(self):
self.client.close()
+241
View File
@@ -0,0 +1,241 @@
"""Application telemetry with bounded export and an explicit attribute allowlist."""
import json
import logging
import os
import resource
import time
from contextlib import contextmanager
from importlib.metadata import version
from uuid import uuid4
from opentelemetry import trace
from opentelemetry._logs import LogRecord, SeverityNumber
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
from opentelemetry.metrics import Observation
from opentelemetry.sdk._logs import LoggerProvider
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
from opentelemetry.sdk.metrics import MeterProvider
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
from opentelemetry.sdk.metrics.view import ExplicitBucketHistogramAggregation, View
from opentelemetry.sdk.resources import Resource
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor
from opentelemetry.sdk.trace.sampling import ParentBased, TraceIdRatioBased
from opentelemetry.trace import SpanKind, Status, StatusCode
from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
logger = logging.getLogger("messenger")
PROPAGATOR = TraceContextTextMapPropagator()
BUCKETS = (0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 120)
HTTP_METHODS = frozenset(("GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS"))
JOB_TYPE = "attachment.process.v1"
runtime = None
class Telemetry:
def __init__(self, role, instance, metrics):
resource_attrs = Resource(
{
"service.name": f"messenger-{role}",
"service.version": version("mtusi-messenger"),
"service.instance.id": instance,
"deployment.environment.name": "lab",
}
)
ratio = float(os.getenv("OTEL_SAMPLE_RATIO", "1"))
self.traces = TracerProvider(
resource=resource_attrs, sampler=ParentBased(TraceIdRatioBased(ratio))
)
self.traces.add_span_processor(
BatchSpanProcessor(
OTLPSpanExporter(timeout=1),
max_queue_size=512,
max_export_batch_size=64,
schedule_delay_millis=500,
)
)
self.tracer = self.traces.get_tracer("messenger")
self.logs = LoggerProvider(resource=resource_attrs)
self.logs.add_log_record_processor(
BatchLogRecordProcessor(
OTLPLogExporter(timeout=1),
max_queue_size=512,
max_export_batch_size=64,
schedule_delay_millis=500,
)
)
self.log = self.logs.get_logger("messenger")
self.meters = None
self.instruments = {}
if metrics:
self.meters = MeterProvider(
resource=resource_attrs,
metric_readers=[
PeriodicExportingMetricReader(
OTLPMetricExporter(timeout=1),
export_interval_millis=2000,
export_timeout_millis=1000,
)
],
views=[
View(
instrument_name="messenger.*.duration",
aggregation=ExplicitBucketHistogramAggregation(BUCKETS),
)
],
)
self.meter = self.meters.get_meter("messenger")
self.meter.create_observable_counter(
"messenger.process.cpu", unit="s", callbacks=[self.cpu_seconds]
)
self.meter.create_observable_gauge(
"messenger.process.rss", unit="By", callbacks=[self.rss_bytes]
)
@staticmethod
def cpu_seconds(_):
usage = resource.getrusage(resource.RUSAGE_CHILDREN)
yield Observation(time.process_time() + usage.ru_utime + usage.ru_stime)
@staticmethod
def rss_bytes(_):
try:
with open("/proc/self/statm", encoding="ascii") as stats:
yield Observation(int(stats.read().split()[1]) * os.sysconf("SC_PAGE_SIZE"))
except OSError:
return
def shutdown(self):
if self.meters:
self.meters.shutdown(timeout_millis=1500)
self.logs.shutdown()
self.traces.shutdown()
@contextmanager
def active(role, instance=None, *, metrics=True):
global runtime
if os.getenv("TELEMETRY_ENABLED", "0") != "1":
yield
return
runtime = Telemetry(role, instance or uuid4().hex, metrics)
try:
yield
finally:
state, runtime = runtime, None
state.shutdown()
@contextmanager
def span(name, *, context=None, kind=SpanKind.INTERNAL, **attrs):
if runtime is None:
yield trace.INVALID_SPAN
return
with runtime.tracer.start_as_current_span(
name,
context=context,
kind=kind,
attributes=attrs,
record_exception=False,
set_status_on_exception=False,
) as current:
try:
yield current
except Exception as exc:
current.set_attribute("error.type", type(exc).__name__)
current.set_status(Status(StatusCode.ERROR))
raise
def carrier():
result = {}
if runtime is not None:
PROPAGATOR.inject(result)
return result
def context(headers):
return PROPAGATOR.extract(
{k: headers[k] for k in ("traceparent", "tracestate") if headers.get(k)}
)
def record(name, value, *, kind="counter", unit="1", **attrs):
if runtime is None or runtime.meters is None:
return
instrument = runtime.instruments.get(name)
if instrument is None:
create = {
"counter": runtime.meter.create_counter,
"histogram": runtime.meter.create_histogram,
"gauge": runtime.meter.create_gauge,
}[kind]
instrument = runtime.instruments.setdefault(name, create(name, unit=unit))
if kind == "counter":
instrument.add(value, attrs)
elif kind == "histogram":
instrument.record(value, attrs)
else:
instrument.set(value, attrs)
def duration(name, seconds, **attrs):
record(name, seconds, kind="histogram", unit="s", **attrs)
def gauge(name, value, *, unit="", **attrs):
record(name, value, kind="gauge", unit=unit, **attrs)
def pool(pool):
if runtime is not None and runtime.meters is not None:
stats = pool.get_stats()
gauge(
"messenger.db.pool.used",
max(0, stats["pool_size"] - stats["pool_available"]),
)
gauge("messenger.db.pool.waiting", stats["requests_waiting"])
def event(event_name, *, error=False, **attrs):
fields = {"event": event_name, **attrs}
current = trace.get_current_span().get_span_context()
if current.is_valid:
fields["trace_id"] = format(current.trace_id, "032x")
fields["span_id"] = format(current.span_id, "016x")
logger.log(logging.ERROR if error else logging.INFO, json.dumps(fields))
if runtime is not None:
runtime.log.emit(
LogRecord(
timestamp=time.time_ns(),
observed_timestamp=time.time_ns(),
severity_text="ERROR" if error else "INFO",
severity_number=SeverityNumber.ERROR if error else SeverityNumber.INFO,
body=event_name,
attributes=fields,
)
)
def http(current, method, route, status, elapsed, request_id):
method = method if method in HTTP_METHODS else "_OTHER"
attrs = {"method": method, "route": route, "status_class": f"{status // 100}xx"}
current.update_name(f"{method} {route}")
current.set_attributes(
{"http.request.method": method, "http.route": route, "http.response.status_code": status}
)
if status >= 500:
current.set_status(Status(StatusCode.ERROR))
record("messenger.http.requests", 1, **attrs)
duration("messenger.http.duration", elapsed, **attrs)
event(
"http_request",
error=status >= 500,
request_id=request_id,
status=status,
duration_ms=round(elapsed * 1000, 3),
**attrs,
)
+270
View File
@@ -0,0 +1,270 @@
"""Bounded supervisor: one disposable OS process per job, hard wall-clock and RAM limits."""
import hashlib
import io
import multiprocessing
import resource
import signal
import tempfile
import time
from datetime import UTC, datetime
from pathlib import Path
from uuid import uuid4
import redis
from opentelemetry.trace import SpanKind
from pydantic import ValidationError
from messenger import telemetry
from messenger.attachments import image_type
from messenger.broker import GROUP, STREAM, Broker, Job
from messenger.domain import DomainError
from messenger.faults import PermanentFailure, checkpoint
from messenger.images import variants
from messenger.pipeline import Pipeline, event
from messenger.repository import PostgresRepository
from messenger.settings import Settings
from messenger.storage import ObjectStore
def verify_effect(store, row):
records = [row, *row["variants"]]
if len(records) != 4 or {v["name"] for v in row["variants"]} != {
"thumbnail",
"crop",
"optimized",
}:
raise DomainError(503, "unavailable", "Incomplete result")
for item in records:
head = store.head(item["object_key"])
if (
not head
or head["ContentLength"] != item["size_bytes"]
or head["Metadata"].get("sha256") != item["sha256"]
):
raise DomainError(503, "unavailable", "Incomplete result")
def handle(db, broker, store, settings, id, payload):
try:
job = Job.model_validate_json(payload)
except ValidationError:
broker.quarantine(id, "invalid_envelope")
return "invalid", None
context = telemetry.context(job.model_dump(include={"traceparent", "tracestate"}))
with telemetry.span(
"jobs.consume",
context=context,
kind=SpanKind.CONSUMER,
**{"messaging.destination.name": STREAM},
) as span:
outcome, age = process_job(db, broker, store, settings, id, job)
span.set_attribute("job.outcome", outcome)
telemetry.event(
"job_processed",
job_id=job.job_id,
attempt=job.attempt,
outcome=outcome,
error=outcome in {"retry", "failed"},
)
return outcome, age
def process_job(db, broker, store, settings, id, job):
pipeline = Pipeline(db, settings)
outcome, row = pipeline.claim(job)
if outcome == "busy":
return "busy", None # Watchdog recovers the lease.
if outcome == "unknown":
broker.quarantine(id, "unknown_job")
return "unknown", None
if outcome == "ready":
verify_effect(store, row)
broker.ack(id)
return "duplicate", None
if outcome != "claimed":
broker.ack(id) # Failed registry / obsolete version / durable retry owns this work.
return outcome, None
try:
checkpoint(db, settings, job.job_id, "before_effect")
with tempfile.TemporaryFile() as original:
body = store.download({**row, "object_key": row["source_key"]})
size, digest = 0, hashlib.sha256()
try:
while chunk := body.read(65536):
size += len(chunk)
if size > settings.max_upload_bytes:
raise PermanentFailure
digest.update(chunk)
original.write(chunk)
finally:
body.close()
if size != row["size_bytes"] or digest.hexdigest() != row["sha256"]:
raise PermanentFailure
image_type(original, settings.max_image_pixels)
prefix = f"objects/{row['id']}/v{row['pipeline_version']}/"
original_key = prefix + "original"
store.put(original_key, original, size, row["content_type"], digest.hexdigest())
checkpoint(db, settings, job.job_id, "after_effect")
records = []
for name, data, (width, height), media in variants(original, row["pipeline_version"]):
checksum = hashlib.sha256(data).hexdigest()
key = prefix + name
store.put(key, io.BytesIO(data), len(data), media, checksum)
records.append(
{
"name": name,
"content_type": media,
"size_bytes": len(data),
"sha256": checksum,
"width": width,
"height": height,
"object_key": key,
}
)
with db.connection(write=True) as conn:
event(conn, job.job_id, "objects_written", version=row["pipeline_version"])
committed = pipeline.publish(row, original_key, records)
age = max(0.0, (datetime.now(UTC) - row["accepted_at"]).total_seconds())
if committed:
checkpoint(db, settings, job.job_id, "after_commit")
broker.ack(id)
return ("ready", age) if committed else ("fenced", None)
except PermanentFailure:
outcome = pipeline.failure(row, "invalid_image", permanent=True)
broker.ack(id)
return outcome, None
except MemoryError:
outcome = pipeline.failure(row, "resource_limit", permanent=True)
broker.ack(id)
return outcome, None
except DomainError as exc:
outcome = pipeline.failure(
row,
"invalid_image" if exc.status in (413, 415) else "storage_unavailable",
permanent=exc.status in (413, 415),
)
broker.ack(id)
return outcome, None
except redis.RedisError:
# Result may already be committed; retain the pending entry for a verified no-op.
raise
except Exception:
outcome = pipeline.failure(row, "processing_failed")
broker.ack(id)
return outcome, None
def child(settings, id, payload, result, instance):
start = time.monotonic()
try:
with telemetry.active("worker", instance, metrics=False):
outcome, age = run_child(settings, id, payload)
result.send((outcome, age, time.monotonic() - start))
finally:
result.close()
def run_child(settings, id, payload):
# Linux hard limit includes Pillow buffers and native allocations. Supervisor kills hangs.
resource.setrlimit(resource.RLIMIT_AS, (768 * 1024**2, 768 * 1024**2))
db = PostgresRepository(settings.database_url, pool_size=2)
db.open()
broker, store = Broker(settings), ObjectStore(settings)
try:
return handle(db, broker, store, settings, id, payload)
except redis.RedisError, DomainError:
return "dependency_error", None # Leave pending; omit driver details.
finally:
store.close()
broker.close()
db.close()
def observe(result, elapsed, timed_out):
try:
outcome, age, duration = result.recv() if result.poll() else ("crashed", None, elapsed)
except EOFError:
outcome, age, duration = "crashed", None, elapsed
finally:
result.close()
if timed_out:
outcome = "timeout"
attrs = {"job_type": telemetry.JOB_TYPE, "outcome": outcome}
telemetry.record("messenger.jobs.processed", 1, **attrs)
telemetry.duration("messenger.jobs.duration", duration, **attrs)
if age is not None:
telemetry.duration("messenger.ready.duration", age, job_type=telemetry.JOB_TYPE)
def main():
instance = uuid4().hex
with telemetry.active("worker", instance):
run(instance)
def run(instance):
settings = Settings.from_env()
broker = Broker(settings)
consumer, cursor = uuid4().hex, "0-0"
context = multiprocessing.get_context("spawn")
active, running = [], True
def stop(*_):
nonlocal running
running = False
signal.signal(signal.SIGTERM, stop)
signal.signal(signal.SIGINT, stop)
try:
while running:
for task in active[:]:
process, result, started = task
elapsed = time.monotonic() - started
alive = process.is_alive()
if not alive or elapsed > settings.job_timeout:
if alive:
process.kill()
process.join()
observe(result, elapsed, alive)
active.remove(task)
process.close()
telemetry.gauge("messenger.worker.busy", len(active))
telemetry.gauge("messenger.worker.capacity", settings.worker_concurrency)
Path("/tmp/worker-heartbeat").touch()
if len(active) >= settings.worker_concurrency:
time.sleep(0.05)
continue
try:
broker.group()
reclaimed = broker.client.xautoclaim(
STREAM, GROUP, consumer, settings.lease_seconds * 1000, cursor, count=1
)
cursor, entries = reclaimed[0], reclaimed[1]
if not entries:
result = broker.client.xreadgroup(
GROUP, consumer, {STREAM: ">"}, count=1, block=200
)
entries = result[0][1] if result else []
if entries:
id, fields = entries[0]
result, sender = context.Pipe(duplex=False)
process = context.Process(
target=child, args=(settings, id, fields.get("job", ""), sender, instance)
)
process.start()
sender.close()
active.append((process, result, time.monotonic()))
except redis.RedisError:
time.sleep(0.5)
finally:
for process, result, _ in active:
result.close()
if process.is_alive():
process.kill()
process.join()
broker.close()
if __name__ == "__main__":
main()
+86
View File
@@ -0,0 +1,86 @@
import base64
from pathlib import Path
from uuid import uuid4
import psycopg
import pytest
from fastapi.testclient import TestClient
from psycopg import sql
from psycopg.conninfo import make_conninfo
from messenger.app import create_app
from messenger.migrate import migrate
from messenger.settings import Settings
ROOT = Path(__file__).resolve().parents[1]
PASSWORD = "Учебный:пароль😀-12345"
@pytest.fixture(scope="session")
def settings():
password = (ROOT / "secrets/postgres_password").read_text().strip()
admin = make_conninfo(
host="127.0.0.1", port=55467, user="messenger", password=password, dbname="messenger"
)
database = "student_tests_" + uuid4().hex
with psycopg.connect(admin, autocommit=True) as conn:
conn.execute(sql.SQL("CREATE DATABASE {}").format(sql.Identifier(database)))
config = Settings(
make_conninfo(admin, dbname=database),
"redis://127.0.0.1:56386/1",
broker_url="redis://127.0.0.1:56387/1",
s3_bucket="messenger-attachments-tests",
s3_access_key=(ROOT / "secrets/s3_test_access_key_id").read_text().strip(),
s3_secret_key=(ROOT / "secrets/s3_test_secret_access_key").read_text().strip(),
)
migrate(config.database_url, ROOT / "migrations")
with pytest.MonkeyPatch.context() as environment:
environment.setenv("S3_ENDPOINT", config.s3_endpoint)
environment.setenv("S3_BUCKET", config.s3_bucket)
environment.setenv("S3_ACCESS_KEY_ID_FILE", str(ROOT / "secrets/s3_test_access_key_id"))
environment.setenv(
"S3_SECRET_ACCESS_KEY_FILE", str(ROOT / "secrets/s3_test_secret_access_key")
)
yield config
with psycopg.connect(admin, autocommit=True) as conn:
conn.execute(sql.SQL("DROP DATABASE {} WITH (FORCE)").format(sql.Identifier(database)))
@pytest.fixture
def client(settings):
with psycopg.connect(settings.database_url) as conn:
conn.execute("TRUNCATE users,chats,chat_members,messages,sessions CASCADE")
with TestClient(create_app(settings), raise_server_exceptions=False) as client:
client.app.state.cache.client.flushdb()
yield client
def basic(username, password=PASSWORD):
return {
"Authorization": "Basic " + base64.b64encode((username + ":" + password).encode()).decode()
}
@pytest.fixture
def actors(client):
result = []
for name in ("alice", "bob", "eve"):
r = client.post(
"/api/v1/users", json={"username": name, "display_name": name, "password": PASSWORD}
)
assert r.status_code == 201
session = client.post("/api/v1/auth/sessions", headers=basic(name))
assert session.status_code == 201
result.append({**r.json(), "token": session.json()["token"]})
return result
@pytest.fixture
def room(client, actors):
response = client.post(
"/api/v1/chats",
headers={"Authorization": "Bearer " + actors[0]["token"]},
json={"title": "Чат 👋", "member_ids": [u["id"] for u in actors[:2]]},
)
assert response.status_code == 201
return response.json()
+299
View File
@@ -0,0 +1,299 @@
import hashlib
import json
import re
import time
from concurrent.futures import ThreadPoolExecutor
from uuid import uuid4
import psycopg
import pytest
from conftest import PASSWORD, ROOT, basic
from fastapi.testclient import TestClient
from messenger.app import create_app
from messenger.auth import basic_credentials
from messenger.migrate import migrate
def headers(actor):
return {"Authorization": "Bearer " + actor["token"], "Idempotency-Key": uuid4().hex}
def error(response, status, code):
assert response.status_code == status, response.text
body = response.json()["error"]
assert set(body) == {"code", "message", "request_id"}
assert body["code"] == code
assert body["request_id"] == response.headers["X-Request-Id"]
assert len(response.headers.get_list("X-Request-Id")) == 1
if status == 401:
assert response.headers["WWW-Authenticate"].startswith(("Basic", "Bearer"))
@pytest.mark.parametrize("body", ["{", "null", "[]", '"text"', '{"username":'])
def test_malformed_json(client, body):
error(
client.post("/api/v1/users", content=body, headers={"Content-Type": "application/json"}),
400,
"invalid_request",
)
@pytest.mark.parametrize(
"change",
[
{"username": "ab"},
{"username": "a" * 33},
{"username": "Upper"},
{"username": "кириллица"},
{"username": "valid\n"},
{"username": 123},
{"username": True},
{"display_name": 123},
{"display_name": None},
{"display_name": " \n\t"},
{"display_name": "😀" * 101},
{"display_name": "\ud800"},
{"password": "short"},
{"password": "😀" * 129},
{"password": 123},
{"password": " " * 12},
{"password": "\ud800" * 12},
{"extra": True},
],
)
def test_strict_registration(client, change):
data = {"username": "valid", "display_name": "Имя", "password": PASSWORD, **change}
response = client.post(
"/api/v1/users", content=json.dumps(data), headers={"Content-Type": "application/json"}
)
error(response, 400, "invalid_request")
def test_unicode_password_hash_and_authentication(client, settings, actors):
for u in actors:
assert "password" not in u and "password_hash" not in u
assert re.fullmatch(r"[A-Za-z0-9_-]{43}", u["token"])
with psycopg.connect(settings.database_url) as conn:
encoded = conn.execute("SELECT password_hash FROM users WHERE username='alice'").fetchone()[
0
]
assert encoded.startswith("$argon2id$") and PASSWORD not in encoded
assert (
conn.execute("SELECT octet_length(token_hash) FROM sessions LIMIT 1").fetchone()[0]
== 32
)
bad = client.post("/api/v1/auth/sessions", headers=basic("alice", "wrong-password"))
absent = client.post("/api/v1/auth/sessions", headers=basic("unknown", "wrong-password"))
error(bad, 401, "unauthorized")
error(absent, 401, "unauthorized")
assert bad.json()["error"]["message"] == absent.json()["error"]["message"]
assert bad.headers["WWW-Authenticate"] == 'Basic realm="messenger", charset="UTF-8"'
for value in (
{},
basic("alice"),
{"X-User-Id": actors[0]["id"]},
{"Authorization": "Bearer unknown"},
):
error(client.get("/api/v1/users/me", headers=value), 401, "unauthorized")
actual = client.get(
"/api/v1/users/me", headers={**headers(actors[0]), "X-User-Id": actors[2]["id"]}
).json()
assert actual["id"] == actors[0]["id"]
error(
client.post("/api/v1/auth/sessions", headers=basic("alice"), json={}),
400,
"invalid_request",
)
@pytest.mark.parametrize("value", [None, "Bearer a", "Basic !", "Basic YWxpY2U=", "Basic /w=="])
def test_invalid_basic(value):
with pytest.raises(Exception) as caught:
basic_credentials(value)
assert caught.value.status == 401
def test_registration_race(client, settings):
def register(_):
return client.post(
"/api/v1/users",
json={"username": "same", "display_name": "Алиса", "password": PASSWORD},
).status_code
with ThreadPoolExecutor(max_workers=2) as pool:
assert sorted(pool.map(register, range(10))) == [201] + [409] * 9
with psycopg.connect(settings.database_url) as conn:
assert conn.execute("SELECT count(*) FROM users WHERE username='same'").fetchone()[0] == 1
def test_acl_validation_and_transaction_rollback(client, actors, room, settings):
path = "/api/v1/chats/" + room["id"]
for suffix in ("", "/messages"):
error(client.get(path + suffix, headers=headers(actors[2])), 404, "not_found")
error(client.get("/api/v1/chats/not-a-uuid"), 401, "unauthorized")
error(
client.get("/api/v1/chats/not-a-uuid", headers=headers(actors[0])), 400, "invalid_request"
)
with psycopg.connect(settings.database_url) as conn:
before = conn.execute("SELECT count(*) FROM chats").fetchone()[0]
for members in ([actors[0]["id"], str(uuid4())], [actors[0]["id"]] * 2, [actors[1]["id"]], []):
error(
client.post(
"/api/v1/chats",
headers=headers(actors[0]),
json={"title": "rollback", "member_ids": members},
),
400,
"invalid_request",
)
with psycopg.connect(settings.database_url) as conn:
assert conn.execute("SELECT count(*) FROM chats").fetchone()[0] == before
assert conn.execute("SELECT count(*) FROM chat_members").fetchone()[0] == 2
for body in (
{"text": ""},
{"text": " \t"},
{"text": "😀" * 4001},
{"text": 123},
{"text": None},
{"text": "ok", "sender_id": actors[1]["id"]},
):
error(
client.post(path + "/messages", headers=headers(actors[0]), json=body),
400,
"invalid_request",
)
for query in (
"limit=0",
"limit=101",
"limit=true",
"limit=1.0",
"limit=1&limit=2",
"cursor=",
"cursor=bad!",
"cursor=e30",
"cursor=x&cursor=y",
):
error(
client.get(path + "/messages?" + query, headers=headers(actors[0])),
400,
"invalid_request",
)
def test_concurrent_messages_and_keyset_ties(client, actors, room, settings):
path = f"/api/v1/chats/{room['id']}/messages"
def send(i):
r = client.post(path, headers=headers(actors[i % 2]), json={"text": f"m{i} 😀"})
assert r.status_code == 201
return r.json()
with ThreadPoolExecutor(max_workers=10) as pool:
sent = list(pool.map(send, range(100)))
with psycopg.connect(settings.database_url) as conn:
conn.execute(
"UPDATE messages SET created_at='2026-10-17T12:00:00Z' WHERE chat_id=%s", (room["id"],)
)
for m in sent:
m["created_at"] = "2026-10-17T12:00:00.000000Z"
found, cursor = [], None
while True:
query = {"limit": 17 if cursor else 1}
if cursor:
query["cursor"] = cursor
response = client.get(path, params=query, headers=headers(actors[1]))
assert response.status_code == 200
page = response.json()
found.extend(page["items"])
cursor = page["next_cursor"]
if not cursor:
break
assert len(found) == len({x["id"] for x in found}) == 100
assert found == sorted(sent, key=lambda x: (x["created_at"], x["id"]))
first_cursor = client.get(path + "?limit=1", headers=headers(actors[0])).json()["next_cursor"]
error(
client.get("/api/v1/chats", params={"cursor": first_cursor}, headers=headers(actors[0])),
400,
"invalid_request",
)
def test_cache_hit_flush_and_revocation(client, actors, monkeypatch):
user = actors[0]
url = "/api/v1/users/me"
assert client.get(url, headers=headers(user)).status_code == 200
repository = client.app.state.repository
original = repository.get_user
def must_be_cached(*args):
raise AssertionError("cache hit should avoid fetching immutable user again")
monkeypatch.setattr(repository, "get_user", must_be_cached)
assert client.get(url, headers=headers(user)).status_code == 200
monkeypatch.setattr(repository, "get_user", original)
cache = client.app.state.cache.client
digest = hashlib.sha256(user["token"].encode()).digest()
key = client.app.state.cache.key(digest)
assert 0 < cache.ttl(key) <= 3600
stale = cache.get(key)
cache.flushdb()
assert client.get(url, headers=headers(user)).status_code == 200
assert client.delete("/api/v1/auth/sessions/current", headers=headers(user)).status_code == 204
cache.set(key, stale, ex=3600) # A resurrected stale cache entry must not resurrect a session.
error(client.get(url, headers=headers(user)), 401, "unauthorized")
error(
client.delete("/api/v1/auth/sessions/current", headers=headers(user)), 401, "unauthorized"
)
def test_expiry_and_api_restart(client, actors, settings, monkeypatch, tmp_path):
database_file, cache_file = tmp_path / "database_url", tmp_path / "cache_url"
database_file.write_text(settings.database_url)
cache_file.write_text(settings.cache_url)
monkeypatch.setenv("DATABASE_URL_FILE", str(database_file))
monkeypatch.setenv("CACHE_URL_FILE", str(cache_file))
monkeypatch.setenv("SESSION_TTL_SECONDS", "2")
with TestClient(create_app()) as short:
response = short.post("/api/v1/auth/sessions", headers=basic("alice"))
assert response.status_code == 201
token = {"Authorization": "Bearer " + response.json()["token"]}
assert short.get("/api/v1/users/me", headers=token).status_code == 200
with TestClient(create_app(settings)) as restarted:
assert restarted.get("/api/v1/users/me", headers=token).status_code == 200
time.sleep(2.1)
error(restarted.get("/api/v1/users/me", headers=token), 401, "unauthorized")
def test_migrations_idempotent_and_detect_change(client, settings, tmp_path):
migrate(settings.database_url, ROOT / "migrations")
modified = tmp_path / "001_initial.sql"
modified.write_text((ROOT / "migrations/001_initial.sql").read_text() + "\n-- changed")
with pytest.raises(RuntimeError, match="checksum changed"):
migrate(settings.database_url, tmp_path)
def test_logs_and_safe_errors(client, monkeypatch, caplog):
canary = "PRIVATE_BODY_CANARY_123"
with caplog.at_level("INFO", logger="messenger"):
r = client.post(
"/api/v1/users",
json={"username": "logging", "display_name": canary, "password": PASSWORD},
)
assert r.status_code == 201
assert canary not in caplog.text and PASSWORD not in caplog.text
def broken(*args):
raise RuntimeError(canary)
monkeypatch.setattr(client.app.state.authentication, "register", broken)
response = client.post(
"/api/v1/users", json={"username": "broken", "display_name": "x", "password": PASSWORD}
)
error(response, 500, "internal_error")
assert canary not in response.text and canary not in caplog.text
error(
client.post("/api/v1/users", content="{}", headers={"Content-Type": "text/plain"}),
415,
"unsupported_media_type",
)
+137
View File
@@ -0,0 +1,137 @@
import json
from concurrent.futures import ThreadPoolExecutor
from dataclasses import replace
from uuid import uuid4
import psycopg
import pytest
from fastapi.testclient import TestClient
from test_api import error, headers
from messenger.app import create_app
@pytest.mark.parametrize("key", [None, "", "a b", "/", "a" * 129])
def test_message_key_required(client, actors, room, key):
h = {"Authorization": "Bearer " + actors[0]["token"]}
if key is not None:
h["Idempotency-Key"] = key
error(
client.post(
f"/api/v1/chats/{room['id']}/messages", headers=h, json={"text": "key validation"}
),
400,
"invalid_request",
)
def test_replay_scope_conflict_and_fresh_authorization(client, actors, room):
path = f"/api/v1/chats/{room['id']}/messages"
key = "Scope.Key:123"
h = {**headers(actors[0]), "Idempotency-Key": key}
first = client.post(path, headers=h, json={"text": "exact whitespace \n"})
assert first.status_code == 201
repeat = client.post(
path,
headers={**h, "Content-Type": "application/json"},
content=json.dumps({"text": "exact whitespace \n"}, indent=3),
)
assert repeat.status_code == 201 and repeat.json() == first.json()
error(client.post(path, headers=h, json={"text": "different"}), 409, "conflict")
outsider = {**headers(actors[2]), "Idempotency-Key": key}
error(
client.post(path, headers=outsider, json={"text": "exact whitespace \n"}), 404, "not_found"
)
other_user = {**headers(actors[1]), "Idempotency-Key": key}
own = client.post(path, headers=other_user, json={"text": "another author"})
assert own.status_code == 201 and own.json()["id"] != first.json()["id"]
chat = client.post(
"/api/v1/chats",
headers=headers(actors[0]),
json={"title": "another", "member_ids": [actors[0]["id"]]},
).json()
other_chat = client.post(
f"/api/v1/chats/{chat['id']}/messages", headers=h, json={"text": "different chat"}
)
assert other_chat.status_code == 201 and other_chat.json()["id"] != first.json()["id"]
assert client.delete("/api/v1/auth/sessions/current", headers=h).status_code == 204
error(client.post(path, headers=h, json={"text": "exact whitespace \n"}), 401, "unauthorized")
def test_concurrent_replay_across_independent_apps(client, actors, room, settings):
path = f"/api/v1/chats/{room['id']}/messages"
h = {**headers(actors[0]), "Idempotency-Key": uuid4().hex}
with TestClient(create_app(settings)) as second:
assert second.app.state.instance_id != client.app.state.instance_id
def send(i):
c = client if i % 2 else second
r = c.post(path, headers=h, json={"text": "same concurrent body"})
assert r.status_code == 201
return r.json(), r.headers["X-Instance-Id"]
with ThreadPoolExecutor(max_workers=10) as pool:
results = list(pool.map(send, range(30)))
assert len({item[0]["id"] for item in results}) == 1
assert len({item[1] for item in results}) == 2
assert len(client.get(path, headers=headers(actors[1])).json()["items"]) == 1
def test_side_effect_and_result_rollback_together(client, actors, room, settings, monkeypatch):
repository = client.app.state.repository
original = repository._save_request
def fail_after_message_insert(*args):
raise RuntimeError("injected between message and result")
monkeypatch.setattr(repository, "_save_request", fail_after_message_insert)
path = f"/api/v1/chats/{room['id']}/messages"
h = {**headers(actors[0]), "Idempotency-Key": uuid4().hex}
error(
client.post(path, headers=h, json={"text": "retry after rollback"}), 500, "internal_error"
)
with psycopg.connect(settings.database_url) as conn:
assert conn.execute("SELECT count(*) FROM messages").fetchone()[0] == 0
assert conn.execute("SELECT count(*) FROM message_requests").fetchone()[0] == 0
monkeypatch.setattr(repository, "_save_request", original)
result = client.post(path, headers=h, json={"text": "retry after rollback"})
assert result.status_code == 201
assert (
client.post(path, headers=h, json={"text": "retry after rollback"}).json() == result.json()
)
def test_expired_key_can_be_reused_without_deleting_old_message(client, actors, room, settings):
path = f"/api/v1/chats/{room['id']}/messages"
h = {**headers(actors[0]), "Idempotency-Key": uuid4().hex}
first = client.post(path, headers=h, json={"text": "old"}).json()
with psycopg.connect(settings.database_url) as conn:
conn.execute(
"UPDATE message_requests SET created_at=clock_timestamp()-interval '2 days', "
"expires_at=clock_timestamp()-interval '1 day'"
)
second = client.post(path, headers=h, json={"text": "new"})
assert second.status_code == 201 and second.json()["id"] != first["id"]
assert len(client.get(path, headers=headers(actors[0])).json()["items"]) == 2
@pytest.mark.parametrize("optimized", [True, False])
def test_read_modes_preserve_public_result(client, actors, room, settings, optimized):
with TestClient(create_app(replace(settings, read_autocommit=optimized))) as c:
c.get("/api/v1/users/me", headers=headers(actors[0]))
pool = c.app.state.repository.pool
before = pool.get_stats().get("requests_num", 0)
response = c.get(f"/api/v1/chats/{room['id']}/messages", headers=headers(actors[0]))
assert response.status_code == 200 and response.json()["items"] == []
assert pool.get_stats()["requests_num"] - before == 3
def test_instance_header_covers_health_errors_and_success(client):
values = [
client.get("/health/live"),
client.get("/health/ready"),
client.get("/api/v1/users/me"),
client.get("/not-a-route"),
]
assert len({r.headers["X-Instance-Id"] for r in values}) == 1
assert all(len(r.headers.get_list("X-Instance-Id")) == 1 for r in values)
+357
View File
@@ -0,0 +1,357 @@
import hashlib
import io
import json
import struct
import zlib
from dataclasses import replace
from datetime import UTC, datetime, timedelta
from types import SimpleNamespace
from urllib.parse import quote
from uuid import uuid4
import jsonschema
import pytest
from conftest import ROOT
from PIL import Image
from test_api import error, headers
from messenger.broker import GROUP, STREAM, Broker, Job
from messenger.cleanup import collect
from messenger.domain import DomainError
from messenger.images import variants
from messenger.pipeline import Pipeline
from messenger.relay import publish_one
from messenger.worker import handle
PNG = (ROOT / "tests/fixtures/sample.png").read_bytes()
def upload(client, actor, room, data=PNG, media="image/png", name="Фото.png"):
return client.post(
f"/api/v1/chats/{room['id']}/attachments?filename={quote(name, safe='')}",
headers={**headers(actor), "Content-Type": media},
content=data,
)
@pytest.fixture
def broker(settings):
b = Broker(settings)
assert settings.broker_url.endswith("/1")
b.client.flushdb()
b.group()
yield b
b.client.flushdb()
b.close()
def process(client, settings, broker):
db = client.app.state.repository
assert publish_one(db, broker, settings)
id, fields = broker.client.xreadgroup(GROUP, "unit", {STREAM: ">"}, count=1)[0][1][0]
jsonschema.validate(
json.loads(fields["job"]),
json.loads((ROOT / "contracts/jobs.schema.json").read_text()),
format_checker=jsonschema.FormatChecker(),
)
handle(db, broker, client.app.state.attachments.store, settings, id, fields["job"])
def test_queued_message_ready_and_all_variants(client, actors, room, settings, broker):
r = upload(client, actors[0], room)
assert r.status_code == 202 and r.json()["status"] == "queued"
a = r.json()
assert not a["variants"] and a["sha256"] == hashlib.sha256(PNG).hexdigest()
path = "/api/v1/attachments/" + a["id"]
error(client.get(path + "/content", headers=headers(actors[0])), 409, "conflict")
key = uuid4().hex
msg = client.post(
f"/api/v1/chats/{room['id']}/messages",
json={"text": "", "attachment_ids": [a["id"]]},
headers={**headers(actors[0]), "Idempotency-Key": key},
)
assert msg.status_code == 201
for suffix in ("", "/content"):
error(client.get(path + suffix, headers=headers(actors[2])), 404, "not_found")
error(client.get(path + suffix), 401, "unauthorized")
process(client, settings, broker)
ready = client.get(path, headers=headers(actors[1])).json()
assert ready["status"] == "ready" and ready["error_code"] is None
assert {v["name"] for v in ready["variants"]} == {"thumbnail", "crop", "optimized"}
assert client.get(path + "/content", headers=headers(actors[1])).content == PNG
for v in ready["variants"]:
content = client.get(path + "/content?variant=" + v["name"], headers=headers(actors[1]))
assert content.status_code == 200 and content.headers["content-type"] == v["content_type"]
assert len(content.content) == v["size_bytes"]
assert hashlib.sha256(content.content).hexdigest() == v["sha256"]
with Image.open(io.BytesIO(content.content)) as image:
assert image.size == (v["width"], v["height"])
assert not image.getexif()
with client.app.state.repository.connection() as conn:
j = conn.execute(
"SELECT * FROM attachment_jobs WHERE attachment_id=%s", (a["id"],)
).fetchone()
job = Job(
schema_version=1,
type="attachment.process.v1",
job_id=str(j["job_id"]),
attachment_id=a["id"],
pipeline_version=1,
attempt=1,
created_at=a["created_at"],
)
sid = broker.publish(job)
handle(
client.app.state.repository,
broker,
client.app.state.attachments.store,
settings,
sid,
job.model_dump_json(exclude_none=True),
)
assert client.get(path, headers=headers(actors[0])).json() == ready
assert broker.client.xlen(STREAM) == 0
@pytest.mark.parametrize("name", ["", "../a.png", "a\\b.png", "a\x00.png", "\n.png", "x" * 256])
def test_bad_filename(client, actors, room, name):
error(upload(client, actors[0], room, name=name), 400, "invalid_request")
@pytest.mark.parametrize(
"data,media",
[
(b"", "image/png"),
(b"bad", "image/png"),
(PNG[:30], "image/png"),
(PNG, "image/jpeg"),
(PNG, "text/plain"),
],
)
def test_bad_image(client, actors, room, data, media):
error(upload(client, actors[0], room, data, media), 415, "unsupported_media_type")
def test_upload_limits_and_pixel_bomb(client, actors, room):
error(upload(client, actors[0], room, b"x" * (10485760 + 1)), 413, "payload_too_large")
error(
upload(client, actors[0], room, (b"x" * 1048576 for _ in range(11))),
413,
"payload_too_large",
)
data = bytearray(PNG)
data[16:24] = struct.pack(">II", 10000, 10000)
data[29:33] = struct.pack(">I", zlib.crc32(data[12:29]) & 0xFFFFFFFF)
error(upload(client, actors[0], room, bytes(data)), 413, "payload_too_large")
@pytest.mark.parametrize("size", [(320, 200), (17, 11), (1, 400), (400, 1), (1281, 713)])
def test_transform_dimensions_alpha_and_exif(size):
raw = io.BytesIO()
Image.new("RGBA", size, (1, 2, 3, 0)).save(raw, "PNG")
results = {name: (data, dims, media) for name, data, dims, media in variants(raw, 1)}
scale = max(size)
assert results["thumbnail"][1] == tuple(max(1, n * min(256, scale) // scale) for n in size)
assert results["crop"][1] == (128, 128)
assert max(results["optimized"][1]) <= 1280
with Image.open(io.BytesIO(results["thumbnail"][0])) as image:
assert image.getpixel((0, 0)) == (255, 255, 255)
def test_exif_orientation_and_original_unchanged():
raw = io.BytesIO()
exif = Image.Exif()
exif[274] = 6
Image.new("RGB", (320, 200), "red").save(raw, "JPEG", exif=exif)
before = raw.getvalue()
result = list(variants(raw, 1))
assert result[0][2] == (160, 256) and raw.getvalue() == before
for _, data, _, _ in result:
with Image.open(io.BytesIO(data)) as image:
assert not image.getexif()
def test_failed_retry_replay_and_ready_replay_rejected(
client, actors, room, settings, broker, monkeypatch
):
a = upload(client, actors[0], room).json()
store, db = client.app.state.attachments.store, client.app.state.repository
original = store.put
monkeypatch.setattr(
store, "put", lambda *_: (_ for _ in ()).throw(DomainError(503, "unavailable", "test"))
)
process(client, replace(settings, job_max_attempts=1), broker)
path = "/api/v1/attachments/" + a["id"]
assert client.get(path, headers=headers(actors[0])).json()["error_code"] == "retry_exhausted"
error(
client.post(
f"/api/v1/chats/{room['id']}/messages",
json={"text": "", "attachment_ids": [a["id"]]},
headers={**headers(actors[0]), "Idempotency-Key": uuid4().hex},
),
409,
"conflict",
)
Pipeline(db, settings).replay(a["id"])
monkeypatch.setattr(store, "put", original)
process(client, settings, broker)
assert client.get(path, headers=headers(actors[0])).json()["status"] == "ready"
with pytest.raises(ValueError):
Pipeline(db, settings).replay(a["id"])
def test_invalid_envelope_quarantined(client, settings, broker):
sid = broker.client.xadd(STREAM, {"job": '{"url":"http://untrusted.invalid/"}'})
handle(
client.app.state.repository,
broker,
client.app.state.attachments.store,
settings,
sid,
'{"url":"http://untrusted.invalid/"}',
)
assert broker.client.xlen("invalid-jobs") == 1
assert "url" not in json.dumps(broker.client.xrange("invalid-jobs"))
def test_staging_failure_has_no_accepted_job(client, actors, room, monkeypatch):
store = client.app.state.attachments.store
monkeypatch.setattr(
store, "put", lambda *_: (_ for _ in ()).throw(DomainError(503, "unavailable", "test"))
)
error(upload(client, actors[0], room), 503, "unavailable")
with client.app.state.repository.connection() as conn:
assert conn.execute("SELECT count(*) AS n FROM attachment_jobs").fetchone()["n"] == 0
def test_gc_minimum_retention(client):
with pytest.raises(ValueError):
collect(client.app.state.repository, client.app.state.attachments.store, 0, True)
def test_gc_fences_paused_upload_before_job_commit(client, actors, room, settings, monkeypatch):
db, store = client.app.state.repository, client.app.state.attachments.store
p = Pipeline(db, settings)
id = str(uuid4())
key = "staging/" + id
p.reserve(id, key)
store.put(key, io.BytesIO(PNG), len(PNG), "image/png", hashlib.sha256(PNG).hexdigest())
# Simulated S3 object age; actual private test-bucket deletion and DB locks are real.
old = datetime.now(UTC) - timedelta(seconds=301)
head = store.head(key)
head["LastModified"] = old
objects = [{"Key": key, "ETag": head["ETag"], "Size": len(PNG), "LastModified": old}]
monkeypatch.setattr(store, "head", lambda _: head)
monkeypatch.setattr(
store.client,
"get_paginator",
lambda _: SimpleNamespace(paginate=lambda **_: [{"Contents": objects}]),
)
assert collect(db, store, 300, True)["protected"] == 1
with db.connection(write=True) as conn:
conn.execute(
"UPDATE upload_reservations SET expires_at=clock_timestamp()-interval '1 second' "
"WHERE id=%s",
(id,),
)
assert collect(db, store, 300, False)["eligible"] == 1
assert collect(db, store, 300, True)["deleted"] == 1
with pytest.raises(DomainError) as expired:
p.accept(
id,
room["id"],
actors[0]["id"],
"paused.png",
"image/png",
len(PNG),
hashlib.sha256(PNG).hexdigest(),
key,
)
assert expired.value.status == 503 and db.attachment(id) is None
def test_version_fence_rejects_old_writer(client, actors, room, settings, broker):
a = upload(client, actors[0], room).json()
db, store = client.app.state.repository, client.app.state.attachments.store
with db.connection() as conn:
j = conn.execute(
"SELECT * FROM attachment_jobs WHERE attachment_id=%s", (a["id"],)
).fetchone()
envelope = Job(
schema_version=1,
type="attachment.process.v1",
job_id=str(j["job_id"]),
attachment_id=a["id"],
pipeline_version=1,
attempt=1,
created_at=a["created_at"],
)
pipeline = Pipeline(db, settings)
outcome, old = pipeline.claim(envelope)
assert outcome == "claimed"
pipeline.replay(a["id"], new_version=True)
process(client, settings, broker)
current = db.attachment(a["id"])
assert current["pipeline_version"] == 2 and current["storage_state"] == "ready"
assert not pipeline.publish(old, "objects/old-result", [])
handle(
db,
broker,
store,
settings,
broker.publish(envelope),
envelope.model_dump_json(exclude_none=True),
)
assert db.attachment(a["id"])["object_key"] == current["object_key"]
def test_foreign_chat_attachment_is_not_linkable(client, actors, room):
a = upload(client, actors[0], room).json()
other = client.post(
"/api/v1/chats",
headers=headers(actors[0]),
json={"title": "other", "member_ids": [actors[0]["id"]]},
).json()
error(
client.post(
f"/api/v1/chats/{other['id']}/messages",
headers={**headers(actors[0]), "Idempotency-Key": uuid4().hex},
json={"text": "", "attachment_ids": [a["id"]]},
),
404,
"not_found",
)
@pytest.mark.parametrize(
"patch",
[
{"schema_version": True},
{"traceparent": None},
{"tracestate": None},
{"url": "https://untrusted.invalid/"},
{"job_id": "not-a-uuid"},
{"pipeline_version": 0},
{"attempt": 0},
{"created_at": "2026-10-17T12:00:00"},
{"traceparent": "invalid"},
{"type": "download.anything"},
],
)
def test_job_schema_rejects_invalid_carrier(client, settings, broker, patch):
payload = {
"schema_version": 1,
"type": "attachment.process.v1",
"job_id": str(uuid4()),
"attachment_id": str(uuid4()),
"pipeline_version": 1,
"attempt": 1,
"created_at": "2026-10-17T12:00:00Z",
**patch,
}
data = json.dumps(payload)
sid = broker.client.xadd(STREAM, {"job": data})
handle(
client.app.state.repository, broker, client.app.state.attachments.store, settings, sid, data
)
assert broker.client.xrange("invalid-jobs")[0][1]["reason"] == "invalid_envelope"
Generated
+1206
View File
File diff suppressed because it is too large Load Diff