forked from mtusi2026/mtusi2026lab6
Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b2e70c9619 | ||
|
|
b2eca3d259 |
@@ -17,4 +17,10 @@ backups
|
||||
__pycache__
|
||||
node_modules
|
||||
evidence
|
||||
student-tests
|
||||
tests
|
||||
research
|
||||
REPORT.md
|
||||
SCHEMA.md
|
||||
requirements-dev.txt
|
||||
.DS_Store
|
||||
|
||||
@@ -28,3 +28,12 @@ OTEL_SERVICE_NAME=messenger-api
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318
|
||||
OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
|
||||
OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab
|
||||
|
||||
# Local ports and telemetry mode.
|
||||
PUBLIC_PORT=8086
|
||||
GRAFANA_PORT=3006
|
||||
PROMETHEUS_PORT=9096
|
||||
LOKI_PORT=3106
|
||||
TEMPO_PORT=3206
|
||||
TELEMETRY_ENABLED=1
|
||||
OTEL_SAMPLE_RATIO=1
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
3.14.7
|
||||
+29
@@ -0,0 +1,29 @@
|
||||
FROM python:3.14.7-slim-bookworm@sha256:82bc3c539b8813ada9d68c63b40158fa002f7f33de9bf3312a3dfdc0620dff56 AS builder
|
||||
WORKDIR /app
|
||||
RUN python -m pip install --no-cache-dir uv==0.12.13
|
||||
ENV UV_PYTHON_DOWNLOADS=never
|
||||
COPY pyproject.toml uv.lock .python-version ./
|
||||
RUN --mount=type=cache,target=/root/.cache/uv uv sync --frozen --no-dev --no-install-project --compile-bytecode
|
||||
COPY src ./src
|
||||
RUN --mount=type=cache,target=/root/.cache/uv uv sync --frozen --no-dev --no-editable --compile-bytecode
|
||||
|
||||
FROM python:3.14.7-slim-bookworm@sha256:82bc3c539b8813ada9d68c63b40158fa002f7f33de9bf3312a3dfdc0620dff56
|
||||
RUN groupadd --gid 10001 app && useradd --uid 10001 --gid app --no-create-home app
|
||||
WORKDIR /app
|
||||
COPY --from=builder /app/.venv /app/.venv
|
||||
COPY migrations /app/migrations
|
||||
ARG VCS_REF=uncommitted
|
||||
LABEL org.opencontainers.image.revision=$VCS_REF
|
||||
ENV PATH="/app/.venv/bin:$PATH" \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
PYTHONDONTWRITEBYTECODE=1 \
|
||||
APP_HOST=0.0.0.0 \
|
||||
APP_PORT=8080 \
|
||||
APP_ENV=lab \
|
||||
APP_LOG_LEVEL=INFO
|
||||
USER 10001:10001
|
||||
EXPOSE 8080
|
||||
STOPSIGNAL SIGTERM
|
||||
HEALTHCHECK --interval=2s --timeout=2s --start-period=5s --retries=3 \
|
||||
CMD ["python", "-c", "import os, urllib.request; urllib.request.urlopen('http://127.0.0.1:' + os.environ.get('APP_PORT', '8080') + '/health/ready', timeout=1).close()"]
|
||||
CMD ["python", "-m", "messenger"]
|
||||
@@ -1,5 +1,5 @@
|
||||
PYTHON ?= python3
|
||||
BASE_URL ?= http://localhost:8080
|
||||
BASE_URL ?= http://127.0.0.1:8086
|
||||
CA_FILE ?=
|
||||
ACTION_SCRIPT ?=
|
||||
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
|
||||
@@ -28,3 +28,21 @@ test-persistence:
|
||||
test-failover:
|
||||
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
|
||||
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
.PHONY: bootstrap up down unit lint verify-cluster verify-pipeline
|
||||
bootstrap:
|
||||
$(PYTHON) scripts/bootstrap.py
|
||||
up:
|
||||
docker compose --env-file .env up -d --build --wait --wait-timeout 180
|
||||
|
||||
down:
|
||||
docker compose --env-file .env stop --timeout 7
|
||||
unit:
|
||||
uv run pytest --tb=short
|
||||
lint:
|
||||
uv run ruff check .
|
||||
uv run ruff format --check .
|
||||
verify-cluster:
|
||||
uv run python scripts/verify_cluster.py --base-url "$(BASE_URL)"
|
||||
verify-pipeline:
|
||||
uv run python scripts/verify_pipeline.py --base-url "$(BASE_URL)"
|
||||
|
||||
@@ -1,37 +1,100 @@
|
||||
# Отчёт — лабораторная 6
|
||||
# ЛР6. Наблюдаемость мессенджера
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
**Горбанюк, БВТ2301.** Целевая оценка 5. Трек 1 «Цена наблюдаемости».
|
||||
|
||||
Использованы материалы задания, документация [OpenTelemetry](https://opentelemetry.io/docs/collector/configuration/), [Loki OTLP](https://grafana.com/docs/loki/latest/send-data/otel/) и [Prometheus](https://prometheus.io/docs/practices/histograms/). За основу взято собственное решение ЛР5. При подготовке реализации, проверок и отчёта использовалась помощь ИИ.
|
||||
|
||||
## Запуск из чистого clone
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
Нужны Docker с Compose и Python 3.14 для локальных проверок. Runtime приложения — Python 3.14.7, зависимости в `uv.lock`, OpenTelemetry Python 1.45.2. Стенд включает Collector Contrib 0.147.0, Prometheus 3.9.1, Loki 3.6.3, Tempo 2.10.0 и Grafana 12.3.3. Образы и лимиты указаны в `compose.yaml`. Docker сам запускает миграции и создаёт приватный bucket.
|
||||
|
||||
## Архитектура и решения
|
||||
```sh
|
||||
make bootstrap
|
||||
make up
|
||||
curl -fsS http://127.0.0.1:8086/health/ready
|
||||
make test
|
||||
uv sync --locked
|
||||
make check
|
||||
make validate PYTHON=.venv/bin/python
|
||||
make lint
|
||||
```
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
Grafana доступна на `http://127.0.0.1:3006`; пользователь `admin`, пароль в локальном `secrets/grafana_password`. Панель **MTUSI Messenger** и три источника появляются при старте. Прямые порты Prometheus 9096, Loki 3106 и Tempo 3206 привязаны к loopback для локальных проверок. `make down` останавливает стенд с сохранением данных. `docker compose down -v` удаляет учебные данные этого проекта.
|
||||
|
||||
## Доказательства критериев
|
||||
Для набора собственных тестов и диагностики с инъекцией одноразового сбоя:
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
```sh
|
||||
docker compose -f compose.yaml -f compose.test.yaml up -d --build --wait
|
||||
make unit
|
||||
make test-persistence ACTION_SCRIPT=scripts/restart.sh
|
||||
make test-failover ACTION_SCRIPT=scripts/stop-one.sh
|
||||
uv run python scripts/verify_pipeline.py --output evidence/pipeline-repeat.json
|
||||
uv run python scripts/observe.py --incidents --retry --output evidence/observe-repeat.json
|
||||
uv run python scripts/telemetry_cost.py --output evidence/cost-repeat.json
|
||||
```
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
## Архитектура
|
||||
|
||||
## Эксперименты и отказы
|
||||
Две реплики FastAPI за HAProxy используют общие PostgreSQL, Valkey и S3. Из ЛР5 сохранены загрузка со статусом 202 после приёма файла и фиксации job/outbox, relay, отдельный durable Valkey Streams и обработка в дочернем процессе worker. PostgreSQL хранит статус и версии, S3 — байты. Повторная доставка ограничена состоянием задачи и транзакционной проверкой fence.
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
API, relay и worker отправляют OTLP/HTTP в Collector. Prometheus опрашивает экспортёр метрик и сам Collector. Loki хранит структурные логи, Tempo хранит трассы. Grafana загружает источники и панель из Git. У хранилищ отдельные volumes, задано удержание примерно 72 часа; для Prometheus также действует предел размера 256 MB.
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
`traceparent` и `tracestate` добавляются к job той же транзакцией, которая фиксирует её приём. Relay создаёт producer span под контекстом из БД и вкладывает свой контекст в envelope; worker создаёт consumer span. Повторная публикация использует исходный сохранённый контекст. Для обработки видны дочерние операции PostgreSQL и S3. Супервизор агрегирует исходы и задержки процессов, включая таймауты, в стабильную серию worker.
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
В метриках HTTP используется шаблон маршрута и класс статуса; health исключён. В логах остаются созданные сервисом request ID, trace ID, span ID и в событиях обработки job ID для поиска. Инструментация не читает заголовки авторизации, содержимое запросов, SQL, имена файлов или ключи объектов. Исключения в span записывают только тип. `service.instance.id` отличает процессы, `service.version` берётся из установленного приложения, окружение `lab`. Нестандартные имена описаны ниже.
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
| Сигнал | Prometheus | Единицы и агрегация |
|
||||
| --- | --- | --- |
|
||||
| Запросы и ошибки | `messenger_http_requests_total` | rate по всем API; ошибки — 5xx / бизнес-запросы |
|
||||
| Длительность HTTP | `messenger_http_duration_seconds_bucket` | p50/p95/p99 через `histogram_quantile` после суммы rate buckets по `le` |
|
||||
| Задачи и время обработки | `messenger_jobs_processed_total`, `messenger_jobs_duration_seconds_bucket` | исходы и гистограмма в секундах |
|
||||
| Backlog и возраст | `messenger_queue_depth`, `messenger_queue_oldest_seconds` | число и возраст незавершённых job; снимок PostgreSQL каждые 2 с |
|
||||
| Пул БД и worker | `messenger_db_pool_used`, `messenger_db_pool_waiting`, `messenger_worker_busy`, `messenger_worker_capacity` | используемые и ожидающие соединения при обращении к пулу, занятые слоты worker |
|
||||
| Время до ready | `messenger_ready_duration_seconds_bucket` | от времени создания записи в транзакции приёма до фиксации результата; секунды |
|
||||
| CPU и память | `messenger_process_cpu_seconds_total`, `messenger_process_rss_bytes` | CPU процесса с завершёнными дочерними процессами, RSS процесса; для полного дерева дополнительно Docker stats |
|
||||
|
||||
## Диагностика
|
||||
|
||||
Учебные цели для окна 5 минут: 99% бизнес-запросов без 5xx, 95% принятых и завершённых изображений готовы за 30 секунд. Для второго SLI доля берётся из buckets, а незавершённые и окончательно failed контролируются отдельно через backlog и исходы задач. Пороги демонстрационных алертов рассчитаны на короткий стенд. HTTP alert требует >5% 5xx в окне 20 секунд с выдержкой 6 секунд. Queue alert требует незавершённые job и возраст >5 секунд с выдержкой 6 секунд. Дополнительный DB alert использует рост ошибок пула в окне 20 секунд.
|
||||
|
||||
1. При `MessengerQueueLag` проверить число и возраст незавершённых job, слоты worker, затем потребление и логи worker. Из лога загрузки перейти к trace ID и искать `jobs.publish` и `jobs.consume`. Рост очереди при исчезнувших consume и стабильном HTTP указывает на остановку обработки.
|
||||
2. При HTTP 5xx проверить `MessengerDatabaseErrors`, взять request ID из ответа и найти лог в Loki. В Tempo сравнить длительность HTTP span и `db.read`. Узкий участок виден внутри БД; для сравнения обычный запрос укладывался в миллисекунды.
|
||||
3. Если данные исчезли из Tempo, сверить accepted/sent/failed у Collector и проверить readiness API. Отдельная отправка из SDK и очередь Collector ограничивают влияние на мессенджер. В SDK до 512 span и 512 log records на процесс; у Collector memory limiter 128 MiB, cgroup 256 MiB и 16 batches на экспортёр, таймаут попытки 2 секунды, окно повторов 5 секунд. RSS процесса отличается от управляемой памяти Go.
|
||||
|
||||
| Воздействие | Наблюдение |
|
||||
| --- | --- |
|
||||
| Остановка worker и 3 загрузки | Алерт firing при глубине 3 и возрасте 16.9 с. После запуска 3/3 ready, очередь 0 и alert resolved. |
|
||||
| Ограниченная блокировка таблицы sessions | 12 запросов 503, максимум около 1.01 с. Длительность локализуется в `db.read`; оба алерта firing, затем resolved. |
|
||||
| Остановка Tempo | 15 записей сообщений успешны, 21 span учтён счётчиком ошибок экспорта. Снимок RSS Collector около 197 MiB. API сохранил готовность. |
|
||||
|
||||
Доказательства запросов к реальным хранилищам и статусов алертов сохранены в [observability.json](evidence/observability.json). [Трасса повтора](evidence/trace-retry.json) показывает две попытки worker под одним trace ID после одноразового сбоя. В [наследуемой матрице pipeline](evidence/pipeline.json) прошли 10 сценариев ЛР5, 11 подтверждённых загрузок сохранены, устаревших перезаписей нет. При отказе метрики и логи остаются доступны; старые трассы, отброшенные после исчерпания повторов, не воспроизводятся автоматически.
|
||||
|
||||
## Трек 1. Цена наблюдаемости
|
||||
|
||||
Гипотеза: пакетный экспорт сохраняет сопоставимую задержку под выбранной нагрузкой; уменьшение head sampling экономит объём трасс и оставляет метрики и логи. На одном Docker Desktop ARM64, VM 18 vCPU и 7.75 GiB, сравнивались четыре режима. API — 2 × 1 CPU/512 MiB, worker — 2 CPU/1536 MiB, Collector — 0.5 CPU/256 MiB. Один и тот же образ и лимиты. В каждом запуске отдельный чат из 25 сообщений, затем 3 с прогрева и 10 с измерения с фиксированной подачей 18 запросов/с. Запросы: страницы истории до 20 сообщений, около 20% новых сообщений, одна загрузка маленького PNG каждые 45 операций. Два повтора с обратным порядком режимов. Отброшены только результаты прогрева; его записи остаются. Сохраняются четыре снимка CPU/RSS на режим.
|
||||
|
||||
Медианы повторов, 180 из 180 успешных операций в каждом из восьми запусков:
|
||||
|
||||
| Режим | p50, мс | p95, мс | p99, мс | Принято span | Байт трасс в Tempo | CPU приложения, % одного ядра | Память приложения, MiB |
|
||||
| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: |
|
||||
| off | 5.15 | 8.28 | 22.17 | 0 | 0 | 41.74 | 308.82 |
|
||||
| 100% | 5.31 | 8.46 | 17.11 | 997 | 142 250 | 79.81 | 402.25 |
|
||||
| 50% | 5.41 | 10.08 | 21.53 | 522 | 80 782 | 76.21 | 391.50 |
|
||||
| 25% | 5.03 | 9.49 | 27.17 | 238 | 41 374 | 76.44 | 365.21 |
|
||||
|
||||
CPU/RAM приложения — сумма API, relay и worker в снимке Docker stats. В [телеметрии эксперимента](evidence/telemetry-cost.json) также есть снимки четырёх компонентов наблюдаемости и фактическое время каждого прогона. Измерено число записей логов и точек метрик, объём байтов — protobuf трасс после ограничений на входе Tempo. Во включённых режимах за прогон принято около 188 логов; меняется sampling трасс, логи и метрики сохраняются. Разность accepted/sent в несколько span объяснима границами scrape и остатком batch. Счётчик ошибок экспорта в этих восьми прогонах не рос; эксперимент с остановкой Tempo отдельно показывает реальный сбой доставки. Измерение не видит потерю записи до приёма Collector и не устанавливает отсутствие потерь во всех условиях.
|
||||
|
||||
Режим 50% уменьшил объём трасс примерно на 43% относительно 100% при сохранении метрик и логов. По CPU основной рост связан с включением всей телеметрии, переключение sampling само по себе не убирает стоимость метрик и логов. По двум коротким повторам нельзя установить выигрыш в задержке: p95 и особенно p99 колеблются. Head sampling выбирает trace до ответа, поэтому пропуск редкого медленного или ошибочного запроса так же вероятен, как пропуск обычного. При 50% вероятность пропуска одного уникального инцидента примерно 1/2, при 25% — 3/4. Для разборов можно временно использовать 100%, для наблюдения при данной нагрузке проверен 50%. Исходный режим защиты — 100%, чтобы учебная трасса воспроизводилась детерминированно.
|
||||
|
||||
## Проверки и границы
|
||||
|
||||
- Публичный smoke, OpenAPI и 84 собственных сценария, включая ACL, сохранение и восстановление pipeline. [Чистый clone](evidence/clean-clone.txt) поднят отдельным Compose project с новыми volumes и локальными credentials.
|
||||
- Метрики и provisioning подтверждены запросами к данным, `scripts/observe.py` проверяет связь Loki/Tempo и восстанавливает затронутые сервисы.
|
||||
- Compose развёрнут на одном хосте; потеря VM или volume не покрывается. Отказ backend телеметрии допускает потерю сигналов. Публичная сеть, внутренние TLS и управление сервисными секретами рассматриваются в ЛР7.
|
||||
|
||||
## Вопросы к защите
|
||||
|
||||
- **Чем отличаются метрика, лог и трасса?** Метрика даёт агрегированный симптом, лог — конкретное событие, трасса — путь и время операции между процессами.
|
||||
- **Как считается p95 нескольких реплик?** Сначала суммируются скорости bucket counters по границе, потом находится квантиль общей гистограммы.
|
||||
- **Как связаны HTTP и worker?** Контекст хранится в job, relay передаёт его в envelope, worker продолжает trace. Request ID называет HTTP-попытку, trace ID объединяет операции.
|
||||
- **Что при отказе хранилища наблюдаемости?** SDK и Collector буферизуют в пределах лимитов. По таймауту экспорт допускает потери, сообщения и задания продолжают жить в прикладных хранилищах.
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
# Only DB/cache test ports; API replicas always stay behind the LB.
|
||||
services:
|
||||
postgres:
|
||||
ports: ["127.0.0.1:${TEST_PG_PORT:-55467}:5432"]
|
||||
valkey:
|
||||
ports: ["127.0.0.1:${TEST_CACHE_PORT:-56386}:6379"]
|
||||
s3:
|
||||
ports: ["127.0.0.1:${TEST_S3_PORT:-59006}:9000"]
|
||||
bucket-init:
|
||||
environment:
|
||||
CREATE_TEST_BUCKET: '1'
|
||||
broker:
|
||||
ports: ["127.0.0.1:${TEST_BROKER_PORT:-56387}:6379"]
|
||||
worker:
|
||||
environment:
|
||||
ENABLE_FAULT_INJECTION: '1'
|
||||
JOB_TIMEOUT_SECONDS: ${TEST_JOB_TIMEOUT_SECONDS:-5}
|
||||
relay:
|
||||
environment:
|
||||
ENABLE_FAULT_INJECTION: '1'
|
||||
JOB_TIMEOUT_SECONDS: ${TEST_JOB_TIMEOUT_SECONDS:-5}
|
||||
+323
@@ -0,0 +1,323 @@
|
||||
name: mtusi-lab6
|
||||
|
||||
x-api: &api
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab6}
|
||||
build:
|
||||
context: .
|
||||
args:
|
||||
VCS_REF: ${VCS_REF:-uncommitted}
|
||||
user: "10001:10001"
|
||||
read_only: true
|
||||
tmpfs: ["/tmp:rw,noexec,nosuid,size=32m"]
|
||||
cap_drop: [ALL]
|
||||
security_opt: [no-new-privileges:true]
|
||||
pids_limit: 96
|
||||
cpus: ${API_CPU:-1}
|
||||
mem_limit: ${API_MEMORY:-512m}
|
||||
memswap_limit: ${API_MEMORY:-512m}
|
||||
stop_grace_period: 7s
|
||||
secrets:
|
||||
- database_url
|
||||
- cache_url
|
||||
- source: s3_api_access_key_id
|
||||
target: s3_access_key_id
|
||||
- source: s3_api_secret_access_key
|
||||
target: s3_secret_access_key
|
||||
environment:
|
||||
APP_HOST: 0.0.0.0
|
||||
APP_PORT: 8080
|
||||
BROKER_URL_FILE: /run/secrets/broker_url
|
||||
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
|
||||
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
||||
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-20}
|
||||
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
|
||||
APP_ENV: lab6
|
||||
TELEMETRY_ENABLED: ${TELEMETRY_ENABLED:-1}
|
||||
OTEL_SAMPLE_RATIO: ${OTEL_SAMPLE_RATIO:-1}
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
|
||||
DATABASE_READ_AUTOCOMMIT: ${DATABASE_READ_AUTOCOMMIT:-1}
|
||||
DATABASE_POOL_SIZE: ${DATABASE_POOL_SIZE:-8}
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
CACHE_URL_FILE: /run/secrets/cache_url
|
||||
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000}
|
||||
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
|
||||
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
|
||||
S3_REGION: ${S3_REGION:-us-east-1}
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
||||
networks: [backend]
|
||||
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:18.3
|
||||
environment:
|
||||
POSTGRES_USER: messenger
|
||||
POSTGRES_DB: messenger
|
||||
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
|
||||
secrets: [postgres_password]
|
||||
volumes: [postgres_data:/var/lib/postgresql]
|
||||
command: [postgres, -c, fsync=on, -c, synchronous_commit=on, -c, full_page_writes=on,
|
||||
-c, shared_preload_libraries=pg_stat_statements]
|
||||
healthcheck:
|
||||
test: [CMD-SHELL, "pg_isready -U messenger -d messenger"]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 30
|
||||
cpus: ${POSTGRES_CPU:-1}
|
||||
mem_limit: 768m
|
||||
memswap_limit: 768m
|
||||
networks: [backend]
|
||||
valkey:
|
||||
image: valkey/valkey:8.1.3-alpine
|
||||
command: [valkey-server, --save, "", --appendonly, "no", --maxmemory, 64mb,
|
||||
--maxmemory-policy, allkeys-lru]
|
||||
healthcheck:
|
||||
test: [CMD, valkey-cli, ping]
|
||||
interval: 2s
|
||||
timeout: 1s
|
||||
retries: 20
|
||||
cpus: 0.5
|
||||
mem_limit: 128m
|
||||
memswap_limit: 128m
|
||||
networks: [backend]
|
||||
migrate:
|
||||
<<: *api
|
||||
command: [python, -m, messenger.migrate]
|
||||
secrets: [database_url]
|
||||
healthcheck: {disable: true}
|
||||
depends_on:
|
||||
postgres: {condition: service_healthy}
|
||||
api:
|
||||
<<: *api
|
||||
deploy:
|
||||
replicas: ${API_REPLICAS:-2}
|
||||
depends_on:
|
||||
migrate: {condition: service_completed_successfully}
|
||||
valkey: {condition: service_healthy}
|
||||
bucket-init: {condition: service_completed_successfully}
|
||||
lb:
|
||||
image: haproxy:3.2.1-alpine
|
||||
volumes: [./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro]
|
||||
ports: ["127.0.0.1:${PUBLIC_PORT:-8086}:8080"]
|
||||
read_only: true
|
||||
tmpfs: ["/tmp:rw,noexec,nosuid,size=4m"]
|
||||
cap_drop: [ALL]
|
||||
security_opt: [no-new-privileges:true]
|
||||
cpus: 0.5
|
||||
mem_limit: 128m
|
||||
memswap_limit: 128m
|
||||
healthcheck:
|
||||
test: [CMD, wget, -q, -O, /dev/null, http://127.0.0.1:8080/health/ready]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 30
|
||||
depends_on:
|
||||
api: {condition: service_healthy}
|
||||
networks: [backend]
|
||||
|
||||
broker:
|
||||
image: valkey/valkey:8.1.3-alpine
|
||||
command: [valkey-server, --appendonly, "yes", --appendfsync, always, --save, "60 1",
|
||||
--maxmemory, 192mb, --maxmemory-policy, noeviction]
|
||||
volumes: [broker_data:/data]
|
||||
cpus: 0.5
|
||||
mem_limit: 384m
|
||||
memswap_limit: 384m
|
||||
healthcheck:
|
||||
test: [CMD, valkey-cli, ping]
|
||||
interval: 2s
|
||||
timeout: 1s
|
||||
retries: 30
|
||||
networks: [backend]
|
||||
relay:
|
||||
<<: *api
|
||||
command: [python, -m, messenger.relay]
|
||||
restart: unless-stopped
|
||||
healthcheck: {disable: true}
|
||||
secrets: [database_url, cache_url, broker_url]
|
||||
depends_on:
|
||||
migrate: {condition: service_completed_successfully}
|
||||
broker: {condition: service_healthy}
|
||||
worker:
|
||||
<<: *api
|
||||
command: [python, -m, messenger.worker]
|
||||
restart: unless-stopped
|
||||
cpus: 2
|
||||
mem_limit: 1536m
|
||||
memswap_limit: 1536m
|
||||
tmpfs: ["/tmp:rw,noexec,nosuid,size=64m"]
|
||||
healthcheck: {disable: true}
|
||||
secrets:
|
||||
- database_url
|
||||
- cache_url
|
||||
- broker_url
|
||||
- source: s3_worker_access_key_id
|
||||
target: s3_access_key_id
|
||||
- source: s3_worker_secret_access_key
|
||||
target: s3_secret_access_key
|
||||
depends_on:
|
||||
migrate: {condition: service_completed_successfully}
|
||||
broker: {condition: service_healthy}
|
||||
bucket-init: {condition: service_completed_successfully}
|
||||
|
||||
s3:
|
||||
image: mtusi-lab6-s3:2025-10-15
|
||||
build: {context: ., dockerfile: infra/s3.Dockerfile, target: server}
|
||||
command: [server, /data, --address, ':9000', --console-address, '127.0.0.1:9001']
|
||||
secrets: [s3_root_user, s3_root_password]
|
||||
volumes: [s3_data:/data]
|
||||
read_only: true
|
||||
tmpfs: ["/tmp:rw,noexec,nosuid,size=32m"]
|
||||
cap_drop: [ALL]
|
||||
cpus: 1
|
||||
mem_limit: 768m
|
||||
memswap_limit: 768m
|
||||
healthcheck:
|
||||
test: [CMD, wget, -q, -O, /dev/null, 'http://127.0.0.1:9000/minio/health/live']
|
||||
interval: 2s
|
||||
timeout: 1s
|
||||
retries: 30
|
||||
networks: [backend]
|
||||
bucket-init:
|
||||
image: mtusi-lab6-mc:2025-08-13
|
||||
build: {context: ., dockerfile: infra/s3.Dockerfile, target: client}
|
||||
command: [/bootstrap-s3.sh]
|
||||
volumes:
|
||||
- ./infra/bootstrap-s3.sh:/bootstrap-s3.sh:ro
|
||||
- ./infra/api-policy.json:/policies/api-policy.json:ro
|
||||
- ./infra/reconciler-policy.json:/policies/reconciler-policy.json:ro
|
||||
- ./infra/worker-policy.json:/policies/worker-policy.json:ro
|
||||
- ./infra/test-policy.json:/policies/test-policy.json:ro
|
||||
secrets: [s3_root_user, s3_root_password, s3_api_access_key_id, s3_api_secret_access_key,
|
||||
s3_reconciler_access_key_id, s3_reconciler_secret_access_key,
|
||||
s3_test_access_key_id, s3_test_secret_access_key,
|
||||
s3_worker_access_key_id, s3_worker_secret_access_key]
|
||||
environment:
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
read_only: true
|
||||
tmpfs: ["/tmp:rw,noexec,nosuid,size=16m"]
|
||||
cpus: 0.5
|
||||
mem_limit: 128m
|
||||
networks: [backend]
|
||||
depends_on:
|
||||
s3: {condition: service_healthy}
|
||||
reconciler:
|
||||
<<: *api
|
||||
profiles: [tools]
|
||||
command: [python, -m, messenger.cleanup]
|
||||
healthcheck: {disable: true}
|
||||
secrets:
|
||||
- database_url
|
||||
- cache_url
|
||||
- source: s3_reconciler_access_key_id
|
||||
target: s3_access_key_id
|
||||
- source: s3_reconciler_secret_access_key
|
||||
target: s3_secret_access_key
|
||||
depends_on:
|
||||
migrate: {condition: service_completed_successfully}
|
||||
bucket-init: {condition: service_completed_successfully}
|
||||
|
||||
otel-collector:
|
||||
image: otel/opentelemetry-collector-contrib:0.147.0
|
||||
command: [--config=/etc/otelcol/config.yaml]
|
||||
volumes: [./infra/otel/collector.yaml:/etc/otelcol/config.yaml:ro]
|
||||
mem_limit: 256m
|
||||
cpus: 0.5
|
||||
read_only: true
|
||||
cap_drop: [ALL]
|
||||
security_opt: [no-new-privileges:true]
|
||||
networks: [backend]
|
||||
prometheus:
|
||||
image: prom/prometheus:v3.9.1
|
||||
command: [--config.file=/etc/prometheus/prometheus.yaml, --storage.tsdb.path=/prometheus, --storage.tsdb.retention.time=3d, --storage.tsdb.retention.size=256MB]
|
||||
volumes:
|
||||
- ./infra/otel/prometheus.yaml:/etc/prometheus/prometheus.yaml:ro
|
||||
- ./infra/otel/alerts.yaml:/etc/prometheus/alerts.yaml:ro
|
||||
- prometheus_data:/prometheus
|
||||
ports: ["127.0.0.1:${PROMETHEUS_PORT:-9096}:9090"]
|
||||
mem_limit: 384m
|
||||
cpus: 0.5
|
||||
networks: [backend]
|
||||
loki:
|
||||
image: grafana/loki:3.6.3
|
||||
command: [-config.file=/etc/loki/local.yaml]
|
||||
volumes:
|
||||
- ./infra/otel/loki.yaml:/etc/loki/local.yaml:ro
|
||||
- loki_data:/loki
|
||||
ports: ["127.0.0.1:${LOKI_PORT:-3106}:3100"]
|
||||
mem_limit: 512m
|
||||
cpus: 0.5
|
||||
networks: [backend]
|
||||
tempo:
|
||||
image: grafana/tempo:2.10.0
|
||||
command: [-config.file=/etc/tempo/local.yaml]
|
||||
volumes:
|
||||
- ./infra/otel/tempo.yaml:/etc/tempo/local.yaml:ro
|
||||
- tempo_data:/var/tempo
|
||||
ports: ["127.0.0.1:${TEMPO_PORT:-3206}:3200"]
|
||||
mem_limit: 512m
|
||||
cpus: 0.5
|
||||
networks: [backend]
|
||||
grafana:
|
||||
image: grafana/grafana:12.3.3
|
||||
environment:
|
||||
GF_SECURITY_ADMIN_PASSWORD__FILE: /run/secrets/grafana_password
|
||||
GF_USERS_ALLOW_SIGN_UP: "false"
|
||||
GF_ANALYTICS_REPORTING_ENABLED: "false"
|
||||
GF_ANALYTICS_CHECK_FOR_UPDATES: "false"
|
||||
GF_ANALYTICS_CHECK_FOR_PLUGIN_UPDATES: "false"
|
||||
secrets: [grafana_password]
|
||||
volumes:
|
||||
- grafana_data:/var/lib/grafana
|
||||
- ./infra/otel/grafana/provisioning:/etc/grafana/provisioning:ro
|
||||
- ./infra/otel/grafana/dashboards:/var/lib/grafana/dashboards:ro
|
||||
ports: ["127.0.0.1:${GRAFANA_PORT:-3006}:3000"]
|
||||
mem_limit: 384m
|
||||
cpus: 0.5
|
||||
networks: [backend]
|
||||
|
||||
networks:
|
||||
backend: {}
|
||||
volumes:
|
||||
prometheus_data:
|
||||
loki_data:
|
||||
tempo_data:
|
||||
grafana_data:
|
||||
broker_data:
|
||||
s3_data:
|
||||
postgres_data:
|
||||
secrets:
|
||||
grafana_password:
|
||||
file: ./secrets/grafana_password
|
||||
postgres_password:
|
||||
file: ./secrets/postgres_password
|
||||
database_url:
|
||||
file: ./secrets/database_url
|
||||
cache_url:
|
||||
file: ./secrets/cache_url
|
||||
s3_root_user:
|
||||
file: ./secrets/s3_root_user
|
||||
s3_root_password:
|
||||
file: ./secrets/s3_root_password
|
||||
s3_api_access_key_id:
|
||||
file: ./secrets/s3_api_access_key_id
|
||||
s3_api_secret_access_key:
|
||||
file: ./secrets/s3_api_secret_access_key
|
||||
s3_reconciler_access_key_id:
|
||||
file: ./secrets/s3_reconciler_access_key_id
|
||||
s3_reconciler_secret_access_key:
|
||||
file: ./secrets/s3_reconciler_secret_access_key
|
||||
s3_test_access_key_id:
|
||||
file: ./secrets/s3_test_access_key_id
|
||||
s3_test_secret_access_key:
|
||||
file: ./secrets/s3_test_secret_access_key
|
||||
|
||||
broker_url:
|
||||
file: ./secrets/broker_url
|
||||
s3_worker_access_key_id:
|
||||
file: ./secrets/s3_worker_access_key_id
|
||||
s3_worker_secret_access_key:
|
||||
file: ./secrets/s3_worker_secret_access_key
|
||||
@@ -0,0 +1,4 @@
|
||||
python3 tests/smoke.py --base-url "http://127.0.0.1:18086"
|
||||
PASS: lab6 public API scenario; infrastructure and grading evidence remain separate.
|
||||
python3 scripts/check_contract.py
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
@@ -0,0 +1,4 @@
|
||||
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://127.0.0.1:8086"
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 128, "failed_attempts": 3, "first_recovery_seconds": 4.763}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
@@ -0,0 +1,57 @@
|
||||
{
|
||||
"correlation": {
|
||||
"request_id": "26db18d7-d97b-488a-b9e5-2e3a3f48fb03",
|
||||
"trace_id": "d72b60b8bdea49a5cff62c0abd43f1e0",
|
||||
"span_names": [
|
||||
"POST /api/v1/chats/{chat_id}/attachments",
|
||||
"db.read",
|
||||
"db.transaction",
|
||||
"jobs.consume",
|
||||
"jobs.publish",
|
||||
"s3.download",
|
||||
"s3.put"
|
||||
],
|
||||
"worker_parent": "jobs.publish",
|
||||
"api_and_worker_loki": true
|
||||
},
|
||||
"worker_stop": {
|
||||
"depth_at_firing": 3.0,
|
||||
"oldest_seconds_at_firing": 16.917512,
|
||||
"state": [
|
||||
"firing",
|
||||
"resolved"
|
||||
],
|
||||
"uploads_ready": 3
|
||||
},
|
||||
"database_lock": {
|
||||
"statuses": [
|
||||
503,
|
||||
503,
|
||||
503,
|
||||
503,
|
||||
503,
|
||||
503,
|
||||
503,
|
||||
503,
|
||||
503,
|
||||
503,
|
||||
503,
|
||||
503
|
||||
],
|
||||
"max_latency_ms": 1007.0772920007585,
|
||||
"trace_id": "d3e8b679ae370a4de65821ead5d74acc",
|
||||
"state": [
|
||||
"firing",
|
||||
"resolved"
|
||||
],
|
||||
"spans": [
|
||||
"GET /api/v1/users/me",
|
||||
"db.read"
|
||||
]
|
||||
},
|
||||
"tempo_stop": {
|
||||
"successful_messages": 15,
|
||||
"failed_export_spans": 21.0,
|
||||
"collector_rss_bytes": 206360576.0
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://127.0.0.1:8086"
|
||||
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
|
||||
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,862 @@
|
||||
{
|
||||
"seconds": 10,
|
||||
"offered_requests_per_s": 18,
|
||||
"runs": [
|
||||
{
|
||||
"mode": "off",
|
||||
"sample_ratio": 0.0,
|
||||
"load": {
|
||||
"requests": 180,
|
||||
"elapsed_s": 9.952042167002219,
|
||||
"latency_ms": {
|
||||
"p50": 4.991917001461843,
|
||||
"p95": 7.5844580023840535,
|
||||
"p99": 16.816375002235873
|
||||
}
|
||||
},
|
||||
"container_samples": [
|
||||
[
|
||||
{
|
||||
"container": "mtusi-lab6-api-1",
|
||||
"cpu_percent": 2.45,
|
||||
"memory": "91.67MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-api-2",
|
||||
"cpu_percent": 2.93,
|
||||
"memory": "91.85MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-loki-1",
|
||||
"cpu_percent": 0.46,
|
||||
"memory": "107.5MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-otel-collector-1",
|
||||
"cpu_percent": 0.2,
|
||||
"memory": "58.3MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-prometheus-1",
|
||||
"cpu_percent": 0.09,
|
||||
"memory": "40.25MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-relay-1",
|
||||
"cpu_percent": 0.38,
|
||||
"memory": "46.33MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-tempo-1",
|
||||
"cpu_percent": 0.13,
|
||||
"memory": "88.37MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-worker-1",
|
||||
"cpu_percent": 60.82,
|
||||
"memory": "75.3MiB"
|
||||
}
|
||||
],
|
||||
[
|
||||
{
|
||||
"container": "mtusi-lab6-api-1",
|
||||
"cpu_percent": 15.95,
|
||||
"memory": "91.73MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-api-2",
|
||||
"cpu_percent": 3.69,
|
||||
"memory": "93.02MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-loki-1",
|
||||
"cpu_percent": 0.45,
|
||||
"memory": "107MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-otel-collector-1",
|
||||
"cpu_percent": 0.19,
|
||||
"memory": "58.55MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-prometheus-1",
|
||||
"cpu_percent": 0.09,
|
||||
"memory": "40.26MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-relay-1",
|
||||
"cpu_percent": 0.59,
|
||||
"memory": "46.33MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-tempo-1",
|
||||
"cpu_percent": 0.14,
|
||||
"memory": "88.38MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-worker-1",
|
||||
"cpu_percent": 0.88,
|
||||
"memory": "78.04MiB"
|
||||
}
|
||||
]
|
||||
],
|
||||
"ingest": {
|
||||
"trace_ingest_proto_bytes": 0.0,
|
||||
"accepted_spans": 0.0,
|
||||
"sent_spans": 0.0,
|
||||
"failed_spans": 0.0,
|
||||
"accepted_logs": 0.0,
|
||||
"accepted_metric_points": 0.0
|
||||
}
|
||||
},
|
||||
{
|
||||
"mode": "full",
|
||||
"sample_ratio": 1.0,
|
||||
"load": {
|
||||
"requests": 180,
|
||||
"elapsed_s": 9.950246666998282,
|
||||
"latency_ms": {
|
||||
"p50": 5.275124996842351,
|
||||
"p95": 8.54145799894468,
|
||||
"p99": 16.04162500007078
|
||||
}
|
||||
},
|
||||
"container_samples": [
|
||||
[
|
||||
{
|
||||
"container": "mtusi-lab6-api-1",
|
||||
"cpu_percent": 15.27,
|
||||
"memory": "94.31MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-api-2",
|
||||
"cpu_percent": 15.43,
|
||||
"memory": "96.71MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-loki-1",
|
||||
"cpu_percent": 0.55,
|
||||
"memory": "107MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-otel-collector-1",
|
||||
"cpu_percent": 0.45,
|
||||
"memory": "59.33MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-prometheus-1",
|
||||
"cpu_percent": 0.01,
|
||||
"memory": "40.25MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-relay-1",
|
||||
"cpu_percent": 0.52,
|
||||
"memory": "56.45MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-tempo-1",
|
||||
"cpu_percent": 0.29,
|
||||
"memory": "94.84MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-worker-1",
|
||||
"cpu_percent": 46.73,
|
||||
"memory": "155.4MiB"
|
||||
}
|
||||
],
|
||||
[
|
||||
{
|
||||
"container": "mtusi-lab6-api-1",
|
||||
"cpu_percent": 15.41,
|
||||
"memory": "93.86MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-api-2",
|
||||
"cpu_percent": 7.99,
|
||||
"memory": "106.5MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-loki-1",
|
||||
"cpu_percent": 0.62,
|
||||
"memory": "107.3MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-otel-collector-1",
|
||||
"cpu_percent": 0.57,
|
||||
"memory": "59.59MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-prometheus-1",
|
||||
"cpu_percent": 0.0,
|
||||
"memory": "40.26MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-relay-1",
|
||||
"cpu_percent": 0.85,
|
||||
"memory": "56.21MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-tempo-1",
|
||||
"cpu_percent": 0.29,
|
||||
"memory": "94.63MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-worker-1",
|
||||
"cpu_percent": 53.06,
|
||||
"memory": "160.2MiB"
|
||||
}
|
||||
]
|
||||
],
|
||||
"ingest": {
|
||||
"trace_ingest_proto_bytes": 142249.0,
|
||||
"accepted_spans": 998.0,
|
||||
"sent_spans": 997.0,
|
||||
"failed_spans": 0.0,
|
||||
"accepted_logs": 188.0,
|
||||
"accepted_metric_points": 271.0
|
||||
}
|
||||
},
|
||||
{
|
||||
"mode": "half",
|
||||
"sample_ratio": 0.5,
|
||||
"load": {
|
||||
"requests": 180,
|
||||
"elapsed_s": 9.95444554200003,
|
||||
"latency_ms": {
|
||||
"p50": 5.507500001840526,
|
||||
"p95": 11.544834000233095,
|
||||
"p99": 24.414750001596985
|
||||
}
|
||||
},
|
||||
"container_samples": [
|
||||
[
|
||||
{
|
||||
"container": "mtusi-lab6-api-1",
|
||||
"cpu_percent": 15.49,
|
||||
"memory": "96.14MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-api-2",
|
||||
"cpu_percent": 16.11,
|
||||
"memory": "95.01MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-loki-1",
|
||||
"cpu_percent": 0.52,
|
||||
"memory": "113.9MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-otel-collector-1",
|
||||
"cpu_percent": 0.52,
|
||||
"memory": "58.38MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-prometheus-1",
|
||||
"cpu_percent": 0.05,
|
||||
"memory": "40.26MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-relay-1",
|
||||
"cpu_percent": 0.42,
|
||||
"memory": "54.59MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-tempo-1",
|
||||
"cpu_percent": 0.3,
|
||||
"memory": "107MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-worker-1",
|
||||
"cpu_percent": 45.37,
|
||||
"memory": "132.9MiB"
|
||||
}
|
||||
],
|
||||
[
|
||||
{
|
||||
"container": "mtusi-lab6-api-1",
|
||||
"cpu_percent": 14.36,
|
||||
"memory": "108.8MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-api-2",
|
||||
"cpu_percent": 6.47,
|
||||
"memory": "104.6MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-loki-1",
|
||||
"cpu_percent": 0.68,
|
||||
"memory": "113.9MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-otel-collector-1",
|
||||
"cpu_percent": 0.62,
|
||||
"memory": "58.88MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-prometheus-1",
|
||||
"cpu_percent": 0.1,
|
||||
"memory": "40.26MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-relay-1",
|
||||
"cpu_percent": 0.83,
|
||||
"memory": "54.34MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-tempo-1",
|
||||
"cpu_percent": 0.32,
|
||||
"memory": "109MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-worker-1",
|
||||
"cpu_percent": 53.37,
|
||||
"memory": "157.9MiB"
|
||||
}
|
||||
]
|
||||
],
|
||||
"ingest": {
|
||||
"trace_ingest_proto_bytes": 80752.0,
|
||||
"accepted_spans": 522.0,
|
||||
"sent_spans": 523.0,
|
||||
"failed_spans": 0.0,
|
||||
"accepted_logs": 188.0,
|
||||
"accepted_metric_points": 287.0
|
||||
}
|
||||
},
|
||||
{
|
||||
"mode": "quarter",
|
||||
"sample_ratio": 0.25,
|
||||
"load": {
|
||||
"requests": 180,
|
||||
"elapsed_s": 9.952545791999,
|
||||
"latency_ms": {
|
||||
"p50": 5.109958998218644,
|
||||
"p95": 8.02891599960276,
|
||||
"p99": 18.22395799899823
|
||||
}
|
||||
},
|
||||
"container_samples": [
|
||||
[
|
||||
{
|
||||
"container": "mtusi-lab6-api-1",
|
||||
"cpu_percent": 14.66,
|
||||
"memory": "94.55MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-api-2",
|
||||
"cpu_percent": 15.32,
|
||||
"memory": "94.49MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-loki-1",
|
||||
"cpu_percent": 0.89,
|
||||
"memory": "114.4MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-otel-collector-1",
|
||||
"cpu_percent": 0.51,
|
||||
"memory": "59.14MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-prometheus-1",
|
||||
"cpu_percent": 0.16,
|
||||
"memory": "40.51MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-relay-1",
|
||||
"cpu_percent": 0.49,
|
||||
"memory": "56.43MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-tempo-1",
|
||||
"cpu_percent": 0.64,
|
||||
"memory": "115.5MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-worker-1",
|
||||
"cpu_percent": 46.14,
|
||||
"memory": "158.4MiB"
|
||||
}
|
||||
],
|
||||
[
|
||||
{
|
||||
"container": "mtusi-lab6-api-1",
|
||||
"cpu_percent": 16.03,
|
||||
"memory": "94.77MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-api-2",
|
||||
"cpu_percent": 3.98,
|
||||
"memory": "95.34MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-loki-1",
|
||||
"cpu_percent": 0.69,
|
||||
"memory": "114.9MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-otel-collector-1",
|
||||
"cpu_percent": 0.52,
|
||||
"memory": "59.16MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-prometheus-1",
|
||||
"cpu_percent": 0.14,
|
||||
"memory": "40.26MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-relay-1",
|
||||
"cpu_percent": 0.77,
|
||||
"memory": "56.18MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-tempo-1",
|
||||
"cpu_percent": 0.33,
|
||||
"memory": "117MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-worker-1",
|
||||
"cpu_percent": 55.48,
|
||||
"memory": "157.7MiB"
|
||||
}
|
||||
]
|
||||
],
|
||||
"ingest": {
|
||||
"trace_ingest_proto_bytes": 41309.0,
|
||||
"accepted_spans": 238.0,
|
||||
"sent_spans": 237.0,
|
||||
"failed_spans": 0.0,
|
||||
"accepted_logs": 188.0,
|
||||
"accepted_metric_points": 273.0
|
||||
}
|
||||
},
|
||||
{
|
||||
"mode": "quarter",
|
||||
"sample_ratio": 0.25,
|
||||
"load": {
|
||||
"requests": 180,
|
||||
"elapsed_s": 9.9528304580017,
|
||||
"latency_ms": {
|
||||
"p50": 4.951999999320833,
|
||||
"p95": 10.958250000840053,
|
||||
"p99": 36.11366699988139
|
||||
}
|
||||
},
|
||||
"container_samples": [
|
||||
[
|
||||
{
|
||||
"container": "mtusi-lab6-api-1",
|
||||
"cpu_percent": 17.98,
|
||||
"memory": "94.93MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-api-2",
|
||||
"cpu_percent": 3.8,
|
||||
"memory": "94.62MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-loki-1",
|
||||
"cpu_percent": 0.78,
|
||||
"memory": "114.9MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-otel-collector-1",
|
||||
"cpu_percent": 0.53,
|
||||
"memory": "59.66MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-prometheus-1",
|
||||
"cpu_percent": 0.31,
|
||||
"memory": "40.51MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-relay-1",
|
||||
"cpu_percent": 0.98,
|
||||
"memory": "54.59MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-tempo-1",
|
||||
"cpu_percent": 0.32,
|
||||
"memory": "117.3MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-worker-1",
|
||||
"cpu_percent": 65.06,
|
||||
"memory": "79.88MiB"
|
||||
}
|
||||
],
|
||||
[
|
||||
{
|
||||
"container": "mtusi-lab6-api-1",
|
||||
"cpu_percent": 3.32,
|
||||
"memory": "95.67MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-api-2",
|
||||
"cpu_percent": 11.75,
|
||||
"memory": "95.2MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-loki-1",
|
||||
"cpu_percent": 0.66,
|
||||
"memory": "113.9MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-otel-collector-1",
|
||||
"cpu_percent": 0.5,
|
||||
"memory": "59.87MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-prometheus-1",
|
||||
"cpu_percent": 0.0,
|
||||
"memory": "40.27MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-relay-1",
|
||||
"cpu_percent": 0.58,
|
||||
"memory": "54.6MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-tempo-1",
|
||||
"cpu_percent": 0.26,
|
||||
"memory": "117.8MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-worker-1",
|
||||
"cpu_percent": 58.04,
|
||||
"memory": "81.07MiB"
|
||||
}
|
||||
]
|
||||
],
|
||||
"ingest": {
|
||||
"trace_ingest_proto_bytes": 41438.0,
|
||||
"accepted_spans": 238.0,
|
||||
"sent_spans": 235.0,
|
||||
"failed_spans": 0.0,
|
||||
"accepted_logs": 188.0,
|
||||
"accepted_metric_points": 287.0
|
||||
}
|
||||
},
|
||||
{
|
||||
"mode": "half",
|
||||
"sample_ratio": 0.5,
|
||||
"load": {
|
||||
"requests": 180,
|
||||
"elapsed_s": 9.95298329200159,
|
||||
"latency_ms": {
|
||||
"p50": 5.302292000123998,
|
||||
"p95": 8.62283300011768,
|
||||
"p99": 18.653041999641573
|
||||
}
|
||||
},
|
||||
"container_samples": [
|
||||
[
|
||||
{
|
||||
"container": "mtusi-lab6-api-1",
|
||||
"cpu_percent": 2.95,
|
||||
"memory": "96.92MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-api-2",
|
||||
"cpu_percent": 3.27,
|
||||
"memory": "95.07MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-loki-1",
|
||||
"cpu_percent": 0.61,
|
||||
"memory": "114.5MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-otel-collector-1",
|
||||
"cpu_percent": 0.47,
|
||||
"memory": "59.41MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-prometheus-1",
|
||||
"cpu_percent": 0.05,
|
||||
"memory": "40.52MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-relay-1",
|
||||
"cpu_percent": 0.49,
|
||||
"memory": "54.61MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-tempo-1",
|
||||
"cpu_percent": 0.27,
|
||||
"memory": "118.1MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-worker-1",
|
||||
"cpu_percent": 15.31,
|
||||
"memory": "78.76MiB"
|
||||
}
|
||||
],
|
||||
[
|
||||
{
|
||||
"container": "mtusi-lab6-api-1",
|
||||
"cpu_percent": 15.06,
|
||||
"memory": "96.86MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-api-2",
|
||||
"cpu_percent": 15.71,
|
||||
"memory": "95.6MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-loki-1",
|
||||
"cpu_percent": 0.57,
|
||||
"memory": "114.7MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-otel-collector-1",
|
||||
"cpu_percent": 0.56,
|
||||
"memory": "59.4MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-prometheus-1",
|
||||
"cpu_percent": 0.06,
|
||||
"memory": "40.27MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-relay-1",
|
||||
"cpu_percent": 0.65,
|
||||
"memory": "54.61MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-tempo-1",
|
||||
"cpu_percent": 0.3,
|
||||
"memory": "117.4MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-worker-1",
|
||||
"cpu_percent": 49.13,
|
||||
"memory": "157.3MiB"
|
||||
}
|
||||
]
|
||||
],
|
||||
"ingest": {
|
||||
"trace_ingest_proto_bytes": 80811.0,
|
||||
"accepted_spans": 521.0,
|
||||
"sent_spans": 520.0,
|
||||
"failed_spans": 0.0,
|
||||
"accepted_logs": 188.0,
|
||||
"accepted_metric_points": 273.0
|
||||
}
|
||||
},
|
||||
{
|
||||
"mode": "full",
|
||||
"sample_ratio": 1.0,
|
||||
"load": {
|
||||
"requests": 180,
|
||||
"elapsed_s": 9.95264620799935,
|
||||
"latency_ms": {
|
||||
"p50": 5.342791999282781,
|
||||
"p95": 8.375000001251465,
|
||||
"p99": 18.184750002546934
|
||||
}
|
||||
},
|
||||
"container_samples": [
|
||||
[
|
||||
{
|
||||
"container": "mtusi-lab6-api-1",
|
||||
"cpu_percent": 16.07,
|
||||
"memory": "94.52MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-api-2",
|
||||
"cpu_percent": 15.33,
|
||||
"memory": "94.31MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-loki-1",
|
||||
"cpu_percent": 0.52,
|
||||
"memory": "114.2MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-otel-collector-1",
|
||||
"cpu_percent": 0.69,
|
||||
"memory": "58.92MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-prometheus-1",
|
||||
"cpu_percent": 0.09,
|
||||
"memory": "40.27MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-relay-1",
|
||||
"cpu_percent": 0.86,
|
||||
"memory": "56.7MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-tempo-1",
|
||||
"cpu_percent": 0.37,
|
||||
"memory": "117.5MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-worker-1",
|
||||
"cpu_percent": 49.41,
|
||||
"memory": "156.1MiB"
|
||||
}
|
||||
],
|
||||
[
|
||||
{
|
||||
"container": "mtusi-lab6-api-1",
|
||||
"cpu_percent": 4.12,
|
||||
"memory": "93.79MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-api-2",
|
||||
"cpu_percent": 15.92,
|
||||
"memory": "94.05MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-loki-1",
|
||||
"cpu_percent": 0.87,
|
||||
"memory": "114.7MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-otel-collector-1",
|
||||
"cpu_percent": 0.52,
|
||||
"memory": "59.92MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-prometheus-1",
|
||||
"cpu_percent": 0.01,
|
||||
"memory": "40.27MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-relay-1",
|
||||
"cpu_percent": 0.77,
|
||||
"memory": "56.45MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-tempo-1",
|
||||
"cpu_percent": 0.31,
|
||||
"memory": "118.2MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-worker-1",
|
||||
"cpu_percent": 61.5,
|
||||
"memory": "153.1MiB"
|
||||
}
|
||||
]
|
||||
],
|
||||
"ingest": {
|
||||
"trace_ingest_proto_bytes": 142250.0,
|
||||
"accepted_spans": 996.0,
|
||||
"sent_spans": 998.0,
|
||||
"failed_spans": 0.0,
|
||||
"accepted_logs": 188.0,
|
||||
"accepted_metric_points": 271.0
|
||||
}
|
||||
},
|
||||
{
|
||||
"mode": "off",
|
||||
"sample_ratio": 0.0,
|
||||
"load": {
|
||||
"requests": 180,
|
||||
"elapsed_s": 9.951078292000602,
|
||||
"latency_ms": {
|
||||
"p50": 5.305624999891734,
|
||||
"p95": 8.97895800153492,
|
||||
"p99": 27.521125000930624
|
||||
}
|
||||
},
|
||||
"container_samples": [
|
||||
[
|
||||
{
|
||||
"container": "mtusi-lab6-api-1",
|
||||
"cpu_percent": 2.74,
|
||||
"memory": "93.45MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-api-2",
|
||||
"cpu_percent": 2.7,
|
||||
"memory": "91.64MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-loki-1",
|
||||
"cpu_percent": 0.41,
|
||||
"memory": "114.9MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-otel-collector-1",
|
||||
"cpu_percent": 0.17,
|
||||
"memory": "58.39MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-prometheus-1",
|
||||
"cpu_percent": 0.13,
|
||||
"memory": "40.27MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-relay-1",
|
||||
"cpu_percent": 0.42,
|
||||
"memory": "46.32MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-tempo-1",
|
||||
"cpu_percent": 0.13,
|
||||
"memory": "116MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-worker-1",
|
||||
"cpu_percent": 56.51,
|
||||
"memory": "77.11MiB"
|
||||
}
|
||||
],
|
||||
[
|
||||
{
|
||||
"container": "mtusi-lab6-api-1",
|
||||
"cpu_percent": 3.66,
|
||||
"memory": "93.92MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-api-2",
|
||||
"cpu_percent": 3.46,
|
||||
"memory": "92.96MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-loki-1",
|
||||
"cpu_percent": 0.47,
|
||||
"memory": "114.2MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-otel-collector-1",
|
||||
"cpu_percent": 0.22,
|
||||
"memory": "58.39MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-prometheus-1",
|
||||
"cpu_percent": 0.08,
|
||||
"memory": "40.52MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-relay-1",
|
||||
"cpu_percent": 0.34,
|
||||
"memory": "46.07MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-tempo-1",
|
||||
"cpu_percent": 0.18,
|
||||
"memory": "116MiB"
|
||||
},
|
||||
{
|
||||
"container": "mtusi-lab6-worker-1",
|
||||
"cpu_percent": 0.22,
|
||||
"memory": "76.44MiB"
|
||||
}
|
||||
]
|
||||
],
|
||||
"ingest": {
|
||||
"trace_ingest_proto_bytes": 0.0,
|
||||
"accepted_spans": 0.0,
|
||||
"sent_spans": 0.0,
|
||||
"failed_spans": 0.0,
|
||||
"accepted_logs": 0.0,
|
||||
"accepted_metric_points": 0.0
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"correlation": {
|
||||
"request_id": "bda1ff17-5d60-4d88-a0c1-2eaf450d5769",
|
||||
"trace_id": "fc3682b58c4df5515088f7fef7f0d559",
|
||||
"span_names": [
|
||||
"POST /api/v1/chats/{chat_id}/attachments",
|
||||
"db.read",
|
||||
"db.transaction",
|
||||
"jobs.consume",
|
||||
"jobs.publish",
|
||||
"s3.download",
|
||||
"s3.put"
|
||||
],
|
||||
"worker_parent": "jobs.publish",
|
||||
"worker_attempts": 2,
|
||||
"api_and_worker_loki": true
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
======================== 84 passed, 1 warning in 13.36s ========================
|
||||
@@ -0,0 +1,33 @@
|
||||
{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:ListBucket",
|
||||
"s3:GetBucketLocation"
|
||||
],
|
||||
"Resource": [
|
||||
"arn:aws:s3:::BUCKET"
|
||||
]
|
||||
},
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:PutObject"
|
||||
],
|
||||
"Resource": [
|
||||
"arn:aws:s3:::BUCKET/staging/*"
|
||||
]
|
||||
},
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:GetObject"
|
||||
],
|
||||
"Resource": [
|
||||
"arn:aws:s3:::BUCKET/objects/*"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
# Never print mc arguments/errors: they can include access key identifiers.
|
||||
exec 3>&1
|
||||
exec > /tmp/bootstrap.log 2>&1
|
||||
trap 'printf "%s\n" "S3 bootstrap failed; credentials suppressed" >&3' EXIT
|
||||
bucket="$S3_BUCKET"
|
||||
case "$bucket" in ''|*[!a-z0-9.-]*) exit 2;; esac
|
||||
mc --config-dir /tmp/mc alias set -- local http://s3:9000 "$(cat /run/secrets/s3_root_user)" "$(cat /run/secrets/s3_root_password)"
|
||||
mc --config-dir /tmp/mc mb --ignore-existing "local/$bucket"
|
||||
mc --config-dir /tmp/mc anonymous set none "local/$bucket"
|
||||
for role in api worker reconciler test; do
|
||||
target_bucket="$bucket"
|
||||
policy_source=/policies/api-policy.json
|
||||
if [ "$role" = reconciler ]; then policy_source=/policies/reconciler-policy.json; fi
|
||||
if [ "$role" = worker ]; then policy_source=/policies/worker-policy.json; fi
|
||||
if [ "$role" = test ]; then
|
||||
if [ "${CREATE_TEST_BUCKET:-0}" != 1 ]; then continue; fi
|
||||
policy_source=/policies/test-policy.json
|
||||
target_bucket="$bucket-tests"
|
||||
mc --config-dir /tmp/mc mb --ignore-existing "local/$target_bucket"
|
||||
mc --config-dir /tmp/mc anonymous set none "local/$target_bucket"
|
||||
fi
|
||||
sed "s/BUCKET/$target_bucket/g" "$policy_source" > "/tmp/$role-policy.json"
|
||||
mc --config-dir /tmp/mc admin policy create local "messenger-$role" "/tmp/$role-policy.json"
|
||||
key="$(cat /run/secrets/s3_${role}_access_key_id)"
|
||||
mc --config-dir /tmp/mc admin user add -- local "$key" "$(cat /run/secrets/s3_${role}_secret_access_key)"
|
||||
mc --config-dir /tmp/mc admin policy attach local "messenger-$role" --user "$key"
|
||||
done
|
||||
trap - EXIT
|
||||
printf '%s\n' 'PASS: private bucket and scoped identities ready' >&3
|
||||
@@ -0,0 +1,44 @@
|
||||
global
|
||||
log stdout format raw local0
|
||||
maxconn 2048
|
||||
|
||||
defaults
|
||||
log global
|
||||
mode http
|
||||
log-format "lb status=%ST backend=%b server=%s duration_ms=%Tt"
|
||||
timeout connect 500ms
|
||||
timeout check 500ms
|
||||
timeout client 20s
|
||||
timeout server 30s
|
||||
timeout http-request 3s
|
||||
retries 0
|
||||
|
||||
resolvers docker
|
||||
nameserver embedded 127.0.0.11:53
|
||||
accepted_payload_size 8192
|
||||
resolve_retries 3
|
||||
timeout resolve 1s
|
||||
timeout retry 1s
|
||||
hold valid 1s
|
||||
hold obsolete 0s
|
||||
hold nx 1s
|
||||
hold refused 1s
|
||||
hold timeout 1s
|
||||
hold other 1s
|
||||
|
||||
frontend public
|
||||
bind :8080
|
||||
default_backend messenger
|
||||
|
||||
backend messenger
|
||||
balance roundrobin
|
||||
option httpchk GET /health/ready
|
||||
http-check expect status 200
|
||||
default-server inter 500ms fastinter 250ms downinter 500ms fall 2 rise 1
|
||||
server-template app 1-8 api:8080 check resolvers docker resolve-prefer ipv4 init-addr none
|
||||
|
||||
listen metrics
|
||||
bind :8404
|
||||
stats enable
|
||||
stats uri /stats
|
||||
stats refresh 1s
|
||||
@@ -0,0 +1,27 @@
|
||||
groups:
|
||||
- name: messenger
|
||||
rules:
|
||||
- alert: MessengerHttpErrors
|
||||
expr: |
|
||||
sum(rate(messenger_http_requests_total{route!~"/health/.*",status_class="5xx"}[20s]))
|
||||
/ clamp_min(sum(rate(messenger_http_requests_total{route!~"/health/.*"}[20s])), 0.01)
|
||||
> 0.05
|
||||
for: 6s
|
||||
labels: {severity: warning}
|
||||
annotations:
|
||||
summary: More than 5% of business requests have failed for six seconds
|
||||
runbook: "REPORT.md#диагностика"
|
||||
- alert: MessengerQueueLag
|
||||
expr: max(messenger_queue_oldest_seconds) > 5 and max(messenger_queue_depth) > 0
|
||||
for: 6s
|
||||
labels: {severity: warning}
|
||||
annotations:
|
||||
summary: The oldest unfinished job is more than five seconds old
|
||||
runbook: "REPORT.md#диагностика"
|
||||
- alert: MessengerDatabaseErrors
|
||||
expr: sum(increase(messenger_db_errors_total[20s])) > 0
|
||||
for: 4s
|
||||
labels: {severity: warning}
|
||||
annotations:
|
||||
summary: PostgreSQL requests are failing
|
||||
runbook: "REPORT.md#диагностика"
|
||||
@@ -0,0 +1,53 @@
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
http:
|
||||
endpoint: 0.0.0.0:4318
|
||||
|
||||
processors:
|
||||
memory_limiter:
|
||||
check_interval: 1s
|
||||
limit_mib: 128
|
||||
spike_limit_mib: 32
|
||||
batch:
|
||||
timeout: 500ms
|
||||
send_batch_size: 128
|
||||
send_batch_max_size: 256
|
||||
|
||||
exporters:
|
||||
prometheus:
|
||||
endpoint: 0.0.0.0:8889
|
||||
metric_expiration: 15s
|
||||
otlp_http/tempo:
|
||||
endpoint: http://tempo:4318
|
||||
timeout: 2s
|
||||
sending_queue: {queue_size: 16, num_consumers: 1}
|
||||
retry_on_failure: {initial_interval: 500ms, max_interval: 2s, max_elapsed_time: 5s}
|
||||
otlp_http/loki:
|
||||
endpoint: http://loki:3100/otlp
|
||||
timeout: 2s
|
||||
sending_queue: {queue_size: 16, num_consumers: 1}
|
||||
retry_on_failure: {initial_interval: 500ms, max_interval: 2s, max_elapsed_time: 5s}
|
||||
|
||||
service:
|
||||
telemetry:
|
||||
metrics:
|
||||
readers:
|
||||
- pull:
|
||||
exporter:
|
||||
prometheus:
|
||||
host: 0.0.0.0
|
||||
port: 8888
|
||||
pipelines:
|
||||
metrics:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, batch]
|
||||
exporters: [prometheus]
|
||||
logs:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, batch]
|
||||
exporters: [otlp_http/loki]
|
||||
traces:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, batch]
|
||||
exporters: [otlp_http/tempo]
|
||||
@@ -0,0 +1,453 @@
|
||||
{
|
||||
"uid": "messenger",
|
||||
"title": "MTUSI Messenger",
|
||||
"schemaVersion": 41,
|
||||
"version": 1,
|
||||
"editable": false,
|
||||
"timezone": "browser",
|
||||
"refresh": "5s",
|
||||
"time": {
|
||||
"from": "now-15m",
|
||||
"to": "now"
|
||||
},
|
||||
"panels": [
|
||||
{
|
||||
"id": 1,
|
||||
"type": "timeseries",
|
||||
"title": "API / запросов в секунду",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 0,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "reqps"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum(rate(messenger_http_requests_total{route!~\"/health/.*\"}[30s]))",
|
||||
"legendFormat": ""
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"type": "timeseries",
|
||||
"title": "API / доля 5xx",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 0,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percentunit"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum(rate(messenger_http_requests_total{status_class=\"5xx\",route!~\"/health/.*\"}[30s])) / clamp_min(sum(rate(messenger_http_requests_total{route!~\"/health/.*\"}[30s])),0.001)",
|
||||
"legendFormat": ""
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"type": "timeseries",
|
||||
"title": "API / p50, p95, p99",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 8,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "histogram_quantile(0.5, sum by (le) (rate(messenger_http_duration_seconds_bucket{route!~\"/health/.*\"}[30s])))",
|
||||
"legendFormat": "p50"
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_http_duration_seconds_bucket{route!~\"/health/.*\"}[30s])))",
|
||||
"legendFormat": "p95"
|
||||
},
|
||||
{
|
||||
"refId": "C",
|
||||
"expr": "histogram_quantile(0.99, sum by (le) (rate(messenger_http_duration_seconds_bucket{route!~\"/health/.*\"}[30s])))",
|
||||
"legendFormat": "p99"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"type": "timeseries",
|
||||
"title": "Worker / исходы в секунду",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 8,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "ops"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum by (outcome) (rate(messenger_jobs_processed_total[30s]))",
|
||||
"legendFormat": "{{outcome}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"type": "timeseries",
|
||||
"title": "Worker / p95 по исходу",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 16,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "histogram_quantile(0.95, sum by (le,outcome) (rate(messenger_jobs_duration_seconds_bucket[30s])))",
|
||||
"legendFormat": ""
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"type": "timeseries",
|
||||
"title": "Upload → ready / p95",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 16,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_ready_duration_seconds_bucket[30s])))",
|
||||
"legendFormat": ""
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 7,
|
||||
"type": "timeseries",
|
||||
"title": "Backlog / заданий",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 24,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "max(messenger_queue_depth)",
|
||||
"legendFormat": ""
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 8,
|
||||
"type": "timeseries",
|
||||
"title": "Возраст старейшего задания",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 24,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "max(messenger_queue_oldest_seconds)",
|
||||
"legendFormat": ""
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 9,
|
||||
"type": "timeseries",
|
||||
"title": "PostgreSQL / занято и ожидает",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 32,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "messenger_db_pool_used",
|
||||
"legendFormat": ""
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"expr": "messenger_db_pool_waiting",
|
||||
"legendFormat": ""
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 10,
|
||||
"type": "timeseries",
|
||||
"title": "Worker / занято и доступно",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 32,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum(messenger_worker_busy)",
|
||||
"legendFormat": ""
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"expr": "sum(messenger_worker_capacity)",
|
||||
"legendFormat": ""
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 11,
|
||||
"type": "timeseries",
|
||||
"title": "CPU процессов",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 40,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "cores"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "rate(messenger_process_cpu_seconds_total[30s])",
|
||||
"legendFormat": ""
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 12,
|
||||
"type": "timeseries",
|
||||
"title": "RSS процессов",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 40,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "bytes"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "messenger_process_rss_bytes",
|
||||
"legendFormat": ""
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 13,
|
||||
"type": "timeseries",
|
||||
"title": "Потери экспорта Collector",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 48,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum by (exporter) (increase(otelcol_exporter_send_failed_spans_total[30s]))",
|
||||
"legendFormat": ""
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"expr": "sum by (exporter) (increase(otelcol_exporter_send_failed_log_records_total[30s]))",
|
||||
"legendFormat": ""
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 14,
|
||||
"type": "timeseries",
|
||||
"title": "Алерты",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 48,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "ALERTS{alertname=~\"Messenger.*\"}",
|
||||
"legendFormat": ""
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 30,
|
||||
"type": "logs",
|
||||
"title": "API, relay и worker / связанные логи",
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "loki"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 56,
|
||||
"w": 24,
|
||||
"h": 10
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "{service_name=~\"messenger-(api|relay|worker)\"}"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
apiVersion: 1
|
||||
providers:
|
||||
- name: messenger
|
||||
folder: MTUSI
|
||||
type: file
|
||||
allowUiUpdates: false
|
||||
options: {path: /var/lib/grafana/dashboards}
|
||||
@@ -0,0 +1,33 @@
|
||||
apiVersion: 1
|
||||
datasources:
|
||||
- name: Prometheus
|
||||
type: prometheus
|
||||
uid: prometheus
|
||||
url: http://prometheus:9090
|
||||
isDefault: true
|
||||
editable: false
|
||||
- name: Loki
|
||||
type: loki
|
||||
uid: loki
|
||||
url: http://loki:3100
|
||||
editable: false
|
||||
jsonData:
|
||||
derivedFields:
|
||||
- name: Trace
|
||||
matcherType: label
|
||||
matcherRegex: trace_id
|
||||
datasourceUid: tempo
|
||||
url: '$${__value.raw}'
|
||||
- name: Tempo
|
||||
type: tempo
|
||||
uid: tempo
|
||||
url: http://tempo:3200
|
||||
editable: false
|
||||
jsonData:
|
||||
tracesToLogsV2:
|
||||
datasourceUid: loki
|
||||
spanStartTimeShift: '-2m'
|
||||
spanEndTimeShift: '2m'
|
||||
filterByTraceID: true
|
||||
tags:
|
||||
- {key: service.name, value: service_name}
|
||||
@@ -0,0 +1,36 @@
|
||||
auth_enabled: false
|
||||
server:
|
||||
http_listen_port: 3100
|
||||
common:
|
||||
instance_addr: 127.0.0.1
|
||||
path_prefix: /loki
|
||||
storage:
|
||||
filesystem:
|
||||
chunks_directory: /loki/chunks
|
||||
rules_directory: /loki/rules
|
||||
replication_factor: 1
|
||||
ring:
|
||||
kvstore:
|
||||
store: inmemory
|
||||
schema_config:
|
||||
configs:
|
||||
- from: 2024-01-01
|
||||
store: tsdb
|
||||
object_store: filesystem
|
||||
schema: v13
|
||||
index: {prefix: index_, period: 24h}
|
||||
compactor:
|
||||
working_directory: /loki/compactor
|
||||
retention_enabled: true
|
||||
delete_request_store: filesystem
|
||||
limits_config:
|
||||
retention_period: 72h
|
||||
allow_structured_metadata: true
|
||||
otlp_config:
|
||||
resource_attributes:
|
||||
ignore_defaults: true
|
||||
attributes_config:
|
||||
- action: index_label
|
||||
attributes: [service.name, deployment.environment.name]
|
||||
analytics:
|
||||
reporting_enabled: false
|
||||
@@ -0,0 +1,13 @@
|
||||
global:
|
||||
scrape_interval: 2s
|
||||
evaluation_interval: 2s
|
||||
|
||||
rule_files:
|
||||
- /etc/prometheus/alerts.yaml
|
||||
|
||||
scrape_configs:
|
||||
- job_name: messenger-otel
|
||||
honor_labels: true
|
||||
static_configs: [{targets: ["otel-collector:8889"]}]
|
||||
- job_name: collector
|
||||
static_configs: [{targets: ["otel-collector:8888"]}]
|
||||
@@ -0,0 +1,18 @@
|
||||
server:
|
||||
http_listen_port: 3200
|
||||
distributor:
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
http:
|
||||
endpoint: 0.0.0.0:4318
|
||||
storage:
|
||||
trace:
|
||||
backend: local
|
||||
wal: {path: /var/tempo/wal}
|
||||
local: {path: /var/tempo/blocks}
|
||||
compactor:
|
||||
compaction:
|
||||
block_retention: 72h
|
||||
usage_report:
|
||||
reporting_enabled: false
|
||||
@@ -0,0 +1,26 @@
|
||||
{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:ListBucket",
|
||||
"s3:GetBucketLocation"
|
||||
],
|
||||
"Resource": [
|
||||
"arn:aws:s3:::BUCKET"
|
||||
]
|
||||
},
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:GetObject",
|
||||
"s3:DeleteObject"
|
||||
],
|
||||
"Resource": [
|
||||
"arn:aws:s3:::BUCKET/staging/*",
|
||||
"arn:aws:s3:::BUCKET/objects/*"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
# Official security-fixed source release; no dependency on retired image tags.
|
||||
FROM golang:1.26.7-alpine3.23@sha256:b17af760035fc2f338eed92d448a6c67f2d45438844fc6c60678fa5f99e44b57 AS builder
|
||||
ENV CGO_ENABLED=0 GOTOOLCHAIN=local GOMAXPROCS=4 GOFLAGS=-p=4
|
||||
RUN go install -ldflags='-s -w' github.com/minio/minio@RELEASE.2025-10-15T17-29-55Z
|
||||
RUN go install -ldflags='-s -w' github.com/minio/mc@RELEASE.2025-08-13T08-35-41Z
|
||||
|
||||
FROM alpine:3.23@sha256:85fe1e81d6758c208f3e1eed4338a1997e19d4be002d4dd32d3100c9a8c010a0 AS server
|
||||
COPY --from=builder /go/bin/minio /usr/local/bin/minio
|
||||
COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
|
||||
RUN addgroup -g 10001 app && adduser -D -u 10001 -G app app && mkdir /data && chown app:app /data
|
||||
COPY --chmod=755 infra/start-s3.sh /usr/local/bin/start-s3.sh
|
||||
USER 10001:10001
|
||||
ENTRYPOINT ["/usr/local/bin/start-s3.sh"]
|
||||
|
||||
FROM alpine:3.23@sha256:85fe1e81d6758c208f3e1eed4338a1997e19d4be002d4dd32d3100c9a8c010a0 AS client
|
||||
COPY --from=builder /go/bin/mc /usr/local/bin/mc
|
||||
COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
|
||||
ENTRYPOINT ["/bin/sh"]
|
||||
@@ -0,0 +1,5 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
export MINIO_ROOT_USER="$(cat /run/secrets/s3_root_user)"
|
||||
export MINIO_ROOT_PASSWORD="$(cat /run/secrets/s3_root_password)"
|
||||
exec minio "$@"
|
||||
@@ -0,0 +1,27 @@
|
||||
{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:ListBucket",
|
||||
"s3:GetBucketLocation"
|
||||
],
|
||||
"Resource": [
|
||||
"arn:aws:s3:::BUCKET"
|
||||
]
|
||||
},
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:GetObject",
|
||||
"s3:PutObject",
|
||||
"s3:DeleteObject"
|
||||
],
|
||||
"Resource": [
|
||||
"arn:aws:s3:::BUCKET/staging/*",
|
||||
"arn:aws:s3:::BUCKET/objects/*"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:ListBucket",
|
||||
"s3:GetBucketLocation"
|
||||
],
|
||||
"Resource": [
|
||||
"arn:aws:s3:::BUCKET"
|
||||
]
|
||||
},
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:GetObject"
|
||||
],
|
||||
"Resource": [
|
||||
"arn:aws:s3:::BUCKET/staging/*",
|
||||
"arn:aws:s3:::BUCKET/objects/*"
|
||||
]
|
||||
},
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:PutObject"
|
||||
],
|
||||
"Resource": [
|
||||
"arn:aws:s3:::BUCKET/objects/*"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
CREATE TABLE users (
|
||||
id uuid PRIMARY KEY,
|
||||
username varchar(32) COLLATE "C" NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'),
|
||||
display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100),
|
||||
password_hash text NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
CREATE TABLE chats (
|
||||
id uuid PRIMARY KEY,
|
||||
title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100),
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
CREATE TABLE chat_members (
|
||||
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
user_id uuid NOT NULL REFERENCES users(id),
|
||||
position smallint NOT NULL CHECK (position BETWEEN 0 AND 99),
|
||||
PRIMARY KEY (chat_id,user_id),
|
||||
UNIQUE (chat_id,position)
|
||||
);
|
||||
CREATE INDEX chat_members_by_user ON chat_members (user_id,chat_id);
|
||||
CREATE TABLE messages (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL REFERENCES chats(id),
|
||||
sender_id uuid NOT NULL REFERENCES users(id),
|
||||
text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000),
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
FOREIGN KEY (chat_id,sender_id) REFERENCES chat_members(chat_id,user_id)
|
||||
);
|
||||
CREATE INDEX messages_history ON messages (chat_id,created_at,id);
|
||||
CREATE TABLE sessions (
|
||||
token_hash bytea PRIMARY KEY CHECK (octet_length(token_hash)=32),
|
||||
user_id uuid NOT NULL REFERENCES users(id),
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
expires_at timestamptz NOT NULL,
|
||||
CHECK (expires_at > created_at)
|
||||
);
|
||||
CREATE INDEX sessions_expiry ON sessions (expires_at);
|
||||
@@ -0,0 +1,14 @@
|
||||
CREATE TABLE message_requests (
|
||||
user_id uuid NOT NULL REFERENCES users(id),
|
||||
http_method text NOT NULL DEFAULT 'POST' CHECK (http_method='POST'),
|
||||
chat_id uuid NOT NULL REFERENCES chats(id),
|
||||
idempotency_key varchar(128) COLLATE "C" NOT NULL
|
||||
CHECK (idempotency_key ~ '^[A-Za-z0-9._:-]{1,128}$'),
|
||||
payload_hash bytea NOT NULL CHECK (octet_length(payload_hash)=32),
|
||||
response_message_id uuid NOT NULL REFERENCES messages(id),
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
expires_at timestamptz NOT NULL,
|
||||
PRIMARY KEY (user_id,http_method,chat_id,idempotency_key),
|
||||
CHECK (expires_at > created_at)
|
||||
);
|
||||
CREATE INDEX message_requests_expiry ON message_requests (expires_at);
|
||||
@@ -0,0 +1,30 @@
|
||||
CREATE TABLE attachments (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL REFERENCES chats(id),
|
||||
owner_id uuid NOT NULL REFERENCES users(id),
|
||||
filename text NOT NULL CHECK (char_length(filename) BETWEEN 1 AND 255),
|
||||
content_type text NOT NULL CHECK (content_type IN ('image/png','image/jpeg')),
|
||||
size_bytes bigint NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760),
|
||||
sha256 text NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'),
|
||||
object_key text NOT NULL UNIQUE,
|
||||
storage_state text NOT NULL CHECK (storage_state IN ('pending','ready','missing')),
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
updated_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
UNIQUE (id,chat_id),
|
||||
FOREIGN KEY (chat_id,owner_id) REFERENCES chat_members(chat_id,user_id)
|
||||
);
|
||||
CREATE INDEX attachments_reconcile ON attachments (updated_at,id);
|
||||
ALTER TABLE messages DROP CONSTRAINT messages_text_check;
|
||||
ALTER TABLE messages ADD CHECK (char_length(text) BETWEEN 0 AND 4000);
|
||||
ALTER TABLE messages ADD UNIQUE (id,chat_id);
|
||||
CREATE TABLE message_attachments (
|
||||
message_id uuid NOT NULL,
|
||||
chat_id uuid NOT NULL,
|
||||
attachment_id uuid NOT NULL,
|
||||
position smallint NOT NULL CHECK (position BETWEEN 0 AND 9),
|
||||
PRIMARY KEY (message_id,attachment_id),
|
||||
UNIQUE (message_id,position),
|
||||
FOREIGN KEY (message_id,chat_id) REFERENCES messages(id,chat_id),
|
||||
FOREIGN KEY (attachment_id,chat_id) REFERENCES attachments(id,chat_id)
|
||||
);
|
||||
CREATE INDEX message_attachments_by_attachment ON message_attachments (attachment_id);
|
||||
@@ -0,0 +1,46 @@
|
||||
-- Preserve earlier migration checksums; transition lab4 storage to versioned jobs.
|
||||
ALTER TABLE attachments DROP CONSTRAINT attachments_storage_state_check;
|
||||
ALTER TABLE attachments ADD CHECK (storage_state IN ('queued','processing','ready','failed')) NOT VALID;
|
||||
UPDATE attachments SET storage_state='queued' WHERE storage_state IN ('pending','missing');
|
||||
ALTER TABLE attachments ADD COLUMN pipeline_version integer NOT NULL DEFAULT 1 CHECK (pipeline_version > 0);
|
||||
ALTER TABLE attachments ADD COLUMN variants jsonb NOT NULL DEFAULT '[]' CHECK (jsonb_typeof(variants)='array');
|
||||
ALTER TABLE attachments ADD COLUMN error_code text;
|
||||
CREATE TABLE attachment_jobs (
|
||||
job_id uuid PRIMARY KEY,
|
||||
attachment_id uuid NOT NULL REFERENCES attachments(id) ON DELETE CASCADE,
|
||||
pipeline_version integer NOT NULL CHECK (pipeline_version > 0),
|
||||
source_key text NOT NULL,
|
||||
state text NOT NULL DEFAULT 'queued' CHECK (state IN ('queued','processing','ready','failed','superseded')),
|
||||
attempt integer NOT NULL DEFAULT 0 CHECK (attempt >= 0),
|
||||
fence uuid,
|
||||
lease_until timestamptz,
|
||||
next_attempt_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
error_code text,
|
||||
source_expired boolean NOT NULL DEFAULT false,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
updated_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
UNIQUE (attachment_id,pipeline_version)
|
||||
);
|
||||
CREATE INDEX jobs_recovery ON attachment_jobs(state,lease_until,next_attempt_at);
|
||||
CREATE TABLE job_outbox (
|
||||
job_id uuid PRIMARY KEY REFERENCES attachment_jobs(job_id) ON DELETE CASCADE,
|
||||
due_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
published_at timestamptz,
|
||||
stream_id text
|
||||
);
|
||||
CREATE INDEX outbox_due ON job_outbox(due_at) WHERE published_at IS NULL;
|
||||
CREATE TABLE pipeline_audit (
|
||||
sequence bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
|
||||
job_id uuid NOT NULL REFERENCES attachment_jobs(job_id) ON DELETE CASCADE,
|
||||
event text NOT NULL,
|
||||
details jsonb NOT NULL DEFAULT '{}',
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
-- Only an operator with DB access can configure faults; production ignores this table.
|
||||
CREATE TABLE pipeline_faults (
|
||||
job_id uuid REFERENCES attachment_jobs(job_id) ON DELETE CASCADE,
|
||||
phase text NOT NULL,
|
||||
action text NOT NULL CHECK (action IN ('crash','hang','transient','permanent')),
|
||||
remaining integer NOT NULL DEFAULT 1 CHECK (remaining >= 0),
|
||||
PRIMARY KEY(job_id,phase)
|
||||
);
|
||||
@@ -0,0 +1,8 @@
|
||||
-- A paused API must not acknowledge an object that GC has already retired.
|
||||
CREATE TABLE upload_reservations (
|
||||
id uuid PRIMARY KEY,
|
||||
object_key text NOT NULL UNIQUE,
|
||||
state text NOT NULL DEFAULT 'uploading' CHECK (state IN ('uploading','accepted','abandoned')),
|
||||
expires_at timestamptz NOT NULL DEFAULT clock_timestamp()+interval '120 seconds',
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
@@ -0,0 +1,5 @@
|
||||
-- Optional W3C context follows the durable job, including outbox retries.
|
||||
ALTER TABLE attachment_jobs ADD COLUMN traceparent text CHECK (
|
||||
traceparent ~ '^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$'
|
||||
);
|
||||
ALTER TABLE attachment_jobs ADD COLUMN tracestate text CHECK (length(tracestate) <= 512);
|
||||
@@ -0,0 +1,48 @@
|
||||
[project]
|
||||
name = "mtusi-messenger"
|
||||
version = "0.6.0"
|
||||
description = "MTUSI lab 6: messenger observability"
|
||||
requires-python = ">=3.14,<3.15"
|
||||
dependencies = [
|
||||
"opentelemetry-sdk==1.45.2",
|
||||
"opentelemetry-exporter-otlp-proto-http==1.45.2",
|
||||
"argon2-cffi==25.1.0",
|
||||
"boto3==1.43.109",
|
||||
"fastapi==0.143.3",
|
||||
"pillow==13.0.0",
|
||||
"psycopg[binary,pool]==3.3.6",
|
||||
"pydantic==2.14.1",
|
||||
"redis==8.1.0",
|
||||
"uvicorn==0.54.0",
|
||||
]
|
||||
|
||||
[build-system]
|
||||
requires = ["hatchling==1.27.0"]
|
||||
build-backend = "hatchling.build"
|
||||
|
||||
[tool.hatch.build.targets.wheel]
|
||||
packages = ["src/messenger"]
|
||||
|
||||
[dependency-groups]
|
||||
dev = [
|
||||
"httpx==0.28.1",
|
||||
"openapi-spec-validator==0.7.2",
|
||||
"pytest==9.1.1",
|
||||
"ruff==0.17.1",
|
||||
]
|
||||
research = ["matplotlib==3.11.2"]
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
testpaths = ["student-tests"]
|
||||
addopts = "-ra --tb=short"
|
||||
|
||||
[tool.ruff]
|
||||
extend-exclude = ["tests", "scripts/check_contract.py", "scripts/validate_openapi.py"]
|
||||
target-version = "py314"
|
||||
line-length = 100
|
||||
|
||||
[tool.ruff.lint]
|
||||
select = ["E", "F", "I", "B", "UP"]
|
||||
|
||||
[tool.ruff.per-file-target-version]
|
||||
"scripts/*.py" = "py39"
|
||||
@@ -0,0 +1,50 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Create local synthetic credentials once, without printing their values."""
|
||||
|
||||
import os
|
||||
import secrets
|
||||
from pathlib import Path
|
||||
|
||||
root = Path(__file__).resolve().parents[1]
|
||||
directory = root / "secrets"
|
||||
directory.mkdir(mode=0o700, exist_ok=True)
|
||||
password_file = directory / "postgres_password"
|
||||
if not password_file.exists():
|
||||
password_file.write_text(secrets.token_urlsafe(32) + "\n")
|
||||
password = password_file.read_text().strip()
|
||||
for name, value in {
|
||||
"database_url": f"postgresql://messenger:{password}@postgres:5432/messenger",
|
||||
"grafana_password": secrets.token_urlsafe(24),
|
||||
"cache_url": "redis://valkey:6379/0",
|
||||
"broker_url": "redis://broker:6379/0",
|
||||
}.items():
|
||||
path = directory / name
|
||||
if not path.exists():
|
||||
path.write_text(value + "\n")
|
||||
# Compose bind-mount secrets must be readable by the non-root container process.
|
||||
# The parent directory remains 0700 on the host.
|
||||
for role in ("root", "api", "worker", "reconciler", "test"):
|
||||
names = (
|
||||
("s3_root_user", "s3_root_password")
|
||||
if role == "root"
|
||||
else ("s3_" + role + "_access_key_id", "s3_" + role + "_secret_access_key")
|
||||
)
|
||||
for index, name in enumerate(names):
|
||||
path = directory / name
|
||||
if not path.exists():
|
||||
path.write_text(
|
||||
(secrets.token_hex(10) if index == 0 else secrets.token_urlsafe(32)) + "\n"
|
||||
)
|
||||
for name in (
|
||||
"postgres_password",
|
||||
"grafana_password",
|
||||
"database_url",
|
||||
"cache_url",
|
||||
"broker_url",
|
||||
*[p.name for p in directory.glob("s3_*")],
|
||||
):
|
||||
os.chmod(directory / name, 0o644)
|
||||
env = root / ".env"
|
||||
if not env.exists():
|
||||
env.write_text((root / ".env.example").read_text())
|
||||
print("Local secret files and environment are ready; existing values preserved.")
|
||||
@@ -0,0 +1,82 @@
|
||||
"""Read-only discovery and scoped Compose operations for the local lab5 stack."""
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def compose(*args, env=None):
|
||||
return subprocess.run(
|
||||
["docker", "compose", "--env-file", ".env", *args],
|
||||
cwd=ROOT,
|
||||
env=env,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=90,
|
||||
).stdout
|
||||
|
||||
|
||||
def inspect(container):
|
||||
template = (
|
||||
'{"id":{{json .Id}},"started_at":{{json .State.StartedAt}},'
|
||||
'"pid":{{.State.Pid}},"running":{{json .State.Running}},'
|
||||
'"image":{{json .Image}},"memory":{{.HostConfig.Memory}},'
|
||||
'"nano_cpus":{{.HostConfig.NanoCpus}},'
|
||||
'"published_ports":{{json .HostConfig.PortBindings}}}'
|
||||
)
|
||||
r = subprocess.run(
|
||||
["docker", "inspect", "--format", template, container],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
timeout=5,
|
||||
)
|
||||
return json.loads(r.stdout)
|
||||
|
||||
|
||||
def instance(container):
|
||||
code = (
|
||||
"import urllib.request; r=urllib.request.urlopen("
|
||||
"'http://127.0.0.1:8080/health/live',timeout=1); print(r.headers['X-Instance-Id'])"
|
||||
)
|
||||
r = subprocess.run(
|
||||
["docker", "exec", container, "python", "-c", code],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
timeout=3,
|
||||
)
|
||||
value = inspect(container)
|
||||
value["instance"] = r.stdout.strip()
|
||||
return value
|
||||
|
||||
|
||||
def instances():
|
||||
ids = compose("ps", "-q", "api").split()
|
||||
with ThreadPoolExecutor(max_workers=8) as pool:
|
||||
return list(pool.map(instance, ids))
|
||||
|
||||
|
||||
def wait_lb(base="http://127.0.0.1:8086", count=2, timeout=45):
|
||||
deadline, seen = time.monotonic() + timeout, set()
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
current = {value["instance"] for value in instances()}
|
||||
if len(current) != count:
|
||||
time.sleep(0.2)
|
||||
continue
|
||||
with urllib.request.urlopen(base + "/health/ready", timeout=2) as response:
|
||||
seen.add(response.headers["X-Instance-Id"])
|
||||
if current <= seen:
|
||||
return sorted(current)
|
||||
except (OSError, urllib.error.URLError, subprocess.SubprocessError):
|
||||
pass
|
||||
time.sleep(0.1)
|
||||
raise RuntimeError("Expected healthy API replicas were not observed through LB")
|
||||
@@ -0,0 +1,279 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Collect sanitized evidence from the local lab6 stand."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
from urllib.error import HTTPError, URLError
|
||||
from urllib.parse import urlencode
|
||||
from urllib.request import urlopen
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client # noqa: E402
|
||||
|
||||
|
||||
def compose(*args, **kwargs):
|
||||
return subprocess.run(["docker", "compose", *args], cwd=ROOT, check=True, **kwargs)
|
||||
|
||||
|
||||
def api_json(port, path, **params):
|
||||
url = f"http://127.0.0.1:{port}{path}"
|
||||
if params:
|
||||
url += "?" + urlencode(params)
|
||||
with urlopen(url, timeout=5) as response:
|
||||
return json.load(response)
|
||||
|
||||
|
||||
def prom(query):
|
||||
data = api_json(9096, "/api/v1/query", query=query)["data"]["result"]
|
||||
return float(data[0]["value"][1]) if data else 0
|
||||
|
||||
|
||||
def until(check, timeout=45):
|
||||
end = time.monotonic() + timeout
|
||||
while time.monotonic() < end:
|
||||
try:
|
||||
result = check()
|
||||
if result:
|
||||
return result
|
||||
except (HTTPError, URLError):
|
||||
pass
|
||||
time.sleep(0.5)
|
||||
raise RuntimeError(f"Timed out waiting for {check.__name__}")
|
||||
|
||||
|
||||
def log_for(request_id):
|
||||
data = api_json(
|
||||
3106,
|
||||
"/loki/api/v1/query_range",
|
||||
query='{service_name="messenger-api"} | request_id="' + request_id + '"',
|
||||
start=str(time.time_ns() - 180_000_000_000),
|
||||
limit=5,
|
||||
)["data"]["result"]
|
||||
for stream in data:
|
||||
for value in stream["values"]:
|
||||
return value[2] if len(value) > 2 else stream["stream"]
|
||||
return None
|
||||
|
||||
|
||||
def span_names(trace_id):
|
||||
data = api_json(3206, "/api/traces/" + trace_id)
|
||||
return [
|
||||
(span["name"], span.get("kind"), span["spanId"], span.get("parentSpanId"))
|
||||
for batch in data.get("batches", data.get("resourceSpans", []))
|
||||
for group in batch.get("scopeSpans", batch.get("instrumentationLibrarySpans", []))
|
||||
for span in group["spans"]
|
||||
]
|
||||
|
||||
|
||||
def alerts():
|
||||
return {
|
||||
a["labels"]["alertname"]: a["state"]
|
||||
for a in api_json(9096, "/api/v1/alerts")["data"]["alerts"]
|
||||
}
|
||||
|
||||
|
||||
def correlation(client, actor, chat, retry=False):
|
||||
if retry:
|
||||
compose("stop", "worker", stdout=subprocess.DEVNULL)
|
||||
try:
|
||||
attachment = client.call(
|
||||
"POST",
|
||||
f"/api/v1/chats/{chat['id']}/attachments?filename=observability.png",
|
||||
202,
|
||||
actor=actor,
|
||||
raw=(ROOT / "tests/fixtures/sample.png").read_bytes(),
|
||||
headers={"Content-Type": "image/png"},
|
||||
schema="Attachment",
|
||||
)
|
||||
if retry:
|
||||
job_attachment = attachment["json"]["id"]
|
||||
compose(
|
||||
"exec",
|
||||
"-T",
|
||||
"postgres",
|
||||
"psql",
|
||||
"-U",
|
||||
"messenger",
|
||||
"-Xq",
|
||||
"-v",
|
||||
"ON_ERROR_STOP=1",
|
||||
"-v",
|
||||
"attachment=" + job_attachment,
|
||||
input=(
|
||||
"INSERT INTO pipeline_faults(job_id,phase,action,remaining) "
|
||||
"SELECT job_id,'before_effect','transient',1 FROM attachment_jobs "
|
||||
"WHERE attachment_id=:'attachment';\n"
|
||||
),
|
||||
text=True,
|
||||
stdout=subprocess.DEVNULL,
|
||||
)
|
||||
finally:
|
||||
if retry:
|
||||
compose("start", "worker", stdout=subprocess.DEVNULL)
|
||||
client.wait_ready(actor, attachment["json"])
|
||||
request_id = attachment["headers"]["X-Request-Id"]
|
||||
log = until(lambda: log_for(request_id))
|
||||
trace_id = log["trace_id"]
|
||||
expected = {"jobs.publish", "jobs.consume", "s3.put", "s3.download", "db.transaction"}
|
||||
|
||||
def connected():
|
||||
spans = span_names(trace_id)
|
||||
names = {row[0] for row in spans}
|
||||
consumers = sum(row[0] == "jobs.consume" for row in spans)
|
||||
return spans if expected <= names and consumers >= (2 if retry else 1) else None
|
||||
|
||||
spans = until(connected)
|
||||
by_id = {row[2]: row for row in spans}
|
||||
consumers = [row for row in spans if row[0] == "jobs.consume"]
|
||||
assert all(by_id[row[3]][0] == "jobs.publish" for row in consumers)
|
||||
logs = until(
|
||||
lambda: api_json(
|
||||
3106,
|
||||
"/loki/api/v1/query_range",
|
||||
query='{service_name="messenger-worker"} | trace_id="' + trace_id + '"',
|
||||
start=str(time.time_ns() - 180_000_000_000),
|
||||
limit=5,
|
||||
)["data"]["result"]
|
||||
)
|
||||
assert any(value[1] == "job_processed" for row in logs for value in row["values"])
|
||||
return {
|
||||
"request_id": request_id,
|
||||
"trace_id": trace_id,
|
||||
"span_names": sorted({r[0] for r in spans}),
|
||||
"worker_parent": by_id[consumers[0][3]][0],
|
||||
"worker_attempts": len(consumers),
|
||||
"api_and_worker_loki": True,
|
||||
}
|
||||
|
||||
|
||||
def worker_incident(client, actor, chat):
|
||||
compose("stop", "worker", stdout=subprocess.DEVNULL)
|
||||
try:
|
||||
uploads = [client.attachment(actor, chat)[0] for _ in range(3)]
|
||||
until(lambda: alerts().get("MessengerQueueLag") == "firing")
|
||||
record = {
|
||||
"depth_at_firing": prom("max(messenger_queue_depth)"),
|
||||
"oldest_seconds_at_firing": prom("max(messenger_queue_oldest_seconds)"),
|
||||
}
|
||||
assert record["depth_at_firing"] >= 3
|
||||
finally:
|
||||
compose("start", "worker", stdout=subprocess.DEVNULL)
|
||||
for attachment in uploads:
|
||||
client.wait_ready(actor, attachment)
|
||||
until(lambda: not prom("max(messenger_queue_depth)") and "MessengerQueueLag" not in alerts())
|
||||
return {**record, "state": ["firing", "resolved"], "uploads_ready": len(uploads)}
|
||||
|
||||
|
||||
def database_incident(client, actor):
|
||||
# Independent, bounded transaction. Lock sessions only; readiness can still use the schema.
|
||||
command = [
|
||||
"docker",
|
||||
"compose",
|
||||
"exec",
|
||||
"-T",
|
||||
"postgres",
|
||||
"psql",
|
||||
"-U",
|
||||
"messenger",
|
||||
"-d",
|
||||
"messenger",
|
||||
"-XqAt",
|
||||
"-v",
|
||||
"ON_ERROR_STOP=1",
|
||||
]
|
||||
start = time.monotonic()
|
||||
lock = subprocess.Popen(
|
||||
command, cwd=ROOT, stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True
|
||||
)
|
||||
lock.stdin.write(
|
||||
"BEGIN;\nLOCK TABLE sessions IN ACCESS EXCLUSIVE MODE;\n"
|
||||
"\\echo locked\nSELECT pg_sleep(42);\nROLLBACK;\n"
|
||||
)
|
||||
lock.stdin.close()
|
||||
try:
|
||||
assert lock.stdout.readline().strip() == "locked"
|
||||
attempts = []
|
||||
while time.monotonic() - start < 34:
|
||||
attempt = client.call("GET", "/api/v1/users/me", [200, 503], actor=actor, schema="User")
|
||||
attempts.append(attempt)
|
||||
states = alerts()
|
||||
if all(
|
||||
states.get(a) == "firing"
|
||||
for a in ("MessengerHttpErrors", "MessengerDatabaseErrors")
|
||||
):
|
||||
break
|
||||
assert any(item["status"] == 503 for item in attempts)
|
||||
active = alerts()
|
||||
assert active.get("MessengerDatabaseErrors") == "firing", active
|
||||
assert active.get("MessengerHttpErrors") == "firing", active
|
||||
finally:
|
||||
lock.wait(timeout=46)
|
||||
lock.stdout.close()
|
||||
client.call("GET", "/api/v1/users/me", actor=actor, schema="User")
|
||||
|
||||
# Keep successful traffic to publish fresh data while the alert window slides.
|
||||
def resolved():
|
||||
client.call("GET", "/api/v1/users/me", actor=actor, schema="User")
|
||||
return not ({"MessengerDatabaseErrors", "MessengerHttpErrors"} & alerts().keys())
|
||||
|
||||
until(resolved)
|
||||
failure = next(item for item in attempts if item["status"] == 503)
|
||||
failure_log = until(lambda: log_for(failure["headers"]["X-Request-Id"]))
|
||||
names = span_names(failure_log["trace_id"])
|
||||
assert "db.read" in {row[0] for row in names}
|
||||
return {
|
||||
"statuses": [item["status"] for item in attempts],
|
||||
"max_latency_ms": max(item["elapsed"] for item in attempts) * 1000,
|
||||
"trace_id": failure_log["trace_id"],
|
||||
"state": ["firing", "resolved"],
|
||||
"spans": sorted({row[0] for row in names}),
|
||||
}
|
||||
|
||||
|
||||
def backend_incident(client, actor, chat):
|
||||
lost = "sum(otelcol_exporter_send_failed_spans_total)"
|
||||
baseline = prom(lost)
|
||||
compose("stop", "tempo", stdout=subprocess.DEVNULL)
|
||||
try:
|
||||
for _ in range(15):
|
||||
client.message(actor, chat, "synthetic telemetry outage probe")
|
||||
time.sleep(0.5)
|
||||
delta = until(lambda: value if (value := prom(lost) - baseline) > 0 else None)
|
||||
memory = prom("max(otelcol_process_memory_rss_bytes)")
|
||||
assert memory > 0
|
||||
finally:
|
||||
compose("start", "tempo", stdout=subprocess.DEVNULL)
|
||||
client.call("GET", "/health/ready", schema="Health")
|
||||
return {"successful_messages": 15, "failed_export_spans": delta, "collector_rss_bytes": memory}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--base-url", default="http://127.0.0.1:8086")
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
parser.add_argument("--incidents", action="store_true")
|
||||
parser.add_argument("--retry", action="store_true")
|
||||
args = parser.parse_args()
|
||||
if args.output.exists():
|
||||
parser.error("choose a new evidence file")
|
||||
client = Client(args.base_url)
|
||||
alice, _, _, chat = client.bootstrap()
|
||||
result = {"correlation": correlation(client, alice, chat, retry=args.retry)}
|
||||
print("PASS: same Tempo trace and Loki logs for upload and worker", flush=True)
|
||||
if args.incidents:
|
||||
result["worker_stop"] = worker_incident(client, alice, chat)
|
||||
print("PASS: queue alert firing and resolved", flush=True)
|
||||
result["database_lock"] = database_incident(client, alice)
|
||||
print("PASS: database and HTTP alerts firing and resolved", flush=True)
|
||||
result["tempo_stop"] = backend_incident(client, alice, chat)
|
||||
print("PASS: messages persisted during Tempo outage", flush=True)
|
||||
args.output.write_text(json.dumps(result, indent=2) + "\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,33 @@
|
||||
"""Local test operator configuration. Credentials stay in ignored files/in memory."""
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
from psycopg.conninfo import make_conninfo
|
||||
|
||||
from messenger.settings import Settings
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def settings(role="worker"):
|
||||
prefix = "s3_root" if role == "root" else "s3_" + role
|
||||
access, secret = (
|
||||
("user", "password") if role == "root" else ("access_key_id", "secret_access_key")
|
||||
)
|
||||
return Settings(
|
||||
make_conninfo(
|
||||
host="127.0.0.1",
|
||||
port=int(os.getenv("TEST_PG_PORT", "55467")),
|
||||
user="messenger",
|
||||
dbname="messenger",
|
||||
password=(ROOT / "secrets/postgres_password").read_text().strip(),
|
||||
),
|
||||
"redis://127.0.0.1:" + os.getenv("TEST_CACHE_PORT", "56386") + "/0",
|
||||
s3_endpoint="http://127.0.0.1:" + os.getenv("TEST_S3_PORT", "59006"),
|
||||
s3_access_key=(ROOT / ("secrets/" + prefix + "_" + access)).read_text().strip(),
|
||||
s3_secret_key=(ROOT / ("secrets/" + prefix + "_" + secret)).read_text().strip(),
|
||||
broker_url="redis://127.0.0.1:" + os.getenv("TEST_BROKER_PORT", "56387") + "/0",
|
||||
job_timeout=int(os.getenv("TEST_JOB_TIMEOUT_SECONDS", "5")),
|
||||
fault_injection=True,
|
||||
)
|
||||
Executable
+5
@@ -0,0 +1,5 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
# All running application/state services; no migration, down -v, flush or seed.
|
||||
docker compose --env-file .env restart --timeout 7 api postgres valkey s3 broker relay worker lb
|
||||
@@ -0,0 +1,49 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Sequential process replacement; at least one existing API stays running."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import subprocess
|
||||
|
||||
from cluster import compose, instances, wait_lb
|
||||
|
||||
|
||||
def replace_all(base="http://127.0.0.1:8084"):
|
||||
original = instances()
|
||||
if len(original) < 2:
|
||||
raise RuntimeError("Rolling replacement requires at least two healthy replicas")
|
||||
wait_lb(base, len(original))
|
||||
replacements = []
|
||||
for current in original:
|
||||
subprocess.run(
|
||||
["docker", "stop", "--time", "5", current["id"]],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
timeout=8,
|
||||
)
|
||||
subprocess.run(["docker", "rm", current["id"]], check=True, capture_output=True)
|
||||
compose(
|
||||
"up", "-d", "--no-deps", "--no-recreate", "--scale", "api=" + str(len(original)), "api"
|
||||
)
|
||||
aliases = wait_lb(base, len(original))
|
||||
if current["instance"] in aliases:
|
||||
raise RuntimeError("Old API process is still serving after replacement")
|
||||
replacements.append(
|
||||
{
|
||||
"old_container": current["id"],
|
||||
"old_instance": current["instance"],
|
||||
"live_instances": aliases,
|
||||
}
|
||||
)
|
||||
return replacements
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--base-url", default="http://127.0.0.1:8084")
|
||||
args = parser.parse_args()
|
||||
print(json.dumps(replace_all(args.base_url)))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,48 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Map a process alias to ONE container belonging to this Compose service."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def alias(container):
|
||||
code = (
|
||||
"import urllib.request; r=urllib.request.urlopen("
|
||||
"'http://127.0.0.1:8080/health/live',timeout=1); print(r.headers['X-Instance-Id'])"
|
||||
)
|
||||
r = subprocess.run(
|
||||
["docker", "exec", container, "python", "-c", code],
|
||||
cwd=ROOT,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=3,
|
||||
check=True,
|
||||
)
|
||||
return container, r.stdout.strip()
|
||||
|
||||
|
||||
def main():
|
||||
target = os.environ.get("TARGET_INSTANCE_ID", "")
|
||||
if not re.fullmatch(r"[a-f0-9]{32}", target):
|
||||
raise SystemExit("TARGET_INSTANCE_ID must name a live API process")
|
||||
ids = subprocess.check_output(
|
||||
["docker", "compose", "ps", "-q", "api"], cwd=ROOT, text=True
|
||||
).split()
|
||||
with ThreadPoolExecutor(max_workers=8) as pool:
|
||||
matches = [cid for cid, identity in pool.map(alias, ids) if identity == target]
|
||||
if len(matches) != 1:
|
||||
raise SystemExit("Expected exactly one matching API in this project")
|
||||
subprocess.run(
|
||||
["docker", "stop", "--time", "5", matches[0]], check=True, capture_output=True, timeout=7
|
||||
)
|
||||
print(json.dumps({"stopped_container": matches[0], "instance": target}))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+4
@@ -0,0 +1,4 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
exec python3 scripts/stop-one.py
|
||||
@@ -0,0 +1,184 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Fixed offered-load comparison. Stops only named services in the current lab6 project."""
|
||||
|
||||
import argparse
|
||||
import concurrent.futures
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import time
|
||||
from pathlib import Path
|
||||
from urllib.request import urlopen
|
||||
|
||||
from observe import Client, compose, prom, until
|
||||
|
||||
MODES = (("off", "0", "0"), ("full", "1", "1"), ("half", "1", "0.5"), ("quarter", "1", "0.25"))
|
||||
COUNTERS = {
|
||||
"accepted_spans": "sum(otelcol_receiver_accepted_spans_total)",
|
||||
"sent_spans": "sum(otelcol_exporter_sent_spans_total)",
|
||||
"failed_spans": "sum(otelcol_exporter_send_failed_spans_total)",
|
||||
"accepted_logs": "sum(otelcol_receiver_accepted_log_records_total)",
|
||||
"accepted_metric_points": "sum(otelcol_receiver_accepted_metric_points_total)",
|
||||
}
|
||||
|
||||
|
||||
def configure(enabled, ratio):
|
||||
compose(
|
||||
"up",
|
||||
"-d",
|
||||
"--no-build",
|
||||
"--no-deps",
|
||||
"--force-recreate",
|
||||
"api",
|
||||
"worker",
|
||||
"relay",
|
||||
env={**os.environ, "TELEMETRY_ENABLED": enabled, "OTEL_SAMPLE_RATIO": ratio},
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
)
|
||||
|
||||
def ready():
|
||||
try:
|
||||
with urlopen("http://127.0.0.1:8086/health/ready", timeout=2) as response:
|
||||
return response.status == 200
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
until(ready)
|
||||
|
||||
|
||||
def snapshot():
|
||||
with urlopen("http://127.0.0.1:3206/metrics", timeout=4) as response:
|
||||
metrics = response.read().decode()
|
||||
received_bytes = sum(
|
||||
float(value)
|
||||
for value in re.findall(
|
||||
r"^tempo_distributor_bytes_received_total\{[^}]+\} (\S+)", metrics, re.M
|
||||
)
|
||||
)
|
||||
return {
|
||||
"trace_ingest_proto_bytes": received_bytes,
|
||||
**{name: prom(expression) for name, expression in COUNTERS.items()},
|
||||
}
|
||||
|
||||
|
||||
def workload(client, actor, chat, seconds, rate):
|
||||
def operation(n):
|
||||
if n % 45 == 0:
|
||||
started = time.perf_counter()
|
||||
client.attachment(actor, chat)
|
||||
return (time.perf_counter() - started) * 1000
|
||||
if n % 5 == 0:
|
||||
response = client.message(actor, chat, "synthetic cost experiment")
|
||||
else:
|
||||
response = client.call(
|
||||
"GET",
|
||||
f"/api/v1/chats/{chat['id']}/messages?limit=20",
|
||||
actor=actor,
|
||||
schema="MessagePage",
|
||||
)
|
||||
return response["elapsed"] * 1000
|
||||
|
||||
start = time.perf_counter()
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=12) as pool:
|
||||
futures = []
|
||||
for number in range(seconds * rate):
|
||||
delay = start + number / rate - time.perf_counter()
|
||||
if delay > 0:
|
||||
time.sleep(delay)
|
||||
futures.append(pool.submit(operation, number))
|
||||
observations = sorted(future.result() for future in futures)
|
||||
elapsed = time.perf_counter() - start
|
||||
return {
|
||||
"requests": len(observations),
|
||||
"elapsed_s": elapsed,
|
||||
"latency_ms": {
|
||||
f"p{int(q * 100)}": observations[math.ceil(q * len(observations)) - 1]
|
||||
for q in (0.50, 0.95, 0.99)
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def stats():
|
||||
services = ("api", "worker", "relay", "otel-collector", "prometheus", "loki", "tempo")
|
||||
ids = compose("ps", "-q", *services, capture_output=True, text=True).stdout.split()
|
||||
raw = subprocess.check_output(
|
||||
["docker", "stats", "--no-stream", "--format", "{{json .}}", *ids], text=True
|
||||
)
|
||||
result = []
|
||||
for line in raw.splitlines():
|
||||
row = json.loads(line)
|
||||
result.append(
|
||||
{
|
||||
"container": row["Name"],
|
||||
"cpu_percent": float(row["CPUPerc"].rstrip("%")),
|
||||
"memory": row["MemUsage"].split(" / ")[0],
|
||||
}
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def run_one(mode, seconds, rate):
|
||||
label, enabled, ratio = mode
|
||||
configure(enabled, ratio)
|
||||
client = Client("http://127.0.0.1:8086")
|
||||
actor, _, _, chat = client.bootstrap()
|
||||
for _ in range(25):
|
||||
client.message(actor, chat, "seed")
|
||||
workload(client, actor, chat, 3, rate)
|
||||
time.sleep(3)
|
||||
initial = snapshot()
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool:
|
||||
run = pool.submit(workload, client, actor, chat, seconds, rate)
|
||||
samples = []
|
||||
for _ in range(2):
|
||||
time.sleep(2)
|
||||
samples.append(stats())
|
||||
receipt = run.result()
|
||||
# Collector emits batches and Prometheus scrapes every two seconds.
|
||||
time.sleep(4)
|
||||
final = snapshot()
|
||||
return {
|
||||
"mode": label,
|
||||
"sample_ratio": float(ratio),
|
||||
"load": receipt,
|
||||
"container_samples": samples,
|
||||
"ingest": {k: final[k] - v for k, v in initial.items()},
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--seconds", type=int, default=10)
|
||||
parser.add_argument("--rate", type=int, default=18)
|
||||
parser.add_argument("--repeats", type=int, default=2)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
if args.output.exists() or args.seconds < 8 or args.rate < 1 or args.repeats < 1:
|
||||
parser.error("use a new output and a positive load for at least 8 s")
|
||||
results = []
|
||||
try:
|
||||
for repeat in range(args.repeats):
|
||||
for mode in MODES if repeat % 2 == 0 else MODES[::-1]:
|
||||
row = run_one(mode, args.seconds, args.rate)
|
||||
print(row["mode"], row["load"]["latency_ms"], row["ingest"], flush=True)
|
||||
results.append(row)
|
||||
args.output.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"seconds": args.seconds,
|
||||
"offered_requests_per_s": args.rate,
|
||||
"runs": results,
|
||||
},
|
||||
indent=2,
|
||||
)
|
||||
+ "\n"
|
||||
)
|
||||
finally:
|
||||
configure("1", "1")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,172 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Two-phase real-age GC check: seed, run other checks, then verify after >=300 seconds."""
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
from cluster import ROOT, compose
|
||||
from pipeline_lab import settings
|
||||
|
||||
from messenger.cleanup import collect
|
||||
from messenger.pipeline import Pipeline, intent
|
||||
from messenger.repository import PostgresRepository
|
||||
from messenger.storage import ObjectStore
|
||||
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client, check # noqa: E402
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(__doc__)
|
||||
parser.add_argument("action", choices=("seed", "verify"))
|
||||
parser.add_argument("--seed", type=Path, required=True)
|
||||
parser.add_argument("--output", type=Path)
|
||||
parser.add_argument("--base-url", default="http://127.0.0.1:8086")
|
||||
args = parser.parse_args()
|
||||
config = settings("reconciler")
|
||||
db = PostgresRepository(config.database_url, pool_size=3)
|
||||
db.open()
|
||||
store = ObjectStore(config)
|
||||
try:
|
||||
if args.action == "seed":
|
||||
check(not args.seed.exists(), "Seed already exists")
|
||||
compose("stop", "--timeout", "2", "worker", "relay")
|
||||
try:
|
||||
c = Client(args.base_url)
|
||||
alice, _, _, room = c.bootstrap()
|
||||
active, data = c.attachment(alice, room)
|
||||
failed, _ = c.attachment(alice, room)
|
||||
orphan = str(uuid4())
|
||||
Pipeline(db, config).reserve(orphan, "staging/" + orphan)
|
||||
uploader = ObjectStore(settings("api"))
|
||||
try:
|
||||
uploader.put(
|
||||
"staging/" + orphan,
|
||||
io.BytesIO(data),
|
||||
len(data),
|
||||
"image/png",
|
||||
hashlib.sha256(data).hexdigest(),
|
||||
)
|
||||
finally:
|
||||
uploader.close()
|
||||
with db.connection(write=True) as conn:
|
||||
j = conn.execute(
|
||||
"SELECT * FROM attachment_jobs WHERE attachment_id=%s", (active["id"],)
|
||||
).fetchone()
|
||||
conn.execute(
|
||||
"UPDATE attachment_jobs SET "
|
||||
"next_attempt_at=clock_timestamp()+interval '1 hour' "
|
||||
"WHERE job_id=%s",
|
||||
(j["job_id"],),
|
||||
)
|
||||
conn.execute(
|
||||
"UPDATE job_outbox SET due_at=clock_timestamp()+interval '1 "
|
||||
"hour' WHERE job_id=%s",
|
||||
(j["job_id"],),
|
||||
)
|
||||
f = conn.execute(
|
||||
"SELECT job_id FROM attachment_jobs WHERE attachment_id=%s", (failed["id"],)
|
||||
).fetchone()
|
||||
conn.execute(
|
||||
"INSERT INTO pipeline_faults(job_id,phase,action) VALUES "
|
||||
"(%s,'before_effect','permanent')",
|
||||
(f["job_id"],),
|
||||
)
|
||||
seed = {
|
||||
"created_at": datetime.now(UTC).isoformat(),
|
||||
"active_id": active["id"],
|
||||
"failed_id": failed["id"],
|
||||
"orphan_id": orphan,
|
||||
}
|
||||
args.seed.write_text(json.dumps(seed, indent=2) + "\n")
|
||||
print(
|
||||
"PASS: active job, failed candidate and unaccepted orphan seeded; no "
|
||||
"credentials saved"
|
||||
)
|
||||
finally:
|
||||
compose("start", "worker", "relay")
|
||||
else:
|
||||
check(args.output is not None and not args.output.exists(), "Choose a new output file")
|
||||
seed = json.loads(args.seed.read_text())
|
||||
# Real S3 timestamps and failed retention; no modified clocks.
|
||||
while True:
|
||||
with db.connection() as conn:
|
||||
f = conn.execute(
|
||||
"SELECT * FROM attachment_jobs WHERE attachment_id=%s", (seed["failed_id"],)
|
||||
).fetchone()
|
||||
check(f["state"] == "failed", "Failed seed did not reach terminal state")
|
||||
age = (datetime.now(UTC) - f["updated_at"]).total_seconds()
|
||||
if age >= 301:
|
||||
break
|
||||
check(age >= 0, "Clock mismatch")
|
||||
time.sleep(min(1, 301 - age))
|
||||
with db.connection() as conn:
|
||||
ready = conn.execute(
|
||||
"SELECT * FROM attachments WHERE storage_state='ready' LIMIT 1"
|
||||
).fetchone()
|
||||
check(ready is not None, "Need a published attachment to protect")
|
||||
keys = [ready["object_key"], *[v["object_key"] for v in ready["variants"]]]
|
||||
before = {key: store.head(key)["ETag"] for key in keys}
|
||||
preview = collect(db, store, 300, False)
|
||||
result = collect(db, store, 300, True)
|
||||
check(
|
||||
store.head("staging/" + seed["active_id"]) is not None, "Active staging was deleted"
|
||||
)
|
||||
check(store.head("staging/" + seed["orphan_id"]) is None, "Orphan not deleted")
|
||||
check(
|
||||
store.head("staging/" + seed["failed_id"]) is None,
|
||||
"Expired failed source not deleted",
|
||||
)
|
||||
check(
|
||||
all(store.head(k)["ETag"] == tag for k, tag in before.items()),
|
||||
"Published result changed",
|
||||
)
|
||||
with db.connection(write=True) as conn:
|
||||
expired = conn.execute(
|
||||
"SELECT source_expired FROM attachment_jobs WHERE attachment_id=%s",
|
||||
(seed["failed_id"],),
|
||||
).fetchone()
|
||||
check(expired["source_expired"], "Expired source not registered")
|
||||
j = conn.execute(
|
||||
"UPDATE attachment_jobs SET next_attempt_at=clock_timestamp() WHERE "
|
||||
"attachment_id=%s RETURNING job_id",
|
||||
(seed["active_id"],),
|
||||
).fetchone()
|
||||
intent(conn, j["job_id"])
|
||||
deadline = time.monotonic() + 60
|
||||
while time.monotonic() < deadline:
|
||||
if db.attachment(seed["active_id"])["storage_state"] == "ready":
|
||||
break
|
||||
time.sleep(0.2)
|
||||
check(
|
||||
db.attachment(seed["active_id"])["storage_state"] == "ready",
|
||||
"Protected work failed to complete",
|
||||
)
|
||||
proof = {
|
||||
"retention_seconds": 300,
|
||||
"failed_real_age_seconds": age,
|
||||
"seed": seed,
|
||||
"dry_run": preview,
|
||||
"apply": result,
|
||||
"active_protected_then_ready": True,
|
||||
"orphan_deleted": True,
|
||||
"failed_source_expired": True,
|
||||
"published_objects_unchanged": len(keys),
|
||||
}
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(json.dumps(proof, indent=2) + "\n")
|
||||
print("PASS: real-age cleanup preserves active work and published results")
|
||||
finally:
|
||||
store.close()
|
||||
db.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,253 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Real LB requests during each replica failure and sequential replacement."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
from cluster import ROOT, compose, inspect, instance, instances, wait_lb
|
||||
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client, Failure, check # noqa: E402
|
||||
from resilience import history # noqa: E402
|
||||
|
||||
|
||||
def percentile(values, fraction):
|
||||
values = sorted(values)
|
||||
return values[min(int((len(values) - 1) * fraction), len(values) - 1)] if values else None
|
||||
|
||||
|
||||
class Traffic:
|
||||
def __init__(self, base, actor, room, workers=6):
|
||||
self.base, self.actor, self.room = base, actor, room
|
||||
self.prefix = uuid4().hex
|
||||
self.stop = threading.Event()
|
||||
self.lock = threading.Lock()
|
||||
self.events = []
|
||||
self.executor = ThreadPoolExecutor(max_workers=workers)
|
||||
self.workers = workers
|
||||
|
||||
def worker(self, worker):
|
||||
client, iteration = Client(self.base, timeout=3), 0
|
||||
while not self.stop.is_set():
|
||||
started = time.monotonic()
|
||||
write = iteration % 10 == 9
|
||||
try:
|
||||
if write:
|
||||
key = f"{self.prefix}:{worker}:{iteration}"
|
||||
response = client.message(self.actor, self.room, key, key=key)
|
||||
else:
|
||||
response = client.call(
|
||||
"GET", "/api/v1/users/me", actor=self.actor, schema="User"
|
||||
)
|
||||
event = {
|
||||
"at": time.monotonic(),
|
||||
"ok": True,
|
||||
"instance": response["headers"]["X-Instance-Id"],
|
||||
"latency_ms": (time.monotonic() - started) * 1000,
|
||||
"message": response["json"] if write else None,
|
||||
}
|
||||
iteration += 1
|
||||
except Failure:
|
||||
event = {
|
||||
"at": time.monotonic(),
|
||||
"ok": False,
|
||||
"latency_ms": (time.monotonic() - started) * 1000,
|
||||
}
|
||||
with self.lock:
|
||||
self.events.append(event)
|
||||
|
||||
def __enter__(self):
|
||||
self.started = time.monotonic()
|
||||
self.futures = [self.executor.submit(self.worker, i) for i in range(self.workers)]
|
||||
return self
|
||||
|
||||
def __exit__(self, *args):
|
||||
self.stop.set()
|
||||
self.executor.shutdown(wait=True)
|
||||
for future in self.futures:
|
||||
future.result()
|
||||
self.finished = time.monotonic()
|
||||
|
||||
def report(self, action_start, action_end):
|
||||
events = sorted(self.events, key=lambda row: row["at"])
|
||||
ok = [row for row in events if row["ok"]]
|
||||
after = [row for row in ok if row["at"] >= action_end]
|
||||
check(bool(after), "No successful request after action")
|
||||
elapsed = self.finished - self.started
|
||||
latency = [row["latency_ms"] for row in ok]
|
||||
return {
|
||||
"duration_seconds": elapsed,
|
||||
"workers": self.workers,
|
||||
"requests": len(events),
|
||||
"successes": len(ok),
|
||||
"errors": len(events) - len(ok),
|
||||
"successful_rps": len(ok) / elapsed,
|
||||
"error_fraction": (len(events) - len(ok)) / len(events),
|
||||
"first_success_after_action_seconds": after[0]["at"] - action_start,
|
||||
"action_seconds": action_end - action_start,
|
||||
"p50_ms": percentile(latency, 0.5),
|
||||
"p95_ms": percentile(latency, 0.95),
|
||||
"p99_ms": percentile(latency, 0.99),
|
||||
"instances": sorted({row["instance"] for row in ok}),
|
||||
"tail_instances": sorted({row["instance"] for row in after[-10:]}),
|
||||
"confirmed_writes": sum(row["message"] is not None for row in ok),
|
||||
}
|
||||
|
||||
def verify_messages(self, client, actor, room):
|
||||
actual = history(client, actor, room)
|
||||
by_id = {value["id"]: value for value in actual}
|
||||
check(len(by_id) == len(actual), "History has duplicate IDs")
|
||||
texts = [row["text"] for row in actual if row["text"].startswith(self.prefix)]
|
||||
check(len(texts) == len(set(texts)), "An uncertain retry created a duplicate effect")
|
||||
for event in self.events:
|
||||
if event["ok"] and event["message"]:
|
||||
check(
|
||||
by_id.get(event["message"]["id"]) == event["message"],
|
||||
"A confirmed message was lost or changed",
|
||||
)
|
||||
|
||||
|
||||
def public_test(target, base):
|
||||
result = subprocess.run(
|
||||
[
|
||||
"make",
|
||||
target,
|
||||
"ACTION_SCRIPT=scripts/"
|
||||
+ ("restart.sh" if target == "test-persistence" else "stop-one.sh"),
|
||||
"BASE_URL=" + base,
|
||||
],
|
||||
cwd=ROOT,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=150,
|
||||
)
|
||||
return "\n".join(line.rstrip() for line in result.stdout.splitlines()) + "\n"
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--base-url", default="http://127.0.0.1:8086")
|
||||
parser.add_argument(
|
||||
"--output", type=Path, default=ROOT / f"evidence/cluster-{time.time_ns()}.json"
|
||||
)
|
||||
args = parser.parse_args()
|
||||
if args.output.exists():
|
||||
raise SystemExit("Use a new evidence filename")
|
||||
os.environ["NO_PROXY"] = os.environ["no_proxy"] = "127.0.0.1,localhost"
|
||||
baseline = instances()
|
||||
check(len(baseline) == 2, "This verification expects the default two API replicas")
|
||||
check(not any(row["published_ports"] for row in baseline), "API must not publish host ports")
|
||||
wait_lb(args.base_url, 2)
|
||||
result = {"initial_instances": baseline, "failures": []}
|
||||
client = Client(args.base_url, timeout=4)
|
||||
alice, bob, eve, room = client.bootstrap()
|
||||
key = uuid4().hex
|
||||
|
||||
def same_request(_):
|
||||
response = Client(args.base_url).message(alice, room, "concurrent replay", key=key)
|
||||
return response["json"], response["headers"]["X-Instance-Id"]
|
||||
|
||||
with ThreadPoolExecutor(max_workers=10) as pool:
|
||||
responses = list(pool.map(same_request, range(30)))
|
||||
check(
|
||||
len({row[0]["id"] for row in responses}) == 1, "Cross-replica replay duplicated a message"
|
||||
)
|
||||
check(len({row[1] for row in responses}) == 2, "Concurrent replay did not cross real replicas")
|
||||
result["concurrent_replays"] = 30
|
||||
result["concurrent_effects"] = 1
|
||||
try:
|
||||
for selected in baseline:
|
||||
selected = instance(selected["id"])
|
||||
with Traffic(args.base_url, alice, room) as traffic:
|
||||
time.sleep(1)
|
||||
action_start = time.monotonic()
|
||||
env = {**os.environ, "TARGET_INSTANCE_ID": selected["instance"]}
|
||||
subprocess.run(
|
||||
[str(ROOT / "scripts/stop-one.sh")],
|
||||
cwd=ROOT,
|
||||
env=env,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
timeout=10,
|
||||
)
|
||||
action_end = time.monotonic()
|
||||
check(not inspect(selected["id"])["running"], "Chosen process was not stopped")
|
||||
time.sleep(3)
|
||||
report = traffic.report(action_start, action_end)
|
||||
check(report["first_success_after_action_seconds"] <= 10, "Failover took too long")
|
||||
check(selected["instance"] not in report["tail_instances"], "Stopped API still serves")
|
||||
report["stopped"] = selected
|
||||
traffic.verify_messages(client, bob, room)
|
||||
result["failures"].append(report)
|
||||
subprocess.run(["docker", "start", selected["id"]], check=True, capture_output=True)
|
||||
wait_lb(args.base_url, 2)
|
||||
print("Verified failure and restoration of one API", flush=True)
|
||||
with Traffic(args.base_url, alice, room) as traffic:
|
||||
time.sleep(1)
|
||||
action_start = time.monotonic()
|
||||
replaced = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
str(ROOT / "scripts/rolling-update.py"),
|
||||
"--base-url",
|
||||
args.base_url,
|
||||
],
|
||||
cwd=ROOT,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=120,
|
||||
)
|
||||
action_end = time.monotonic()
|
||||
time.sleep(3)
|
||||
traffic.verify_messages(client, bob, room)
|
||||
result["rolling"] = traffic.report(action_start, action_end)
|
||||
result["rolling"]["replacements"] = json.loads(replaced.stdout)
|
||||
result["rolling"]["confirmed_messages_preserved"] = True
|
||||
print("Verified sequential replacement under traffic", flush=True)
|
||||
(ROOT / "evidence/failover.txt").write_text(public_test("test-failover", args.base_url))
|
||||
for value in compose("ps", "-aq", "api").split():
|
||||
if not inspect(value)["running"]:
|
||||
subprocess.run(["docker", "start", value], check=True, capture_output=True)
|
||||
wait_lb(args.base_url, 2)
|
||||
before = {
|
||||
service: inspect(compose("ps", "-q", service).strip())
|
||||
for service in ("postgres", "valkey", "s3", "broker", "lb")
|
||||
}
|
||||
(ROOT / "evidence/persistence.txt").write_text(
|
||||
public_test("test-persistence", args.base_url)
|
||||
)
|
||||
after = {service: inspect(compose("ps", "-q", service).strip()) for service in before}
|
||||
check(
|
||||
all(before[s]["started_at"] != after[s]["started_at"] for s in before),
|
||||
"Persistence hook did not restart all state/LB services",
|
||||
)
|
||||
result["persistence"] = {"before": before, "after": after, "same_token_public_test": True}
|
||||
result["final_instances"] = instances()
|
||||
logs = compose("logs", "--no-color", "api", "lb")
|
||||
check(
|
||||
not any(
|
||||
a["token"] in logs or a["registration"]["password"] in logs
|
||||
for a in (alice, bob, eve)
|
||||
),
|
||||
"Credential appeared in logs",
|
||||
)
|
||||
args.output.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n")
|
||||
print("PASS: real replica failures, replay, rolling replacement and persistence")
|
||||
finally:
|
||||
for value in compose("ps", "-aq", "api").split():
|
||||
if not inspect(value)["running"]:
|
||||
subprocess.run(["docker", "start", value], check=True, capture_output=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,513 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Real-process fault matrix for the versioned pipeline; test Compose override required."""
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
from botocore.exceptions import ClientError
|
||||
from cluster import ROOT, compose, inspect, instances, wait_lb
|
||||
from PIL import Image
|
||||
from pipeline_lab import settings
|
||||
|
||||
from messenger.broker import STREAM, Broker, Job
|
||||
from messenger.pipeline import Pipeline
|
||||
from messenger.repository import PostgresRepository, stamp
|
||||
from messenger.storage import ObjectStore
|
||||
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client, check # noqa: E402
|
||||
|
||||
|
||||
def until(fn, timeout=60):
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
result = fn()
|
||||
if result:
|
||||
return result
|
||||
time.sleep(0.2)
|
||||
raise AssertionError("Timed out waiting for a durable pipeline condition")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(__doc__)
|
||||
parser.add_argument("--base-url", default="http://127.0.0.1:8086")
|
||||
parser.add_argument("--output", type=Path, default=ROOT / "evidence/pipeline.json")
|
||||
args = parser.parse_args()
|
||||
if args.output.exists():
|
||||
parser.error("Output already exists; choose a new evidence file")
|
||||
config = settings()
|
||||
db = PostgresRepository(config.database_url, pool_size=4)
|
||||
db.open()
|
||||
broker, store = Broker(config), ObjectStore(config)
|
||||
c = Client(args.base_url)
|
||||
alice, bob, eve, room = c.bootstrap()
|
||||
records, accepted = [], []
|
||||
data = (ROOT / "tests/fixtures/sample.png").read_bytes()
|
||||
|
||||
def query(sql, params=()):
|
||||
with db.connection() as conn:
|
||||
return conn.execute(sql, params).fetchall()
|
||||
|
||||
def job(a, version=1):
|
||||
return query(
|
||||
"SELECT * FROM attachment_jobs WHERE attachment_id=%s AND pipeline_version=%s",
|
||||
(a["id"], version),
|
||||
)[0]
|
||||
|
||||
def audit(j):
|
||||
return query(
|
||||
"SELECT event,details,created_at FROM pipeline_audit WHERE job_id=%s ORDER BY sequence",
|
||||
(j["job_id"],),
|
||||
)
|
||||
|
||||
def injected(j, phase):
|
||||
return any(
|
||||
e["event"] == "fault_injected" and e["details"]["phase"] == phase for e in audit(j)
|
||||
)
|
||||
|
||||
def fault(j, phase, action, count=1):
|
||||
with db.connection(write=True) as conn:
|
||||
conn.execute(
|
||||
"INSERT INTO pipeline_faults(job_id,phase,action,remaining) VALUES (%s,%s,%s,%s)",
|
||||
(j["job_id"], phase, action, count),
|
||||
)
|
||||
|
||||
def upload():
|
||||
a, original = c.attachment(alice, room)
|
||||
check(original == data, "Unexpected fixture")
|
||||
accepted.append(a["id"])
|
||||
return a
|
||||
|
||||
def ready(a):
|
||||
value = c.wait_ready(bob, a)
|
||||
path = "/api/v1/attachments/" + a["id"]
|
||||
check(c.call("GET", path + "/content", actor=bob)["bytes"] == data, "Original changed")
|
||||
for v in value["variants"]:
|
||||
content = c.call("GET", path + "/content?variant=" + v["name"], actor=bob)["bytes"]
|
||||
check(hashlib.sha256(content).hexdigest() == v["sha256"], "Variant checksum mismatch")
|
||||
with Image.open(io.BytesIO(content)) as im:
|
||||
check(im.size == (v["width"], v["height"]), "Variant dimensions mismatch")
|
||||
check(len(value["variants"]) == 3, "Missing variants")
|
||||
check(
|
||||
len(query("SELECT 1 FROM message_attachments WHERE attachment_id=%s", (a["id"],))) <= 1,
|
||||
"Unexpected duplicate link",
|
||||
)
|
||||
return value
|
||||
|
||||
def saved(name, a, start, **details):
|
||||
j = job(a, details.get("version", 1))
|
||||
current = query(
|
||||
"SELECT storage_state,pipeline_version FROM attachments WHERE id=%s", (a["id"],)
|
||||
)[0]
|
||||
keys = [
|
||||
o["Key"]
|
||||
for o in store.client.list_objects_v2(
|
||||
Bucket=store.bucket,
|
||||
Prefix="objects/" + a["id"] + "/v" + str(current["pipeline_version"]) + "/",
|
||||
).get("Contents", [])
|
||||
]
|
||||
check(len(keys) == 4, "Repeated deliveries multiplied the published variant set")
|
||||
record = {
|
||||
"scenario": name,
|
||||
"attachment_id": a["id"],
|
||||
"job_id": str(j["job_id"]),
|
||||
"elapsed_seconds": time.monotonic() - start,
|
||||
"final_state": current["storage_state"],
|
||||
"attempts": j["attempt"],
|
||||
"final_version": current["pipeline_version"],
|
||||
"published_objects": len(keys),
|
||||
"audit": audit(j),
|
||||
**details,
|
||||
}
|
||||
records.append(record)
|
||||
print("PASS: " + name, flush=True)
|
||||
|
||||
def envelope(j, attempt=None):
|
||||
return Job(
|
||||
schema_version=1,
|
||||
type="attachment.process.v1",
|
||||
job_id=str(j["job_id"]),
|
||||
attachment_id=str(j["attachment_id"]),
|
||||
pipeline_version=j["pipeline_version"],
|
||||
attempt=attempt or max(1, j["attempt"]),
|
||||
created_at=stamp(j["created_at"]),
|
||||
)
|
||||
|
||||
try:
|
||||
# Actual COMMIT/publish gap: no relay exists while the acknowledged upload is stored.
|
||||
compose("stop", "--timeout", "2", "worker", "relay")
|
||||
start = time.monotonic()
|
||||
a = upload()
|
||||
j = job(a)
|
||||
check(
|
||||
query("SELECT published_at FROM job_outbox WHERE job_id=%s", (j["job_id"],))[0][
|
||||
"published_at"
|
||||
]
|
||||
is None,
|
||||
"Gap was not established",
|
||||
)
|
||||
message = c.message(alice, room, "", attachments=[a["id"]])["json"]
|
||||
c.call("GET", "/api/v1/attachments/" + a["id"] + "/content", 409, actor=bob)
|
||||
before = instances()
|
||||
compose("restart", "--timeout", "3", "api")
|
||||
wait_lb(args.base_url)
|
||||
after = instances()
|
||||
check(
|
||||
{r["instance"] for r in before}.isdisjoint({r["instance"] for r in after}),
|
||||
"API did not restart",
|
||||
)
|
||||
compose("start", "relay")
|
||||
until(
|
||||
lambda: query("SELECT published_at FROM job_outbox WHERE job_id=%s", (j["job_id"],))[0][
|
||||
"published_at"
|
||||
]
|
||||
)
|
||||
entry_ids = [
|
||||
id
|
||||
for id, fields in broker.client.xrange(STREAM)
|
||||
if json.loads(fields["job"])["job_id"] == str(j["job_id"])
|
||||
]
|
||||
check(bool(entry_ids), "Broker has no accepted work")
|
||||
broker_id = compose("ps", "-q", "broker").strip()
|
||||
broker_before = inspect(broker_id)
|
||||
subprocess.run(
|
||||
["docker", "kill", "--signal", "KILL", broker_id], check=True, capture_output=True
|
||||
)
|
||||
subprocess.run(["docker", "start", broker_id], check=True, capture_output=True)
|
||||
|
||||
def broker_restored():
|
||||
try:
|
||||
return broker.client.ping()
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
until(broker_restored)
|
||||
check(
|
||||
set(entry_ids) <= {id for id, _ in broker.client.xrange(STREAM)},
|
||||
"Durable stream lost confirmed work",
|
||||
)
|
||||
compose("start", "worker")
|
||||
ready(a)
|
||||
history = c.call(
|
||||
"GET", "/api/v1/chats/" + room["id"] + "/messages", actor=bob, schema="MessagePage"
|
||||
)["json"]
|
||||
check(message in history["items"], "Queued attachment message lost")
|
||||
saved(
|
||||
"commit_publish_gap_api_restart_broker_sigkill",
|
||||
a,
|
||||
start,
|
||||
api_before=before,
|
||||
api_after=after,
|
||||
broker_before=broker_before,
|
||||
broker_after=inspect(broker_id),
|
||||
broker_entry_ids=entry_ids,
|
||||
message_id=message["id"],
|
||||
)
|
||||
|
||||
# A publisher can lose its COMMIT after XADD; same logical ID is sent twice.
|
||||
compose("stop", "--timeout", "2", "worker", "relay")
|
||||
start = time.monotonic()
|
||||
a = upload()
|
||||
j = job(a)
|
||||
fault(j, "after_publish", "crash")
|
||||
compose("start", "relay")
|
||||
until(lambda: injected(j, "after_publish"))
|
||||
until(
|
||||
lambda: (
|
||||
sum(
|
||||
json.loads(f["job"])["job_id"] == str(j["job_id"])
|
||||
for _, f in broker.client.xrange(STREAM)
|
||||
)
|
||||
>= 2
|
||||
)
|
||||
)
|
||||
compose("start", "worker")
|
||||
ready(a)
|
||||
until(lambda j=j: any(e["event"] == "duplicate_ready" for e in audit(j)))
|
||||
saved("publish_before_outbox_commit_duplicate", a, start)
|
||||
|
||||
for phase in ("before_effect", "after_effect", "after_commit"):
|
||||
compose("stop", "--timeout", "2", "worker")
|
||||
start = time.monotonic()
|
||||
a = upload()
|
||||
j = job(a)
|
||||
fault(j, phase, "crash")
|
||||
compose("start", "worker")
|
||||
ready(a)
|
||||
until(lambda j=j, phase=phase: injected(j, phase))
|
||||
if phase == "after_commit":
|
||||
until(lambda j=j: any(e["event"] == "duplicate_ready" for e in audit(j)))
|
||||
check(job(a)["attempt"] == 1, "Committed job was processed twice")
|
||||
else:
|
||||
check(job(a)["attempt"] == 2, "Expired lease did not recover exactly once")
|
||||
saved("worker_crash_" + phase, a, start)
|
||||
|
||||
for action, count, expected in (
|
||||
("transient", 2, "ready"),
|
||||
("hang", 1, "ready"),
|
||||
("permanent", 1, "failed"),
|
||||
("transient", 5, "failed"),
|
||||
):
|
||||
compose("stop", "--timeout", "2", "worker")
|
||||
start = time.monotonic()
|
||||
a = upload()
|
||||
j = job(a)
|
||||
fault(j, "before_effect", action, count)
|
||||
compose("start", "worker")
|
||||
path = "/api/v1/attachments/" + a["id"]
|
||||
if expected == "failed":
|
||||
until(
|
||||
lambda path=path: (
|
||||
c.call("GET", path, actor=bob, schema="Attachment")["json"]["status"]
|
||||
== "failed"
|
||||
)
|
||||
)
|
||||
terminal = job(a)
|
||||
check(terminal["attempt"] == count, "Wrong retry budget")
|
||||
c.call("GET", path + "/content", 409, actor=bob)
|
||||
c.call(
|
||||
"POST",
|
||||
"/api/v1/chats/" + room["id"] + "/messages",
|
||||
409,
|
||||
actor=alice,
|
||||
body={"text": "", "attachment_ids": [a["id"]]},
|
||||
headers={"Idempotency-Key": uuid4().hex},
|
||||
)
|
||||
# Explicit CLI replay, same logical job; no public administrator endpoint.
|
||||
replay = json.loads(
|
||||
compose(
|
||||
"exec",
|
||||
"-T",
|
||||
"worker",
|
||||
"python",
|
||||
"-m",
|
||||
"messenger.operator",
|
||||
"replay",
|
||||
a["id"],
|
||||
)
|
||||
)
|
||||
check(replay["job_id"] == str(j["job_id"]), "Replay changed logical job")
|
||||
else:
|
||||
terminal = None
|
||||
ready(a)
|
||||
events = audit(j)
|
||||
delays = [e["details"]["delay_seconds"] for e in events if e["event"] == "queued"]
|
||||
check(all(0.25 <= delay <= 30 for delay in delays), "Invalid bounded jitter")
|
||||
rejected = subprocess.run(
|
||||
[
|
||||
"docker",
|
||||
"compose",
|
||||
"--env-file",
|
||||
".env",
|
||||
"exec",
|
||||
"-T",
|
||||
"worker",
|
||||
"python",
|
||||
"-m",
|
||||
"messenger.operator",
|
||||
"replay",
|
||||
a["id"],
|
||||
],
|
||||
cwd=ROOT,
|
||||
capture_output=True,
|
||||
)
|
||||
check(rejected.returncode != 0, "Ready replay was accepted")
|
||||
saved(
|
||||
action + "_" + str(count) + ("_failed_then_explicit_replay" if terminal else ""),
|
||||
a,
|
||||
start,
|
||||
terminal_before_replay={k: terminal[k] for k in ("state", "attempt", "error_code")}
|
||||
if terminal
|
||||
else None,
|
||||
backoff_delays=delays,
|
||||
)
|
||||
|
||||
# Reorder and stale v1 delivered after v2; exercise the DB publication fence directly too.
|
||||
compose("stop", "--timeout", "2", "worker", "relay")
|
||||
start = time.monotonic()
|
||||
a = upload()
|
||||
old = job(a)
|
||||
old_envelope = envelope(old, 1)
|
||||
outcome, old_claim = Pipeline(db, config).claim(old_envelope)
|
||||
check(outcome == "claimed", "Cannot establish old writer")
|
||||
Pipeline(db, config).replay(a["id"], new_version=True)
|
||||
new = job(a, 2)
|
||||
# Publish v2 first, then v1, plus duplicates through both worker slots.
|
||||
for value in (envelope(new, 1), old_envelope, envelope(new, 1), old_envelope):
|
||||
broker.publish(value)
|
||||
compose("start", "worker", "relay")
|
||||
current = ready(a)
|
||||
row = db.attachment(a["id"])
|
||||
check(
|
||||
not Pipeline(db, config).publish(old_claim, "objects/old-writer", []),
|
||||
"Old writer overwrote v2",
|
||||
)
|
||||
for _ in range(20):
|
||||
broker.publish(old_envelope)
|
||||
broker.publish(envelope(new, 1))
|
||||
until(lambda: sum(e["event"] == "stale_delivery" for e in audit(old)) >= 21)
|
||||
until(lambda: sum(e["event"] == "duplicate_ready" for e in audit(new)) >= 20)
|
||||
check(
|
||||
db.attachment(a["id"])["object_key"] == row["object_key"],
|
||||
"Stale delivery changed current pointer",
|
||||
)
|
||||
check(
|
||||
c.call("GET", "/api/v1/attachments/" + a["id"], actor=bob, schema="Attachment")["json"]
|
||||
== current,
|
||||
"Old version mutated ready metadata",
|
||||
)
|
||||
saved(
|
||||
"reorder_stale_v1_after_v2_and_publication_fence",
|
||||
a,
|
||||
start,
|
||||
version=2,
|
||||
old_audit=audit(old),
|
||||
duplicates=44,
|
||||
)
|
||||
|
||||
# ACL remains enforced through both API replicas; S3 is private and role-scoped.
|
||||
a = {"id": accepted[0]}
|
||||
for suffix in ("", "/content", "/content?variant=crop"):
|
||||
c.call("GET", "/api/v1/attachments/" + a["id"] + suffix, 404, actor=eve)
|
||||
# Direct bucket access and write/delete privilege checks use no public credentials.
|
||||
import boto3
|
||||
from botocore import UNSIGNED
|
||||
from botocore.config import Config
|
||||
|
||||
first = db.attachment(accepted[0])
|
||||
unsigned = boto3.client(
|
||||
"s3", endpoint_url=config.s3_endpoint, config=Config(signature_version=UNSIGNED)
|
||||
)
|
||||
try:
|
||||
unsigned.get_object(Bucket=store.bucket, Key=first["object_key"])
|
||||
raise AssertionError("Anonymous S3 read succeeded")
|
||||
except ClientError as exc:
|
||||
check(
|
||||
exc.response["ResponseMetadata"]["HTTPStatusCode"] == 403, "Wrong anonymous status"
|
||||
)
|
||||
finally:
|
||||
unsigned.close()
|
||||
denied = []
|
||||
for role, action in (
|
||||
("api", "put_object"),
|
||||
("api", "delete_object"),
|
||||
("worker", "delete_object"),
|
||||
("reconciler", "put_object"),
|
||||
):
|
||||
scoped = ObjectStore(settings(role))
|
||||
try:
|
||||
kw = {"Bucket": store.bucket, "Key": "objects/" + uuid4().hex + "/forbidden"}
|
||||
if action == "put_object":
|
||||
kw["Body"] = b"forbidden"
|
||||
try:
|
||||
getattr(scoped.client, action)(**kw)
|
||||
raise AssertionError("IAM action unexpectedly allowed")
|
||||
except ClientError as exc:
|
||||
check(
|
||||
exc.response["ResponseMetadata"]["HTTPStatusCode"] == 403,
|
||||
"Wrong IAM status",
|
||||
)
|
||||
denied.append(role + ":" + action)
|
||||
finally:
|
||||
scoped.close()
|
||||
# Stop the actual uploader, then read original and thumbnail through another replica.
|
||||
response = c.call(
|
||||
"POST",
|
||||
"/api/v1/chats/" + room["id"] + "/attachments?filename=cross.png",
|
||||
202,
|
||||
actor=alice,
|
||||
raw=data,
|
||||
headers={"Content-Type": "image/png"},
|
||||
schema="Attachment",
|
||||
)
|
||||
cross = response["json"]
|
||||
accepted.append(cross["id"])
|
||||
ready(cross)
|
||||
uploader = response["headers"]["X-Instance-Id"]
|
||||
process = next(v for v in instances() if v["instance"] == uploader)
|
||||
subprocess.run(
|
||||
["docker", "stop", "--time", "3", process["id"]], check=True, capture_output=True
|
||||
)
|
||||
try:
|
||||
wait_lb(args.base_url, count=1)
|
||||
download = c.call("GET", "/api/v1/attachments/" + cross["id"] + "/content", actor=bob)
|
||||
check(
|
||||
download["bytes"] == data and download["headers"]["X-Instance-Id"] != uploader,
|
||||
"Other replica could not serve the original",
|
||||
)
|
||||
c.call(
|
||||
"GET",
|
||||
"/api/v1/attachments/" + cross["id"] + "/content?variant=thumbnail",
|
||||
actor=bob,
|
||||
)
|
||||
cross_proof = {
|
||||
"uploader": uploader,
|
||||
"stopped": inspect(process["id"]),
|
||||
"reader": download["headers"]["X-Instance-Id"],
|
||||
}
|
||||
finally:
|
||||
subprocess.run(["docker", "start", process["id"]], check=True, capture_output=True)
|
||||
wait_lb(args.base_url)
|
||||
logs = compose("logs", "--no-color", "api", "worker", "relay", "lb")
|
||||
check(
|
||||
not any(
|
||||
actor["token"] in logs or actor["registration"]["password"] in logs
|
||||
for actor in (alice, bob, eve)
|
||||
),
|
||||
"Credential in logs",
|
||||
)
|
||||
states = query(
|
||||
"SELECT id,storage_state FROM attachments WHERE id=ANY(%s::uuid[])", (accepted,)
|
||||
)
|
||||
check(
|
||||
len(states) == len(accepted) and all(s["storage_state"] == "ready" for s in states),
|
||||
"Confirmed work lost",
|
||||
)
|
||||
stats = compose(
|
||||
"exec",
|
||||
"-T",
|
||||
"worker",
|
||||
"python",
|
||||
"-c",
|
||||
"import json;from pathlib import Path;print(json.dumps({"
|
||||
"n:Path('/sys/fs/cgroup/'+n).read_text().strip() "
|
||||
"for n in ('memory.current','memory.peak','memory.events','cpu.stat')}))",
|
||||
)
|
||||
result = {
|
||||
"anonymous_s3_status": 403,
|
||||
"denied_iam_actions": denied,
|
||||
"cross_replica": cross_proof,
|
||||
"track": 2,
|
||||
"delivery": "at-least-once",
|
||||
"effect_key": ["attachment_id", "pipeline_version", "variant"],
|
||||
"worker_concurrency": config.worker_concurrency,
|
||||
"prefetch_per_slot": 1,
|
||||
"job_timeout_seconds": config.job_timeout,
|
||||
"lease_seconds": config.lease_seconds,
|
||||
"max_attempts": config.job_max_attempts,
|
||||
"scenarios": records,
|
||||
"accepted": len(accepted),
|
||||
"lost_confirmed": 0,
|
||||
"stale_overwrites": 0,
|
||||
"worker_cgroup": json.loads(stats),
|
||||
"logs_credentials_absent": True,
|
||||
}
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(json.dumps(result, ensure_ascii=False, indent=2, default=str) + "\n")
|
||||
print("PASS: all acknowledged work ready; no stale overwrites", flush=True)
|
||||
finally:
|
||||
compose("start", "worker", "relay", "broker", "api")
|
||||
broker.close()
|
||||
store.close()
|
||||
db.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1 @@
|
||||
"""One-process implementation of the lab 1 messenger contract."""
|
||||
@@ -0,0 +1,27 @@
|
||||
import logging
|
||||
import os
|
||||
|
||||
import uvicorn
|
||||
|
||||
from messenger import telemetry
|
||||
|
||||
|
||||
def main():
|
||||
level = os.getenv("APP_LOG_LEVEL", "INFO").upper()
|
||||
logging.basicConfig(level=level, format="%(message)s")
|
||||
with telemetry.active("api"):
|
||||
uvicorn.run(
|
||||
"messenger.app:create_app",
|
||||
factory=True,
|
||||
host=os.getenv("APP_HOST", "0.0.0.0"),
|
||||
port=int(os.getenv("APP_PORT", "8080")),
|
||||
workers=1,
|
||||
access_log=False,
|
||||
log_level=level.lower(),
|
||||
timeout_keep_alive=2,
|
||||
timeout_graceful_shutdown=5,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,350 @@
|
||||
"""HTTP adapter. The instructor's OpenAPI, not FastAPI defaults, defines errors."""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from contextlib import asynccontextmanager
|
||||
from time import perf_counter
|
||||
from typing import Annotated
|
||||
from urllib.parse import quote
|
||||
from uuid import uuid4
|
||||
|
||||
from fastapi import Depends, FastAPI, Request
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from fastapi.responses import JSONResponse, Response, StreamingResponse
|
||||
from opentelemetry.context import Context
|
||||
from opentelemetry.trace import INVALID_SPAN, SpanKind
|
||||
from starlette.exceptions import HTTPException
|
||||
from starlette.types import ASGIApp, Message, Receive, Scope, Send
|
||||
|
||||
from messenger import telemetry
|
||||
from messenger.attachments import Attachments
|
||||
from messenger.auth import AuthService, Identity, basic_credentials, bearer_digest, unauthorized
|
||||
from messenger.cache import SessionCache
|
||||
from messenger.domain import DomainError, User, invalid, not_found
|
||||
from messenger.repository import PostgresRepository
|
||||
from messenger.schemas import (
|
||||
AttachmentView,
|
||||
ChatPage,
|
||||
ChatView,
|
||||
CreateChat,
|
||||
CreateMessage,
|
||||
CreateUser,
|
||||
MessagePage,
|
||||
MessageView,
|
||||
SessionView,
|
||||
UserView,
|
||||
canonical_uuid,
|
||||
)
|
||||
from messenger.service import Messenger
|
||||
from messenger.settings import Settings
|
||||
from messenger.storage import ObjectStore
|
||||
|
||||
logger = logging.getLogger("messenger")
|
||||
|
||||
|
||||
class RequestContext:
|
||||
def __init__(self, app: ASGIApp, instance_id: str):
|
||||
self.app = app
|
||||
self.instance_id = instance_id
|
||||
|
||||
async def __call__(self, scope: Scope, receive: Receive, send: Send):
|
||||
if scope["type"] != "http":
|
||||
return await self.app(scope, receive, send)
|
||||
if scope.get("path") in {"/health/live", "/health/ready"}:
|
||||
await self.request(scope, receive, send, INVALID_SPAN)
|
||||
return
|
||||
with telemetry.span("http.request", context=Context(), kind=SpanKind.SERVER) as span:
|
||||
await self.request(scope, receive, send, span)
|
||||
|
||||
async def request(self, scope, receive, send, span):
|
||||
request_id = str(uuid4())
|
||||
scope.setdefault("state", {})["request_id"] = request_id
|
||||
start, status, response_started = perf_counter(), 500, False
|
||||
|
||||
async def with_id(message: Message):
|
||||
nonlocal status, response_started
|
||||
if message["type"] == "http.response.start":
|
||||
response_started = True
|
||||
status = message["status"]
|
||||
message["headers"] = [
|
||||
pair
|
||||
for pair in message.get("headers", [])
|
||||
if pair[0].lower() not in (b"x-request-id", b"x-instance-id")
|
||||
] + [
|
||||
(b"x-request-id", request_id.encode()),
|
||||
(b"x-instance-id", self.instance_id.encode()),
|
||||
]
|
||||
await send(message)
|
||||
|
||||
try:
|
||||
await self.app(scope, receive, with_id)
|
||||
except Exception as exc:
|
||||
# Do not let Uvicorn print exception values that could contain user data.
|
||||
if response_started:
|
||||
raise
|
||||
logger.error(json.dumps({"event": "internal_error", "type": type(exc).__name__}))
|
||||
response = error_response(
|
||||
Request(scope), 500, "internal_error", "Внутренняя ошибка сервера"
|
||||
)
|
||||
await response(scope, receive, with_id)
|
||||
finally:
|
||||
route = getattr(scope.get("route"), "path", "unmatched")
|
||||
if route not in {"/health/live", "/health/ready"}:
|
||||
telemetry.http(
|
||||
span, scope["method"], route, status, perf_counter() - start, request_id
|
||||
)
|
||||
|
||||
|
||||
def error_response(
|
||||
request: Request, status: int, code: str, message: str, headers: dict | None = None
|
||||
) -> JSONResponse:
|
||||
identity = getattr(request.state, "request_id", str(uuid4()))
|
||||
return JSONResponse(
|
||||
status_code=status,
|
||||
content={"error": {"code": code, "message": message, "request_id": identity}},
|
||||
headers={"X-Request-Id": identity, **(headers or {})},
|
||||
)
|
||||
|
||||
|
||||
def create_app(settings: Settings | None = None) -> FastAPI:
|
||||
settings = settings or Settings.from_env()
|
||||
repository = PostgresRepository(
|
||||
settings.database_url,
|
||||
settings.pool_size,
|
||||
settings.idempotency_ttl,
|
||||
settings.read_autocommit,
|
||||
)
|
||||
cache = SessionCache(settings.cache_url)
|
||||
store = ObjectStore(settings)
|
||||
authentication = AuthService(repository, cache, settings.session_ttl)
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
repository.open()
|
||||
app.state.ready = True
|
||||
logger.info(json.dumps({"event": "started", "environment": os.getenv("APP_ENV", "lab")}))
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
app.state.ready = False
|
||||
repository.close()
|
||||
cache.close()
|
||||
store.close()
|
||||
logger.info(json.dumps({"event": "stopped"}))
|
||||
|
||||
app = FastAPI(lifespan=lifespan, docs_url=None, redoc_url=None, openapi_url=None)
|
||||
app.state.instance_id = uuid4().hex
|
||||
app.add_middleware(RequestContext, instance_id=app.state.instance_id)
|
||||
app.state.messenger = Messenger(repository)
|
||||
app.state.attachments = Attachments(repository, app.state.messenger, store, settings)
|
||||
app.state.repository = repository
|
||||
app.state.authentication = authentication
|
||||
app.state.cache = cache
|
||||
app.state.ready = False
|
||||
|
||||
@app.exception_handler(DomainError)
|
||||
async def domain_error(request: Request, exc: DomainError):
|
||||
return error_response(request, exc.status, exc.code, exc.message, exc.headers)
|
||||
|
||||
@app.exception_handler(RequestValidationError)
|
||||
async def validation_error(request: Request, exc: RequestValidationError):
|
||||
return error_response(request, 400, "invalid_request", "Некорректные поля или JSON")
|
||||
|
||||
@app.exception_handler(HTTPException)
|
||||
async def http_error(request: Request, exc: HTTPException):
|
||||
code = "not_found" if exc.status_code == 404 else "invalid_request"
|
||||
return error_response(request, exc.status_code, code, "Некорректный HTTP-запрос")
|
||||
|
||||
@app.exception_handler(Exception)
|
||||
async def unexpected_error(request: Request, exc: Exception):
|
||||
logger.error(json.dumps({"event": "internal_error", "type": type(exc).__name__}))
|
||||
return error_response(request, 500, "internal_error", "Внутренняя ошибка сервера")
|
||||
|
||||
def service(request: Request) -> Messenger:
|
||||
return request.app.state.messenger
|
||||
|
||||
def token_digest(request: Request) -> bytes:
|
||||
if len(request.headers.getlist("authorization")) != 1:
|
||||
raise unauthorized()
|
||||
return bearer_digest(request.headers.get("authorization"))
|
||||
|
||||
def identity(
|
||||
digest: Annotated[bytes, Depends(token_digest)],
|
||||
) -> Identity:
|
||||
return authentication.identify(digest)
|
||||
|
||||
def actor(current: Annotated[Identity, Depends(identity)]) -> User:
|
||||
return current.user
|
||||
|
||||
async def credentials(request: Request) -> tuple[str, str]:
|
||||
if len(request.headers.getlist("authorization")) != 1:
|
||||
raise unauthorized("Basic")
|
||||
result = basic_credentials(request.headers.get("authorization"))
|
||||
if await request.body():
|
||||
raise invalid("Вход не принимает тело запроса")
|
||||
return result
|
||||
|
||||
def resource_id(chat_id: str) -> str:
|
||||
try:
|
||||
return canonical_uuid(chat_id)
|
||||
except ValueError:
|
||||
raise invalid("Некорректный UUID чата") from None
|
||||
|
||||
def pagination(request: Request) -> tuple[int, str | None]:
|
||||
for key in ("limit", "cursor"):
|
||||
if len(request.query_params.getlist(key)) > 1:
|
||||
raise invalid("Параметр пагинации указан несколько раз")
|
||||
raw = request.query_params.get("limit", "50")
|
||||
try:
|
||||
if not re.fullmatch(r"[0-9]+", raw):
|
||||
raise ValueError
|
||||
limit = int(raw)
|
||||
if not 1 <= limit <= 100:
|
||||
raise ValueError
|
||||
except ValueError:
|
||||
raise invalid("limit должен быть целым от 1 до 100") from None
|
||||
return limit, request.query_params.get("cursor")
|
||||
|
||||
def json_input(request: Request):
|
||||
media = request.headers.get("content-type", "").split(";", 1)[0].strip().lower()
|
||||
if media != "application/json":
|
||||
raise DomainError(415, "unsupported_media_type", "Требуется application/json")
|
||||
|
||||
def message_key(request: Request) -> str:
|
||||
values = request.headers.getlist("idempotency-key")
|
||||
if len(values) != 1 or not re.fullmatch(r"[A-Za-z0-9._:-]{1,128}", values[0]):
|
||||
raise invalid("Требуется корректный Idempotency-Key")
|
||||
return values[0]
|
||||
|
||||
CurrentUser = Annotated[User, Depends(actor)]
|
||||
Service = Annotated[Messenger, Depends(service)]
|
||||
ChatId = Annotated[str, Depends(resource_id)]
|
||||
Pagination = Annotated[tuple[int, str | None], Depends(pagination)]
|
||||
|
||||
@app.get("/health/live")
|
||||
def live():
|
||||
return {"status": "ok"}
|
||||
|
||||
@app.get("/health/ready")
|
||||
def ready(request: Request):
|
||||
if not request.app.state.ready:
|
||||
raise DomainError(503, "unavailable", "Приложение не готово")
|
||||
repository.ready()
|
||||
store.ready()
|
||||
return {"status": "ok"}
|
||||
|
||||
@app.post(
|
||||
"/api/v1/users",
|
||||
status_code=201,
|
||||
response_model=UserView,
|
||||
dependencies=[Depends(json_input)],
|
||||
)
|
||||
def register(body: CreateUser):
|
||||
return authentication.register(body.username, body.display_name, body.password)
|
||||
|
||||
@app.post("/api/v1/auth/sessions", status_code=201, response_model=SessionView)
|
||||
def login(value: Annotated[tuple[str, str], Depends(credentials)]):
|
||||
return authentication.login(*value)
|
||||
|
||||
@app.delete("/api/v1/auth/sessions/current", status_code=204)
|
||||
def logout(current: Annotated[Identity, Depends(identity)]):
|
||||
authentication.logout(current)
|
||||
return Response(status_code=204)
|
||||
|
||||
@app.get("/api/v1/users/me", response_model=UserView)
|
||||
def me(current: CurrentUser):
|
||||
return current
|
||||
|
||||
@app.post(
|
||||
"/api/v1/chats",
|
||||
status_code=201,
|
||||
response_model=ChatView,
|
||||
dependencies=[Depends(json_input)],
|
||||
)
|
||||
def create_chat(body: CreateChat, current: CurrentUser, messenger: Service):
|
||||
return messenger.create_chat(current, body.title, body.member_ids)
|
||||
|
||||
@app.get("/api/v1/chats", response_model=ChatPage)
|
||||
def chats(current: CurrentUser, messenger: Service, page: Pagination):
|
||||
return messenger.chats(current, *page)
|
||||
|
||||
@app.get("/api/v1/chats/{chat_id}", response_model=ChatView)
|
||||
def chat(current: CurrentUser, chat: ChatId, messenger: Service):
|
||||
return messenger.chat(current, chat)
|
||||
|
||||
@app.post(
|
||||
"/api/v1/chats/{chat_id}/messages",
|
||||
status_code=201,
|
||||
response_model=MessageView,
|
||||
dependencies=[Depends(json_input)],
|
||||
)
|
||||
def send_message(
|
||||
body: CreateMessage,
|
||||
current: CurrentUser,
|
||||
chat: ChatId,
|
||||
messenger: Service,
|
||||
key: Annotated[str, Depends(message_key)],
|
||||
):
|
||||
return messenger.send(current, chat, body.text, key, tuple(body.attachment_ids))
|
||||
|
||||
@app.get("/api/v1/chats/{chat_id}/messages", response_model=MessagePage)
|
||||
def messages(current: CurrentUser, chat: ChatId, messenger: Service, page: Pagination):
|
||||
return messenger.messages(current, chat, *page)
|
||||
|
||||
def attachment_id(attachment_id: str) -> str:
|
||||
try:
|
||||
return canonical_uuid(attachment_id)
|
||||
except ValueError:
|
||||
raise invalid("Некорректный UUID вложения") from None
|
||||
|
||||
AttachmentId = Annotated[str, Depends(attachment_id)]
|
||||
|
||||
@app.post("/api/v1/chats/{chat_id}/attachments", status_code=202, response_model=AttachmentView)
|
||||
async def upload(request: Request, current: CurrentUser, chat: ChatId):
|
||||
return await app.state.attachments.upload(request, current, chat)
|
||||
|
||||
@app.get("/api/v1/attachments/{attachment_id}", response_model=AttachmentView)
|
||||
def attachment(current: CurrentUser, id: AttachmentId):
|
||||
return app.state.attachments.metadata(current, id)
|
||||
|
||||
@app.get("/api/v1/attachments/{attachment_id}/content")
|
||||
def content(request: Request, current: CurrentUser, id: AttachmentId):
|
||||
row = app.state.attachments.record(current, id)
|
||||
if row["storage_state"] != "ready":
|
||||
raise DomainError(409, "conflict", "Вложение пока недоступно")
|
||||
values = request.query_params.getlist("variant")
|
||||
if (
|
||||
len(values) > 1
|
||||
or values
|
||||
and values[0] not in ("original", "thumbnail", "crop", "optimized")
|
||||
):
|
||||
raise invalid("Некорректный вариант вложения")
|
||||
if values and values[0] != "original":
|
||||
row = next((v for v in row["variants"] if v["name"] == values[0]), None)
|
||||
if row is None:
|
||||
raise not_found()
|
||||
row = {**row, "filename": values[0]}
|
||||
body = store.download(row)
|
||||
|
||||
def chunks():
|
||||
try:
|
||||
while data := body.read(65536):
|
||||
yield data
|
||||
finally:
|
||||
body.close()
|
||||
|
||||
return StreamingResponse(
|
||||
chunks(),
|
||||
media_type=row["content_type"],
|
||||
headers={
|
||||
"Content-Length": str(row["size_bytes"]),
|
||||
"Content-Disposition": "attachment; filename*=UTF-8''"
|
||||
+ quote(row["filename"], safe=""),
|
||||
"Cache-Control": "private, no-store",
|
||||
"X-Content-Type-Options": "nosniff",
|
||||
},
|
||||
)
|
||||
|
||||
return app
|
||||
@@ -0,0 +1,127 @@
|
||||
"""Bounded upload: verified staging and transactional job intent precede HTTP 202."""
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import tempfile
|
||||
import unicodedata
|
||||
import warnings
|
||||
|
||||
from anyio import to_thread
|
||||
from PIL import Image, UnidentifiedImageError
|
||||
from starlette.requests import ClientDisconnect
|
||||
|
||||
from messenger.domain import DomainError, invalid, new_id, not_found
|
||||
from messenger.pipeline import Pipeline
|
||||
from messenger.repository import attachment_value
|
||||
|
||||
Image.MAX_IMAGE_PIXELS = 40_000_000
|
||||
warnings.filterwarnings("error", category=Image.DecompressionBombWarning)
|
||||
|
||||
|
||||
def filename(value):
|
||||
if (
|
||||
not value
|
||||
or not value.strip()
|
||||
or len(value) > 255
|
||||
or any(c in "/\\" or unicodedata.category(c).startswith("C") for c in value)
|
||||
):
|
||||
raise invalid("Некорректное имя файла")
|
||||
return value
|
||||
|
||||
|
||||
def image_type(file, maximum):
|
||||
try:
|
||||
file.seek(0)
|
||||
with Image.open(file) as image:
|
||||
if image.width * image.height > maximum:
|
||||
raise DomainError(413, "payload_too_large", "Слишком много пикселей")
|
||||
if image.format not in ("PNG", "JPEG"):
|
||||
raise ValueError
|
||||
media = "image/png" if image.format == "PNG" else "image/jpeg"
|
||||
image.verify()
|
||||
file.seek(0)
|
||||
with Image.open(file) as image:
|
||||
image.load() # Reject truncated pixel streams, not just an intact header.
|
||||
file.seek(0)
|
||||
return media
|
||||
except Image.DecompressionBombError, Image.DecompressionBombWarning:
|
||||
raise DomainError(413, "payload_too_large", "Слишком много пикселей") from None
|
||||
except UnidentifiedImageError, OSError, ValueError, SyntaxError:
|
||||
raise DomainError(415, "unsupported_media_type", "Повреждённое изображение") from None
|
||||
|
||||
|
||||
class Attachments:
|
||||
def __init__(self, repository, messenger, store, settings):
|
||||
self.repository, self.messenger, self.store = repository, messenger, store
|
||||
self.settings = settings
|
||||
self.slots = asyncio.Semaphore(settings.upload_slots)
|
||||
|
||||
async def upload(self, request, actor, chat_id):
|
||||
await to_thread.run_sync(self.messenger.chat, actor, chat_id)
|
||||
values = request.query_params.getlist("filename")
|
||||
if len(values) != 1:
|
||||
raise invalid("Требуется одно имя файла")
|
||||
name = filename(values[0])
|
||||
media = request.headers.get("content-type", "").split(";", 1)[0].strip().lower()
|
||||
if media not in ("image/png", "image/jpeg"):
|
||||
raise DomainError(415, "unsupported_media_type", "Требуется PNG или JPEG")
|
||||
length = request.headers.get("content-length")
|
||||
if length is not None:
|
||||
if not length.isascii() or not length.isdecimal():
|
||||
raise invalid("Некорректный Content-Length")
|
||||
if int(length) > self.settings.max_upload_bytes:
|
||||
raise DomainError(413, "payload_too_large", "Файл больше 10 MiB")
|
||||
try:
|
||||
await asyncio.wait_for(self.slots.acquire(), timeout=5)
|
||||
except TimeoutError:
|
||||
raise DomainError(429, "rate_limited", "Слишком много загрузок") from None
|
||||
try:
|
||||
with tempfile.TemporaryFile() as file:
|
||||
size, digest = 0, hashlib.sha256()
|
||||
try:
|
||||
async with asyncio.timeout(15):
|
||||
async for chunk in request.stream():
|
||||
size += len(chunk)
|
||||
if size > self.settings.max_upload_bytes:
|
||||
raise DomainError(413, "payload_too_large", "Файл больше 10 MiB")
|
||||
digest.update(chunk)
|
||||
await to_thread.run_sync(file.write, chunk)
|
||||
except TimeoutError:
|
||||
raise DomainError(400, "invalid_request", "Время загрузки истекло") from None
|
||||
except ClientDisconnect:
|
||||
raise invalid("Загрузка прервана") from None
|
||||
actual = await to_thread.run_sync(image_type, file, self.settings.max_image_pixels)
|
||||
if actual != media:
|
||||
raise DomainError(415, "unsupported_media_type", "MIME не совпадает с файлом")
|
||||
id = new_id()
|
||||
key = "staging/" + id
|
||||
await to_thread.run_sync(Pipeline(self.repository, self.settings).reserve, id, key)
|
||||
await to_thread.run_sync(
|
||||
self.store.put, key, file, size, actual, digest.hexdigest()
|
||||
)
|
||||
# A failure before this COMMIT leaves only an unaccepted staging orphan.
|
||||
# The same transaction creates attachment, job and outbox intent.
|
||||
row = await to_thread.run_sync(
|
||||
Pipeline(self.repository, self.settings).accept,
|
||||
id,
|
||||
chat_id,
|
||||
actor.id,
|
||||
name,
|
||||
actual,
|
||||
size,
|
||||
digest.hexdigest(),
|
||||
key,
|
||||
)
|
||||
return attachment_value(row)
|
||||
finally:
|
||||
self.slots.release()
|
||||
|
||||
def record(self, actor, id):
|
||||
row = self.repository.attachment(id)
|
||||
if row is None:
|
||||
raise not_found()
|
||||
self.messenger.chat(actor, str(row["chat_id"]))
|
||||
return row
|
||||
|
||||
def metadata(self, actor, id):
|
||||
return attachment_value(self.record(actor, id))
|
||||
@@ -0,0 +1,127 @@
|
||||
"""Argon2id passwords and opaque sessions. Cache hits never authorize a request."""
|
||||
|
||||
import base64
|
||||
import binascii
|
||||
import hashlib
|
||||
import re
|
||||
import secrets
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass
|
||||
from threading import BoundedSemaphore
|
||||
|
||||
from argon2 import PasswordHasher
|
||||
from argon2.exceptions import InvalidHashError, VerificationError
|
||||
from argon2.profiles import RFC_9106_LOW_MEMORY
|
||||
|
||||
from messenger.cache import SessionCache
|
||||
from messenger.domain import DomainError, User
|
||||
from messenger.repository import PostgresRepository
|
||||
|
||||
|
||||
def unauthorized(scheme: str = "Bearer") -> DomainError:
|
||||
challenge = scheme + ' realm="messenger"'
|
||||
if scheme == "Basic":
|
||||
challenge += ', charset="UTF-8"'
|
||||
return DomainError(
|
||||
401,
|
||||
"unauthorized",
|
||||
"Требуется действительная идентификация",
|
||||
{"WWW-Authenticate": challenge},
|
||||
)
|
||||
|
||||
|
||||
def basic_credentials(header: str | None) -> tuple[str, str]:
|
||||
try:
|
||||
if header is None or len(header) > 2048:
|
||||
raise ValueError
|
||||
scheme, encoded = header.split(" ", 1)
|
||||
if scheme.lower() != "basic":
|
||||
raise ValueError
|
||||
raw = base64.b64decode(encoded, validate=True).decode("utf-8")
|
||||
username, password = raw.split(":", 1)
|
||||
if not re.fullmatch(r"[a-z0-9_]{3,32}", username):
|
||||
raise ValueError
|
||||
if not 12 <= len(password) <= 128 or not password.strip():
|
||||
raise ValueError
|
||||
return username, password
|
||||
except ValueError, UnicodeError, binascii.Error:
|
||||
raise unauthorized("Basic") from None
|
||||
|
||||
|
||||
def bearer_digest(header: str | None) -> bytes:
|
||||
if header:
|
||||
parts = header.split(" ")
|
||||
if len(parts) == 2 and parts[0].lower() == "bearer":
|
||||
if re.fullmatch(r"[A-Za-z0-9_-]{43}", parts[1]):
|
||||
return hashlib.sha256(parts[1].encode()).digest()
|
||||
raise unauthorized()
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Identity:
|
||||
user: User
|
||||
token_hash: bytes
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Session:
|
||||
token: str
|
||||
token_type: str
|
||||
expires_at: str
|
||||
user: User
|
||||
|
||||
|
||||
class AuthService:
|
||||
def __init__(self, repository: PostgresRepository, cache: SessionCache, ttl: int):
|
||||
self.repository, self.cache, self.ttl = repository, cache, ttl
|
||||
self.hasher = PasswordHasher.from_parameters(RFC_9106_LOW_MEMORY)
|
||||
self.kdf_slots = BoundedSemaphore(2)
|
||||
self.dummy_hash = self.hasher.hash(secrets.token_urlsafe(32))
|
||||
|
||||
@contextmanager
|
||||
def kdf(self):
|
||||
if not self.kdf_slots.acquire(timeout=0.5):
|
||||
raise DomainError(503, "unavailable", "Обработка пароля временно занята")
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
self.kdf_slots.release()
|
||||
|
||||
def register(self, username: str, display_name: str, password: str) -> User:
|
||||
with self.kdf():
|
||||
encoded = self.hasher.hash(password)
|
||||
return self.repository.add_user(username, display_name, encoded)
|
||||
|
||||
def login(self, username: str, password: str) -> Session:
|
||||
credentials = self.repository.credentials(username)
|
||||
encoded = credentials[1] if credentials else self.dummy_hash
|
||||
try:
|
||||
with self.kdf():
|
||||
self.hasher.verify(encoded, password)
|
||||
except VerificationError, InvalidHashError:
|
||||
raise unauthorized("Basic") from None
|
||||
if credentials is None:
|
||||
raise unauthorized("Basic")
|
||||
token = secrets.token_urlsafe(32)
|
||||
digest = hashlib.sha256(token.encode()).digest()
|
||||
user = credentials[0]
|
||||
expires = self.repository.create_session(digest, user.id, self.ttl)
|
||||
return Session(token, "Bearer", expires, user)
|
||||
|
||||
def identify(self, digest: bytes) -> Identity:
|
||||
session = self.repository.active_session(digest)
|
||||
if session is None:
|
||||
raise unauthorized()
|
||||
user_id = str(session["user_id"])
|
||||
user = self.cache.get(digest, user_id)
|
||||
if user is None:
|
||||
user = self.repository.get_user(user_id)
|
||||
if user is None:
|
||||
raise unauthorized()
|
||||
ttl = int((session["expires_at"] - session["checked_at"]).total_seconds())
|
||||
self.cache.put(digest, user, ttl)
|
||||
return Identity(user, digest)
|
||||
|
||||
def logout(self, identity: Identity):
|
||||
self.repository.revoke_session(identity.token_hash)
|
||||
self.cache.delete(identity.token_hash)
|
||||
@@ -0,0 +1,96 @@
|
||||
"""Dedicated durable Valkey Stream, one consumer group; never evict/trim pending work."""
|
||||
|
||||
import re
|
||||
from datetime import datetime
|
||||
from typing import Literal
|
||||
from uuid import UUID
|
||||
|
||||
import redis
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
|
||||
|
||||
STREAM = "attachment-jobs"
|
||||
GROUP = "image-workers-v1"
|
||||
DLQ = "invalid-jobs"
|
||||
|
||||
|
||||
class Job(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", strict=True)
|
||||
schema_version: Literal[1]
|
||||
job_id: str
|
||||
type: Literal["attachment.process.v1"]
|
||||
attachment_id: str
|
||||
pipeline_version: int = Field(ge=1)
|
||||
attempt: int = Field(ge=1)
|
||||
created_at: str
|
||||
traceparent: str | None = Field(
|
||||
default=None, pattern=r"^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$"
|
||||
)
|
||||
tracestate: str | None = Field(default=None, max_length=512)
|
||||
|
||||
@model_validator(mode="before")
|
||||
@classmethod
|
||||
def strict_carrier(cls, value):
|
||||
if isinstance(value, dict):
|
||||
if type(value.get("schema_version")) is not int:
|
||||
raise ValueError("Integer schema version required")
|
||||
if any(k in value and value[k] is None for k in ("traceparent", "tracestate")):
|
||||
raise ValueError("Optional carrier fields must be strings when provided")
|
||||
return value
|
||||
|
||||
@field_validator("job_id", "attachment_id")
|
||||
@classmethod
|
||||
def uuid(cls, value):
|
||||
if str(UUID(value)) != value:
|
||||
raise ValueError("Canonical UUID required")
|
||||
return value
|
||||
|
||||
@field_validator("created_at")
|
||||
@classmethod
|
||||
def timestamp(cls, value):
|
||||
if not re.fullmatch(r"\d{4}-\d\d-\d\dT.+(?:Z|[+-]\d\d:\d\d)", value):
|
||||
raise ValueError("RFC3339 required")
|
||||
datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||
return value
|
||||
|
||||
|
||||
class Broker:
|
||||
def __init__(self, settings):
|
||||
self.client = redis.Redis.from_url(
|
||||
settings.broker_url,
|
||||
decode_responses=True,
|
||||
socket_timeout=2,
|
||||
socket_connect_timeout=1,
|
||||
retry_on_timeout=False,
|
||||
)
|
||||
|
||||
def group(self):
|
||||
try:
|
||||
self.client.xgroup_create(STREAM, GROUP, id="0", mkstream=True)
|
||||
except redis.ResponseError as exc:
|
||||
if "BUSYGROUP" not in str(exc):
|
||||
raise
|
||||
|
||||
def publish(self, job):
|
||||
# XADD reply is the publisher boundary; appendfsync=always precedes that reply.
|
||||
return self.client.xadd(STREAM, {"job": job.model_dump_json(exclude_none=True)})
|
||||
|
||||
def ack(self, id):
|
||||
# A single consumer group only. Atomically remove completed entries, never MAXLEN.
|
||||
self.client.eval(
|
||||
"redis.call('XACK',KEYS[1],ARGV[1],ARGV[2]); return redis.call('XDEL',KEYS[1],ARGV[2])",
|
||||
1,
|
||||
STREAM,
|
||||
GROUP,
|
||||
id,
|
||||
)
|
||||
|
||||
def quarantine(self, id, reason):
|
||||
# Never copy arbitrary attacker-supplied fields into DLQ/logs.
|
||||
with self.client.pipeline(transaction=True) as tx:
|
||||
tx.xadd(DLQ, {"stream_id": id, "reason": reason}, maxlen=1000)
|
||||
tx.xack(STREAM, GROUP, id)
|
||||
tx.xdel(STREAM, id)
|
||||
tx.execute()
|
||||
|
||||
def close(self):
|
||||
self.client.close()
|
||||
@@ -0,0 +1,57 @@
|
||||
"""Disposable session-user cache; PostgreSQL always decides token validity."""
|
||||
|
||||
import json
|
||||
import logging
|
||||
from dataclasses import asdict
|
||||
|
||||
from redis import Redis
|
||||
from redis.backoff import NoBackoff
|
||||
from redis.exceptions import RedisError
|
||||
from redis.retry import Retry
|
||||
|
||||
from messenger.domain import User
|
||||
|
||||
logger = logging.getLogger("messenger")
|
||||
|
||||
|
||||
class SessionCache:
|
||||
def __init__(self, url: str):
|
||||
self.client = Redis.from_url(
|
||||
url,
|
||||
socket_connect_timeout=0.2,
|
||||
socket_timeout=0.2,
|
||||
max_connections=8,
|
||||
decode_responses=True,
|
||||
retry_on_timeout=False,
|
||||
retry=Retry(NoBackoff(), 0),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def key(digest: bytes) -> str:
|
||||
return "session-user:" + digest.hex()
|
||||
|
||||
def get(self, digest: bytes, user_id: str) -> User | None:
|
||||
try:
|
||||
raw = self.client.get(self.key(digest))
|
||||
if raw:
|
||||
value = User(**json.loads(raw))
|
||||
if value.id == user_id:
|
||||
return value
|
||||
except RedisError, ValueError, TypeError:
|
||||
logger.warning('{"event":"session_cache_unavailable","operation":"get"}')
|
||||
return None
|
||||
|
||||
def put(self, digest: bytes, user: User, ttl: int):
|
||||
try:
|
||||
self.client.set(self.key(digest), json.dumps(asdict(user)), ex=max(1, ttl))
|
||||
except RedisError:
|
||||
logger.warning('{"event":"session_cache_unavailable","operation":"set"}')
|
||||
|
||||
def delete(self, digest: bytes):
|
||||
try:
|
||||
self.client.delete(self.key(digest))
|
||||
except RedisError:
|
||||
logger.warning('{"event":"session_cache_unavailable","operation":"delete"}')
|
||||
|
||||
def close(self):
|
||||
self.client.close()
|
||||
@@ -0,0 +1,122 @@
|
||||
"""Dry-run by default. Retention GC protects all active jobs and every published object."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from datetime import UTC, datetime
|
||||
from uuid import UUID
|
||||
|
||||
from messenger.pipeline import event
|
||||
from messenger.repository import PostgresRepository
|
||||
from messenger.settings import Settings
|
||||
from messenger.storage import ObjectStore
|
||||
|
||||
|
||||
def collect(db, store, retention, apply=False):
|
||||
if retention < 300:
|
||||
raise ValueError("Retention must be at least 300 seconds")
|
||||
counts = {"scanned": 0, "eligible": 0, "deleted": 0, "protected": 0}
|
||||
# Serialize collectors; API handoff is bounded < retention and all owned keys have UUID prefix.
|
||||
with db.connection() as lock:
|
||||
if not lock.execute("SELECT pg_try_advisory_lock(2301005) AS ok").fetchone()["ok"]:
|
||||
return {**counts, "busy": True}
|
||||
try:
|
||||
for page in store.client.get_paginator("list_objects_v2").paginate(Bucket=store.bucket):
|
||||
for obj in page.get("Contents", []):
|
||||
counts["scanned"] += 1
|
||||
key = obj["Key"]
|
||||
parts = key.split("/")
|
||||
try:
|
||||
if (
|
||||
parts[0] not in ("staging", "objects")
|
||||
or str(UUID(parts[1])) != parts[1]
|
||||
):
|
||||
continue
|
||||
except ValueError, IndexError:
|
||||
continue
|
||||
if (datetime.now(UTC) - obj["LastModified"]).total_seconds() < retention:
|
||||
continue
|
||||
with db.connection(write=True) as conn:
|
||||
reservation = conn.execute(
|
||||
"SELECT * FROM upload_reservations WHERE id=%s FOR UPDATE", (parts[1],)
|
||||
).fetchone()
|
||||
if (
|
||||
reservation
|
||||
and reservation["state"] == "uploading"
|
||||
and reservation["expires_at"] > datetime.now(UTC)
|
||||
):
|
||||
counts["protected"] += 1
|
||||
continue
|
||||
# This row lock serializes replay/new-version with deletion eligibility.
|
||||
a = conn.execute(
|
||||
"SELECT * FROM attachments WHERE id=%s FOR UPDATE", (parts[1],)
|
||||
).fetchone()
|
||||
active = conn.execute(
|
||||
"SELECT 1 FROM attachment_jobs WHERE attachment_id=%s AND "
|
||||
"state IN ('queued','processing') LIMIT 1",
|
||||
(parts[1],),
|
||||
).fetchone()
|
||||
published = (
|
||||
a
|
||||
and a["storage_state"] == "ready"
|
||||
and (
|
||||
key == a["object_key"]
|
||||
or any(v["object_key"] == key for v in a["variants"])
|
||||
)
|
||||
)
|
||||
recent_failed = conn.execute(
|
||||
"SELECT 1 FROM attachment_jobs WHERE attachment_id=%s AND "
|
||||
"state='failed' "
|
||||
"AND updated_at>clock_timestamp()-(%s*interval '1 second') LIMIT 1",
|
||||
(parts[1], retention),
|
||||
).fetchone()
|
||||
if active or published or recent_failed:
|
||||
counts["protected"] += 1
|
||||
continue
|
||||
head = store.head(key)
|
||||
if (
|
||||
not head
|
||||
or head["ETag"] != obj["ETag"]
|
||||
or head["ContentLength"] != obj["Size"]
|
||||
or (datetime.now(UTC) - head["LastModified"]).total_seconds()
|
||||
< retention
|
||||
):
|
||||
continue
|
||||
counts["eligible"] += 1
|
||||
if apply:
|
||||
if reservation and reservation["state"] == "uploading":
|
||||
conn.execute(
|
||||
"UPDATE upload_reservations SET state='abandoned' WHERE id=%s",
|
||||
(parts[1],),
|
||||
)
|
||||
store.client.delete_object(Bucket=store.bucket, Key=key)
|
||||
counts["deleted"] += 1
|
||||
rows = conn.execute(
|
||||
"UPDATE attachment_jobs SET source_expired=true WHERE "
|
||||
"source_key=%s "
|
||||
"AND state='failed' RETURNING job_id",
|
||||
(key,),
|
||||
).fetchall()
|
||||
for row in rows:
|
||||
event(conn, row["job_id"], "source_expired")
|
||||
finally:
|
||||
lock.execute("SELECT pg_advisory_unlock(2301005)")
|
||||
return counts
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--apply", action="store_true")
|
||||
args = parser.parse_args()
|
||||
settings = Settings.from_env()
|
||||
db = PostgresRepository(settings.database_url, pool_size=2)
|
||||
db.open()
|
||||
store = ObjectStore(settings)
|
||||
try:
|
||||
print(json.dumps(collect(db, store, settings.staging_retention, args.apply)))
|
||||
finally:
|
||||
store.close()
|
||||
db.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,41 @@
|
||||
"""Versioned keyset position, bound to a list and its resource, never an offset."""
|
||||
|
||||
import base64
|
||||
import binascii
|
||||
import json
|
||||
from datetime import UTC, datetime
|
||||
from uuid import UUID
|
||||
|
||||
from messenger.domain import HistoryKey, invalid
|
||||
|
||||
|
||||
def encode_cursor(scope: str, position: HistoryKey) -> str:
|
||||
value = {"v": 1, "scope": scope, "created_at": position[0], "id": position[1]}
|
||||
return (
|
||||
base64.urlsafe_b64encode(json.dumps(value, separators=(",", ":")).encode())
|
||||
.decode()
|
||||
.rstrip("=")
|
||||
)
|
||||
|
||||
|
||||
def decode_cursor(value: str | None, scope: str) -> HistoryKey | None:
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
if not value or len(value) > 1024:
|
||||
raise ValueError
|
||||
raw = base64.b64decode(value + "=" * (-len(value) % 4), altchars=b"-_", validate=True)
|
||||
data = json.loads(raw)
|
||||
if not isinstance(data, dict) or set(data) != {"v", "scope", "created_at", "id"}:
|
||||
raise ValueError
|
||||
if type(data["v"]) is not int or data["v"] != 1 or data["scope"] != scope:
|
||||
raise ValueError
|
||||
if not isinstance(data["created_at"], str) or not data["created_at"].endswith("Z"):
|
||||
raise ValueError
|
||||
stamp = datetime.fromisoformat(data["created_at"])
|
||||
if stamp.tzinfo != UTC or not isinstance(data["id"], str):
|
||||
raise ValueError
|
||||
identity = str(UUID(data["id"]))
|
||||
return stamp.isoformat(timespec="microseconds").replace("+00:00", "Z"), identity
|
||||
except ValueError, TypeError, KeyError, UnicodeError, binascii.Error:
|
||||
raise invalid("Некорректный cursor или другая область пагинации") from None
|
||||
@@ -0,0 +1,79 @@
|
||||
"""Immutable domain values; no dependency on HTTP or a storage technology."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime
|
||||
from uuid import uuid4
|
||||
|
||||
|
||||
def utc_now() -> str:
|
||||
return datetime.now(UTC).isoformat(timespec="microseconds").replace("+00:00", "Z")
|
||||
|
||||
|
||||
def new_id() -> str:
|
||||
return str(uuid4())
|
||||
|
||||
|
||||
class DomainError(Exception):
|
||||
def __init__(self, status: int, code: str, message: str, headers: dict | None = None):
|
||||
super().__init__(message)
|
||||
self.status = status
|
||||
self.code = code
|
||||
self.message = message
|
||||
self.headers = headers or {}
|
||||
|
||||
|
||||
def invalid(message: str = "Некорректный запрос") -> DomainError:
|
||||
return DomainError(400, "invalid_request", message)
|
||||
|
||||
|
||||
def not_found() -> DomainError:
|
||||
return DomainError(404, "not_found", "Объект не найден")
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class User:
|
||||
id: str
|
||||
username: str
|
||||
display_name: str
|
||||
created_at: str
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Chat:
|
||||
id: str
|
||||
title: str
|
||||
member_ids: tuple[str, ...]
|
||||
created_at: str
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Message:
|
||||
id: str
|
||||
chat_id: str
|
||||
sender_id: str
|
||||
text: str
|
||||
created_at: str
|
||||
attachment_ids: tuple[str, ...] = ()
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Attachment:
|
||||
id: str
|
||||
chat_id: str
|
||||
owner_id: str
|
||||
filename: str
|
||||
content_type: str
|
||||
size_bytes: int
|
||||
sha256: str
|
||||
created_at: str
|
||||
updated_at: str
|
||||
status: str = "ready"
|
||||
variants: tuple = ()
|
||||
error_code: str | None = None
|
||||
|
||||
|
||||
type HistoryKey = tuple[str, str]
|
||||
|
||||
|
||||
def history_key(item: Chat | Message) -> HistoryKey:
|
||||
return item.created_at, item.id
|
||||
@@ -0,0 +1,35 @@
|
||||
"""Operator-only, single-use fault barriers. Disabled in the normal Compose stack."""
|
||||
|
||||
import os
|
||||
import time
|
||||
|
||||
from messenger.domain import DomainError
|
||||
from messenger.pipeline import event
|
||||
|
||||
|
||||
class PermanentFailure(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def checkpoint(db, settings, job, phase):
|
||||
if not settings.fault_injection:
|
||||
return
|
||||
with db.connection(write=True) as conn:
|
||||
row = conn.execute(
|
||||
"UPDATE pipeline_faults SET remaining=remaining-1 WHERE job_id=%s AND phase=%s "
|
||||
"AND remaining>0 RETURNING action",
|
||||
(job, phase),
|
||||
).fetchone()
|
||||
if row:
|
||||
event(conn, job, "fault_injected", phase=phase, action=row["action"], pid=os.getpid())
|
||||
if not row:
|
||||
return
|
||||
match row["action"]:
|
||||
case "crash":
|
||||
os._exit(87)
|
||||
case "hang":
|
||||
time.sleep(settings.job_timeout * 3)
|
||||
case "transient":
|
||||
raise DomainError(503, "unavailable", "Synthetic transient failure")
|
||||
case "permanent":
|
||||
raise PermanentFailure
|
||||
@@ -0,0 +1,51 @@
|
||||
"""Pure, versioned image transforms. No network, filenames, or EXIF in output."""
|
||||
|
||||
import io
|
||||
|
||||
from PIL import Image, ImageOps
|
||||
|
||||
|
||||
def fit(image, maximum):
|
||||
scale = max(image.size)
|
||||
if scale <= maximum:
|
||||
return image.copy()
|
||||
size = tuple(max(1, n * maximum // scale) for n in image.size)
|
||||
return image.resize(size, Image.Resampling.LANCZOS)
|
||||
|
||||
|
||||
def variants(source, version):
|
||||
if version not in (1, 2):
|
||||
raise ValueError("Unsupported pipeline version")
|
||||
with Image.open(source) as raw:
|
||||
oriented = ImageOps.exif_transpose(raw)
|
||||
rgba = oriented.convert("RGBA")
|
||||
image = Image.new("RGB", rgba.size, "white")
|
||||
image.paste(rgba, mask=rgba.getchannel("A"))
|
||||
rgba.close()
|
||||
oriented.close()
|
||||
try:
|
||||
for name, output, format, options in (
|
||||
("thumbnail", fit(image, 256), "JPEG", {"quality": 85, "subsampling": 0}),
|
||||
(
|
||||
"crop",
|
||||
ImageOps.fit(image, (128, 128), method=Image.Resampling.LANCZOS),
|
||||
"JPEG",
|
||||
{"quality": 85, "subsampling": 0},
|
||||
),
|
||||
(
|
||||
"optimized",
|
||||
fit(image, 1280),
|
||||
"WEBP",
|
||||
{"quality": 82 if version == 1 else 76, "method": 4},
|
||||
),
|
||||
):
|
||||
with output, io.BytesIO() as buffer:
|
||||
output.save(buffer, format=format, exif=b"", **options)
|
||||
yield (
|
||||
name,
|
||||
buffer.getvalue(),
|
||||
output.size,
|
||||
"image/webp" if format == "WEBP" else "image/jpeg",
|
||||
)
|
||||
finally:
|
||||
image.close()
|
||||
@@ -0,0 +1,46 @@
|
||||
"""Transactional, checksummed migrations, serialized across concurrent starts."""
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import psycopg
|
||||
|
||||
from messenger.settings import secret
|
||||
|
||||
|
||||
def migrate(conninfo: str, directory: Path):
|
||||
with psycopg.connect(conninfo, connect_timeout=5) as conn:
|
||||
conn.execute("SELECT pg_advisory_xact_lock(2301002)")
|
||||
conn.execute(
|
||||
"CREATE TABLE IF NOT EXISTS schema_migrations ("
|
||||
"version text PRIMARY KEY, checksum text NOT NULL, "
|
||||
"applied_at timestamptz NOT NULL DEFAULT clock_timestamp())"
|
||||
)
|
||||
for path in sorted(directory.glob("*.sql")):
|
||||
content = path.read_text()
|
||||
checksum = hashlib.sha256(content.encode()).hexdigest()
|
||||
current = conn.execute(
|
||||
"SELECT checksum FROM schema_migrations WHERE version=%s", (path.stem,)
|
||||
).fetchone()
|
||||
if current:
|
||||
if current[0] != checksum:
|
||||
raise RuntimeError("Applied migration checksum changed: " + path.stem)
|
||||
continue
|
||||
conn.execute(content)
|
||||
conn.execute(
|
||||
"INSERT INTO schema_migrations (version,checksum) VALUES (%s,%s)",
|
||||
(path.stem, checksum),
|
||||
)
|
||||
print("Applied migration " + path.stem)
|
||||
|
||||
|
||||
def main():
|
||||
migrate(
|
||||
secret("DATABASE_URL_FILE", "/run/secrets/database_url"),
|
||||
Path(os.getenv("MIGRATIONS_DIR", "/app/migrations")),
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,60 @@
|
||||
"""Explicit local operator actions, never exposed as a public HTTP endpoint."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
|
||||
from messenger.pipeline import Pipeline
|
||||
from messenger.repository import PostgresRepository
|
||||
from messenger.settings import Settings
|
||||
from messenger.storage import ObjectStore
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("action", choices=("failed", "replay", "new-version"))
|
||||
parser.add_argument("attachment_id", nargs="?")
|
||||
args = parser.parse_args()
|
||||
settings = Settings.from_env()
|
||||
db = PostgresRepository(settings.database_url, pool_size=2)
|
||||
db.open()
|
||||
try:
|
||||
if args.action == "failed":
|
||||
with db.connection() as conn:
|
||||
rows = conn.execute(
|
||||
"SELECT job_id,attachment_id,pipeline_version,attempt,error_code,"
|
||||
"source_expired FROM attachment_jobs WHERE state='failed' "
|
||||
"ORDER BY updated_at LIMIT 100"
|
||||
).fetchall()
|
||||
print(json.dumps(rows, default=str))
|
||||
else:
|
||||
if not args.attachment_id:
|
||||
parser.error("attachment_id is required")
|
||||
with db.connection() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT j.source_key,a.object_key,a.storage_state FROM attachment_jobs j "
|
||||
"JOIN attachments a ON a.id=j.attachment_id AND "
|
||||
"a.pipeline_version=j.pipeline_version "
|
||||
"WHERE a.id=%s",
|
||||
(args.attachment_id,),
|
||||
).fetchone()
|
||||
if not row:
|
||||
parser.error("Unknown attachment")
|
||||
store = ObjectStore(settings)
|
||||
try:
|
||||
key = row["object_key"] if row["storage_state"] == "ready" else row["source_key"]
|
||||
if not store.head(key):
|
||||
parser.error("Source is no longer retained")
|
||||
finally:
|
||||
store.close()
|
||||
job = Pipeline(db, settings).replay(
|
||||
args.attachment_id, new_version=args.action == "new-version"
|
||||
)
|
||||
print(json.dumps({"job_id": job, "action": args.action}))
|
||||
except ValueError as exc:
|
||||
parser.error(str(exc))
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,266 @@
|
||||
"""Job state machine. PostgreSQL fences publication; the stream is at least once."""
|
||||
|
||||
import os
|
||||
import random
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
from psycopg.types.json import Jsonb
|
||||
|
||||
from messenger import telemetry
|
||||
from messenger.domain import DomainError, new_id
|
||||
|
||||
|
||||
def event(conn, job_id, name, **details):
|
||||
conn.execute(
|
||||
"INSERT INTO pipeline_audit(job_id,event,details) VALUES (%s,%s,%s)",
|
||||
(job_id, name, Jsonb(details)),
|
||||
)
|
||||
|
||||
|
||||
def intent(conn, job_id, when=None):
|
||||
conn.execute(
|
||||
"INSERT INTO job_outbox(job_id,due_at) VALUES (%s,coalesce(%s,clock_timestamp())) "
|
||||
"ON CONFLICT(job_id) DO UPDATE SET due_at=EXCLUDED.due_at,published_at=NULL,stream_id=NULL",
|
||||
(job_id, when),
|
||||
)
|
||||
|
||||
|
||||
class Pipeline:
|
||||
def __init__(self, repository, settings):
|
||||
self.db, self.settings = repository, settings
|
||||
|
||||
def reserve(self, id, key):
|
||||
with self.db.connection(write=True) as conn:
|
||||
conn.execute("INSERT INTO upload_reservations(id,object_key) VALUES (%s,%s)", (id, key))
|
||||
|
||||
def accept(self, id, chat, owner, name, media, size, digest, key):
|
||||
with self.db.connection(write=True) as conn:
|
||||
reservation = conn.execute(
|
||||
"SELECT * FROM upload_reservations WHERE id=%s FOR UPDATE", (id,)
|
||||
).fetchone()
|
||||
if (
|
||||
not reservation
|
||||
or reservation["state"] != "uploading"
|
||||
or reservation["object_key"] != key
|
||||
or reservation["expires_at"] <= datetime.now(UTC)
|
||||
):
|
||||
raise DomainError(503, "unavailable", "Срок приёма файла истёк")
|
||||
conn.execute("UPDATE upload_reservations SET state='accepted' WHERE id=%s", (id,))
|
||||
row = conn.execute(
|
||||
"INSERT INTO attachments "
|
||||
"(id,chat_id,owner_id,filename,content_type,size_bytes,sha256,"
|
||||
"object_key,storage_state) "
|
||||
"VALUES (%s,%s,%s,%s,%s,%s,%s,%s,'queued') RETURNING *",
|
||||
(id, chat, owner, name, media, size, digest, key),
|
||||
).fetchone()
|
||||
job = new_id()
|
||||
trace = telemetry.carrier()
|
||||
conn.execute(
|
||||
"INSERT INTO attachment_jobs(job_id,attachment_id,pipeline_version,source_key,"
|
||||
"traceparent,tracestate) VALUES (%s,%s,1,%s,%s,%s)",
|
||||
(job, id, key, trace.get("traceparent"), trace.get("tracestate")),
|
||||
)
|
||||
intent(conn, job)
|
||||
event(conn, job, "accepted")
|
||||
return row
|
||||
|
||||
def claim(self, envelope):
|
||||
with self.db.connection(write=True) as conn:
|
||||
# All writers lock attachment before job, including publication and operator replay.
|
||||
a = conn.execute(
|
||||
"SELECT * FROM attachments WHERE id=%s FOR UPDATE", (envelope.attachment_id,)
|
||||
).fetchone()
|
||||
if not a:
|
||||
return "unknown", None
|
||||
j = conn.execute(
|
||||
"SELECT * FROM attachment_jobs WHERE job_id=%s FOR NO KEY UPDATE",
|
||||
(envelope.job_id,),
|
||||
).fetchone()
|
||||
if (
|
||||
not j
|
||||
or j["attachment_id"] != a["id"]
|
||||
or j["pipeline_version"] != envelope.pipeline_version
|
||||
):
|
||||
return "unknown", None
|
||||
if a["pipeline_version"] != j["pipeline_version"] or j["state"] == "superseded":
|
||||
event(conn, j["job_id"], "stale_delivery")
|
||||
return "stale", None
|
||||
if j["state"] == "ready":
|
||||
event(conn, j["job_id"], "duplicate_ready")
|
||||
return "ready", a
|
||||
if j["state"] == "failed":
|
||||
return "failed", None
|
||||
now = datetime.now(UTC)
|
||||
if j["state"] == "processing" or j["next_attempt_at"] > now:
|
||||
return "busy", None
|
||||
if envelope.attempt != j["attempt"] + 1:
|
||||
# An older envelope cannot steal a scheduled retry or an explicit replay.
|
||||
return "obsolete_attempt", None
|
||||
fence = new_id()
|
||||
conn.execute(
|
||||
"UPDATE attachment_jobs SET state='processing',attempt=attempt+1,fence=%s,"
|
||||
"lease_until=clock_timestamp()+(%s*interval '1 "
|
||||
"second'),updated_at=clock_timestamp() "
|
||||
"WHERE job_id=%s",
|
||||
(fence, self.settings.lease_seconds, j["job_id"]),
|
||||
)
|
||||
conn.execute(
|
||||
"UPDATE attachments SET storage_state='processing',"
|
||||
"updated_at=clock_timestamp() WHERE id=%s",
|
||||
(a["id"],),
|
||||
)
|
||||
event(
|
||||
conn, j["job_id"], "claimed", attempt=j["attempt"] + 1, fence=fence, pid=os.getpid()
|
||||
)
|
||||
return "claimed", {
|
||||
**a,
|
||||
"job_id": str(j["job_id"]),
|
||||
"source_key": j["source_key"],
|
||||
"fence": fence,
|
||||
"attempt": j["attempt"] + 1,
|
||||
"accepted_at": j["created_at"],
|
||||
}
|
||||
|
||||
def publish(self, row, original_key, variants):
|
||||
with self.db.connection(write=True) as conn:
|
||||
a = conn.execute(
|
||||
"SELECT * FROM attachments WHERE id=%s FOR UPDATE", (row["id"],)
|
||||
).fetchone()
|
||||
j = conn.execute(
|
||||
"SELECT * FROM attachment_jobs WHERE job_id=%s FOR NO KEY UPDATE", (row["job_id"],)
|
||||
).fetchone()
|
||||
if not self.owns(a, j, row):
|
||||
event(conn, row["job_id"], "publication_fenced")
|
||||
return False
|
||||
conn.execute(
|
||||
"UPDATE attachments SET storage_state='ready',object_key=%s,variants=%s,"
|
||||
"error_code=NULL,updated_at=clock_timestamp() WHERE id=%s",
|
||||
(original_key, Jsonb(variants), row["id"]),
|
||||
)
|
||||
conn.execute(
|
||||
"UPDATE attachment_jobs SET state='ready',lease_until=NULL,error_code=NULL,"
|
||||
"updated_at=clock_timestamp() WHERE job_id=%s",
|
||||
(row["job_id"],),
|
||||
)
|
||||
event(conn, row["job_id"], "result_committed", version=row["pipeline_version"])
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def owns(a, j, row):
|
||||
return (
|
||||
a
|
||||
and j
|
||||
and a["pipeline_version"] == row["pipeline_version"]
|
||||
and j["state"] == "processing"
|
||||
and str(j["fence"]) == row["fence"]
|
||||
and j["lease_until"] > datetime.now(UTC)
|
||||
)
|
||||
|
||||
def failure(self, row, code, permanent=False):
|
||||
with self.db.connection(write=True) as conn:
|
||||
a = conn.execute(
|
||||
"SELECT * FROM attachments WHERE id=%s FOR UPDATE", (row["id"],)
|
||||
).fetchone()
|
||||
j = conn.execute(
|
||||
"SELECT * FROM attachment_jobs WHERE job_id=%s FOR NO KEY UPDATE", (row["job_id"],)
|
||||
).fetchone()
|
||||
if self.owns(a, j, row):
|
||||
return self._retry(conn, j, code, permanent)
|
||||
return "fenced"
|
||||
|
||||
def _retry(self, conn, j, code, permanent=False):
|
||||
failed = permanent or j["attempt"] >= self.settings.job_max_attempts
|
||||
state = "failed" if failed else "queued"
|
||||
if failed and not permanent:
|
||||
code = "retry_exhausted"
|
||||
# Full jitter; durable absolute time survives worker/relay restarts.
|
||||
delay = random.uniform(0.25, min(30, 2 ** min(j["attempt"], 5)))
|
||||
due = datetime.now(UTC) + timedelta(seconds=delay)
|
||||
conn.execute(
|
||||
"UPDATE attachment_jobs SET state=%s,error_code=%s,next_attempt_at=%s,lease_until=NULL,"
|
||||
"updated_at=clock_timestamp() WHERE job_id=%s",
|
||||
(state, code, due, j["job_id"]),
|
||||
)
|
||||
conn.execute(
|
||||
"UPDATE attachments SET storage_state=%s,error_code=%s,updated_at=clock_timestamp() "
|
||||
"WHERE id=%s",
|
||||
(state, code if failed else None, j["attachment_id"]),
|
||||
)
|
||||
event(conn, j["job_id"], state, reason=code, attempt=j["attempt"], delay_seconds=delay)
|
||||
if not failed:
|
||||
intent(conn, j["job_id"], due)
|
||||
return "failed" if failed else "retry"
|
||||
|
||||
def recover(self):
|
||||
# Bounded SKIP LOCKED batch; multiple relays. No transaction held during Pillow.
|
||||
with self.db.connection(write=True) as conn:
|
||||
rows = conn.execute(
|
||||
"SELECT a.id FROM attachments a JOIN attachment_jobs j ON j.attachment_id=a.id "
|
||||
"AND j.pipeline_version=a.pipeline_version WHERE j.state='processing' "
|
||||
"AND j.lease_until<clock_timestamp() ORDER BY a.id LIMIT 32 "
|
||||
"FOR UPDATE OF a SKIP LOCKED"
|
||||
).fetchall()
|
||||
for row in rows:
|
||||
j = conn.execute(
|
||||
"SELECT j.* FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id "
|
||||
"AND a.pipeline_version=j.pipeline_version WHERE a.id=%s FOR NO KEY "
|
||||
"UPDATE OF j",
|
||||
(row["id"],),
|
||||
).fetchone()
|
||||
if j["state"] == "processing" and j["lease_until"] < datetime.now(UTC):
|
||||
event(conn, j["job_id"], "lease_expired")
|
||||
self._retry(conn, j, "worker_timeout")
|
||||
|
||||
def replay(self, attachment, *, new_version=False):
|
||||
with self.db.connection(write=True) as conn:
|
||||
a = conn.execute(
|
||||
"SELECT * FROM attachments WHERE id=%s FOR UPDATE", (attachment,)
|
||||
).fetchone()
|
||||
if not a:
|
||||
raise ValueError("Unknown attachment")
|
||||
j = conn.execute(
|
||||
"SELECT * FROM attachment_jobs WHERE attachment_id=%s "
|
||||
"AND pipeline_version=%s FOR UPDATE",
|
||||
(attachment, a["pipeline_version"]),
|
||||
).fetchone()
|
||||
if new_version:
|
||||
if a["pipeline_version"] != 1:
|
||||
raise ValueError("Only pipeline v1 -> v2 is implemented")
|
||||
conn.execute(
|
||||
"UPDATE attachment_jobs SET state='superseded' WHERE job_id=%s "
|
||||
"AND state IN ('queued','processing','failed')",
|
||||
(j["job_id"],),
|
||||
)
|
||||
key = a["object_key"] if a["storage_state"] == "ready" else j["source_key"]
|
||||
job = new_id()
|
||||
conn.execute(
|
||||
"INSERT INTO attachment_jobs(job_id,attachment_id,pipeline_version,source_key) "
|
||||
"VALUES (%s,%s,2,%s)",
|
||||
(job, attachment, key),
|
||||
)
|
||||
conn.execute(
|
||||
"UPDATE attachments SET "
|
||||
"pipeline_version=2,storage_state='queued',variants='[]',"
|
||||
"error_code=NULL,updated_at=clock_timestamp() WHERE id=%s",
|
||||
(attachment,),
|
||||
)
|
||||
event(conn, job, "new_version", previous_job=str(j["job_id"]))
|
||||
else:
|
||||
if j["state"] != "failed" or j["source_expired"]:
|
||||
raise ValueError("Replay requires failed job with retained source")
|
||||
job = str(j["job_id"])
|
||||
conn.execute(
|
||||
"UPDATE attachment_jobs SET "
|
||||
"state='queued',attempt=0,fence=NULL,error_code=NULL,"
|
||||
"next_attempt_at=clock_timestamp(),updated_at=clock_timestamp() WHERE "
|
||||
"job_id=%s",
|
||||
(job,),
|
||||
)
|
||||
conn.execute(
|
||||
"UPDATE attachments SET storage_state='queued',error_code=NULL,"
|
||||
"updated_at=clock_timestamp() WHERE id=%s",
|
||||
(attachment,),
|
||||
)
|
||||
event(conn, job, "operator_replay")
|
||||
intent(conn, job)
|
||||
return job
|
||||
@@ -0,0 +1,106 @@
|
||||
"""Transactional outbox publisher and expired-lease watchdog, independently restartable."""
|
||||
|
||||
import signal
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import redis
|
||||
from opentelemetry.trace import SpanKind
|
||||
|
||||
from messenger import telemetry
|
||||
from messenger.broker import Broker, Job
|
||||
from messenger.domain import DomainError
|
||||
from messenger.faults import checkpoint
|
||||
from messenger.pipeline import Pipeline, event
|
||||
from messenger.repository import PostgresRepository, stamp
|
||||
from messenger.settings import Settings
|
||||
|
||||
|
||||
def publish_one(db, broker, settings):
|
||||
with db.connection(write=True) as conn:
|
||||
row = conn.execute(
|
||||
"SELECT o.job_id,j.attachment_id,j.pipeline_version,j.attempt,j.created_at,"
|
||||
"j.traceparent,j.tracestate "
|
||||
"FROM job_outbox o JOIN attachment_jobs j USING(job_id) "
|
||||
"WHERE o.published_at IS NULL AND o.due_at<=clock_timestamp() AND j.state='queued' "
|
||||
"ORDER BY o.due_at LIMIT 1 FOR UPDATE OF o SKIP LOCKED"
|
||||
).fetchone()
|
||||
if not row:
|
||||
return False
|
||||
with telemetry.span("jobs.publish", context=telemetry.context(row), kind=SpanKind.PRODUCER):
|
||||
job = Job(
|
||||
schema_version=1,
|
||||
type=telemetry.JOB_TYPE,
|
||||
job_id=str(row["job_id"]),
|
||||
attachment_id=str(row["attachment_id"]),
|
||||
pipeline_version=row["pipeline_version"],
|
||||
attempt=row["attempt"] + 1,
|
||||
created_at=stamp(row["created_at"]),
|
||||
**telemetry.carrier(),
|
||||
)
|
||||
stream_id = broker.publish(job)
|
||||
telemetry.event("job_published", job_id=job.job_id, attempt=job.attempt)
|
||||
# Commit-on-return may be lost here; repeat sends the same logical job_id.
|
||||
checkpoint(db, settings, row["job_id"], "after_publish")
|
||||
conn.execute(
|
||||
"UPDATE job_outbox SET published_at=clock_timestamp(),stream_id=%s WHERE job_id=%s",
|
||||
(stream_id, row["job_id"]),
|
||||
)
|
||||
event(conn, row["job_id"], "published", stream_id=stream_id, attempt=job.attempt)
|
||||
return True
|
||||
|
||||
|
||||
def sample_backlog(db):
|
||||
with db.connection() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT count(*) AS depth, "
|
||||
"coalesce(extract(epoch FROM clock_timestamp()-min(created_at)),0) AS age "
|
||||
"FROM attachment_jobs WHERE state IN ('queued','processing')"
|
||||
).fetchone()
|
||||
telemetry.gauge("messenger.queue.depth", row["depth"], queue="attachment-jobs")
|
||||
telemetry.gauge(
|
||||
"messenger.queue.oldest", max(0, float(row["age"])), unit="s", queue="attachment-jobs"
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
with telemetry.active("relay"):
|
||||
run()
|
||||
|
||||
|
||||
def run():
|
||||
settings = Settings.from_env()
|
||||
db = PostgresRepository(settings.database_url, pool_size=4)
|
||||
db.open()
|
||||
broker = Broker(settings)
|
||||
pipeline = Pipeline(db, settings)
|
||||
running = True
|
||||
next_sample = 0.0
|
||||
|
||||
def stop(*_):
|
||||
nonlocal running
|
||||
running = False
|
||||
|
||||
signal.signal(signal.SIGTERM, stop)
|
||||
signal.signal(signal.SIGINT, stop)
|
||||
try:
|
||||
while running:
|
||||
try:
|
||||
if telemetry.runtime is not None and time.monotonic() >= next_sample:
|
||||
sample_backlog(db)
|
||||
next_sample = time.monotonic() + 2
|
||||
pipeline.recover()
|
||||
sent = publish_one(db, broker, settings)
|
||||
Path("/tmp/relay-heartbeat").touch()
|
||||
if not sent:
|
||||
time.sleep(0.2)
|
||||
except redis.RedisError, DomainError:
|
||||
# Keep the transaction uncommitted and retry. No credentials/SQL in logs.
|
||||
time.sleep(0.5)
|
||||
finally:
|
||||
broker.close()
|
||||
db.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,351 @@
|
||||
"""PostgreSQL adapter: bounded connections, parameterized SQL, commit before return."""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from contextlib import contextmanager, nullcontext
|
||||
from datetime import datetime
|
||||
from typing import Protocol
|
||||
|
||||
import psycopg
|
||||
from psycopg import errors
|
||||
from psycopg.rows import dict_row
|
||||
from psycopg_pool import ConnectionPool, PoolTimeout, TooManyRequests
|
||||
|
||||
from messenger import telemetry
|
||||
from messenger.domain import (
|
||||
Attachment,
|
||||
Chat,
|
||||
DomainError,
|
||||
HistoryKey,
|
||||
Message,
|
||||
User,
|
||||
invalid,
|
||||
new_id,
|
||||
)
|
||||
|
||||
|
||||
def stamp(value: datetime) -> str:
|
||||
return value.isoformat(timespec="microseconds").replace("+00:00", "Z")
|
||||
|
||||
|
||||
def user_value(row) -> User:
|
||||
return User(str(row["id"]), row["username"], row["display_name"], stamp(row["created_at"]))
|
||||
|
||||
|
||||
def chat_value(row) -> Chat:
|
||||
return Chat(
|
||||
str(row["id"]),
|
||||
row["title"],
|
||||
tuple(str(x) for x in row["member_ids"]),
|
||||
stamp(row["created_at"]),
|
||||
)
|
||||
|
||||
|
||||
def message_value(row) -> Message:
|
||||
return Message(
|
||||
str(row["id"]),
|
||||
str(row["chat_id"]),
|
||||
str(row["sender_id"]),
|
||||
row["text"],
|
||||
stamp(row["created_at"]),
|
||||
tuple(str(x) for x in row.get("attachment_ids", ())),
|
||||
)
|
||||
|
||||
|
||||
def attachment_value(row) -> Attachment:
|
||||
return Attachment(
|
||||
*(str(row[k]) for k in ("id", "chat_id", "owner_id")),
|
||||
row["filename"],
|
||||
row["content_type"],
|
||||
row["size_bytes"],
|
||||
row["sha256"],
|
||||
stamp(row["created_at"]),
|
||||
stamp(row["updated_at"]),
|
||||
row["storage_state"],
|
||||
tuple(
|
||||
{k: v for k, v in item.items() if k != "object_key"} for item in row.get("variants", [])
|
||||
),
|
||||
row.get("error_code"),
|
||||
)
|
||||
|
||||
|
||||
class Repository(Protocol):
|
||||
def get_user(self, user_id: str) -> User | None: ...
|
||||
def add_chat(self, title: str, member_ids: tuple[str, ...]) -> Chat: ...
|
||||
def get_chat(self, chat_id: str) -> Chat | None: ...
|
||||
def page_chats(self, user_id: str, after: HistoryKey | None, limit: int) -> list[Chat]: ...
|
||||
def add_message(self, chat_id: str, sender_id: str, text: str) -> Message: ...
|
||||
def idempotent_message(
|
||||
self,
|
||||
chat_id: str,
|
||||
sender_id: str,
|
||||
text: str,
|
||||
key: str,
|
||||
attachment_ids: tuple[str, ...] = (),
|
||||
) -> Message: ...
|
||||
def page_messages(
|
||||
self, chat_id: str, after: HistoryKey | None, limit: int
|
||||
) -> list[Message]: ...
|
||||
|
||||
|
||||
CHAT_COLUMNS = """c.id, c.title, c.created_at,
|
||||
ARRAY(SELECT user_id FROM chat_members WHERE chat_id=c.id ORDER BY position) AS member_ids"""
|
||||
|
||||
|
||||
MESSAGE_COLUMNS = """m.*, ARRAY(SELECT attachment_id FROM message_attachments
|
||||
WHERE message_id=m.id ORDER BY position) AS attachment_ids"""
|
||||
|
||||
|
||||
class PostgresRepository:
|
||||
def __init__(
|
||||
self,
|
||||
conninfo: str,
|
||||
pool_size: int = 8,
|
||||
idempotency_ttl: int = 86400,
|
||||
read_autocommit: bool = True,
|
||||
):
|
||||
self.read_autocommit = read_autocommit
|
||||
self.idempotency_ttl = idempotency_ttl
|
||||
self.pool = ConnectionPool(
|
||||
conninfo,
|
||||
min_size=1,
|
||||
max_size=pool_size,
|
||||
timeout=1,
|
||||
max_waiting=32,
|
||||
open=False,
|
||||
check=ConnectionPool.check_connection,
|
||||
kwargs={
|
||||
"row_factory": dict_row,
|
||||
"autocommit": True,
|
||||
"connect_timeout": 2,
|
||||
"options": "-c statement_timeout=2500 -c lock_timeout=1000 "
|
||||
"-c idle_in_transaction_session_timeout=5000 -c timezone=UTC",
|
||||
},
|
||||
)
|
||||
|
||||
def open(self):
|
||||
self.pool.open(wait=True, timeout=15)
|
||||
|
||||
def close(self):
|
||||
self.pool.close(timeout=5)
|
||||
|
||||
@contextmanager
|
||||
def connection(self, *, write: bool = False):
|
||||
with telemetry.span(
|
||||
"db.transaction" if write else "db.read", **{"db.system.name": "postgresql"}
|
||||
):
|
||||
with self._connection(write=write) as conn:
|
||||
yield conn
|
||||
|
||||
@contextmanager
|
||||
def _connection(self, *, write):
|
||||
try:
|
||||
with self.pool.connection() as conn:
|
||||
transaction = (
|
||||
conn.transaction() if write or not self.read_autocommit else nullcontext()
|
||||
)
|
||||
with transaction:
|
||||
telemetry.pool(self.pool)
|
||||
yield conn
|
||||
except (
|
||||
psycopg.OperationalError,
|
||||
psycopg.InterfaceError,
|
||||
PoolTimeout,
|
||||
TooManyRequests,
|
||||
errors.QueryCanceled,
|
||||
errors.LockNotAvailable,
|
||||
):
|
||||
telemetry.pool(self.pool)
|
||||
telemetry.record("messenger.db.errors", 1)
|
||||
raise DomainError(503, "unavailable", "Хранилище временно недоступно") from None
|
||||
|
||||
def ready(self):
|
||||
with self.connection() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT count(*) AS n FROM schema_migrations "
|
||||
"WHERE version IN "
|
||||
"('001_initial','002_idempotency','003_attachments',"
|
||||
"'004_pipeline','005_upload_handoff','006_trace_context')"
|
||||
).fetchone()
|
||||
if row["n"] != 6:
|
||||
raise DomainError(503, "unavailable", "Схема ещё не готова")
|
||||
|
||||
def add_user(self, username: str, display_name: str, password_hash: str) -> User:
|
||||
try:
|
||||
with self.connection(write=True) as conn:
|
||||
row = conn.execute(
|
||||
"INSERT INTO users (id,username,display_name,password_hash) "
|
||||
"VALUES (%s,%s,%s,%s) RETURNING *",
|
||||
(new_id(), username, display_name, password_hash),
|
||||
).fetchone()
|
||||
return user_value(row)
|
||||
except errors.UniqueViolation:
|
||||
raise DomainError(409, "conflict", "Имя пользователя уже занято") from None
|
||||
|
||||
def credentials(self, username: str):
|
||||
with self.connection() as conn:
|
||||
row = conn.execute("SELECT * FROM users WHERE username=%s", (username,)).fetchone()
|
||||
return (user_value(row), row["password_hash"]) if row else None
|
||||
|
||||
def get_user(self, user_id: str) -> User | None:
|
||||
with self.connection() as conn:
|
||||
row = conn.execute("SELECT * FROM users WHERE id=%s", (user_id,)).fetchone()
|
||||
return user_value(row) if row else None
|
||||
|
||||
def add_chat(self, title: str, member_ids: tuple[str, ...]) -> Chat:
|
||||
try:
|
||||
with self.connection(write=True) as conn:
|
||||
row = conn.execute(
|
||||
"INSERT INTO chats (id,title) VALUES (%s,%s) RETURNING *", (new_id(), title)
|
||||
).fetchone()
|
||||
with conn.cursor() as cur:
|
||||
cur.executemany(
|
||||
"INSERT INTO chat_members (chat_id,user_id,position) VALUES (%s,%s,%s)",
|
||||
[(row["id"], member, i) for i, member in enumerate(member_ids)],
|
||||
)
|
||||
return Chat(str(row["id"]), title, member_ids, stamp(row["created_at"]))
|
||||
except errors.ForeignKeyViolation:
|
||||
raise invalid("Указан неизвестный участник") from None
|
||||
|
||||
def get_chat(self, chat_id: str) -> Chat | None:
|
||||
with self.connection() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT " + CHAT_COLUMNS + " FROM chats c WHERE c.id=%s", (chat_id,)
|
||||
).fetchone()
|
||||
return chat_value(row) if row else None
|
||||
|
||||
def page_chats(self, user_id: str, after: HistoryKey | None, limit: int) -> list[Chat]:
|
||||
condition = " AND (c.created_at,c.id) > (%s::timestamptz,%s::uuid)" if after else ""
|
||||
params = (user_id, *after, limit) if after else (user_id, limit)
|
||||
with self.connection() as conn:
|
||||
rows = conn.execute(
|
||||
"SELECT " + CHAT_COLUMNS + " FROM chats c WHERE EXISTS "
|
||||
"(SELECT 1 FROM chat_members m WHERE m.chat_id=c.id AND m.user_id=%s)"
|
||||
+ condition
|
||||
+ " ORDER BY c.created_at,c.id LIMIT %s",
|
||||
params,
|
||||
).fetchall()
|
||||
return [chat_value(row) for row in rows]
|
||||
|
||||
def add_message(self, chat_id: str, sender_id: str, text: str) -> Message:
|
||||
with self.connection(write=True) as conn:
|
||||
row = conn.execute(
|
||||
"INSERT INTO messages (id,chat_id,sender_id,text) VALUES (%s,%s,%s,%s) RETURNING *",
|
||||
(new_id(), chat_id, sender_id, text),
|
||||
).fetchone()
|
||||
return message_value(row)
|
||||
|
||||
def page_messages(self, chat_id: str, after: HistoryKey | None, limit: int) -> list[Message]:
|
||||
condition = " AND (m.created_at,m.id) > (%s::timestamptz,%s::uuid)" if after else ""
|
||||
params = (chat_id, *after, limit) if after else (chat_id, limit)
|
||||
with self.connection() as conn:
|
||||
rows = conn.execute(
|
||||
"SELECT "
|
||||
+ MESSAGE_COLUMNS
|
||||
+ " FROM messages m WHERE m.chat_id=%s"
|
||||
+ condition
|
||||
+ " ORDER BY m.created_at,m.id LIMIT %s",
|
||||
params,
|
||||
).fetchall()
|
||||
return [message_value(row) for row in rows]
|
||||
|
||||
def idempotent_message(
|
||||
self,
|
||||
chat_id: str,
|
||||
sender_id: str,
|
||||
text: str,
|
||||
key: str,
|
||||
attachment_ids: tuple[str, ...] = (),
|
||||
) -> Message:
|
||||
scope = json.dumps([sender_id, "POST", chat_id, key], separators=(",", ":"))
|
||||
lock = int.from_bytes(hashlib.sha256(scope.encode()).digest()[:8], "big", signed=True)
|
||||
payload = json.dumps(
|
||||
{"text": text, "attachment_ids": list(attachment_ids)},
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
ensure_ascii=False,
|
||||
)
|
||||
digest = hashlib.sha256(payload.encode()).digest()
|
||||
with self.connection(write=True) as conn:
|
||||
conn.execute("SELECT pg_advisory_xact_lock(%s)", (lock,))
|
||||
previous = conn.execute(
|
||||
"SELECT r.payload_hash," + MESSAGE_COLUMNS + " FROM message_requests r "
|
||||
"JOIN messages m ON m.id=r.response_message_id "
|
||||
"WHERE r.user_id=%s AND r.http_method='POST' AND r.chat_id=%s "
|
||||
"AND r.idempotency_key=%s AND r.expires_at > clock_timestamp()",
|
||||
(sender_id, chat_id, key),
|
||||
).fetchone()
|
||||
if previous:
|
||||
if previous["payload_hash"] != digest:
|
||||
raise DomainError(409, "conflict", "Ключ уже использован с другим сообщением")
|
||||
return message_value(previous)
|
||||
conn.execute(
|
||||
"DELETE FROM message_requests WHERE user_id=%s AND http_method='POST' "
|
||||
"AND chat_id=%s AND idempotency_key=%s",
|
||||
(sender_id, chat_id, key),
|
||||
)
|
||||
if attachment_ids:
|
||||
rows = conn.execute(
|
||||
"SELECT id,storage_state FROM attachments WHERE chat_id=%s "
|
||||
"AND id=ANY(%s::uuid[]) ORDER BY id FOR UPDATE",
|
||||
(chat_id, list(attachment_ids)),
|
||||
).fetchall()
|
||||
if len(rows) != len(attachment_ids):
|
||||
raise DomainError(404, "not_found", "Объект не найден")
|
||||
if any(row["storage_state"] == "failed" for row in rows):
|
||||
raise DomainError(409, "conflict", "Вложение пока недоступно")
|
||||
message = message_value(
|
||||
conn.execute(
|
||||
"INSERT INTO messages (id,chat_id,sender_id,text) "
|
||||
"VALUES (%s,%s,%s,%s) RETURNING *",
|
||||
(new_id(), chat_id, sender_id, text),
|
||||
).fetchone()
|
||||
)
|
||||
for position, attachment in enumerate(attachment_ids):
|
||||
conn.execute(
|
||||
"INSERT INTO message_attachments VALUES (%s,%s,%s,%s)",
|
||||
(message.id, chat_id, attachment, position),
|
||||
)
|
||||
message = Message(
|
||||
message.id,
|
||||
message.chat_id,
|
||||
message.sender_id,
|
||||
message.text,
|
||||
message.created_at,
|
||||
attachment_ids,
|
||||
)
|
||||
self._save_request(conn, sender_id, chat_id, key, digest, message.id)
|
||||
return message
|
||||
|
||||
def _save_request(self, conn, user_id, chat_id, key, digest, message_id):
|
||||
conn.execute(
|
||||
"INSERT INTO message_requests "
|
||||
"(user_id,chat_id,idempotency_key,payload_hash,response_message_id,expires_at) "
|
||||
"VALUES (%s,%s,%s,%s,%s,clock_timestamp()+make_interval(secs => %s))",
|
||||
(user_id, chat_id, key, digest, message_id, self.idempotency_ttl),
|
||||
)
|
||||
|
||||
def create_session(self, token_hash: bytes, user_id: str, ttl: int) -> str:
|
||||
with self.connection(write=True) as conn:
|
||||
row = conn.execute(
|
||||
"INSERT INTO sessions (token_hash,user_id,expires_at) "
|
||||
"VALUES (%s,%s,clock_timestamp() + make_interval(secs => %s)) RETURNING expires_at",
|
||||
(token_hash, user_id, ttl),
|
||||
).fetchone()
|
||||
return stamp(row["expires_at"])
|
||||
|
||||
def active_session(self, token_hash: bytes):
|
||||
with self.connection() as conn:
|
||||
return conn.execute(
|
||||
"SELECT user_id,expires_at,clock_timestamp() AS checked_at FROM sessions "
|
||||
"WHERE token_hash=%s AND expires_at > clock_timestamp()",
|
||||
(token_hash,),
|
||||
).fetchone()
|
||||
|
||||
def revoke_session(self, token_hash: bytes):
|
||||
with self.connection(write=True) as conn:
|
||||
conn.execute("DELETE FROM sessions WHERE token_hash=%s", (token_hash,))
|
||||
|
||||
def attachment(self, id):
|
||||
with self.connection() as conn:
|
||||
return conn.execute("SELECT * FROM attachments WHERE id=%s", (id,)).fetchone()
|
||||
@@ -0,0 +1,120 @@
|
||||
"""Strict lab 5 wire types; password never belongs to a response User."""
|
||||
|
||||
import re
|
||||
from typing import Annotated
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import AfterValidator, BaseModel, ConfigDict, Field, field_validator, model_validator
|
||||
|
||||
|
||||
def canonical_uuid(value: str) -> str:
|
||||
if not re.fullmatch(r"[0-9a-fA-F]{8}(?:-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}", value):
|
||||
raise ValueError("Ожидается UUID")
|
||||
return str(UUID(value))
|
||||
|
||||
|
||||
def nonblank(value: str) -> str:
|
||||
if not value.strip() or any(0xD800 <= ord(char) <= 0xDFFF for char in value):
|
||||
raise ValueError("Ожидается непустая строка Unicode")
|
||||
return value
|
||||
|
||||
|
||||
type Identifier = Annotated[str, AfterValidator(canonical_uuid)]
|
||||
type ShortText = Annotated[str, Field(min_length=1, max_length=100), AfterValidator(nonblank)]
|
||||
|
||||
|
||||
class Input(BaseModel):
|
||||
model_config = ConfigDict(strict=True, extra="forbid")
|
||||
|
||||
|
||||
class CreateUser(Input):
|
||||
username: str = Field(min_length=3, max_length=32)
|
||||
display_name: ShortText
|
||||
password: Annotated[str, Field(min_length=12, max_length=128), AfterValidator(nonblank)]
|
||||
|
||||
@field_validator("username")
|
||||
@classmethod
|
||||
def valid_username(cls, value: str) -> str:
|
||||
if not re.fullmatch(r"[a-z0-9_]{3,32}", value):
|
||||
raise ValueError("Недопустимое имя пользователя")
|
||||
return value
|
||||
|
||||
|
||||
class CreateChat(Input):
|
||||
title: ShortText
|
||||
member_ids: list[Identifier] = Field(min_length=1, max_length=100)
|
||||
|
||||
|
||||
class CreateMessage(Input):
|
||||
text: str = Field(max_length=4000)
|
||||
attachment_ids: list[Identifier] = Field(default_factory=list, max_length=10)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def valid_content(self):
|
||||
if (
|
||||
not self.text.strip()
|
||||
and not self.attachment_ids
|
||||
or any(0xD800 <= ord(c) <= 0xDFFF for c in self.text)
|
||||
or len(set(self.attachment_ids)) != len(self.attachment_ids)
|
||||
):
|
||||
raise ValueError("Некорректное содержимое сообщения")
|
||||
return self
|
||||
|
||||
|
||||
class View(BaseModel):
|
||||
model_config = ConfigDict(from_attributes=True, extra="forbid")
|
||||
|
||||
|
||||
class UserView(View):
|
||||
id: str
|
||||
username: str
|
||||
display_name: str
|
||||
created_at: str
|
||||
|
||||
|
||||
class SessionView(View):
|
||||
token: str
|
||||
token_type: str
|
||||
expires_at: str
|
||||
user: UserView
|
||||
|
||||
|
||||
class ChatView(View):
|
||||
id: str
|
||||
title: str
|
||||
member_ids: list[str]
|
||||
created_at: str
|
||||
|
||||
|
||||
class MessageView(View):
|
||||
attachment_ids: list[str]
|
||||
id: str
|
||||
chat_id: str
|
||||
sender_id: str
|
||||
text: str
|
||||
created_at: str
|
||||
|
||||
|
||||
class ChatPage(View):
|
||||
items: list[ChatView]
|
||||
next_cursor: str | None
|
||||
|
||||
|
||||
class MessagePage(View):
|
||||
items: list[MessageView]
|
||||
next_cursor: str | None
|
||||
|
||||
|
||||
class AttachmentView(View):
|
||||
id: str
|
||||
chat_id: str
|
||||
owner_id: str
|
||||
filename: str
|
||||
content_type: str
|
||||
size_bytes: int
|
||||
sha256: str
|
||||
status: str
|
||||
variants: list
|
||||
error_code: str | None
|
||||
created_at: str
|
||||
updated_at: str
|
||||
@@ -0,0 +1,52 @@
|
||||
"""Use cases shared by HTTP and tests. Authorization precedes resource access."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
from messenger.cursor import decode_cursor, encode_cursor
|
||||
from messenger.domain import Chat, Message, User, history_key, invalid, not_found
|
||||
from messenger.repository import Repository
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Page[T]:
|
||||
items: list[T]
|
||||
next_cursor: str | None
|
||||
|
||||
|
||||
class Messenger:
|
||||
def __init__(self, repository: Repository):
|
||||
self.repository = repository
|
||||
|
||||
def create_chat(self, actor: User, title: str, member_ids: list[str]) -> Chat:
|
||||
if len(member_ids) != len(set(member_ids)) or actor.id not in member_ids:
|
||||
raise invalid("Участники не должны повторяться; вызывающий должен входить в чат")
|
||||
return self.repository.add_chat(title, tuple(member_ids))
|
||||
|
||||
def chat(self, actor: User, chat_id: str) -> Chat:
|
||||
chat = self.repository.get_chat(chat_id)
|
||||
if chat is None or actor.id not in chat.member_ids:
|
||||
raise not_found()
|
||||
return chat
|
||||
|
||||
def chats(self, actor: User, limit: int, cursor: str | None) -> Page[Chat]:
|
||||
scope = "chats:" + actor.id
|
||||
after = decode_cursor(cursor, scope)
|
||||
return self._page(self.repository.page_chats(actor.id, after, limit + 1), limit, scope)
|
||||
|
||||
def send(
|
||||
self, actor: User, chat_id: str, text: str, key: str, attachments: tuple[str, ...] = ()
|
||||
) -> Message:
|
||||
self.chat(actor, chat_id)
|
||||
return self.repository.idempotent_message(chat_id, actor.id, text, key, attachments)
|
||||
|
||||
def messages(self, actor: User, chat_id: str, limit: int, cursor: str | None) -> Page[Message]:
|
||||
self.chat(actor, chat_id)
|
||||
scope = "messages:" + chat_id
|
||||
after = decode_cursor(cursor, scope)
|
||||
return self._page(self.repository.page_messages(chat_id, after, limit + 1), limit, scope)
|
||||
|
||||
@staticmethod
|
||||
def _page[T: Chat | Message](items: list[T], limit: int, scope: str) -> Page[T]:
|
||||
visible = items[:limit]
|
||||
cursor = encode_cursor(scope, history_key(visible[-1])) if len(items) > limit else None
|
||||
return Page(visible, cursor)
|
||||
@@ -0,0 +1,85 @@
|
||||
"""Explicit configuration; credentials are read from untracked secret files."""
|
||||
|
||||
import os
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def secret(name: str, default: str, optional=False) -> str:
|
||||
path = Path(os.getenv(name, default))
|
||||
return path.read_text().strip() if path.exists() or not optional else ""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Settings:
|
||||
database_url: str = field(repr=False)
|
||||
cache_url: str = field(repr=False)
|
||||
session_ttl: int = 3600
|
||||
pool_size: int = 8
|
||||
idempotency_ttl: int = 86400
|
||||
read_autocommit: bool = True
|
||||
|
||||
s3_endpoint: str = "http://127.0.0.1:59006"
|
||||
s3_region: str = "us-east-1"
|
||||
s3_bucket: str = "messenger-attachments"
|
||||
s3_access_key: str = field(default="", repr=False)
|
||||
s3_secret_key: str = field(default="", repr=False)
|
||||
max_upload_bytes: int = 10485760
|
||||
max_image_pixels: int = 40000000
|
||||
upload_slots: int = 1
|
||||
|
||||
broker_url: str = field(default="redis://broker:6379/0", repr=False)
|
||||
worker_concurrency: int = 2
|
||||
job_max_attempts: int = 5
|
||||
job_timeout: int = 20
|
||||
staging_retention: int = 86400
|
||||
fault_injection: bool = False
|
||||
|
||||
@property
|
||||
def lease_seconds(self):
|
||||
return self.job_timeout + 3
|
||||
|
||||
@classmethod
|
||||
def from_env(cls):
|
||||
ttl = int(os.getenv("SESSION_TTL_SECONDS", "3600"))
|
||||
size = int(os.getenv("DATABASE_POOL_SIZE", "8"))
|
||||
idempotency_ttl = int(os.getenv("IDEMPOTENCY_TTL_SECONDS", "86400"))
|
||||
if idempotency_ttl < 86400:
|
||||
raise ValueError("Idempotency retention must be at least 24 hours")
|
||||
if not 1 <= ttl <= 2_592_000 or not 1 <= size <= 32:
|
||||
raise ValueError("Invalid session lifetime or connection pool size")
|
||||
maximum = int(os.getenv("MAX_UPLOAD_BYTES", "10485760"))
|
||||
pixels = int(os.getenv("MAX_IMAGE_PIXELS", "40000000"))
|
||||
if not 1 <= maximum <= 10485760 or not 1 <= pixels <= 40000000:
|
||||
raise ValueError("Invalid upload limits")
|
||||
concurrency = int(os.getenv("WORKER_CONCURRENCY", "2"))
|
||||
attempts = int(os.getenv("JOB_MAX_ATTEMPTS", "5"))
|
||||
timeout = int(os.getenv("JOB_TIMEOUT_SECONDS", "20"))
|
||||
retention = int(os.getenv("STAGING_RETENTION_SECONDS", "86400"))
|
||||
if not 1 <= concurrency <= 4 or not 1 <= attempts <= 10 or not 2 <= timeout <= 120:
|
||||
raise ValueError("Invalid worker limits")
|
||||
if retention < 300:
|
||||
raise ValueError("Staging retention must be at least 300 seconds")
|
||||
return cls(
|
||||
secret("DATABASE_URL_FILE", "/run/secrets/database_url"),
|
||||
secret("CACHE_URL_FILE", "/run/secrets/cache_url"),
|
||||
ttl,
|
||||
size,
|
||||
idempotency_ttl,
|
||||
os.getenv("DATABASE_READ_AUTOCOMMIT", "1") == "1",
|
||||
os.getenv("S3_ENDPOINT", "http://s3:9000"),
|
||||
os.getenv("S3_REGION", "us-east-1"),
|
||||
os.getenv("S3_BUCKET", "messenger-attachments"),
|
||||
secret("S3_ACCESS_KEY_ID_FILE", "/run/secrets/s3_access_key_id", optional=True),
|
||||
secret("S3_SECRET_ACCESS_KEY_FILE", "/run/secrets/s3_secret_access_key", optional=True),
|
||||
maximum,
|
||||
pixels,
|
||||
broker_url=secret("BROKER_URL_FILE", "/run/secrets/broker_url")
|
||||
if os.path.exists(os.getenv("BROKER_URL_FILE", "/run/secrets/broker_url"))
|
||||
else "redis://broker:6379/0",
|
||||
worker_concurrency=concurrency,
|
||||
job_max_attempts=attempts,
|
||||
job_timeout=timeout,
|
||||
staging_retention=retention,
|
||||
fault_injection=os.getenv("ENABLE_FAULT_INJECTION", "0") == "1",
|
||||
)
|
||||
@@ -0,0 +1,82 @@
|
||||
"""S3 adapter: bounded sockets, least-privilege credentials, no automatic PUT retries."""
|
||||
|
||||
import boto3
|
||||
from botocore.config import Config
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
|
||||
from messenger import telemetry
|
||||
from messenger.domain import DomainError
|
||||
|
||||
|
||||
def unavailable():
|
||||
return DomainError(503, "unavailable", "Файловое хранилище временно недоступно")
|
||||
|
||||
|
||||
class ObjectStore:
|
||||
def __init__(self, settings):
|
||||
self.bucket = settings.s3_bucket
|
||||
self.client = boto3.client(
|
||||
"s3",
|
||||
endpoint_url=settings.s3_endpoint,
|
||||
region_name=settings.s3_region,
|
||||
aws_access_key_id=settings.s3_access_key,
|
||||
aws_secret_access_key=settings.s3_secret_key,
|
||||
config=Config(
|
||||
connect_timeout=0.5,
|
||||
read_timeout=2,
|
||||
max_pool_connections=8,
|
||||
retries={"total_max_attempts": 1},
|
||||
s3={"addressing_style": "path"},
|
||||
request_checksum_calculation="when_required",
|
||||
response_checksum_validation="when_required",
|
||||
),
|
||||
)
|
||||
|
||||
def ready(self):
|
||||
try:
|
||||
self.client.head_bucket(Bucket=self.bucket)
|
||||
except BotoCoreError, ClientError:
|
||||
raise unavailable() from None
|
||||
|
||||
def put(self, key, file, size, media, digest):
|
||||
with telemetry.span("s3.put"):
|
||||
try:
|
||||
self.client.put_object(
|
||||
Bucket=self.bucket,
|
||||
Key=key,
|
||||
Body=file,
|
||||
ContentLength=size,
|
||||
ContentType=media,
|
||||
Metadata={"sha256": digest},
|
||||
)
|
||||
except BotoCoreError, ClientError:
|
||||
raise unavailable() from None
|
||||
|
||||
def head(self, key):
|
||||
with telemetry.span("s3.head"):
|
||||
try:
|
||||
return self.client.head_object(Bucket=self.bucket, Key=key)
|
||||
except ClientError as exc:
|
||||
if str(exc.response.get("Error", {}).get("Code")) in (
|
||||
"404",
|
||||
"NoSuchKey",
|
||||
"NotFound",
|
||||
):
|
||||
return None
|
||||
raise unavailable() from None
|
||||
except BotoCoreError:
|
||||
raise unavailable() from None
|
||||
|
||||
def download(self, row):
|
||||
with telemetry.span("s3.download"):
|
||||
try:
|
||||
response = self.client.get_object(Bucket=self.bucket, Key=row["object_key"])
|
||||
if response["ContentLength"] != row["size_bytes"]:
|
||||
response["Body"].close()
|
||||
raise unavailable()
|
||||
return response["Body"]
|
||||
except BotoCoreError, ClientError:
|
||||
raise unavailable() from None
|
||||
|
||||
def close(self):
|
||||
self.client.close()
|
||||
@@ -0,0 +1,241 @@
|
||||
"""Application telemetry with bounded export and an explicit attribute allowlist."""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import resource
|
||||
import time
|
||||
from contextlib import contextmanager
|
||||
from importlib.metadata import version
|
||||
from uuid import uuid4
|
||||
|
||||
from opentelemetry import trace
|
||||
from opentelemetry._logs import LogRecord, SeverityNumber
|
||||
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
|
||||
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
|
||||
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
|
||||
from opentelemetry.metrics import Observation
|
||||
from opentelemetry.sdk._logs import LoggerProvider
|
||||
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
|
||||
from opentelemetry.sdk.metrics import MeterProvider
|
||||
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
|
||||
from opentelemetry.sdk.metrics.view import ExplicitBucketHistogramAggregation, View
|
||||
from opentelemetry.sdk.resources import Resource
|
||||
from opentelemetry.sdk.trace import TracerProvider
|
||||
from opentelemetry.sdk.trace.export import BatchSpanProcessor
|
||||
from opentelemetry.sdk.trace.sampling import ParentBased, TraceIdRatioBased
|
||||
from opentelemetry.trace import SpanKind, Status, StatusCode
|
||||
from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
|
||||
|
||||
logger = logging.getLogger("messenger")
|
||||
PROPAGATOR = TraceContextTextMapPropagator()
|
||||
BUCKETS = (0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 120)
|
||||
HTTP_METHODS = frozenset(("GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS"))
|
||||
JOB_TYPE = "attachment.process.v1"
|
||||
runtime = None
|
||||
|
||||
|
||||
class Telemetry:
|
||||
def __init__(self, role, instance, metrics):
|
||||
resource_attrs = Resource(
|
||||
{
|
||||
"service.name": f"messenger-{role}",
|
||||
"service.version": version("mtusi-messenger"),
|
||||
"service.instance.id": instance,
|
||||
"deployment.environment.name": "lab",
|
||||
}
|
||||
)
|
||||
ratio = float(os.getenv("OTEL_SAMPLE_RATIO", "1"))
|
||||
self.traces = TracerProvider(
|
||||
resource=resource_attrs, sampler=ParentBased(TraceIdRatioBased(ratio))
|
||||
)
|
||||
self.traces.add_span_processor(
|
||||
BatchSpanProcessor(
|
||||
OTLPSpanExporter(timeout=1),
|
||||
max_queue_size=512,
|
||||
max_export_batch_size=64,
|
||||
schedule_delay_millis=500,
|
||||
)
|
||||
)
|
||||
self.tracer = self.traces.get_tracer("messenger")
|
||||
self.logs = LoggerProvider(resource=resource_attrs)
|
||||
self.logs.add_log_record_processor(
|
||||
BatchLogRecordProcessor(
|
||||
OTLPLogExporter(timeout=1),
|
||||
max_queue_size=512,
|
||||
max_export_batch_size=64,
|
||||
schedule_delay_millis=500,
|
||||
)
|
||||
)
|
||||
self.log = self.logs.get_logger("messenger")
|
||||
self.meters = None
|
||||
self.instruments = {}
|
||||
if metrics:
|
||||
self.meters = MeterProvider(
|
||||
resource=resource_attrs,
|
||||
metric_readers=[
|
||||
PeriodicExportingMetricReader(
|
||||
OTLPMetricExporter(timeout=1),
|
||||
export_interval_millis=2000,
|
||||
export_timeout_millis=1000,
|
||||
)
|
||||
],
|
||||
views=[
|
||||
View(
|
||||
instrument_name="messenger.*.duration",
|
||||
aggregation=ExplicitBucketHistogramAggregation(BUCKETS),
|
||||
)
|
||||
],
|
||||
)
|
||||
self.meter = self.meters.get_meter("messenger")
|
||||
self.meter.create_observable_counter(
|
||||
"messenger.process.cpu", unit="s", callbacks=[self.cpu_seconds]
|
||||
)
|
||||
self.meter.create_observable_gauge(
|
||||
"messenger.process.rss", unit="By", callbacks=[self.rss_bytes]
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def cpu_seconds(_):
|
||||
usage = resource.getrusage(resource.RUSAGE_CHILDREN)
|
||||
yield Observation(time.process_time() + usage.ru_utime + usage.ru_stime)
|
||||
|
||||
@staticmethod
|
||||
def rss_bytes(_):
|
||||
try:
|
||||
with open("/proc/self/statm", encoding="ascii") as stats:
|
||||
yield Observation(int(stats.read().split()[1]) * os.sysconf("SC_PAGE_SIZE"))
|
||||
except OSError:
|
||||
return
|
||||
|
||||
def shutdown(self):
|
||||
if self.meters:
|
||||
self.meters.shutdown(timeout_millis=1500)
|
||||
self.logs.shutdown()
|
||||
self.traces.shutdown()
|
||||
|
||||
|
||||
@contextmanager
|
||||
def active(role, instance=None, *, metrics=True):
|
||||
global runtime
|
||||
if os.getenv("TELEMETRY_ENABLED", "0") != "1":
|
||||
yield
|
||||
return
|
||||
runtime = Telemetry(role, instance or uuid4().hex, metrics)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
state, runtime = runtime, None
|
||||
state.shutdown()
|
||||
|
||||
|
||||
@contextmanager
|
||||
def span(name, *, context=None, kind=SpanKind.INTERNAL, **attrs):
|
||||
if runtime is None:
|
||||
yield trace.INVALID_SPAN
|
||||
return
|
||||
with runtime.tracer.start_as_current_span(
|
||||
name,
|
||||
context=context,
|
||||
kind=kind,
|
||||
attributes=attrs,
|
||||
record_exception=False,
|
||||
set_status_on_exception=False,
|
||||
) as current:
|
||||
try:
|
||||
yield current
|
||||
except Exception as exc:
|
||||
current.set_attribute("error.type", type(exc).__name__)
|
||||
current.set_status(Status(StatusCode.ERROR))
|
||||
raise
|
||||
|
||||
|
||||
def carrier():
|
||||
result = {}
|
||||
if runtime is not None:
|
||||
PROPAGATOR.inject(result)
|
||||
return result
|
||||
|
||||
|
||||
def context(headers):
|
||||
return PROPAGATOR.extract(
|
||||
{k: headers[k] for k in ("traceparent", "tracestate") if headers.get(k)}
|
||||
)
|
||||
|
||||
|
||||
def record(name, value, *, kind="counter", unit="1", **attrs):
|
||||
if runtime is None or runtime.meters is None:
|
||||
return
|
||||
instrument = runtime.instruments.get(name)
|
||||
if instrument is None:
|
||||
create = {
|
||||
"counter": runtime.meter.create_counter,
|
||||
"histogram": runtime.meter.create_histogram,
|
||||
"gauge": runtime.meter.create_gauge,
|
||||
}[kind]
|
||||
instrument = runtime.instruments.setdefault(name, create(name, unit=unit))
|
||||
if kind == "counter":
|
||||
instrument.add(value, attrs)
|
||||
elif kind == "histogram":
|
||||
instrument.record(value, attrs)
|
||||
else:
|
||||
instrument.set(value, attrs)
|
||||
|
||||
|
||||
def duration(name, seconds, **attrs):
|
||||
record(name, seconds, kind="histogram", unit="s", **attrs)
|
||||
|
||||
|
||||
def gauge(name, value, *, unit="", **attrs):
|
||||
record(name, value, kind="gauge", unit=unit, **attrs)
|
||||
|
||||
|
||||
def pool(pool):
|
||||
if runtime is not None and runtime.meters is not None:
|
||||
stats = pool.get_stats()
|
||||
gauge(
|
||||
"messenger.db.pool.used",
|
||||
max(0, stats["pool_size"] - stats["pool_available"]),
|
||||
)
|
||||
gauge("messenger.db.pool.waiting", stats["requests_waiting"])
|
||||
|
||||
|
||||
def event(event_name, *, error=False, **attrs):
|
||||
fields = {"event": event_name, **attrs}
|
||||
current = trace.get_current_span().get_span_context()
|
||||
if current.is_valid:
|
||||
fields["trace_id"] = format(current.trace_id, "032x")
|
||||
fields["span_id"] = format(current.span_id, "016x")
|
||||
logger.log(logging.ERROR if error else logging.INFO, json.dumps(fields))
|
||||
if runtime is not None:
|
||||
runtime.log.emit(
|
||||
LogRecord(
|
||||
timestamp=time.time_ns(),
|
||||
observed_timestamp=time.time_ns(),
|
||||
severity_text="ERROR" if error else "INFO",
|
||||
severity_number=SeverityNumber.ERROR if error else SeverityNumber.INFO,
|
||||
body=event_name,
|
||||
attributes=fields,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def http(current, method, route, status, elapsed, request_id):
|
||||
method = method if method in HTTP_METHODS else "_OTHER"
|
||||
attrs = {"method": method, "route": route, "status_class": f"{status // 100}xx"}
|
||||
current.update_name(f"{method} {route}")
|
||||
current.set_attributes(
|
||||
{"http.request.method": method, "http.route": route, "http.response.status_code": status}
|
||||
)
|
||||
if status >= 500:
|
||||
current.set_status(Status(StatusCode.ERROR))
|
||||
record("messenger.http.requests", 1, **attrs)
|
||||
duration("messenger.http.duration", elapsed, **attrs)
|
||||
event(
|
||||
"http_request",
|
||||
error=status >= 500,
|
||||
request_id=request_id,
|
||||
status=status,
|
||||
duration_ms=round(elapsed * 1000, 3),
|
||||
**attrs,
|
||||
)
|
||||
@@ -0,0 +1,270 @@
|
||||
"""Bounded supervisor: one disposable OS process per job, hard wall-clock and RAM limits."""
|
||||
|
||||
import hashlib
|
||||
import io
|
||||
import multiprocessing
|
||||
import resource
|
||||
import signal
|
||||
import tempfile
|
||||
import time
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
import redis
|
||||
from opentelemetry.trace import SpanKind
|
||||
from pydantic import ValidationError
|
||||
|
||||
from messenger import telemetry
|
||||
from messenger.attachments import image_type
|
||||
from messenger.broker import GROUP, STREAM, Broker, Job
|
||||
from messenger.domain import DomainError
|
||||
from messenger.faults import PermanentFailure, checkpoint
|
||||
from messenger.images import variants
|
||||
from messenger.pipeline import Pipeline, event
|
||||
from messenger.repository import PostgresRepository
|
||||
from messenger.settings import Settings
|
||||
from messenger.storage import ObjectStore
|
||||
|
||||
|
||||
def verify_effect(store, row):
|
||||
records = [row, *row["variants"]]
|
||||
if len(records) != 4 or {v["name"] for v in row["variants"]} != {
|
||||
"thumbnail",
|
||||
"crop",
|
||||
"optimized",
|
||||
}:
|
||||
raise DomainError(503, "unavailable", "Incomplete result")
|
||||
for item in records:
|
||||
head = store.head(item["object_key"])
|
||||
if (
|
||||
not head
|
||||
or head["ContentLength"] != item["size_bytes"]
|
||||
or head["Metadata"].get("sha256") != item["sha256"]
|
||||
):
|
||||
raise DomainError(503, "unavailable", "Incomplete result")
|
||||
|
||||
|
||||
def handle(db, broker, store, settings, id, payload):
|
||||
try:
|
||||
job = Job.model_validate_json(payload)
|
||||
except ValidationError:
|
||||
broker.quarantine(id, "invalid_envelope")
|
||||
return "invalid", None
|
||||
context = telemetry.context(job.model_dump(include={"traceparent", "tracestate"}))
|
||||
with telemetry.span(
|
||||
"jobs.consume",
|
||||
context=context,
|
||||
kind=SpanKind.CONSUMER,
|
||||
**{"messaging.destination.name": STREAM},
|
||||
) as span:
|
||||
outcome, age = process_job(db, broker, store, settings, id, job)
|
||||
span.set_attribute("job.outcome", outcome)
|
||||
telemetry.event(
|
||||
"job_processed",
|
||||
job_id=job.job_id,
|
||||
attempt=job.attempt,
|
||||
outcome=outcome,
|
||||
error=outcome in {"retry", "failed"},
|
||||
)
|
||||
return outcome, age
|
||||
|
||||
|
||||
def process_job(db, broker, store, settings, id, job):
|
||||
pipeline = Pipeline(db, settings)
|
||||
outcome, row = pipeline.claim(job)
|
||||
if outcome == "busy":
|
||||
return "busy", None # Watchdog recovers the lease.
|
||||
if outcome == "unknown":
|
||||
broker.quarantine(id, "unknown_job")
|
||||
return "unknown", None
|
||||
if outcome == "ready":
|
||||
verify_effect(store, row)
|
||||
broker.ack(id)
|
||||
return "duplicate", None
|
||||
if outcome != "claimed":
|
||||
broker.ack(id) # Failed registry / obsolete version / durable retry owns this work.
|
||||
return outcome, None
|
||||
try:
|
||||
checkpoint(db, settings, job.job_id, "before_effect")
|
||||
with tempfile.TemporaryFile() as original:
|
||||
body = store.download({**row, "object_key": row["source_key"]})
|
||||
size, digest = 0, hashlib.sha256()
|
||||
try:
|
||||
while chunk := body.read(65536):
|
||||
size += len(chunk)
|
||||
if size > settings.max_upload_bytes:
|
||||
raise PermanentFailure
|
||||
digest.update(chunk)
|
||||
original.write(chunk)
|
||||
finally:
|
||||
body.close()
|
||||
if size != row["size_bytes"] or digest.hexdigest() != row["sha256"]:
|
||||
raise PermanentFailure
|
||||
image_type(original, settings.max_image_pixels)
|
||||
prefix = f"objects/{row['id']}/v{row['pipeline_version']}/"
|
||||
original_key = prefix + "original"
|
||||
store.put(original_key, original, size, row["content_type"], digest.hexdigest())
|
||||
checkpoint(db, settings, job.job_id, "after_effect")
|
||||
records = []
|
||||
for name, data, (width, height), media in variants(original, row["pipeline_version"]):
|
||||
checksum = hashlib.sha256(data).hexdigest()
|
||||
key = prefix + name
|
||||
store.put(key, io.BytesIO(data), len(data), media, checksum)
|
||||
records.append(
|
||||
{
|
||||
"name": name,
|
||||
"content_type": media,
|
||||
"size_bytes": len(data),
|
||||
"sha256": checksum,
|
||||
"width": width,
|
||||
"height": height,
|
||||
"object_key": key,
|
||||
}
|
||||
)
|
||||
with db.connection(write=True) as conn:
|
||||
event(conn, job.job_id, "objects_written", version=row["pipeline_version"])
|
||||
committed = pipeline.publish(row, original_key, records)
|
||||
age = max(0.0, (datetime.now(UTC) - row["accepted_at"]).total_seconds())
|
||||
if committed:
|
||||
checkpoint(db, settings, job.job_id, "after_commit")
|
||||
broker.ack(id)
|
||||
return ("ready", age) if committed else ("fenced", None)
|
||||
except PermanentFailure:
|
||||
outcome = pipeline.failure(row, "invalid_image", permanent=True)
|
||||
broker.ack(id)
|
||||
return outcome, None
|
||||
except MemoryError:
|
||||
outcome = pipeline.failure(row, "resource_limit", permanent=True)
|
||||
broker.ack(id)
|
||||
return outcome, None
|
||||
except DomainError as exc:
|
||||
outcome = pipeline.failure(
|
||||
row,
|
||||
"invalid_image" if exc.status in (413, 415) else "storage_unavailable",
|
||||
permanent=exc.status in (413, 415),
|
||||
)
|
||||
broker.ack(id)
|
||||
return outcome, None
|
||||
except redis.RedisError:
|
||||
# Result may already be committed; retain the pending entry for a verified no-op.
|
||||
raise
|
||||
except Exception:
|
||||
outcome = pipeline.failure(row, "processing_failed")
|
||||
broker.ack(id)
|
||||
return outcome, None
|
||||
|
||||
|
||||
def child(settings, id, payload, result, instance):
|
||||
start = time.monotonic()
|
||||
try:
|
||||
with telemetry.active("worker", instance, metrics=False):
|
||||
outcome, age = run_child(settings, id, payload)
|
||||
result.send((outcome, age, time.monotonic() - start))
|
||||
finally:
|
||||
result.close()
|
||||
|
||||
|
||||
def run_child(settings, id, payload):
|
||||
# Linux hard limit includes Pillow buffers and native allocations. Supervisor kills hangs.
|
||||
resource.setrlimit(resource.RLIMIT_AS, (768 * 1024**2, 768 * 1024**2))
|
||||
db = PostgresRepository(settings.database_url, pool_size=2)
|
||||
db.open()
|
||||
broker, store = Broker(settings), ObjectStore(settings)
|
||||
try:
|
||||
return handle(db, broker, store, settings, id, payload)
|
||||
except redis.RedisError, DomainError:
|
||||
return "dependency_error", None # Leave pending; omit driver details.
|
||||
finally:
|
||||
store.close()
|
||||
broker.close()
|
||||
db.close()
|
||||
|
||||
|
||||
def observe(result, elapsed, timed_out):
|
||||
try:
|
||||
outcome, age, duration = result.recv() if result.poll() else ("crashed", None, elapsed)
|
||||
except EOFError:
|
||||
outcome, age, duration = "crashed", None, elapsed
|
||||
finally:
|
||||
result.close()
|
||||
if timed_out:
|
||||
outcome = "timeout"
|
||||
attrs = {"job_type": telemetry.JOB_TYPE, "outcome": outcome}
|
||||
telemetry.record("messenger.jobs.processed", 1, **attrs)
|
||||
telemetry.duration("messenger.jobs.duration", duration, **attrs)
|
||||
if age is not None:
|
||||
telemetry.duration("messenger.ready.duration", age, job_type=telemetry.JOB_TYPE)
|
||||
|
||||
|
||||
def main():
|
||||
instance = uuid4().hex
|
||||
with telemetry.active("worker", instance):
|
||||
run(instance)
|
||||
|
||||
|
||||
def run(instance):
|
||||
settings = Settings.from_env()
|
||||
broker = Broker(settings)
|
||||
consumer, cursor = uuid4().hex, "0-0"
|
||||
context = multiprocessing.get_context("spawn")
|
||||
active, running = [], True
|
||||
|
||||
def stop(*_):
|
||||
nonlocal running
|
||||
running = False
|
||||
|
||||
signal.signal(signal.SIGTERM, stop)
|
||||
signal.signal(signal.SIGINT, stop)
|
||||
try:
|
||||
while running:
|
||||
for task in active[:]:
|
||||
process, result, started = task
|
||||
elapsed = time.monotonic() - started
|
||||
alive = process.is_alive()
|
||||
if not alive or elapsed > settings.job_timeout:
|
||||
if alive:
|
||||
process.kill()
|
||||
process.join()
|
||||
observe(result, elapsed, alive)
|
||||
active.remove(task)
|
||||
process.close()
|
||||
telemetry.gauge("messenger.worker.busy", len(active))
|
||||
telemetry.gauge("messenger.worker.capacity", settings.worker_concurrency)
|
||||
Path("/tmp/worker-heartbeat").touch()
|
||||
if len(active) >= settings.worker_concurrency:
|
||||
time.sleep(0.05)
|
||||
continue
|
||||
try:
|
||||
broker.group()
|
||||
reclaimed = broker.client.xautoclaim(
|
||||
STREAM, GROUP, consumer, settings.lease_seconds * 1000, cursor, count=1
|
||||
)
|
||||
cursor, entries = reclaimed[0], reclaimed[1]
|
||||
if not entries:
|
||||
result = broker.client.xreadgroup(
|
||||
GROUP, consumer, {STREAM: ">"}, count=1, block=200
|
||||
)
|
||||
entries = result[0][1] if result else []
|
||||
if entries:
|
||||
id, fields = entries[0]
|
||||
result, sender = context.Pipe(duplex=False)
|
||||
process = context.Process(
|
||||
target=child, args=(settings, id, fields.get("job", ""), sender, instance)
|
||||
)
|
||||
process.start()
|
||||
sender.close()
|
||||
active.append((process, result, time.monotonic()))
|
||||
except redis.RedisError:
|
||||
time.sleep(0.5)
|
||||
finally:
|
||||
for process, result, _ in active:
|
||||
result.close()
|
||||
if process.is_alive():
|
||||
process.kill()
|
||||
process.join()
|
||||
broker.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,86 @@
|
||||
import base64
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
import psycopg
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
from psycopg import sql
|
||||
from psycopg.conninfo import make_conninfo
|
||||
|
||||
from messenger.app import create_app
|
||||
from messenger.migrate import migrate
|
||||
from messenger.settings import Settings
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
PASSWORD = "Учебный:пароль😀-12345"
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def settings():
|
||||
password = (ROOT / "secrets/postgres_password").read_text().strip()
|
||||
admin = make_conninfo(
|
||||
host="127.0.0.1", port=55467, user="messenger", password=password, dbname="messenger"
|
||||
)
|
||||
database = "student_tests_" + uuid4().hex
|
||||
with psycopg.connect(admin, autocommit=True) as conn:
|
||||
conn.execute(sql.SQL("CREATE DATABASE {}").format(sql.Identifier(database)))
|
||||
config = Settings(
|
||||
make_conninfo(admin, dbname=database),
|
||||
"redis://127.0.0.1:56386/1",
|
||||
broker_url="redis://127.0.0.1:56387/1",
|
||||
s3_bucket="messenger-attachments-tests",
|
||||
s3_access_key=(ROOT / "secrets/s3_test_access_key_id").read_text().strip(),
|
||||
s3_secret_key=(ROOT / "secrets/s3_test_secret_access_key").read_text().strip(),
|
||||
)
|
||||
migrate(config.database_url, ROOT / "migrations")
|
||||
with pytest.MonkeyPatch.context() as environment:
|
||||
environment.setenv("S3_ENDPOINT", config.s3_endpoint)
|
||||
environment.setenv("S3_BUCKET", config.s3_bucket)
|
||||
environment.setenv("S3_ACCESS_KEY_ID_FILE", str(ROOT / "secrets/s3_test_access_key_id"))
|
||||
environment.setenv(
|
||||
"S3_SECRET_ACCESS_KEY_FILE", str(ROOT / "secrets/s3_test_secret_access_key")
|
||||
)
|
||||
yield config
|
||||
with psycopg.connect(admin, autocommit=True) as conn:
|
||||
conn.execute(sql.SQL("DROP DATABASE {} WITH (FORCE)").format(sql.Identifier(database)))
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client(settings):
|
||||
with psycopg.connect(settings.database_url) as conn:
|
||||
conn.execute("TRUNCATE users,chats,chat_members,messages,sessions CASCADE")
|
||||
with TestClient(create_app(settings), raise_server_exceptions=False) as client:
|
||||
client.app.state.cache.client.flushdb()
|
||||
yield client
|
||||
|
||||
|
||||
def basic(username, password=PASSWORD):
|
||||
return {
|
||||
"Authorization": "Basic " + base64.b64encode((username + ":" + password).encode()).decode()
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def actors(client):
|
||||
result = []
|
||||
for name in ("alice", "bob", "eve"):
|
||||
r = client.post(
|
||||
"/api/v1/users", json={"username": name, "display_name": name, "password": PASSWORD}
|
||||
)
|
||||
assert r.status_code == 201
|
||||
session = client.post("/api/v1/auth/sessions", headers=basic(name))
|
||||
assert session.status_code == 201
|
||||
result.append({**r.json(), "token": session.json()["token"]})
|
||||
return result
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def room(client, actors):
|
||||
response = client.post(
|
||||
"/api/v1/chats",
|
||||
headers={"Authorization": "Bearer " + actors[0]["token"]},
|
||||
json={"title": "Чат 👋", "member_ids": [u["id"] for u in actors[:2]]},
|
||||
)
|
||||
assert response.status_code == 201
|
||||
return response.json()
|
||||
@@ -0,0 +1,299 @@
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from uuid import uuid4
|
||||
|
||||
import psycopg
|
||||
import pytest
|
||||
from conftest import PASSWORD, ROOT, basic
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from messenger.app import create_app
|
||||
from messenger.auth import basic_credentials
|
||||
from messenger.migrate import migrate
|
||||
|
||||
|
||||
def headers(actor):
|
||||
return {"Authorization": "Bearer " + actor["token"], "Idempotency-Key": uuid4().hex}
|
||||
|
||||
|
||||
def error(response, status, code):
|
||||
assert response.status_code == status, response.text
|
||||
body = response.json()["error"]
|
||||
assert set(body) == {"code", "message", "request_id"}
|
||||
assert body["code"] == code
|
||||
assert body["request_id"] == response.headers["X-Request-Id"]
|
||||
assert len(response.headers.get_list("X-Request-Id")) == 1
|
||||
if status == 401:
|
||||
assert response.headers["WWW-Authenticate"].startswith(("Basic", "Bearer"))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("body", ["{", "null", "[]", '"text"', '{"username":'])
|
||||
def test_malformed_json(client, body):
|
||||
error(
|
||||
client.post("/api/v1/users", content=body, headers={"Content-Type": "application/json"}),
|
||||
400,
|
||||
"invalid_request",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"change",
|
||||
[
|
||||
{"username": "ab"},
|
||||
{"username": "a" * 33},
|
||||
{"username": "Upper"},
|
||||
{"username": "кириллица"},
|
||||
{"username": "valid\n"},
|
||||
{"username": 123},
|
||||
{"username": True},
|
||||
{"display_name": 123},
|
||||
{"display_name": None},
|
||||
{"display_name": " \n\t"},
|
||||
{"display_name": "😀" * 101},
|
||||
{"display_name": "\ud800"},
|
||||
{"password": "short"},
|
||||
{"password": "😀" * 129},
|
||||
{"password": 123},
|
||||
{"password": " " * 12},
|
||||
{"password": "\ud800" * 12},
|
||||
{"extra": True},
|
||||
],
|
||||
)
|
||||
def test_strict_registration(client, change):
|
||||
data = {"username": "valid", "display_name": "Имя", "password": PASSWORD, **change}
|
||||
response = client.post(
|
||||
"/api/v1/users", content=json.dumps(data), headers={"Content-Type": "application/json"}
|
||||
)
|
||||
error(response, 400, "invalid_request")
|
||||
|
||||
|
||||
def test_unicode_password_hash_and_authentication(client, settings, actors):
|
||||
for u in actors:
|
||||
assert "password" not in u and "password_hash" not in u
|
||||
assert re.fullmatch(r"[A-Za-z0-9_-]{43}", u["token"])
|
||||
with psycopg.connect(settings.database_url) as conn:
|
||||
encoded = conn.execute("SELECT password_hash FROM users WHERE username='alice'").fetchone()[
|
||||
0
|
||||
]
|
||||
assert encoded.startswith("$argon2id$") and PASSWORD not in encoded
|
||||
assert (
|
||||
conn.execute("SELECT octet_length(token_hash) FROM sessions LIMIT 1").fetchone()[0]
|
||||
== 32
|
||||
)
|
||||
bad = client.post("/api/v1/auth/sessions", headers=basic("alice", "wrong-password"))
|
||||
absent = client.post("/api/v1/auth/sessions", headers=basic("unknown", "wrong-password"))
|
||||
error(bad, 401, "unauthorized")
|
||||
error(absent, 401, "unauthorized")
|
||||
assert bad.json()["error"]["message"] == absent.json()["error"]["message"]
|
||||
assert bad.headers["WWW-Authenticate"] == 'Basic realm="messenger", charset="UTF-8"'
|
||||
for value in (
|
||||
{},
|
||||
basic("alice"),
|
||||
{"X-User-Id": actors[0]["id"]},
|
||||
{"Authorization": "Bearer unknown"},
|
||||
):
|
||||
error(client.get("/api/v1/users/me", headers=value), 401, "unauthorized")
|
||||
actual = client.get(
|
||||
"/api/v1/users/me", headers={**headers(actors[0]), "X-User-Id": actors[2]["id"]}
|
||||
).json()
|
||||
assert actual["id"] == actors[0]["id"]
|
||||
error(
|
||||
client.post("/api/v1/auth/sessions", headers=basic("alice"), json={}),
|
||||
400,
|
||||
"invalid_request",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value", [None, "Bearer a", "Basic !", "Basic YWxpY2U=", "Basic /w=="])
|
||||
def test_invalid_basic(value):
|
||||
with pytest.raises(Exception) as caught:
|
||||
basic_credentials(value)
|
||||
assert caught.value.status == 401
|
||||
|
||||
|
||||
def test_registration_race(client, settings):
|
||||
def register(_):
|
||||
return client.post(
|
||||
"/api/v1/users",
|
||||
json={"username": "same", "display_name": "Алиса", "password": PASSWORD},
|
||||
).status_code
|
||||
|
||||
with ThreadPoolExecutor(max_workers=2) as pool:
|
||||
assert sorted(pool.map(register, range(10))) == [201] + [409] * 9
|
||||
with psycopg.connect(settings.database_url) as conn:
|
||||
assert conn.execute("SELECT count(*) FROM users WHERE username='same'").fetchone()[0] == 1
|
||||
|
||||
|
||||
def test_acl_validation_and_transaction_rollback(client, actors, room, settings):
|
||||
path = "/api/v1/chats/" + room["id"]
|
||||
for suffix in ("", "/messages"):
|
||||
error(client.get(path + suffix, headers=headers(actors[2])), 404, "not_found")
|
||||
error(client.get("/api/v1/chats/not-a-uuid"), 401, "unauthorized")
|
||||
error(
|
||||
client.get("/api/v1/chats/not-a-uuid", headers=headers(actors[0])), 400, "invalid_request"
|
||||
)
|
||||
with psycopg.connect(settings.database_url) as conn:
|
||||
before = conn.execute("SELECT count(*) FROM chats").fetchone()[0]
|
||||
for members in ([actors[0]["id"], str(uuid4())], [actors[0]["id"]] * 2, [actors[1]["id"]], []):
|
||||
error(
|
||||
client.post(
|
||||
"/api/v1/chats",
|
||||
headers=headers(actors[0]),
|
||||
json={"title": "rollback", "member_ids": members},
|
||||
),
|
||||
400,
|
||||
"invalid_request",
|
||||
)
|
||||
with psycopg.connect(settings.database_url) as conn:
|
||||
assert conn.execute("SELECT count(*) FROM chats").fetchone()[0] == before
|
||||
assert conn.execute("SELECT count(*) FROM chat_members").fetchone()[0] == 2
|
||||
for body in (
|
||||
{"text": ""},
|
||||
{"text": " \t"},
|
||||
{"text": "😀" * 4001},
|
||||
{"text": 123},
|
||||
{"text": None},
|
||||
{"text": "ok", "sender_id": actors[1]["id"]},
|
||||
):
|
||||
error(
|
||||
client.post(path + "/messages", headers=headers(actors[0]), json=body),
|
||||
400,
|
||||
"invalid_request",
|
||||
)
|
||||
for query in (
|
||||
"limit=0",
|
||||
"limit=101",
|
||||
"limit=true",
|
||||
"limit=1.0",
|
||||
"limit=1&limit=2",
|
||||
"cursor=",
|
||||
"cursor=bad!",
|
||||
"cursor=e30",
|
||||
"cursor=x&cursor=y",
|
||||
):
|
||||
error(
|
||||
client.get(path + "/messages?" + query, headers=headers(actors[0])),
|
||||
400,
|
||||
"invalid_request",
|
||||
)
|
||||
|
||||
|
||||
def test_concurrent_messages_and_keyset_ties(client, actors, room, settings):
|
||||
path = f"/api/v1/chats/{room['id']}/messages"
|
||||
|
||||
def send(i):
|
||||
r = client.post(path, headers=headers(actors[i % 2]), json={"text": f"m{i} 😀"})
|
||||
assert r.status_code == 201
|
||||
return r.json()
|
||||
|
||||
with ThreadPoolExecutor(max_workers=10) as pool:
|
||||
sent = list(pool.map(send, range(100)))
|
||||
with psycopg.connect(settings.database_url) as conn:
|
||||
conn.execute(
|
||||
"UPDATE messages SET created_at='2026-10-17T12:00:00Z' WHERE chat_id=%s", (room["id"],)
|
||||
)
|
||||
for m in sent:
|
||||
m["created_at"] = "2026-10-17T12:00:00.000000Z"
|
||||
found, cursor = [], None
|
||||
while True:
|
||||
query = {"limit": 17 if cursor else 1}
|
||||
if cursor:
|
||||
query["cursor"] = cursor
|
||||
response = client.get(path, params=query, headers=headers(actors[1]))
|
||||
assert response.status_code == 200
|
||||
page = response.json()
|
||||
found.extend(page["items"])
|
||||
cursor = page["next_cursor"]
|
||||
if not cursor:
|
||||
break
|
||||
assert len(found) == len({x["id"] for x in found}) == 100
|
||||
assert found == sorted(sent, key=lambda x: (x["created_at"], x["id"]))
|
||||
first_cursor = client.get(path + "?limit=1", headers=headers(actors[0])).json()["next_cursor"]
|
||||
error(
|
||||
client.get("/api/v1/chats", params={"cursor": first_cursor}, headers=headers(actors[0])),
|
||||
400,
|
||||
"invalid_request",
|
||||
)
|
||||
|
||||
|
||||
def test_cache_hit_flush_and_revocation(client, actors, monkeypatch):
|
||||
user = actors[0]
|
||||
url = "/api/v1/users/me"
|
||||
assert client.get(url, headers=headers(user)).status_code == 200
|
||||
repository = client.app.state.repository
|
||||
original = repository.get_user
|
||||
|
||||
def must_be_cached(*args):
|
||||
raise AssertionError("cache hit should avoid fetching immutable user again")
|
||||
|
||||
monkeypatch.setattr(repository, "get_user", must_be_cached)
|
||||
assert client.get(url, headers=headers(user)).status_code == 200
|
||||
monkeypatch.setattr(repository, "get_user", original)
|
||||
cache = client.app.state.cache.client
|
||||
digest = hashlib.sha256(user["token"].encode()).digest()
|
||||
key = client.app.state.cache.key(digest)
|
||||
assert 0 < cache.ttl(key) <= 3600
|
||||
stale = cache.get(key)
|
||||
cache.flushdb()
|
||||
assert client.get(url, headers=headers(user)).status_code == 200
|
||||
assert client.delete("/api/v1/auth/sessions/current", headers=headers(user)).status_code == 204
|
||||
cache.set(key, stale, ex=3600) # A resurrected stale cache entry must not resurrect a session.
|
||||
error(client.get(url, headers=headers(user)), 401, "unauthorized")
|
||||
error(
|
||||
client.delete("/api/v1/auth/sessions/current", headers=headers(user)), 401, "unauthorized"
|
||||
)
|
||||
|
||||
|
||||
def test_expiry_and_api_restart(client, actors, settings, monkeypatch, tmp_path):
|
||||
database_file, cache_file = tmp_path / "database_url", tmp_path / "cache_url"
|
||||
database_file.write_text(settings.database_url)
|
||||
cache_file.write_text(settings.cache_url)
|
||||
monkeypatch.setenv("DATABASE_URL_FILE", str(database_file))
|
||||
monkeypatch.setenv("CACHE_URL_FILE", str(cache_file))
|
||||
monkeypatch.setenv("SESSION_TTL_SECONDS", "2")
|
||||
with TestClient(create_app()) as short:
|
||||
response = short.post("/api/v1/auth/sessions", headers=basic("alice"))
|
||||
assert response.status_code == 201
|
||||
token = {"Authorization": "Bearer " + response.json()["token"]}
|
||||
assert short.get("/api/v1/users/me", headers=token).status_code == 200
|
||||
with TestClient(create_app(settings)) as restarted:
|
||||
assert restarted.get("/api/v1/users/me", headers=token).status_code == 200
|
||||
time.sleep(2.1)
|
||||
error(restarted.get("/api/v1/users/me", headers=token), 401, "unauthorized")
|
||||
|
||||
|
||||
def test_migrations_idempotent_and_detect_change(client, settings, tmp_path):
|
||||
migrate(settings.database_url, ROOT / "migrations")
|
||||
modified = tmp_path / "001_initial.sql"
|
||||
modified.write_text((ROOT / "migrations/001_initial.sql").read_text() + "\n-- changed")
|
||||
with pytest.raises(RuntimeError, match="checksum changed"):
|
||||
migrate(settings.database_url, tmp_path)
|
||||
|
||||
|
||||
def test_logs_and_safe_errors(client, monkeypatch, caplog):
|
||||
canary = "PRIVATE_BODY_CANARY_123"
|
||||
with caplog.at_level("INFO", logger="messenger"):
|
||||
r = client.post(
|
||||
"/api/v1/users",
|
||||
json={"username": "logging", "display_name": canary, "password": PASSWORD},
|
||||
)
|
||||
assert r.status_code == 201
|
||||
assert canary not in caplog.text and PASSWORD not in caplog.text
|
||||
|
||||
def broken(*args):
|
||||
raise RuntimeError(canary)
|
||||
|
||||
monkeypatch.setattr(client.app.state.authentication, "register", broken)
|
||||
response = client.post(
|
||||
"/api/v1/users", json={"username": "broken", "display_name": "x", "password": PASSWORD}
|
||||
)
|
||||
error(response, 500, "internal_error")
|
||||
assert canary not in response.text and canary not in caplog.text
|
||||
error(
|
||||
client.post("/api/v1/users", content="{}", headers={"Content-Type": "text/plain"}),
|
||||
415,
|
||||
"unsupported_media_type",
|
||||
)
|
||||
@@ -0,0 +1,137 @@
|
||||
import json
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from dataclasses import replace
|
||||
from uuid import uuid4
|
||||
|
||||
import psycopg
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
from test_api import error, headers
|
||||
|
||||
from messenger.app import create_app
|
||||
|
||||
|
||||
@pytest.mark.parametrize("key", [None, "", "a b", "/", "a" * 129])
|
||||
def test_message_key_required(client, actors, room, key):
|
||||
h = {"Authorization": "Bearer " + actors[0]["token"]}
|
||||
if key is not None:
|
||||
h["Idempotency-Key"] = key
|
||||
error(
|
||||
client.post(
|
||||
f"/api/v1/chats/{room['id']}/messages", headers=h, json={"text": "key validation"}
|
||||
),
|
||||
400,
|
||||
"invalid_request",
|
||||
)
|
||||
|
||||
|
||||
def test_replay_scope_conflict_and_fresh_authorization(client, actors, room):
|
||||
path = f"/api/v1/chats/{room['id']}/messages"
|
||||
key = "Scope.Key:123"
|
||||
h = {**headers(actors[0]), "Idempotency-Key": key}
|
||||
first = client.post(path, headers=h, json={"text": "exact whitespace \n"})
|
||||
assert first.status_code == 201
|
||||
repeat = client.post(
|
||||
path,
|
||||
headers={**h, "Content-Type": "application/json"},
|
||||
content=json.dumps({"text": "exact whitespace \n"}, indent=3),
|
||||
)
|
||||
assert repeat.status_code == 201 and repeat.json() == first.json()
|
||||
error(client.post(path, headers=h, json={"text": "different"}), 409, "conflict")
|
||||
outsider = {**headers(actors[2]), "Idempotency-Key": key}
|
||||
error(
|
||||
client.post(path, headers=outsider, json={"text": "exact whitespace \n"}), 404, "not_found"
|
||||
)
|
||||
other_user = {**headers(actors[1]), "Idempotency-Key": key}
|
||||
own = client.post(path, headers=other_user, json={"text": "another author"})
|
||||
assert own.status_code == 201 and own.json()["id"] != first.json()["id"]
|
||||
chat = client.post(
|
||||
"/api/v1/chats",
|
||||
headers=headers(actors[0]),
|
||||
json={"title": "another", "member_ids": [actors[0]["id"]]},
|
||||
).json()
|
||||
other_chat = client.post(
|
||||
f"/api/v1/chats/{chat['id']}/messages", headers=h, json={"text": "different chat"}
|
||||
)
|
||||
assert other_chat.status_code == 201 and other_chat.json()["id"] != first.json()["id"]
|
||||
assert client.delete("/api/v1/auth/sessions/current", headers=h).status_code == 204
|
||||
error(client.post(path, headers=h, json={"text": "exact whitespace \n"}), 401, "unauthorized")
|
||||
|
||||
|
||||
def test_concurrent_replay_across_independent_apps(client, actors, room, settings):
|
||||
path = f"/api/v1/chats/{room['id']}/messages"
|
||||
h = {**headers(actors[0]), "Idempotency-Key": uuid4().hex}
|
||||
with TestClient(create_app(settings)) as second:
|
||||
assert second.app.state.instance_id != client.app.state.instance_id
|
||||
|
||||
def send(i):
|
||||
c = client if i % 2 else second
|
||||
r = c.post(path, headers=h, json={"text": "same concurrent body"})
|
||||
assert r.status_code == 201
|
||||
return r.json(), r.headers["X-Instance-Id"]
|
||||
|
||||
with ThreadPoolExecutor(max_workers=10) as pool:
|
||||
results = list(pool.map(send, range(30)))
|
||||
assert len({item[0]["id"] for item in results}) == 1
|
||||
assert len({item[1] for item in results}) == 2
|
||||
assert len(client.get(path, headers=headers(actors[1])).json()["items"]) == 1
|
||||
|
||||
|
||||
def test_side_effect_and_result_rollback_together(client, actors, room, settings, monkeypatch):
|
||||
repository = client.app.state.repository
|
||||
original = repository._save_request
|
||||
|
||||
def fail_after_message_insert(*args):
|
||||
raise RuntimeError("injected between message and result")
|
||||
|
||||
monkeypatch.setattr(repository, "_save_request", fail_after_message_insert)
|
||||
path = f"/api/v1/chats/{room['id']}/messages"
|
||||
h = {**headers(actors[0]), "Idempotency-Key": uuid4().hex}
|
||||
error(
|
||||
client.post(path, headers=h, json={"text": "retry after rollback"}), 500, "internal_error"
|
||||
)
|
||||
with psycopg.connect(settings.database_url) as conn:
|
||||
assert conn.execute("SELECT count(*) FROM messages").fetchone()[0] == 0
|
||||
assert conn.execute("SELECT count(*) FROM message_requests").fetchone()[0] == 0
|
||||
monkeypatch.setattr(repository, "_save_request", original)
|
||||
result = client.post(path, headers=h, json={"text": "retry after rollback"})
|
||||
assert result.status_code == 201
|
||||
assert (
|
||||
client.post(path, headers=h, json={"text": "retry after rollback"}).json() == result.json()
|
||||
)
|
||||
|
||||
|
||||
def test_expired_key_can_be_reused_without_deleting_old_message(client, actors, room, settings):
|
||||
path = f"/api/v1/chats/{room['id']}/messages"
|
||||
h = {**headers(actors[0]), "Idempotency-Key": uuid4().hex}
|
||||
first = client.post(path, headers=h, json={"text": "old"}).json()
|
||||
with psycopg.connect(settings.database_url) as conn:
|
||||
conn.execute(
|
||||
"UPDATE message_requests SET created_at=clock_timestamp()-interval '2 days', "
|
||||
"expires_at=clock_timestamp()-interval '1 day'"
|
||||
)
|
||||
second = client.post(path, headers=h, json={"text": "new"})
|
||||
assert second.status_code == 201 and second.json()["id"] != first["id"]
|
||||
assert len(client.get(path, headers=headers(actors[0])).json()["items"]) == 2
|
||||
|
||||
|
||||
@pytest.mark.parametrize("optimized", [True, False])
|
||||
def test_read_modes_preserve_public_result(client, actors, room, settings, optimized):
|
||||
with TestClient(create_app(replace(settings, read_autocommit=optimized))) as c:
|
||||
c.get("/api/v1/users/me", headers=headers(actors[0]))
|
||||
pool = c.app.state.repository.pool
|
||||
before = pool.get_stats().get("requests_num", 0)
|
||||
response = c.get(f"/api/v1/chats/{room['id']}/messages", headers=headers(actors[0]))
|
||||
assert response.status_code == 200 and response.json()["items"] == []
|
||||
assert pool.get_stats()["requests_num"] - before == 3
|
||||
|
||||
|
||||
def test_instance_header_covers_health_errors_and_success(client):
|
||||
values = [
|
||||
client.get("/health/live"),
|
||||
client.get("/health/ready"),
|
||||
client.get("/api/v1/users/me"),
|
||||
client.get("/not-a-route"),
|
||||
]
|
||||
assert len({r.headers["X-Instance-Id"] for r in values}) == 1
|
||||
assert all(len(r.headers.get_list("X-Instance-Id")) == 1 for r in values)
|
||||
@@ -0,0 +1,357 @@
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import struct
|
||||
import zlib
|
||||
from dataclasses import replace
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from types import SimpleNamespace
|
||||
from urllib.parse import quote
|
||||
from uuid import uuid4
|
||||
|
||||
import jsonschema
|
||||
import pytest
|
||||
from conftest import ROOT
|
||||
from PIL import Image
|
||||
from test_api import error, headers
|
||||
|
||||
from messenger.broker import GROUP, STREAM, Broker, Job
|
||||
from messenger.cleanup import collect
|
||||
from messenger.domain import DomainError
|
||||
from messenger.images import variants
|
||||
from messenger.pipeline import Pipeline
|
||||
from messenger.relay import publish_one
|
||||
from messenger.worker import handle
|
||||
|
||||
PNG = (ROOT / "tests/fixtures/sample.png").read_bytes()
|
||||
|
||||
|
||||
def upload(client, actor, room, data=PNG, media="image/png", name="Фото.png"):
|
||||
return client.post(
|
||||
f"/api/v1/chats/{room['id']}/attachments?filename={quote(name, safe='')}",
|
||||
headers={**headers(actor), "Content-Type": media},
|
||||
content=data,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def broker(settings):
|
||||
b = Broker(settings)
|
||||
assert settings.broker_url.endswith("/1")
|
||||
b.client.flushdb()
|
||||
b.group()
|
||||
yield b
|
||||
b.client.flushdb()
|
||||
b.close()
|
||||
|
||||
|
||||
def process(client, settings, broker):
|
||||
db = client.app.state.repository
|
||||
assert publish_one(db, broker, settings)
|
||||
id, fields = broker.client.xreadgroup(GROUP, "unit", {STREAM: ">"}, count=1)[0][1][0]
|
||||
jsonschema.validate(
|
||||
json.loads(fields["job"]),
|
||||
json.loads((ROOT / "contracts/jobs.schema.json").read_text()),
|
||||
format_checker=jsonschema.FormatChecker(),
|
||||
)
|
||||
handle(db, broker, client.app.state.attachments.store, settings, id, fields["job"])
|
||||
|
||||
|
||||
def test_queued_message_ready_and_all_variants(client, actors, room, settings, broker):
|
||||
r = upload(client, actors[0], room)
|
||||
assert r.status_code == 202 and r.json()["status"] == "queued"
|
||||
a = r.json()
|
||||
assert not a["variants"] and a["sha256"] == hashlib.sha256(PNG).hexdigest()
|
||||
path = "/api/v1/attachments/" + a["id"]
|
||||
error(client.get(path + "/content", headers=headers(actors[0])), 409, "conflict")
|
||||
key = uuid4().hex
|
||||
msg = client.post(
|
||||
f"/api/v1/chats/{room['id']}/messages",
|
||||
json={"text": "", "attachment_ids": [a["id"]]},
|
||||
headers={**headers(actors[0]), "Idempotency-Key": key},
|
||||
)
|
||||
assert msg.status_code == 201
|
||||
for suffix in ("", "/content"):
|
||||
error(client.get(path + suffix, headers=headers(actors[2])), 404, "not_found")
|
||||
error(client.get(path + suffix), 401, "unauthorized")
|
||||
process(client, settings, broker)
|
||||
ready = client.get(path, headers=headers(actors[1])).json()
|
||||
assert ready["status"] == "ready" and ready["error_code"] is None
|
||||
assert {v["name"] for v in ready["variants"]} == {"thumbnail", "crop", "optimized"}
|
||||
assert client.get(path + "/content", headers=headers(actors[1])).content == PNG
|
||||
for v in ready["variants"]:
|
||||
content = client.get(path + "/content?variant=" + v["name"], headers=headers(actors[1]))
|
||||
assert content.status_code == 200 and content.headers["content-type"] == v["content_type"]
|
||||
assert len(content.content) == v["size_bytes"]
|
||||
assert hashlib.sha256(content.content).hexdigest() == v["sha256"]
|
||||
with Image.open(io.BytesIO(content.content)) as image:
|
||||
assert image.size == (v["width"], v["height"])
|
||||
assert not image.getexif()
|
||||
with client.app.state.repository.connection() as conn:
|
||||
j = conn.execute(
|
||||
"SELECT * FROM attachment_jobs WHERE attachment_id=%s", (a["id"],)
|
||||
).fetchone()
|
||||
job = Job(
|
||||
schema_version=1,
|
||||
type="attachment.process.v1",
|
||||
job_id=str(j["job_id"]),
|
||||
attachment_id=a["id"],
|
||||
pipeline_version=1,
|
||||
attempt=1,
|
||||
created_at=a["created_at"],
|
||||
)
|
||||
sid = broker.publish(job)
|
||||
handle(
|
||||
client.app.state.repository,
|
||||
broker,
|
||||
client.app.state.attachments.store,
|
||||
settings,
|
||||
sid,
|
||||
job.model_dump_json(exclude_none=True),
|
||||
)
|
||||
assert client.get(path, headers=headers(actors[0])).json() == ready
|
||||
assert broker.client.xlen(STREAM) == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", ["", "../a.png", "a\\b.png", "a\x00.png", "\n.png", "x" * 256])
|
||||
def test_bad_filename(client, actors, room, name):
|
||||
error(upload(client, actors[0], room, name=name), 400, "invalid_request")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"data,media",
|
||||
[
|
||||
(b"", "image/png"),
|
||||
(b"bad", "image/png"),
|
||||
(PNG[:30], "image/png"),
|
||||
(PNG, "image/jpeg"),
|
||||
(PNG, "text/plain"),
|
||||
],
|
||||
)
|
||||
def test_bad_image(client, actors, room, data, media):
|
||||
error(upload(client, actors[0], room, data, media), 415, "unsupported_media_type")
|
||||
|
||||
|
||||
def test_upload_limits_and_pixel_bomb(client, actors, room):
|
||||
error(upload(client, actors[0], room, b"x" * (10485760 + 1)), 413, "payload_too_large")
|
||||
error(
|
||||
upload(client, actors[0], room, (b"x" * 1048576 for _ in range(11))),
|
||||
413,
|
||||
"payload_too_large",
|
||||
)
|
||||
data = bytearray(PNG)
|
||||
data[16:24] = struct.pack(">II", 10000, 10000)
|
||||
data[29:33] = struct.pack(">I", zlib.crc32(data[12:29]) & 0xFFFFFFFF)
|
||||
error(upload(client, actors[0], room, bytes(data)), 413, "payload_too_large")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("size", [(320, 200), (17, 11), (1, 400), (400, 1), (1281, 713)])
|
||||
def test_transform_dimensions_alpha_and_exif(size):
|
||||
raw = io.BytesIO()
|
||||
Image.new("RGBA", size, (1, 2, 3, 0)).save(raw, "PNG")
|
||||
results = {name: (data, dims, media) for name, data, dims, media in variants(raw, 1)}
|
||||
scale = max(size)
|
||||
assert results["thumbnail"][1] == tuple(max(1, n * min(256, scale) // scale) for n in size)
|
||||
assert results["crop"][1] == (128, 128)
|
||||
assert max(results["optimized"][1]) <= 1280
|
||||
with Image.open(io.BytesIO(results["thumbnail"][0])) as image:
|
||||
assert image.getpixel((0, 0)) == (255, 255, 255)
|
||||
|
||||
|
||||
def test_exif_orientation_and_original_unchanged():
|
||||
raw = io.BytesIO()
|
||||
exif = Image.Exif()
|
||||
exif[274] = 6
|
||||
Image.new("RGB", (320, 200), "red").save(raw, "JPEG", exif=exif)
|
||||
before = raw.getvalue()
|
||||
result = list(variants(raw, 1))
|
||||
assert result[0][2] == (160, 256) and raw.getvalue() == before
|
||||
for _, data, _, _ in result:
|
||||
with Image.open(io.BytesIO(data)) as image:
|
||||
assert not image.getexif()
|
||||
|
||||
|
||||
def test_failed_retry_replay_and_ready_replay_rejected(
|
||||
client, actors, room, settings, broker, monkeypatch
|
||||
):
|
||||
a = upload(client, actors[0], room).json()
|
||||
store, db = client.app.state.attachments.store, client.app.state.repository
|
||||
original = store.put
|
||||
monkeypatch.setattr(
|
||||
store, "put", lambda *_: (_ for _ in ()).throw(DomainError(503, "unavailable", "test"))
|
||||
)
|
||||
process(client, replace(settings, job_max_attempts=1), broker)
|
||||
path = "/api/v1/attachments/" + a["id"]
|
||||
assert client.get(path, headers=headers(actors[0])).json()["error_code"] == "retry_exhausted"
|
||||
error(
|
||||
client.post(
|
||||
f"/api/v1/chats/{room['id']}/messages",
|
||||
json={"text": "", "attachment_ids": [a["id"]]},
|
||||
headers={**headers(actors[0]), "Idempotency-Key": uuid4().hex},
|
||||
),
|
||||
409,
|
||||
"conflict",
|
||||
)
|
||||
Pipeline(db, settings).replay(a["id"])
|
||||
monkeypatch.setattr(store, "put", original)
|
||||
process(client, settings, broker)
|
||||
assert client.get(path, headers=headers(actors[0])).json()["status"] == "ready"
|
||||
with pytest.raises(ValueError):
|
||||
Pipeline(db, settings).replay(a["id"])
|
||||
|
||||
|
||||
def test_invalid_envelope_quarantined(client, settings, broker):
|
||||
sid = broker.client.xadd(STREAM, {"job": '{"url":"http://untrusted.invalid/"}'})
|
||||
handle(
|
||||
client.app.state.repository,
|
||||
broker,
|
||||
client.app.state.attachments.store,
|
||||
settings,
|
||||
sid,
|
||||
'{"url":"http://untrusted.invalid/"}',
|
||||
)
|
||||
assert broker.client.xlen("invalid-jobs") == 1
|
||||
assert "url" not in json.dumps(broker.client.xrange("invalid-jobs"))
|
||||
|
||||
|
||||
def test_staging_failure_has_no_accepted_job(client, actors, room, monkeypatch):
|
||||
store = client.app.state.attachments.store
|
||||
monkeypatch.setattr(
|
||||
store, "put", lambda *_: (_ for _ in ()).throw(DomainError(503, "unavailable", "test"))
|
||||
)
|
||||
error(upload(client, actors[0], room), 503, "unavailable")
|
||||
with client.app.state.repository.connection() as conn:
|
||||
assert conn.execute("SELECT count(*) AS n FROM attachment_jobs").fetchone()["n"] == 0
|
||||
|
||||
|
||||
def test_gc_minimum_retention(client):
|
||||
with pytest.raises(ValueError):
|
||||
collect(client.app.state.repository, client.app.state.attachments.store, 0, True)
|
||||
|
||||
|
||||
def test_gc_fences_paused_upload_before_job_commit(client, actors, room, settings, monkeypatch):
|
||||
db, store = client.app.state.repository, client.app.state.attachments.store
|
||||
p = Pipeline(db, settings)
|
||||
id = str(uuid4())
|
||||
key = "staging/" + id
|
||||
p.reserve(id, key)
|
||||
store.put(key, io.BytesIO(PNG), len(PNG), "image/png", hashlib.sha256(PNG).hexdigest())
|
||||
# Simulated S3 object age; actual private test-bucket deletion and DB locks are real.
|
||||
old = datetime.now(UTC) - timedelta(seconds=301)
|
||||
head = store.head(key)
|
||||
head["LastModified"] = old
|
||||
objects = [{"Key": key, "ETag": head["ETag"], "Size": len(PNG), "LastModified": old}]
|
||||
monkeypatch.setattr(store, "head", lambda _: head)
|
||||
monkeypatch.setattr(
|
||||
store.client,
|
||||
"get_paginator",
|
||||
lambda _: SimpleNamespace(paginate=lambda **_: [{"Contents": objects}]),
|
||||
)
|
||||
assert collect(db, store, 300, True)["protected"] == 1
|
||||
with db.connection(write=True) as conn:
|
||||
conn.execute(
|
||||
"UPDATE upload_reservations SET expires_at=clock_timestamp()-interval '1 second' "
|
||||
"WHERE id=%s",
|
||||
(id,),
|
||||
)
|
||||
assert collect(db, store, 300, False)["eligible"] == 1
|
||||
assert collect(db, store, 300, True)["deleted"] == 1
|
||||
with pytest.raises(DomainError) as expired:
|
||||
p.accept(
|
||||
id,
|
||||
room["id"],
|
||||
actors[0]["id"],
|
||||
"paused.png",
|
||||
"image/png",
|
||||
len(PNG),
|
||||
hashlib.sha256(PNG).hexdigest(),
|
||||
key,
|
||||
)
|
||||
assert expired.value.status == 503 and db.attachment(id) is None
|
||||
|
||||
|
||||
def test_version_fence_rejects_old_writer(client, actors, room, settings, broker):
|
||||
a = upload(client, actors[0], room).json()
|
||||
db, store = client.app.state.repository, client.app.state.attachments.store
|
||||
with db.connection() as conn:
|
||||
j = conn.execute(
|
||||
"SELECT * FROM attachment_jobs WHERE attachment_id=%s", (a["id"],)
|
||||
).fetchone()
|
||||
envelope = Job(
|
||||
schema_version=1,
|
||||
type="attachment.process.v1",
|
||||
job_id=str(j["job_id"]),
|
||||
attachment_id=a["id"],
|
||||
pipeline_version=1,
|
||||
attempt=1,
|
||||
created_at=a["created_at"],
|
||||
)
|
||||
pipeline = Pipeline(db, settings)
|
||||
outcome, old = pipeline.claim(envelope)
|
||||
assert outcome == "claimed"
|
||||
pipeline.replay(a["id"], new_version=True)
|
||||
process(client, settings, broker)
|
||||
current = db.attachment(a["id"])
|
||||
assert current["pipeline_version"] == 2 and current["storage_state"] == "ready"
|
||||
assert not pipeline.publish(old, "objects/old-result", [])
|
||||
handle(
|
||||
db,
|
||||
broker,
|
||||
store,
|
||||
settings,
|
||||
broker.publish(envelope),
|
||||
envelope.model_dump_json(exclude_none=True),
|
||||
)
|
||||
assert db.attachment(a["id"])["object_key"] == current["object_key"]
|
||||
|
||||
|
||||
def test_foreign_chat_attachment_is_not_linkable(client, actors, room):
|
||||
a = upload(client, actors[0], room).json()
|
||||
other = client.post(
|
||||
"/api/v1/chats",
|
||||
headers=headers(actors[0]),
|
||||
json={"title": "other", "member_ids": [actors[0]["id"]]},
|
||||
).json()
|
||||
error(
|
||||
client.post(
|
||||
f"/api/v1/chats/{other['id']}/messages",
|
||||
headers={**headers(actors[0]), "Idempotency-Key": uuid4().hex},
|
||||
json={"text": "", "attachment_ids": [a["id"]]},
|
||||
),
|
||||
404,
|
||||
"not_found",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"patch",
|
||||
[
|
||||
{"schema_version": True},
|
||||
{"traceparent": None},
|
||||
{"tracestate": None},
|
||||
{"url": "https://untrusted.invalid/"},
|
||||
{"job_id": "not-a-uuid"},
|
||||
{"pipeline_version": 0},
|
||||
{"attempt": 0},
|
||||
{"created_at": "2026-10-17T12:00:00"},
|
||||
{"traceparent": "invalid"},
|
||||
{"type": "download.anything"},
|
||||
],
|
||||
)
|
||||
def test_job_schema_rejects_invalid_carrier(client, settings, broker, patch):
|
||||
payload = {
|
||||
"schema_version": 1,
|
||||
"type": "attachment.process.v1",
|
||||
"job_id": str(uuid4()),
|
||||
"attachment_id": str(uuid4()),
|
||||
"pipeline_version": 1,
|
||||
"attempt": 1,
|
||||
"created_at": "2026-10-17T12:00:00Z",
|
||||
**patch,
|
||||
}
|
||||
data = json.dumps(payload)
|
||||
sid = broker.client.xadd(STREAM, {"job": data})
|
||||
handle(
|
||||
client.app.state.repository, broker, client.app.state.attachments.store, settings, sid, data
|
||||
)
|
||||
assert broker.client.xrange("invalid-jobs")[0][1]["reason"] == "invalid_envelope"
|
||||
Reference in New Issue
Block a user