ЛР 6 — БВТ2301 — Васильев — на 5 #28
+7
-20
@@ -1,20 +1,7 @@
|
||||
.git
|
||||
.github
|
||||
.env
|
||||
.env.*
|
||||
!.env.example
|
||||
secrets
|
||||
certs
|
||||
volumes
|
||||
data
|
||||
backups
|
||||
*.key
|
||||
*.pem
|
||||
*.p12
|
||||
*.pfx
|
||||
*.jks
|
||||
.venv
|
||||
__pycache__
|
||||
node_modules
|
||||
evidence
|
||||
.DS_Store
|
||||
*
|
||||
!pyproject.toml
|
||||
!uv.lock
|
||||
!src/
|
||||
!src/**
|
||||
!migrations/
|
||||
!migrations/**
|
||||
|
||||
+5
-29
@@ -1,30 +1,6 @@
|
||||
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
|
||||
# These are logical interface suggestions; map actual framework names in REPORT.md.
|
||||
APP_HOST=0.0.0.0
|
||||
APP_PORT=8080
|
||||
APP_ENV=lab
|
||||
PUBLIC_PORT=8086
|
||||
GRAFANA_PORT=13006
|
||||
SESSION_TTL_SECONDS=3600
|
||||
# Choose credentials locally. File paths contain no secret values.
|
||||
DATABASE_URL_FILE=/run/secrets/database_url
|
||||
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
|
||||
SESSION_STORE=postgres
|
||||
CACHE_URL_FILE=/run/secrets/cache_url
|
||||
API_REPLICAS=2
|
||||
IDEMPOTENCY_TTL_SECONDS=86400
|
||||
S3_ENDPOINT=http://s3:9000
|
||||
S3_REGION=us-east-1
|
||||
S3_BUCKET=messenger-attachments
|
||||
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
|
||||
MAX_UPLOAD_BYTES=10485760
|
||||
MAX_IMAGE_PIXELS=40000000
|
||||
BROKER_URL_FILE=/run/secrets/broker_url
|
||||
WORKER_CONCURRENCY=2
|
||||
JOB_MAX_ATTEMPTS=5
|
||||
JOB_TIMEOUT_SECONDS=60
|
||||
STAGING_RETENTION_SECONDS=86400
|
||||
OTEL_SERVICE_NAME=messenger-api
|
||||
# Worker must set a distinct service.name.
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318
|
||||
OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
|
||||
OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab
|
||||
OTEL_ENABLED=1
|
||||
OTEL_SAMPLE_RATIO=1
|
||||
OTEL_METRIC_INTERVAL_MS=5000
|
||||
|
||||
@@ -30,3 +30,5 @@ coverage/
|
||||
.vscode/
|
||||
# Keep small, sanitized evidence summaries in evidence/ under version control.
|
||||
# Inspect git diff --cached: ignore rules are not a secret scanner.
|
||||
.pytest_cache/
|
||||
.ruff_cache/
|
||||
|
||||
+100
@@ -0,0 +1,100 @@
|
||||
# ЛР6: как объяснить работу на защите
|
||||
|
||||
Васильев Никита, БВТ2301. Работа продолжает тот же мессенджер: пользователи, чаты, сообщения и вложения. Здесь мы добавили способы понять, **что происходит внутри нескольких процессов**, когда запрос медленный или обработка вложения не закончилась.
|
||||
|
||||
## Что изменилось после ЛР5
|
||||
|
||||
Раньше уже работали балансировщик, несколько API, PostgreSQL, приватное S3, диспетчер, брокер и worker. API сохраняет задание в БД вместе с метаданными вложения; диспетчер доставляет уведомление в брокер; worker читает исходное изображение, создаёт результаты и обновляет состояние задания. БД остаётся источником состояния задания, а брокер помогает его доставить.
|
||||
|
||||
В ЛР6 API, диспетчер и worker дополнительно отправляют телеметрию через OpenTelemetry Collector. Метрики хранятся в Prometheus, логи — в Loki, трассировки — в Tempo. Grafana показывает их через заранее описанные источники данных и dashboard. Collector принимает и пересылает сигналы; он не заменяет все три хранилища.
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
U[Клиент] --> LB[Балансировщик]
|
||||
LB --> A[API]
|
||||
A --> DB[(PostgreSQL)]
|
||||
A --> S[(S3)]
|
||||
DB --> D[Диспетчер]
|
||||
D --> B[Брокер]
|
||||
B --> W[Worker]
|
||||
W --> DB
|
||||
W --> S
|
||||
A -. телеметрия .-> C[Collector]
|
||||
D -. телеметрия .-> C
|
||||
W -. телеметрия .-> C
|
||||
C --> P[Prometheus / Loki / Tempo]
|
||||
P --> G[Grafana]
|
||||
```
|
||||
|
||||
Телеметрия не является хранилищем сообщений или заданий. При её отказе мессенджер должен продолжать работу; часть диагностических данных при переполнении ограниченных очередей может потеряться.
|
||||
|
||||
## Три сигнала на одном примере
|
||||
|
||||
Пользователь загрузил картинку, но долго видит `processing`.
|
||||
|
||||
- **Метрика** показывает масштаб: очередь растёт, возраст самого старого задания увеличивается, скорость завершения уменьшилась. Это числа за интервал, удобные для графика и алерта.
|
||||
- **Лог** показывает событие: worker встретил временную ошибку и назначил повтор. У события есть время, сервис, severity и идентификаторы трассировки.
|
||||
- **Trace** показывает путь конкретной загрузки: HTTP API → публикация в брокер → получение worker → SQL/S3. Каждый участок — span со своим временем и связью с родительским участком.
|
||||
|
||||
`trace_id` связывает участки одной операции, `span_id` обозначает отдельный участок. Один trace может проходить через разные процессы и несколько попыток выполнения задания. Это не означает, что все участки происходят одновременно.
|
||||
|
||||
## Как очередь сохраняет причинную связь
|
||||
|
||||
При входящем HTTP-запросе извлекается W3C `traceparent` и при наличии `tracestate`. HTTP span имеет тип SERVER. Контекст сохраняется вместе с заданием в PostgreSQL, поэтому переживает завершение HTTP-запроса и перезапуск диспетчера. При публикации контекст попадает в сообщение брокера, а worker извлекает его перед созданием CONSUMER span. Для SQL и S3 создаются дочерние spans.
|
||||
|
||||
Сохранение trace-контекста помогает диагностике. Оно не заменяет транзакции, outbox, идемпотентность и контроль повторов: эти механизмы обеспечивают корректность обработки.
|
||||
|
||||
## Какие показатели мы наблюдаем
|
||||
|
||||
- Частота HTTP-запросов, доля ответов 5xx и распределение времени ответа.
|
||||
- Успехи, ошибки и повторы заданий; длительность обработки worker.
|
||||
- Время от принятия задания до готовности вложения. `accepted_at` устанавливается при добавлении задания после записи исходника в S3; это отметка перед commit транзакции, с небольшим зазором до фактического подтверждения.
|
||||
- Количество незавершённых заданий и возраст самого старого. Они позволяют заметить остановку worker даже без новых ошибок.
|
||||
- Занятые соединения и ожидающие обращения к пулу БД; занятость worker.
|
||||
- CPU, память и счётчики отправки телеметрии.
|
||||
|
||||
Актуальные SLI, пороги SLO, окна, результаты срабатывания алертов и измеренные числа приведены в REPORT.md и первичных файлах экспериментов. Их нельзя заменять цифрами из другого запуска.
|
||||
|
||||
## Что требовалось для оценки 5
|
||||
|
||||
Нужны работающие возможности предыдущей работы, реальные три сигнала и Grafana provisioning, сквозной контекст между API и worker, подходящие гистограммы, SLI/SLO, проверяемые алерты и ограниченное поведение exporter при отказе backend.
|
||||
|
||||
Выбран дополнительный трек **«Цена наблюдаемости»**: сравнить выключенную телеметрию, полное семплирование и несколько меньших долей при одинаковой предлагаемой нагрузке и ресурсах. Измеряются задержки и ошибки клиентов, CPU/RAM, объём отправленных данных и spans. Отдельно проверяется, сохранились ли редкие ошибки и медленные запросы.
|
||||
|
||||
Sampling уменьшает число записанных traces; при этом метрики запросов должны продолжать учитывать запросы независимо от выбранных traces. Входящий sampled-контекст учитывается ParentBased sampler: доля корневых traces не обязательно равна доле всех запросов с контекстом извне.
|
||||
|
||||
## Ответы на контрольные и частые вопросы
|
||||
|
||||
**1. Чем лог, метрика и trace отвечают на разные вопросы об одном инциденте?** Метрика показывает, сколько и как часто; лог — какое событие произошло; trace — какие связанные участки конкретной операции заняли время или завершились ошибкой.
|
||||
|
||||
**2. Почему `chat_id` в labels может исчерпать хранилище?** Каждый новый набор значений labels создаёт временной ряд. Тысячи чатов × маршруты × статусы × реплики дают множество рядов; гистограмма дополнительно создаёт ряд на каждый bucket. Идентификаторы конкретных операций уместнее в разрешённых полях логов и traces. Для метрик используем шаблон маршрута, метод и класс статуса.
|
||||
|
||||
**3. Как очередь переносит причинную связь между процессами?** Передаём trace-контекст в сообщении; в нашей реализации он также хранится в задании БД. Worker извлекает контекст и создаёт связанный span. Обычная передача данных через брокер сама по себе этого не делает.
|
||||
|
||||
**4. Почему среднее latency не заменяет p99?** Среднее может скрыть редкие очень медленные запросы. p99 — граница, в которую укладывается примерно 99% наблюдений выбранного интервала; оставшийся хвост может быть важен для пользователей.
|
||||
|
||||
**5. Почему нельзя усреднять p99 реплик?** Реплики могли обработать разное число запросов и иметь разные распределения. Среднее двух квантилей не даёт квантиль объединённых данных. Складываем скорости bucket-счётчиков по репликам с сохранением `le`, затем вычисляем `histogram_quantile`.
|
||||
|
||||
**6. Что такое bucket?** Счётчик наблюдений до определённой границы. В обычной гистограмме Prometheus buckets накопительные: запрос в 8 мс учитывается во всех границах ≥8 мс. Квантиль внутри bucket оценивается приближённо, поэтому нужны границы, подходящие нашим миллисекундным задержкам и SLO.
|
||||
|
||||
**7. Чем SLI отличается от SLO и алерта?** SLI — измеряемый показатель, например доля успешных запросов за окно. SLO — целевая граница этого показателя. Алерт — правило уведомления, например длительное превышение доли ошибок или возраста очереди. Правило может предупредить об угрозе SLO ещё до большого числа жалоб.
|
||||
|
||||
**8. Почему нельзя судить об очереди только по числу ошибок?** Остановленный worker не выполняет задания и не создаёт новые ошибки. Очередь и её возраст при этом растут. Поэтому наблюдаем и состояние ожидающей работы.
|
||||
|
||||
**9. Как различить остановку worker и проблему БД?** Сопоставляем очередь, занятость worker, HTTP-ошибки, пул БД и spans зависимостей. Растущая очередь при остановленном worker отличается от таймаутов SQL, которые затрагивают также API. Один график не доказывает причину.
|
||||
|
||||
**10. Что будет при падении Tempo или Loki?** Collector и SDK повторяют отправку в пределах таймаутов и ограниченных очередей; после исчерпания лимитов диагностические данные теряются. Бизнес-запросы и durable задания не должны зависеть от успешной записи телеметрии. Это проверяется реальным отключением backend.
|
||||
|
||||
**11. Что sampling не гарантирует?** Head sampling принимает решение в начале trace и может не сохранить редкую ошибку, обнаруженную позднее. Доля 10% не означает «все ошибки сохранены». Для этого нужны другие правила или tail sampling с отдельными затратами и ограничениями.
|
||||
|
||||
**12. Что именно означает измеренный объём экспорта?** В исследовании учитываются байты protobuf-тел запросов SDK, включая фактические повторные отправки. Это не полный трафик сети: HTTP/TLS-заголовки и служебные пакеты в эту величину не входят. Принятые Collector spans и сохранённые Tempo traces — отдельные стадии.
|
||||
|
||||
**13. Sampling и потеря spans — одно и то же?** Нет. Несохранённые sampler traces исключены намеренно. Потеря при переполнении очереди или неудачном экспорте происходит после решения записывать. Счётчики и выводы должны различать эти причины.
|
||||
|
||||
**14. Что запрещено писать в телеметрию?** Пароли, токены, заголовки авторизации, содержимое сообщений и изображений, приватные ключи и подписанные URL. Для диагностики достаточно технических имён операций, исходов, задержек и безопасных идентификаторов связи.
|
||||
|
||||
**15. Что эта работа не делает отказоустойчивым?** Она помогает обнаружить и объяснить сбой. Единственная БД, балансировщик и хранилища всё ещё могут отказать. Dashboard, alert и trace сами по себе не создают репликацию или резервную копию.
|
||||
|
||||
## Короткий ответ «что ты сделал»
|
||||
|
||||
«Я добавил в мессенджер метрики, структурные логи и распределённые трассировки через OpenTelemetry. Связал HTTP-загрузку с диспетчером и worker, подготовил Grafana и алерты. Проверил остановку worker, задержку БД и отказ хранилищ телеметрии. Для дополнительного трека сравнил стоимость разных режимов sampling; конкретные результаты и границы измерений находятся в отчёте».
|
||||
+19
@@ -0,0 +1,19 @@
|
||||
FROM python:3.12-slim@sha256:646fb0bca3dd3ea1bcc6feb72c17ed16eed6e10cffc732fcc1478bd3e7f02d7b AS build
|
||||
COPY --from=ghcr.io/astral-sh/uv:0.11.7 /uv /usr/local/bin/uv
|
||||
WORKDIR /app
|
||||
COPY pyproject.toml uv.lock ./
|
||||
RUN uv sync --locked --no-dev --no-install-project --python /usr/local/bin/python
|
||||
|
||||
FROM python:3.12-slim@sha256:646fb0bca3dd3ea1bcc6feb72c17ed16eed6e10cffc732fcc1478bd3e7f02d7b
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends libjpeg-turbo-progs=1:2.1.5-4 && rm -rf /var/lib/apt/lists/*
|
||||
ARG VCS_REF=uncommitted
|
||||
LABEL org.opencontainers.image.revision=$VCS_REF
|
||||
ENV PYTHONUNBUFFERED=1 PYTHONDONTWRITEBYTECODE=1 PYTHONPATH=/app/src PATH=/app/.venv/bin:$PATH
|
||||
WORKDIR /app
|
||||
COPY --from=build /app/.venv /app/.venv
|
||||
COPY src ./src
|
||||
COPY migrations ./migrations
|
||||
USER 10001:10001
|
||||
EXPOSE 8080
|
||||
HEALTHCHECK --interval=3s --timeout=2s --start-period=5s --retries=5 CMD ["python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=1)"]
|
||||
CMD ["python", "-m", "messenger"]
|
||||
@@ -28,3 +28,20 @@ test-persistence:
|
||||
test-failover:
|
||||
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
|
||||
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
# Student implementation commands; teacher targets remain unchanged.
|
||||
PUBLIC_PORT ?= 8086
|
||||
BASE_URL = http://127.0.0.1:$(PUBLIC_PORT)
|
||||
bootstrap:
|
||||
python3 scripts/bootstrap.py
|
||||
up: bootstrap
|
||||
VCS_REF=$$(git rev-parse HEAD) docker compose up -d --build --wait
|
||||
migrate:
|
||||
docker compose run --rm migrate
|
||||
down:
|
||||
docker compose down
|
||||
student-test:
|
||||
BASE_URL=$(BASE_URL) uv run --locked pytest -q
|
||||
lint:
|
||||
uv run --locked ruff check src student-tests research
|
||||
.PHONY: bootstrap up down migrate student-test lint
|
||||
|
||||
@@ -1,37 +1,264 @@
|
||||
# Отчёт — лабораторная 6
|
||||
# ЛР6. Наблюдаемость мессенджера
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
**Васильев Никита · БВТ2301 · целевая оценка 5 · трек 1 «Цена наблюдаемости».**
|
||||
|
||||
Перенесена собственная реализация ЛР5: PostgreSQL, две API за HAProxy, приватное S3, транзакционный outbox, dispatcher, отдельные Valkey broker и worker. Обязательные README/COURSE/contracts/tests сохранены. Реализация и анализ подготовлены с помощью ИИ; результаты получены реальными локальными запусками. Публикация выполняется отдельно.
|
||||
|
||||
## Запуск из чистого clone
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
Нужны Docker Engine с Compose v2, Git, uv и Python 3.12. Ориентир для Docker: 4 CPU, 8 GiB RAM. Bootstrap не требует секретов из соседнего репозитория.
|
||||
|
||||
## Архитектура и решения
|
||||
```sh
|
||||
uv sync --locked --python 3.12
|
||||
make up
|
||||
make check validate PYTHON=.venv/bin/python
|
||||
make test
|
||||
make student-test
|
||||
make lint
|
||||
```
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
`make up` создаёт ignored `secrets/` и `.env`, сохраняя уже существующие значения, собирает образ, выполняет отдельный migrator и поднимает весь стенд. Нужные данные хранятся в volumes. Обычная остановка — `make down`, без удаления volumes. Повтор миграций — `make migrate`. Не редактируйте уже применённые SQL-файлы.
|
||||
|
||||
## Доказательства критериев
|
||||
| Интерфейс | Локальный адрес |
|
||||
|---|---|
|
||||
| Бизнес-API через LB | http://127.0.0.1:8086 |
|
||||
| Grafana | http://127.0.0.1:13006 |
|
||||
| Prometheus | http://127.0.0.1:19096 |
|
||||
| Loki API | http://127.0.0.1:13106 |
|
||||
| Tempo API | http://127.0.0.1:13206 |
|
||||
| Collector OTLP HTTP | http://127.0.0.1:14318 |
|
||||
| Collector application metrics | http://127.0.0.1:19464/metrics |
|
||||
| Collector self metrics | http://127.0.0.1:18888/metrics |
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
Административные/диагностические порты привязаны к loopback. API, БД, broker и S3 не публикуют собственные порты. Grafana предоставляет локальный anonymous Viewer; пароль административного пользователя генерируется в `secrets/grafana_password`, login form отключён. Входные TLS/mTLS относятся к следующей лабораторной.
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
## Схема сигналов и роль компонентов
|
||||
|
||||
## Эксперименты и отказы
|
||||
```text
|
||||
API / dispatcher / worker → OTLP HTTP protobuf → Collector
|
||||
├→ Prometheus: numeric metrics
|
||||
├→ Loki: structured logs
|
||||
└→ Tempo: traces
|
||||
Grafana → queries к трём backend по datasource UID
|
||||
```
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
Collector принимает, ограничивает память/буферизацию и экспортирует. Он не является долговременной БД. Grafana визуализирует и запрашивает backend, а не заменяет их хранение. Datasource UID зафиксированы: `prometheus`, `loki`, `tempo`. Provisioning и dashboard JSON находятся в `infra/grafana/`; dashboard UID — `messenger-overview`. Ручная сборка панелей не требуется.
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
SDK/providers создаются отдельно в каждом процессе. Resource attributes: `service.name` (`messenger-api`, `messenger-worker`, `messenger-dispatcher`), `service.version=lab6-v1`, случайный `service.instance.id`, `deployment.environment.name=lab`. Публичный `X-Instance-Id` API совпадает с её resource instance ID, но `X-Request-Id` остаётся идентификатором конкретного обращения.
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
Версии Python-зависимостей закреплены `uv.lock`: OTel SDK и OTLP HTTP exporter 1.45.2, semantic conventions 0.66b2, Pillow 13.0.0. Collector contrib 0.123.0, Prometheus 3.2.1, Loki 3.4.2, Tempo 2.7.2, Grafana 11.6.0. Предыдущие PostgreSQL 17.10, Valkey 8.1.3, SeaweedFS 4.15 и HAProxy 3.2 сохранены; image refs и реальные digests фиксируются в runtime receipts. Базовый Python и часть инфраструктурных образов закреплены digest, остальные — явным версионным тегом.
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
## Метрики: реальные имена, единицы и агрегация
|
||||
|
||||
SDK-инструменты `messenger.http.*` переводятся Prometheus exporter в имена ниже. Дублирующего HTTP auto-instrumentation нет.
|
||||
|
||||
| Prometheus family | Значение / единица |
|
||||
|---|---|
|
||||
| `messenger_http_requests_total` | Counter обслуженных бизнес-HTTP запросов; method, route, status_class |
|
||||
| `messenger_http_duration_seconds` | Histogram HTTP handler duration, секунды |
|
||||
| `messenger_jobs_total` | Counter реальных попыток worker по job_type/outcome: success/retry/failed/superseded; дубли не считаются успешной работой |
|
||||
| `messenger_job_duration_seconds` | Histogram длительности попытки, секунды |
|
||||
| `messenger_attachment_ready_duration_seconds` | Histogram от принятия job до устойчивой публикации ready, секунды |
|
||||
| `messenger_queue_depth` | Gauge outstanding queued+processing из PostgreSQL, jobs |
|
||||
| `messenger_queue_oldest_age_seconds` | Gauge возраста старейшей outstanding job, секунды |
|
||||
| `messenger_db_pool_used`, `messenger_db_pool_waiting` | Gauge занятых соединений и текущего ожидания пула |
|
||||
| `messenger_worker_busy`, `messenger_worker_capacity` | Gauge 0/1 и capacity=1 для serial worker |
|
||||
| `messenger_process_cpu_percent` | CPU процесса, 100% = один логический CPU |
|
||||
| `messenger_process_memory_bytes` | RSS родительского процесса, bytes; не RSS всех дочерних decoder процессов |
|
||||
| `messenger_http_trace_decisions_total` | Counter sampled/not_sampled решений HTTP spans |
|
||||
| `messenger_telemetry_export` | Диагностические cumulative observations SDK: названия `*_wire_bytes` в bytes, `spans_*` в spans, `*_http_sends` в отправках; значение различается label `observation` |
|
||||
|
||||
CPU/RAM дочерних процессов и cgroup всего контейнера измеряются отдельно Docker/cgroup receipts. Родительский RSS нельзя выдавать за пик памяти всего worker/API.
|
||||
|
||||
Для HTTP histogram заданы границы 0.001/0.0025/0.005/0.01/0.025/0.05/0.1/0.25/0.5/1/2.5/5/10 секунд. Для job/ready — 0.01/0.025/0.05/0.1/0.25/0.5/1/2.5/5/10/20/45/60/120 секунд. При агрегации сначала суммируются bucket rates нужных реплик по `le`, затем вызывается `histogram_quantile`; готовые p95 реплик не усредняются. Примеры фактических PromQL — в dashboard JSON.
|
||||
|
||||
`accepted_at` job устанавливается `clock_timestamp()` при INSERT после S3 PUT. Timestamp сохраняется одной транзакцией с job/outbox, непосредственно перед COMMIT; метрика включает небольшой интервал от записи timestamp до COMMIT, а точный commit timestamp не измеряется. Она не начинается с начала передачи HTTP-тела и не включает всю загрузку клиента. Исторические jobs при миграции получают старый created_at; эксперименты создают новые jobs.
|
||||
|
||||
Queue age — возраст старейшей outstanding job, а не p95 обработки и не число сообщений в stream. Наблюдение выполняет dispatcher независимо от worker. Молчание worker не скрывается за нулевым счётчиком ошибок; при остановке dispatcher сами наблюдения могут устареть, что требует отдельной проверки источника telemetry.
|
||||
|
||||
Метрики не имеют labels user_id/chat_id/message_id/attachment_id/request_id/filename/raw URL. Маршруты берутся из шаблонов FastAPI; unknown route фиксирован, неизвестные HTTP methods нормализуются в OTHER. Resource instance ID нужен для различения процессов; рост истории instance IDs ограничивается сроками хранения.
|
||||
|
||||
## Логи и распределённые traces
|
||||
|
||||
Structured JSON logs содержат timestamp, severity, service.name, event, trace_id/span_id при активном span. HTTP logs сохраняют request_id, method, route template, status и duration_ms. Worker пишет outcome попытки. Не записываются Authorization, session token, пароль, DSN, broker URL, содержимое сообщений/картинок, filename, S3 object key или signed URL.
|
||||
|
||||
HTTP middleware извлекает W3C traceparent/tracestate до вызова обработчика и создаёт **SERVER** span с тем же trace ID. Реальные PostgreSQL/S3 операции создают дочерние CLIENT spans без SQL-текста и секретных атрибутов. Исключения помечаются безопасным классом/кодом, без сериализации exception payload.
|
||||
|
||||
Upload сохраняет traceparent/tracestate в job (миграция 007). Dispatcher извлекает устойчивый контекст, создаёт PRODUCER span, inject выполняется в envelope перед XADD. Worker извлекает envelope, создаёт CONSUMER span; S3 чтение/запись и DB публикация находятся под ним. Retry не теряет контекст исходного upload. Consumer outcome=failed/retry помечается ERROR; duplicate/superseded не выдаются за success.
|
||||
|
||||
Публичные дополнительные заголовки `X-Trace-Id` и `X-Trace-Sampled` позволяют сравнить клиентское наблюдение с решением sampling. Они не являются токенами доступа и не заменяют X-Request-Id. Health probes и фоновое получение gauge не создают поток лишних бизнес-traces/logs.
|
||||
|
||||
В Loki по trace_id находится событие API/worker; derived field Grafana ссылается на Tempo. Trace может поступать несколькими batch, поэтому проверка ждёт всю цепочку SERVER/PRODUCER/CONSUMER, а не считает первый частичный ответ окончательным.
|
||||
|
||||
## SLI/SLO и alerts
|
||||
|
||||
SLI API = `1 - 5xx / все обслуженные бизнес-запросы API`, исключая health. Учебная цель — не менее 99% за скользящие 5 минут. 4xx относятся к корректно обработанным пользовательским ошибкам и не считаются отказом сервера. Ответы, сгенерированные самим LB до обращения к API, этой метрикой не покрываются; это не полная end-user availability.
|
||||
|
||||
Rules находятся в `infra/observability/alerts.yaml`, evaluation/scrape — каждые 5 секунд:
|
||||
|
||||
- MessengerHighErrorRate: 5xx ratio >5% за 1 минуту, `for: 10s`.
|
||||
- MessengerQueueLag: outstanding depth >0 и oldest age >15s, `for: 10s`.
|
||||
- MessengerTelemetryUnavailable: scrape Collector application metrics недоступен, `for: 10s`.
|
||||
|
||||
Это symptom alerts уровня 4, не multi-window burn-rate исследование другого трека. Локальное firing/resolved проверяется через Prometheus API; наружная доставка не настроена. Диагностика, восстановление и ограничения — [RUNBOOK.md](RUNBOOK.md).
|
||||
|
||||
## Ограниченная буферизация и отказ telemetry
|
||||
|
||||
SDK: очереди traces/logs по 256, batch до 64, задержка 500ms, сетевое ожидание 1s; metrics interval 5s. Collector: memory limiter160MiB/spike32MiB, контейнер256MiB, batch максимум256, очереди exporter по128, 1consumer, retry budget5s. Loki 384 MiB, Tempo 768 MiB с GOMEMLIMIT 512 MiB, Prometheus/Grafana по 256 MiB; CPU-лимиты и остальные роли заданы в compose.yaml.
|
||||
|
||||
Наблюдаемость не находится в транзакции бизнес-записи. Экспорт асинхронный; отказ backend не превращает успешное сообщение в ошибку. При длительной недоступности возможны потери telemetry. `sampled-out`, ошибки HTTP экспорта SDK, переполнение SDK очереди, отказы приёма Collector и неуспешные отправки Collector в backend — разные величины.
|
||||
|
||||
При длительном хранении маленьких блоков первоначальный лимит Tempo 384 MiB оказался недостаточным для compaction (реальный OOM). Он увеличен до 768 MiB с мягким Go memory budget 512 MiB. Collector отправляет traces по OTLP HTTP в Tempo 4318: HTTP повторно разрешает DNS после восстановления backend и не удерживает отрицательный gRPC DNS cache.
|
||||
|
||||
SDK считает реальные HTTP sends и длину отправляемого protobuf body, включая повторные sends; это не полный сетевой трафик с заголовками/TLS и не размер хранения в Tempo. Число sampled spans сверяется с exported/failed/in-flight/pending после quiescence; остаток отражает вытеснение ограниченной очереди. Для read-only наблюдения pending используется поле pinned SDK 1.45.2; отдельный regression test создаёт 1200 spans при отсутствующем Collector и проверяет bounded queue и учёт потерь.
|
||||
|
||||
Остановка приложения ограничивает ожидание observer и shutdown providers; зависимости закрываются после остановки observer. При исчерпании shutdown budget не обещается сохранение всех накопленных spans/logs.
|
||||
|
||||
## Проверки и исследование
|
||||
|
||||
Публичные smoke, собственные API/session/image/job tests, схема OpenAPI и lint выполняются с включённой telemetry. Отдельно проверяются реальные chains/logs/метрики/Grafana provisioning и приватность. Команды воспроизведения:
|
||||
|
||||
```sh
|
||||
uv run --locked python research/incident_suite.py
|
||||
uv run --locked python research/observability_cost.py
|
||||
make test-persistence ACTION_SCRIPT=scripts/restart.sh
|
||||
make test-failover ACTION_SCRIPT=scripts/stop-one.sh
|
||||
docker compose start api
|
||||
```
|
||||
|
||||
Incident driver временно останавливает worker, pause/unpause PostgreSQL и telemetry backend; finally восстанавливает зависимости. Для retry и редких ошибок используются только локальные SQL fault hooks с явным overlay, без публичных fault endpoint. Нагрузочные/жизненные циклы запускать последовательно без соседних исследований.
|
||||
|
||||
Методика цены наблюдаемости: off, on100%, on25%, on10%, по три повтора с вращением порядка. Один образ и одинаковые CPU/RAM, предложенная нагрузка40/с, 10s измерения после одинакового прогрева. Две upload ошибки500 на повтор создаются после устойчивого COMMIT через локальный hook; задания остаются и обрабатываются. Один краткий DB lock350ms создаёт редкие медленные чтения. Учитываются все статусы, клиентские p50/p95/p99, not-sent слоты, CPU/RAM, protobuf HTTP bytes, принятые Collector spans и SDK потери. Границы RPS не включают join sampler-потока Docker stats.
|
||||
|
||||
В режиме off отключены OTel providers, exporter и поток наблюдений; технические JSON stdout logs и no-op вызовы обёрток остаются. Это измерение цены включённого SDK и доставки сигналов относительно этого приложения, а не удаление всего logging-кода.
|
||||
|
||||
Перед baseline и после нагрузки выдерживается 10s quiescence; SDK counters читаются напрямую из Collector без дополнительного scrape delay Prometheus и фильтруются по двум текущим API resource instance IDs. При sampling видно, какие реальные редкие slow/error traces сохранены, а какие отсеяны. Все исходные результаты сохраняются с параметрами; локальные короткие серии не являются production SLA или максимальной пропускной способностью.
|
||||
|
||||
## Границы реализации
|
||||
|
||||
Один host, один LB, PostgreSQL, S3 и broker остаются SPOF. Volumes не заменяют независимый backup. Контекст tracing не служит аутентификацией. Head sampling может потерять редкую ошибку/медленный запрос, поскольку решение принято до знания исхода. Tail sampling и кластер backend не реализованы. Полный доступный бизнес-API и авторизация предыдущих работ сохранены; исправлена обработка одиночных Unicode surrogates без потери допустимого NUL/emoji.
|
||||
|
||||
## Первичные материалы
|
||||
|
||||
- [Collector configuration](https://opentelemetry.io/docs/collector/configuration/)
|
||||
- [Python SDK](https://opentelemetry.io/docs/languages/python/)
|
||||
- [Context propagation](https://opentelemetry.io/docs/concepts/context-propagation/)
|
||||
- [W3C Trace Context](https://www.w3.org/TR/trace-context/)
|
||||
- [Prometheus histograms](https://prometheus.io/docs/practices/histograms/)
|
||||
- [Loki native OTLP ingestion](https://grafana.com/docs/loki/latest/send-data/otel/)
|
||||
- [Grafana provisioning](https://grafana.com/docs/grafana/latest/administration/provisioning/)
|
||||
|
||||
## Проверенные отказы
|
||||
|
||||
Реальный прогон: [evidence/incidents.json](evidence/incidents.json), все четыре сценария `passed=true`.
|
||||
|
||||
| Сценарий | Что наблюдалось | Восстановление |
|
||||
|---|---|---|
|
||||
| Повторная обработка вложения | Одна контролируемая ошибка до эффекта; два CONSUMER spans в том же входящем trace, первый ERROR, затем ready; API/worker найдены в Loki | Автоматический retry, публикация результата один раз |
|
||||
| Worker остановлен, три принятых вложения | QueueLag firing через 34.739 с; depth 3, oldest 29.868 с, HTTP error ratio 0 | Все три ready; alert inactive через 14.131 с после запуска worker |
|
||||
| PostgreSQL приостановлен | 12 ограниченных по времени HTTP503; HighErrorRate firing через 29.150 с, наблюдаемый error ratio 0.533 | После unpause успешные запросы; alert inactive через 35.220 с |
|
||||
| Tempo и Loki остановлены | 200/200 HTTP200, вложение ready; failed span sends Collector 0→118, без OOM API/worker/Collector | После запуска backend свежее вложение даёт доступный trace; длительность всего сценария 39.125 с |
|
||||
|
||||
Время detection включает клиентское ожидание, scrape/evaluation и `for:10s`; это локальные
|
||||
наблюдения, не гарантированные верхние границы. Пауза БД приводит к ожидаемому503,
|
||||
а не к успешному ответу с потерянными данными. При telemetry outage потеря spans
|
||||
допустима и измерена; бизнес-очередь и DB остаются отдельными компонентами.
|
||||
|
||||
## Измерения трека 1: цена наблюдаемости
|
||||
|
||||
Итог: **12/12 завершённых профилей PASS**, по три повтора каждого режима.
|
||||
Raw: [observability-cost.json](evidence/observability-cost.json);
|
||||
производная сводка: [observability-cost-summary.json](evidence/observability-cost-summary.json);
|
||||
методика и команды: [research/README.md](research/README.md).
|
||||
Контрольные fault500 — ожидаемые ошибки эксперимента, 2/400=0.5% в каждом прогоне;
|
||||
остальные 398 ответов200, `not_sent=0`, неожиданных статусов нет.
|
||||
|
||||
Таблица: медианы трёх RPS/перцентилей; CPU — среднее выборок суммы всех контейнеров,
|
||||
RAM — максимум выборок суммы всех контейнеров среди трёх прогонов. CPU 100%=одно ядро.
|
||||
|
||||
| Режим | Успешных RPS | p50, мс | p95, мс | p99, мс | CPU, % | RAM sampled peak, MiB |
|
||||
|---|---:|---:|---:|---:|---:|---:|
|
||||
| off | 39.85 | 5.39 | 36.84 | 288.09 | 45.95 | 1175.8 |
|
||||
| full | 39.85 | 5.51 | 35.05 | 285.96 | 52.06 | 983.8 |
|
||||
| sample25 | 39.78 | 5.81 | 45.10 | 272.20 | 50.21 | 1232.9 |
|
||||
| sample10 | 39.84 | 6.04 | 30.33 | 286.31 | 53.40 | 1079.9 |
|
||||
|
||||
Поток SDK относится **только к двум текущим API**, а accepted/refused spans Collector —
|
||||
**ко всем сервисам**. Эти числа нельзя напрямую вычитать для вычисления потерь.
|
||||
Байты — медианы фактически отправленных protobuf bodies за профиль с завершением экспорта
|
||||
после нагрузки, включая HTTP retries; это не размер хранения и не сетевые заголовки.
|
||||
Metrics bytes включают периодические экспорты в окне ожидания.
|
||||
|
||||
| Режим | Trace bytes API | Log bytes API | Metric bytes API | Accepted spans Collector | Refused spans |
|
||||
|---|---:|---:|---:|---:|---:|
|
||||
| off | 0 | 0 | 0 | 0 | 0 |
|
||||
| full | 134268 | 228873 | 19200 | 840 | 0 |
|
||||
| sample25 | 39431 | 229445 | 18935 | 194 | 0 |
|
||||
| sample10 | 20769 | 228854 | 18694 | 82 | 0 |
|
||||
|
||||
Для нормировки только trace-потока на десятисекундное окно нагрузки: full 13.43 KB/s,
|
||||
25% — 3.94 KB/s, 10% — 2.08 KB/s (десятичные KB). Передача завершается асинхронно после окна.
|
||||
Это не измерение пропускной способности Collector и не суммарный ingest всех ролей.
|
||||
|
||||
Ниже суммы трёх прогонов; rare slow — GET с клиентской latency≥150мс, errors — два
|
||||
контролируемых500 в каждом прогоне. Число затронутых блокировкой запросов меняется
|
||||
в зависимости от очередей и планирования. Sampling решение наблюдается по заголовку
|
||||
SDK, а наличие trace отдельно проверяется в Tempo.
|
||||
|
||||
| Режим | Найдены rare traces | Найдены ошибки500 | Найдены slow reads |
|
||||
|---|---:|---:|---:|
|
||||
| off | 0/32 | 0/6 | 0/26 |
|
||||
| full | 39/39 | 6/6 | 33/33 |
|
||||
| sample25 | 10/31 | 2/6 | 8/25 |
|
||||
| sample10 | 5/32 | 1/6 | 4/26 |
|
||||
|
||||
Во всех завершённых профилях SDK export_failed=0, queue_lost=0 и Collector refused=0.
|
||||
Отсутствующие unsampled rare traces — сознательный head sampling, не ошибка exporter.
|
||||
При полном sampling найдены 39/39, при25% — 10/31, при10% — 5/32.
|
||||
|
||||
Выбран **25% head sampling** как измеренный компромисс: trace bytes меньше full на
|
||||
70.6%, но сохранено больше rare traces, чем при10%. Медиана суммарных
|
||||
API SDK body bytes (traces+logs+metrics) уменьшается с382335 до287666
|
||||
байт, то есть на24.8%; это существенно меньше снижения
|
||||
только traces, потому что log export остаётся большим. Снижение до10% даёт ещё экономию
|
||||
trace bytes, но в этих прогонах сохранило только одну из шести ошибок.
|
||||
|
||||
Сбалансированный профиль проверен тремя прогонами и поставляется как
|
||||
`compose.balanced.yaml`. По умолчанию остаётся100%, чтобы на защите можно было
|
||||
показать любой свежий trace. Ни 25%, ни 10% не гарантируют сохранения ошибки:
|
||||
решение принимается до знания исхода. Для такой гарантии понадобилась бы отдельная
|
||||
политика tail sampling, которая здесь не реализована.
|
||||
|
||||
RPS ограничен генератором40/с и не является максимальной пропускной способностью.
|
||||
В CPU/RAM и latency нет устойчивого монотонного улучшения от sampling: на коротких
|
||||
окнах влияют DB lock, scheduling, cache и compaction backend. Данные подтверждают
|
||||
снижение trace ingest, а не доказанное ускорение API или уменьшение RAM.
|
||||
|
||||
Максимумы cgroup memory.peak среди записанных after snapshots: API 129.1 MiB,
|
||||
worker 113.1 MiB, Collector 64.2 MiB, Tempo 421.2 MiB; OOM kills 0. Peak относится к жизни
|
||||
контейнера, не только десятисекундному окну. Tempo peak объясняет, почему первоначальный
|
||||
лимит 384 MiB был недостаточен.
|
||||
|
||||
Срез размера данных во время исследования:367 users,140 chats,271 messages,55 attachments
|
||||
и55 jobs — [dataset-during-benchmark.json](evidence/dataset-during-benchmark.json).
|
||||
Данные добавляются синтетическими проверками; каждый профиль добавляет два вложения.
|
||||
Это маленькая учебная БД, не эксперимент на миллионах сообщений.
|
||||
|
||||
## Проверки и идентичность измеренного runtime
|
||||
|
||||
- [183 собственных теста PASS](evidence/tests-patched.txt), 58.39с, после обновления SDK 1.45.2/Pillow 13.0.0. После этого менялись bounded shutdown/readiness и инфраструктурная конфигурация; этот лог не выдаётся за финальный clean retest.
|
||||
- [Публичный smoke PASS](evidence/smoke-initial.txt) на первоначальном полном стенде.
|
||||
- [Проверка обязательного контракта](evidence/contract-final.txt), [OpenAPI](evidence/openapi-final.txt), [ruff](evidence/lint-final.txt) — PASS.
|
||||
- Реальные image IDs, лимиты и конфигурация Docker host: [runtime-environment.json](evidence/runtime-environment.json).
|
||||
- Измеренный image revision `548855e2c1a34bf8567a6245de0126dac8ed06a0`; конфигурация восстановления backend — commit `a531af3`. Драйверы исследования сохранены отдельными последующими коммитами, runtime Python при этом не менялся.
|
||||
- [measured-runtime-hashes.json](evidence/measured-runtime-hashes.json): SHA256 всех `src/**/*.py` в обеих API, worker и dispatcher совпадают с исходниками. [measured-source-hashes.json](evidence/measured-source-hashes.json) включает исходники и миграции. Это связывает измерения с реализацией, а не только с именем тега.
|
||||
|
||||
Прогоны исследования сохранялись по checkpoint и возобновлялись. Raw содержит ровно 12
|
||||
завершённых пар profile/repeat; незавершённые диагностические попытки не объявляются
|
||||
успешными измерениями. UTC started/finished включает паузы между продолжениями;
|
||||
latency и длительность каждого окна измеряются monotonic clock.
|
||||
|
||||
Для повторения на чистых volumes после bootstrap: `make up`, `make student-test`,
|
||||
`make test`, затем persistence/failover по teacher tests README и команды из research README.
|
||||
Последующие независимые проверки чистого финального commit следует читать в отдельном
|
||||
receipt вместе с его commit ID; заранее заявленного PASS для будущих запусков нет.
|
||||
|
||||
+31
@@ -0,0 +1,31 @@
|
||||
# Диагностика мессенджера
|
||||
|
||||
Все примеры относятся к локальному учебному стенду. Публичный API: http://127.0.0.1:8086. Grafana: http://127.0.0.1:13006, dashboard `Messenger: API and image pipeline`.
|
||||
|
||||
## API errors
|
||||
|
||||
1. На dashboard проверь Business RPS и долю 5xx. Health-запросы не входят в знаменатель. Пользовательские 4xx не являются отказом сервера.
|
||||
2. Найди структурный лог API с событием request, статусом 5xx и request_id; переход по trace_id открывает соответствующую трассу Tempo.
|
||||
3. Сравни server span и postgres.transaction/s3 child spans. Долгая DB span вместе с ростом pool waiting указывает на ожидание БД; медленный handler без долгой зависимости требует другого объяснения.
|
||||
4. Проверь состояние PostgreSQL. Для контролируемого `docker compose pause postgres` восстановление — `docker compose unpause postgres`.
|
||||
5. После восстановления убедись, что запросы успешны и MessengerHighErrorRate перешёл firing→inactive. Окно 1m сохраняет историю ошибок, поэтому alert не обязан исчезнуть мгновенно.
|
||||
|
||||
## Queue lag
|
||||
|
||||
1. Проверь Outstanding jobs и Oldest outstanding age. Это состояние БД, включающее queued/processing, а не скорость успешных worker.
|
||||
2. Найди worker logs и consumer traces; отсутствие новых consumer spans при растущем backlog вместе с остановленным worker объясняет инцидент.
|
||||
3. Проверь `docker compose ps worker dispatcher broker`. При контролируемой остановке: `docker compose start worker`.
|
||||
4. Убедись, что те же принятые attachment становятся ready, глубина падает до нуля, возраст обнуляется и MessengerQueueLag resolved.
|
||||
5. Если worker активен, проверь длительные S3/DB spans, failed/retry outcomes, процесс CPU/RAM и budget. Повторная попытка использует сохранённый trace context; parent producer/consumer связывают её с upload.
|
||||
|
||||
## Telemetry unavailable
|
||||
|
||||
1. Проверь Prometheus targets и Collector. Grafana сама не хранит метрики, логи и трассы.
|
||||
2. Для отсутствующих trace/log backend проверь Collector send_failed/queue и logs. SDK и Collector имеют ограниченные очереди; не обещается сохранение всей telemetry при долгом outage.
|
||||
3. Продолжи бизнес-запросы через API. Ошибка наблюдаемости не должна превращать успешную отправку сообщения в 5xx.
|
||||
4. Восстанови остановленные backend. Свежая telemetry должна появиться; потерянные записи не восстанавливаются из воздуха.
|
||||
5. Сравни cgroup memory.current/peak/events, реальные счётчики отправок и экспортных ошибок. Sampled-out span — решение sampling, а rejected/export_failed — проблема доставки; не смешивай их.
|
||||
|
||||
## Границы
|
||||
|
||||
Alerts локальные, отправки наружу нет. Один host и одиночные хранилища остаются SPOF. SLO — проверяемая учебная цель, а не production SLA. Для приватности не экспортируются содержимое сообщений/картинок, filename, авторизационные заголовки, пароли, database/broker URLs и S3 object keys.
|
||||
@@ -0,0 +1,11 @@
|
||||
# Данные и telemetry context ЛР6
|
||||
|
||||
Базовые таблицы пользователей, чатов, участников, сообщений, сессий, вложений и jobs/outbox сохранены из собственной ЛР5. PostgreSQL хранит устойчивое бизнес-состояние, S3 — исходные и производные bytes, broker — повторяемые уведомления.
|
||||
|
||||
Миграция007 добавляет nullable traceparent/tracestate в attachment_jobs. При enqueue текущий W3C context сохраняется в одной транзакции с job/outbox. Dispatcher не зависит от памяти API: даже после её crash извлекает parent context из job и inject контекст producer span в envelope. Повтор публикации/обработки сохраняет связь с upload. Текст сообщений/файлов и credentials через очередь и telemetry не передаются.
|
||||
|
||||
Миграция008 добавляет accepted_at; исторические rows получают created_at, новые jobs фиксируют clock_timestamp после S3 PUT непосредственно перед COMMIT. End-to-end ready histogram использует этот момент, включая небольшой pre-COMMIT gap; не начало передачи файла клиентом.
|
||||
|
||||
Telemetry context не влияет на бизнес-идемпотентность, lease/fencing, ACL или ключ S3. Claim/complete по-прежнему проверяют attachment/job под row locks. При retry исходный сохранённый context остаётся; новый producer/consumer span имеет другой span ID в той же причинной цепочке.
|
||||
|
||||
Метрики, логи и traces живут соответственно в Prometheus/Loki/Tempo volumes. Grafana хранит только свои настройки/состояние; dashboard/datasources provisioned из Git. Lifetime resource instance ID отличается при перезапуске процесса и используется для точного сравнения текущих API в исследовании.
|
||||
@@ -0,0 +1,12 @@
|
||||
# Optional measured profile: keep metrics/logs; retain 25% of new root traces.
|
||||
# ParentBased keeps the upstream sampling decision for distributed traces.
|
||||
services:
|
||||
api:
|
||||
environment:
|
||||
OTEL_SAMPLE_RATIO: '0.25'
|
||||
worker:
|
||||
environment:
|
||||
OTEL_SAMPLE_RATIO: '0.25'
|
||||
dispatcher:
|
||||
environment:
|
||||
OTEL_SAMPLE_RATIO: '0.25'
|
||||
@@ -0,0 +1,16 @@
|
||||
# Explicit local fault-injection overlay; normal compose has no enabled fault hooks.
|
||||
services:
|
||||
api:
|
||||
# Only the explicit test overlay permits API-container operator tools to inject broker messages.
|
||||
secrets: [database_url, cache_url, broker_url, s3_access_key, s3_secret_key]
|
||||
environment:
|
||||
ENABLE_FAULT_INJECTION: "1"
|
||||
JOB_LEASE_SECONDS: "8"
|
||||
worker:
|
||||
environment:
|
||||
ENABLE_FAULT_INJECTION: "1"
|
||||
JOB_LEASE_SECONDS: "8"
|
||||
dispatcher:
|
||||
environment:
|
||||
ENABLE_FAULT_INJECTION: "1"
|
||||
JOB_LEASE_SECONDS: "8"
|
||||
@@ -0,0 +1,8 @@
|
||||
# Explicit local experiment only; no public fault endpoint.
|
||||
services:
|
||||
worker:
|
||||
environment:
|
||||
ENABLE_FAULT_INJECTION: "1"
|
||||
api:
|
||||
environment:
|
||||
ENABLE_FAULT_INJECTION: "1"
|
||||
+422
@@ -0,0 +1,422 @@
|
||||
name: mtusi-vasilev-lab6
|
||||
x-application:
|
||||
image: ${API_IMAGE:-mtusi-vasilev:lab6}
|
||||
build: &id001
|
||||
context: .
|
||||
args:
|
||||
VCS_REF: ${VCS_REF:-uncommitted}
|
||||
environment: &id002
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
CACHE_URL_FILE: /run/secrets/cache_url
|
||||
BROKER_URL_FILE: /run/secrets/broker_url
|
||||
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
||||
JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-45}
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
S3_ENDPOINT: http://s3:8333
|
||||
S3_BUCKET: messenger
|
||||
S3_ACCESS_KEY_FILE: /run/secrets/s3_access_key
|
||||
S3_SECRET_KEY_FILE: /run/secrets/s3_secret_key
|
||||
POOL_MAX_SIZE: ${POOL_MAX_SIZE:-8}
|
||||
OTEL_ENABLED: ${OTEL_ENABLED:-1}
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT: http://collector:4318
|
||||
OTEL_SAMPLE_RATIO: ${OTEL_SAMPLE_RATIO:-1}
|
||||
OTEL_METRIC_INTERVAL_MS: ${OTEL_METRIC_INTERVAL_MS:-5000}
|
||||
secrets: &id006
|
||||
- database_url
|
||||
- cache_url
|
||||
- broker_url
|
||||
- s3_access_key
|
||||
- s3_secret_key
|
||||
cpus: ${API_CPUS:-0.5}
|
||||
mem_limit: ${API_MEMORY:-512m}
|
||||
pids_limit: 96
|
||||
read_only: true
|
||||
tmpfs: &id003
|
||||
- /tmp:size=16m,noexec,nosuid
|
||||
cap_drop: &id004
|
||||
- ALL
|
||||
security_opt: &id005
|
||||
- no-new-privileges:true
|
||||
stop_grace_period: 8s
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:17.10-alpine
|
||||
environment:
|
||||
POSTGRES_USER: messenger
|
||||
POSTGRES_DB: messenger
|
||||
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
|
||||
secrets:
|
||||
- postgres_password
|
||||
volumes:
|
||||
- postgres_data:/var/lib/postgresql/data
|
||||
healthcheck:
|
||||
test:
|
||||
- CMD-SHELL
|
||||
- pg_isready -U messenger -d messenger
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 15
|
||||
cpus: 1
|
||||
mem_limit: 768m
|
||||
command:
|
||||
- postgres
|
||||
- -c
|
||||
- fsync=on
|
||||
- -c
|
||||
- synchronous_commit=on
|
||||
- -c
|
||||
- full_page_writes=on
|
||||
cache:
|
||||
image: valkey/valkey:8.1.3-alpine
|
||||
command:
|
||||
- valkey-server
|
||||
- --save
|
||||
- ''
|
||||
- --appendonly
|
||||
- 'no'
|
||||
- --maxmemory
|
||||
- 64mb
|
||||
- --maxmemory-policy
|
||||
- allkeys-lru
|
||||
healthcheck:
|
||||
test:
|
||||
- CMD
|
||||
- valkey-cli
|
||||
- ping
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 15
|
||||
cpus: 0.5
|
||||
mem_limit: 128m
|
||||
broker:
|
||||
image: valkey/valkey:8.1.3-alpine
|
||||
command:
|
||||
- valkey-server
|
||||
- /run/secrets/broker_config
|
||||
secrets:
|
||||
- broker_config
|
||||
- broker_password
|
||||
volumes:
|
||||
- broker_data:/data
|
||||
cpus: 0.5
|
||||
mem_limit: 192m
|
||||
healthcheck:
|
||||
test:
|
||||
- CMD-SHELL
|
||||
- REDISCLI_AUTH=$$(cat /run/secrets/broker_password) valkey-cli ping
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 15
|
||||
dispatcher:
|
||||
image: ${API_IMAGE:-mtusi-vasilev:lab6}
|
||||
build: *id001
|
||||
environment: *id002
|
||||
secrets:
|
||||
- database_url
|
||||
- cache_url
|
||||
- broker_url
|
||||
cpus: 0.25
|
||||
mem_limit: 192m
|
||||
pids_limit: 96
|
||||
read_only: true
|
||||
tmpfs: *id003
|
||||
cap_drop: *id004
|
||||
security_opt: *id005
|
||||
stop_grace_period: 8s
|
||||
command:
|
||||
- python
|
||||
- -m
|
||||
- messenger.worker
|
||||
- --dispatcher
|
||||
healthcheck:
|
||||
disable: true
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
broker:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
worker:
|
||||
image: ${API_IMAGE:-mtusi-vasilev:lab6}
|
||||
build: *id001
|
||||
environment: *id002
|
||||
secrets: *id006
|
||||
cpus: 1
|
||||
mem_limit: 896m
|
||||
pids_limit: 96
|
||||
read_only: true
|
||||
tmpfs:
|
||||
- /tmp:size=32m,noexec,nosuid
|
||||
cap_drop: *id004
|
||||
security_opt: *id005
|
||||
stop_grace_period: 30s
|
||||
command:
|
||||
- python
|
||||
- -m
|
||||
- messenger.worker
|
||||
healthcheck:
|
||||
disable: true
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
broker:
|
||||
condition: service_healthy
|
||||
s3-bootstrap:
|
||||
condition: service_completed_successfully
|
||||
restart: unless-stopped
|
||||
migrate:
|
||||
image: ${API_IMAGE:-mtusi-vasilev:lab6}
|
||||
build: *id001
|
||||
environment: *id002
|
||||
secrets:
|
||||
- database_url
|
||||
cpus: ${API_CPUS:-0.5}
|
||||
mem_limit: ${API_MEMORY:-512m}
|
||||
pids_limit: 96
|
||||
read_only: true
|
||||
tmpfs: *id003
|
||||
cap_drop: *id004
|
||||
security_opt: *id005
|
||||
stop_grace_period: 8s
|
||||
command:
|
||||
- python
|
||||
- -m
|
||||
- messenger.migrate
|
||||
healthcheck:
|
||||
disable: true
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
api:
|
||||
image: ${API_IMAGE:-mtusi-vasilev:lab6}
|
||||
build: *id001
|
||||
environment: *id002
|
||||
secrets:
|
||||
- database_url
|
||||
- cache_url
|
||||
- s3_access_key
|
||||
- s3_secret_key
|
||||
cpus: ${API_CPUS:-0.5}
|
||||
mem_limit: ${API_MEMORY:-512m}
|
||||
pids_limit: 96
|
||||
read_only: true
|
||||
tmpfs: *id003
|
||||
cap_drop: *id004
|
||||
security_opt: *id005
|
||||
stop_grace_period: 8s
|
||||
deploy:
|
||||
replicas: ${API_REPLICAS:-2}
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
cache:
|
||||
condition: service_healthy
|
||||
s3-bootstrap:
|
||||
condition: service_completed_successfully
|
||||
storage:
|
||||
image: chrislusf/seaweedfs:4.15@sha256:1593bd2a98964f5d4df83fa7db2b3d4ee7b20190813cc5bf8c3200e0e70d346e
|
||||
command:
|
||||
- server
|
||||
- -dir=/data
|
||||
- -ip=storage
|
||||
- -ip.bind=0.0.0.0
|
||||
- -filer
|
||||
- -volume.max=32
|
||||
- -volume.preStopSeconds=0
|
||||
- -master.volumeSizeLimitMB=64
|
||||
volumes:
|
||||
- s3_data:/data
|
||||
- ./infra/filer.toml:/etc/seaweedfs/filer.toml:ro
|
||||
networks:
|
||||
- storage_backend
|
||||
cpus: 1
|
||||
mem_limit: 512m
|
||||
healthcheck:
|
||||
test:
|
||||
- CMD-SHELL
|
||||
- wget -q -O /dev/null http://127.0.0.1:9333/dir/status
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 20
|
||||
s3:
|
||||
image: chrislusf/seaweedfs:4.15@sha256:1593bd2a98964f5d4df83fa7db2b3d4ee7b20190813cc5bf8c3200e0e70d346e
|
||||
command:
|
||||
- s3
|
||||
- -filer=storage:8888
|
||||
- -ip.bind=0.0.0.0
|
||||
- -config=/run/secrets/s3_config
|
||||
- -port=8333
|
||||
- -port.iceberg=0
|
||||
secrets:
|
||||
- s3_config
|
||||
networks:
|
||||
- default
|
||||
- storage_backend
|
||||
depends_on:
|
||||
storage:
|
||||
condition: service_healthy
|
||||
cpus: 0.5
|
||||
mem_limit: 256m
|
||||
s3-bootstrap:
|
||||
image: ${API_IMAGE:-mtusi-vasilev:lab6}
|
||||
build: *id001
|
||||
environment:
|
||||
S3_ENDPOINT: http://s3:8333
|
||||
S3_ADMIN_ACCESS_KEY_FILE: /run/secrets/s3_admin_access_key
|
||||
S3_ADMIN_SECRET_KEY_FILE: /run/secrets/s3_admin_secret_key
|
||||
secrets:
|
||||
- s3_admin_access_key
|
||||
- s3_admin_secret_key
|
||||
cpus: ${API_CPUS:-0.5}
|
||||
mem_limit: ${API_MEMORY:-512m}
|
||||
pids_limit: 96
|
||||
read_only: true
|
||||
tmpfs: *id003
|
||||
cap_drop: *id004
|
||||
security_opt: *id005
|
||||
stop_grace_period: 8s
|
||||
command:
|
||||
- python
|
||||
- -m
|
||||
- messenger.storage_bootstrap
|
||||
healthcheck:
|
||||
disable: true
|
||||
depends_on:
|
||||
s3:
|
||||
condition: service_started
|
||||
lb:
|
||||
image: haproxy:3.2-alpine@sha256:96ec3c75a9264ba94e733eeacb44051ed6e7b79bb77a73c2c36f236b6423f1f0
|
||||
ports:
|
||||
- 127.0.0.1:${PUBLIC_PORT:-8086}:8080
|
||||
volumes:
|
||||
- ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
|
||||
depends_on:
|
||||
api:
|
||||
condition: service_healthy
|
||||
cpus: 0.5
|
||||
mem_limit: 128m
|
||||
read_only: true
|
||||
tmpfs:
|
||||
- /tmp:size=8m
|
||||
cap_drop:
|
||||
- ALL
|
||||
security_opt:
|
||||
- no-new-privileges:true
|
||||
healthcheck:
|
||||
test:
|
||||
- CMD-SHELL
|
||||
- wget -q -O /dev/null http://127.0.0.1:8080/health/ready
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 10
|
||||
collector:
|
||||
image: otel/opentelemetry-collector-contrib:0.123.0
|
||||
command:
|
||||
- --config=/etc/otelcol/config.yaml
|
||||
volumes:
|
||||
- ./infra/observability/collector.yaml:/etc/otelcol/config.yaml:ro
|
||||
ports:
|
||||
- 127.0.0.1:14318:4318
|
||||
- 127.0.0.1:19464:9464
|
||||
- 127.0.0.1:18888:8888
|
||||
cpus: 0.5
|
||||
mem_limit: 256m
|
||||
pids_limit: 96
|
||||
cap_drop:
|
||||
- ALL
|
||||
security_opt:
|
||||
- no-new-privileges:true
|
||||
prometheus:
|
||||
image: prom/prometheus:v3.2.1
|
||||
command:
|
||||
- --config.file=/etc/prometheus/prometheus.yml
|
||||
- --storage.tsdb.path=/prometheus
|
||||
- --storage.tsdb.retention.time=24h
|
||||
volumes:
|
||||
- ./infra/observability/prometheus.yaml:/etc/prometheus/prometheus.yml:ro
|
||||
- ./infra/observability/alerts.yaml:/etc/prometheus/alerts.yaml:ro
|
||||
- prometheus_data:/prometheus
|
||||
ports:
|
||||
- 127.0.0.1:19096:9090
|
||||
cpus: 0.5
|
||||
mem_limit: 256m
|
||||
loki:
|
||||
image: grafana/loki:3.4.2
|
||||
command:
|
||||
- -config.file=/etc/loki/config.yaml
|
||||
volumes:
|
||||
- ./infra/observability/loki.yaml:/etc/loki/config.yaml:ro
|
||||
- loki_data:/loki
|
||||
ports:
|
||||
- 127.0.0.1:13106:3100
|
||||
cpus: 0.5
|
||||
mem_limit: 384m
|
||||
tempo:
|
||||
image: grafana/tempo:2.7.2
|
||||
environment:
|
||||
GOMEMLIMIT: 512MiB
|
||||
command:
|
||||
- -config.file=/etc/tempo/config.yaml
|
||||
volumes:
|
||||
- ./infra/observability/tempo.yaml:/etc/tempo/config.yaml:ro
|
||||
- tempo_data:/var/tempo
|
||||
ports:
|
||||
- 127.0.0.1:13206:3200
|
||||
cpus: 0.5
|
||||
mem_limit: 768m
|
||||
grafana:
|
||||
image: grafana/grafana:11.6.0
|
||||
environment:
|
||||
GF_AUTH_ANONYMOUS_ENABLED: 'true'
|
||||
GF_AUTH_ANONYMOUS_ORG_ROLE: Viewer
|
||||
GF_AUTH_DISABLE_LOGIN_FORM: 'true'
|
||||
GF_USERS_ALLOW_SIGN_UP: 'false'
|
||||
GF_ANALYTICS_REPORTING_ENABLED: 'false'
|
||||
GF_ANALYTICS_CHECK_FOR_UPDATES: 'false'
|
||||
GF_SECURITY_ADMIN_PASSWORD__FILE: /run/secrets/grafana_password
|
||||
volumes:
|
||||
- ./infra/grafana/provisioning:/etc/grafana/provisioning:ro
|
||||
- ./infra/grafana/dashboards:/var/lib/grafana/dashboards:ro
|
||||
- grafana_data:/var/lib/grafana
|
||||
ports:
|
||||
- 127.0.0.1:${GRAFANA_PORT:-13006}:3000
|
||||
cpus: 0.5
|
||||
mem_limit: 256m
|
||||
secrets:
|
||||
- grafana_password
|
||||
volumes:
|
||||
postgres_data: null
|
||||
s3_data: null
|
||||
broker_data: null
|
||||
prometheus_data: {}
|
||||
loki_data: {}
|
||||
tempo_data: {}
|
||||
grafana_data: {}
|
||||
networks:
|
||||
storage_backend:
|
||||
internal: true
|
||||
secrets:
|
||||
postgres_password:
|
||||
file: ./secrets/postgres_password
|
||||
database_url:
|
||||
file: ./secrets/database_url
|
||||
cache_url:
|
||||
file: ./secrets/cache_url
|
||||
s3_access_key:
|
||||
file: ./secrets/s3_access_key
|
||||
s3_secret_key:
|
||||
file: ./secrets/s3_secret_key
|
||||
s3_admin_access_key:
|
||||
file: ./secrets/s3_admin_access_key
|
||||
s3_admin_secret_key:
|
||||
file: ./secrets/s3_admin_secret_key
|
||||
s3_config:
|
||||
file: ./secrets/s3_config
|
||||
broker_url:
|
||||
file: ./secrets/broker_url
|
||||
broker_password:
|
||||
file: ./secrets/broker_password
|
||||
broker_config:
|
||||
file: ./secrets/broker_config
|
||||
grafana_password:
|
||||
file: ./secrets/grafana_password
|
||||
@@ -0,0 +1,234 @@
|
||||
Image mtusi-vasilev:lab6 Building
|
||||
Image mtusi-vasilev:lab6 Building
|
||||
Image mtusi-vasilev:lab6 Building
|
||||
Image mtusi-vasilev:lab6 Building
|
||||
Image mtusi-vasilev:lab6 Building
|
||||
#1 [internal] load local bake definitions
|
||||
#1 reading from stdin 3.14kB done
|
||||
#1 DONE 0.0s
|
||||
|
||||
#2 [migrate internal] load build definition from Dockerfile
|
||||
#2 transferring dockerfile: 1.07kB done
|
||||
#2 DONE 0.0s
|
||||
|
||||
#3 [migrate internal] load metadata for ghcr.io/astral-sh/uv:0.11.7
|
||||
#3 DONE 0.0s
|
||||
|
||||
#4 [migrate internal] load metadata for docker.io/library/python:3.12-slim@sha256:646fb0bca3dd3ea1bcc6feb72c17ed16eed6e10cffc732fcc1478bd3e7f02d7b
|
||||
#4 DONE 0.0s
|
||||
|
||||
#5 [migrate internal] load .dockerignore
|
||||
#5 transferring context: 109B done
|
||||
#5 DONE 0.0s
|
||||
|
||||
#6 [api internal] load build context
|
||||
#6 transferring context: 28.04kB done
|
||||
#6 DONE 0.0s
|
||||
|
||||
#7 [dispatcher build 1/5] FROM docker.io/library/python:3.12-slim@sha256:646fb0bca3dd3ea1bcc6feb72c17ed16eed6e10cffc732fcc1478bd3e7f02d7b
|
||||
#7 resolve docker.io/library/python:3.12-slim@sha256:646fb0bca3dd3ea1bcc6feb72c17ed16eed6e10cffc732fcc1478bd3e7f02d7b done
|
||||
#7 DONE 0.0s
|
||||
|
||||
#8 [s3-bootstrap] FROM ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
|
||||
#8 resolve ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a done
|
||||
#8 DONE 0.0s
|
||||
|
||||
#9 [api build 4/5] COPY pyproject.toml uv.lock ./
|
||||
#9 CACHED
|
||||
|
||||
#10 [api stage-1 3/6] WORKDIR /app
|
||||
#10 CACHED
|
||||
|
||||
#11 [api build 5/5] RUN uv sync --locked --no-dev --no-install-project --python /usr/local/bin/python
|
||||
#11 CACHED
|
||||
|
||||
#12 [api build 3/5] WORKDIR /app
|
||||
#12 CACHED
|
||||
|
||||
#13 [api build 2/5] COPY --from=ghcr.io/astral-sh/uv:0.11.7 /uv /usr/local/bin/uv
|
||||
#13 CACHED
|
||||
|
||||
#14 [api stage-1 2/6] RUN apt-get update && apt-get install -y --no-install-recommends libjpeg-turbo-progs=1:2.1.5-4 && rm -rf /var/lib/apt/lists/*
|
||||
#14 CACHED
|
||||
|
||||
#15 [dispatcher stage-1 4/6] COPY --from=build /app/.venv /app/.venv
|
||||
#15 CACHED
|
||||
|
||||
#16 [worker stage-1 5/6] COPY src ./src
|
||||
#16 DONE 0.0s
|
||||
|
||||
#17 [api stage-1 6/6] COPY migrations ./migrations
|
||||
#17 DONE 0.0s
|
||||
|
||||
#18 [migrate] exporting to image
|
||||
#18 exporting layers 0.0s done
|
||||
#18 exporting manifest sha256:eaaedf4c6882dbdea34146fff7162d8ef60c2e0928e8e0ace9fb2d32d4d35b45 done
|
||||
#18 exporting config sha256:edbb694fa8e8e3ce614ee6bd69f8f905a4034d2272fd661ff00ed4679190cd23 0.0s done
|
||||
#18 exporting attestation manifest sha256:5ea04d6ae9de1951c6fd4fa7ba7466fd17f38b8073418b8fcd7b36059fdee746 0.0s done
|
||||
#18 exporting manifest list sha256:27a8245d8fe54b02a99556efd6e6aeff5036b3eee6639f4fdcacad1c878d0b9b 0.0s done
|
||||
#18 naming to docker.io/library/mtusi-vasilev:lab6 done
|
||||
#18 unpacking to docker.io/library/mtusi-vasilev:lab6
|
||||
#18 unpacking to docker.io/library/mtusi-vasilev:lab6 0.0s done
|
||||
#18 DONE 0.1s
|
||||
|
||||
#19 [dispatcher] exporting to image
|
||||
#19 exporting layers 0.0s done
|
||||
#19 exporting manifest sha256:d3da07a6fdda64b3618c601f7e2f097488b13767fa78e7aa633caf7ebba7cf0f 0.0s done
|
||||
#19 exporting config sha256:75ac75b785fb5836659578621f4fe38554f6d76b18623db5c26c2ea1efdc8f0c 0.0s done
|
||||
#19 exporting attestation manifest sha256:56353292db76d317ba87f8b4fa216f0bd3ae3a4daff460acded208a49d6871cb 0.0s done
|
||||
#19 exporting manifest list sha256:a392951c40267bc16428d809b7cf2936a3f7d112705ffd276372dbbcc1d68867 0.0s done
|
||||
#19 naming to docker.io/library/mtusi-vasilev:lab6 done
|
||||
#19 unpacking to docker.io/library/mtusi-vasilev:lab6 0.0s done
|
||||
#19 DONE 0.1s
|
||||
|
||||
#20 [s3-bootstrap] exporting to image
|
||||
#20 exporting layers 0.0s done
|
||||
#20 exporting manifest sha256:f8afc946dee98b3ab5b238d588f98b3727228a152b77cf5b33c5268e2d1d118c 0.0s done
|
||||
#20 exporting config sha256:8dba14c01d7488a389d2892935806cf25379175ae443ef6376bb8a8f2fdd242e 0.0s done
|
||||
#20 exporting attestation manifest sha256:a72640ebb9575c7d0a86c89650defd4eb707e47aaf8da77757d4057578d80cd7 0.0s done
|
||||
#20 exporting manifest list sha256:adf5d66d029c412d164a492c3196d1f67c47c8a0918a700844771d613a77c98d 0.0s done
|
||||
#20 naming to docker.io/library/mtusi-vasilev:lab6 done
|
||||
#20 unpacking to docker.io/library/mtusi-vasilev:lab6 0.0s done
|
||||
#20 DONE 0.1s
|
||||
|
||||
#21 [api] exporting to image
|
||||
#21 exporting layers 0.0s done
|
||||
#21 exporting manifest sha256:032bd231dd762fe10a19be37cd71daf8076d3436fc62b6788fbbcfac7728acd6 0.0s done
|
||||
#21 exporting config sha256:4c3020a2b95b6b9f0f37a6629bf17e8e8dd409f38b6090677c121ca04e7bf69a 0.0s done
|
||||
#21 exporting attestation manifest sha256:60c24c58c4c4f2aed80aeaf15f4a6c0699aed0f5e33309db1536902b2f4ce918 0.0s done
|
||||
#21 exporting manifest list sha256:a582478f8f798682db8825193c88a6e7f8d9cd746fe25eb0c3b86af62d2218fc 0.0s done
|
||||
#21 naming to docker.io/library/mtusi-vasilev:lab6 done
|
||||
#21 unpacking to docker.io/library/mtusi-vasilev:lab6 0.0s done
|
||||
#21 DONE 0.1s
|
||||
|
||||
#22 [worker] exporting to image
|
||||
#22 exporting layers 0.0s done
|
||||
#22 exporting manifest sha256:7f4cee04d6c050130a6a782e58225c7a4994b55e7c2022556b8adb3ca7a8818a 0.0s done
|
||||
#22 exporting config sha256:64b56b6a22e6232afd3088acacb29903968a596b0da245ce3537b0558cd4f4e5 0.0s done
|
||||
#22 exporting attestation manifest sha256:cb79a6cf722f860c48cf67c7c0dc9dcfa35ca0052f5f5b47cab876a4ad344e37 0.0s done
|
||||
#22 exporting manifest list sha256:3fc41f43557e885100458a5c245d79119c72330a20c765f66fc4e64bea2fc68e 0.0s done
|
||||
#22 naming to docker.io/library/mtusi-vasilev:lab6 done
|
||||
#22 unpacking to docker.io/library/mtusi-vasilev:lab6 0.0s done
|
||||
#22 DONE 0.2s
|
||||
|
||||
#23 [worker] resolving provenance for metadata file
|
||||
#23 DONE 0.0s
|
||||
|
||||
#24 [dispatcher] resolving provenance for metadata file
|
||||
#24 DONE 0.0s
|
||||
|
||||
#25 [migrate] resolving provenance for metadata file
|
||||
#25 DONE 0.0s
|
||||
|
||||
#26 [s3-bootstrap] resolving provenance for metadata file
|
||||
#26 DONE 0.0s
|
||||
|
||||
#27 [api] resolving provenance for metadata file
|
||||
#27 DONE 0.0s
|
||||
Image mtusi-vasilev:lab6 Built
|
||||
Image mtusi-vasilev:lab6 Built
|
||||
Image mtusi-vasilev:lab6 Built
|
||||
Image mtusi-vasilev:lab6 Built
|
||||
Image mtusi-vasilev:lab6 Built
|
||||
Container mtusi-vasilev-lab6-postgres-1 Running
|
||||
Container mtusi-vasilev-lab6-cache-1 Running
|
||||
Container mtusi-vasilev-lab6-prometheus-1 Running
|
||||
Container mtusi-vasilev-lab6-grafana-1 Running
|
||||
Container mtusi-vasilev-lab6-s3-1 Running
|
||||
Container mtusi-vasilev-lab6-broker-1 Running
|
||||
Container mtusi-vasilev-lab6-lb-1 Running
|
||||
Container mtusi-vasilev-lab6-storage-1 Running
|
||||
Container mtusi-vasilev-lab6-collector-1 Recreate
|
||||
Container mtusi-vasilev-lab6-tempo-1 Recreate
|
||||
Container mtusi-vasilev-lab6-s3-bootstrap-1 Recreate
|
||||
Container mtusi-vasilev-lab6-migrate-1 Recreate
|
||||
Container mtusi-vasilev-lab6-loki-1 Recreate
|
||||
Container mtusi-vasilev-lab6-migrate-1 Recreated
|
||||
Container mtusi-vasilev-lab6-dispatcher-1 Recreate
|
||||
Container mtusi-vasilev-lab6-s3-bootstrap-1 Recreated
|
||||
Container mtusi-vasilev-lab6-worker-1 Recreate
|
||||
Container mtusi-vasilev-lab6-api-1 Recreate
|
||||
Container mtusi-vasilev-lab6-api-2 Recreate
|
||||
Container mtusi-vasilev-lab6-collector-1 Recreated
|
||||
Container mtusi-vasilev-lab6-dispatcher-1 Recreated
|
||||
Container mtusi-vasilev-lab6-tempo-1 Recreated
|
||||
Container mtusi-vasilev-lab6-worker-1 Recreated
|
||||
Container mtusi-vasilev-lab6-api-2 Recreated
|
||||
Container mtusi-vasilev-lab6-api-1 Recreated
|
||||
Container mtusi-vasilev-lab6-loki-1 Recreated
|
||||
Container mtusi-vasilev-lab6-tempo-1 Starting
|
||||
Container mtusi-vasilev-lab6-collector-1 Starting
|
||||
Container mtusi-vasilev-lab6-postgres-1 Waiting
|
||||
Container mtusi-vasilev-lab6-loki-1 Starting
|
||||
Container mtusi-vasilev-lab6-storage-1 Waiting
|
||||
Container mtusi-vasilev-lab6-loki-1 Started
|
||||
Container mtusi-vasilev-lab6-collector-1 Started
|
||||
Container mtusi-vasilev-lab6-tempo-1 Started
|
||||
Container mtusi-vasilev-lab6-postgres-1 Healthy
|
||||
Container mtusi-vasilev-lab6-storage-1 Healthy
|
||||
Container mtusi-vasilev-lab6-migrate-1 Starting
|
||||
Container mtusi-vasilev-lab6-s3-bootstrap-1 Starting
|
||||
Container mtusi-vasilev-lab6-migrate-1 Started
|
||||
Container mtusi-vasilev-lab6-broker-1 Waiting
|
||||
Container mtusi-vasilev-lab6-migrate-1 Waiting
|
||||
Container mtusi-vasilev-lab6-s3-bootstrap-1 Started
|
||||
Container mtusi-vasilev-lab6-migrate-1 Waiting
|
||||
Container mtusi-vasilev-lab6-cache-1 Waiting
|
||||
Container mtusi-vasilev-lab6-s3-bootstrap-1 Waiting
|
||||
Container mtusi-vasilev-lab6-migrate-1 Waiting
|
||||
Container mtusi-vasilev-lab6-broker-1 Waiting
|
||||
Container mtusi-vasilev-lab6-s3-bootstrap-1 Waiting
|
||||
Container mtusi-vasilev-lab6-broker-1 Healthy
|
||||
Container mtusi-vasilev-lab6-cache-1 Healthy
|
||||
Container mtusi-vasilev-lab6-broker-1 Healthy
|
||||
Container mtusi-vasilev-lab6-migrate-1 Exited
|
||||
Container mtusi-vasilev-lab6-dispatcher-1 Starting
|
||||
Container mtusi-vasilev-lab6-migrate-1 Exited
|
||||
Container mtusi-vasilev-lab6-migrate-1 Exited
|
||||
Container mtusi-vasilev-lab6-dispatcher-1 Started
|
||||
Container mtusi-vasilev-lab6-s3-bootstrap-1 Exited
|
||||
Container mtusi-vasilev-lab6-s3-bootstrap-1 Exited
|
||||
Container mtusi-vasilev-lab6-api-2 Starting
|
||||
Container mtusi-vasilev-lab6-worker-1 Starting
|
||||
Container mtusi-vasilev-lab6-worker-1 Started
|
||||
Container mtusi-vasilev-lab6-api-2 Started
|
||||
Container mtusi-vasilev-lab6-api-1 Starting
|
||||
Container mtusi-vasilev-lab6-api-1 Started
|
||||
Container mtusi-vasilev-lab6-api-2 Waiting
|
||||
Container mtusi-vasilev-lab6-api-1 Waiting
|
||||
Container mtusi-vasilev-lab6-api-2 Healthy
|
||||
Container mtusi-vasilev-lab6-api-1 Healthy
|
||||
Container mtusi-vasilev-lab6-worker-1 Waiting
|
||||
Container mtusi-vasilev-lab6-s3-1 Waiting
|
||||
Container mtusi-vasilev-lab6-broker-1 Waiting
|
||||
Container mtusi-vasilev-lab6-storage-1 Waiting
|
||||
Container mtusi-vasilev-lab6-cache-1 Waiting
|
||||
Container mtusi-vasilev-lab6-dispatcher-1 Waiting
|
||||
Container mtusi-vasilev-lab6-loki-1 Waiting
|
||||
Container mtusi-vasilev-lab6-tempo-1 Waiting
|
||||
Container mtusi-vasilev-lab6-lb-1 Waiting
|
||||
Container mtusi-vasilev-lab6-s3-bootstrap-1 Waiting
|
||||
Container mtusi-vasilev-lab6-prometheus-1 Waiting
|
||||
Container mtusi-vasilev-lab6-migrate-1 Waiting
|
||||
Container mtusi-vasilev-lab6-postgres-1 Waiting
|
||||
Container mtusi-vasilev-lab6-grafana-1 Waiting
|
||||
Container mtusi-vasilev-lab6-api-2 Waiting
|
||||
Container mtusi-vasilev-lab6-api-1 Waiting
|
||||
Container mtusi-vasilev-lab6-collector-1 Waiting
|
||||
Container mtusi-vasilev-lab6-lb-1 Healthy
|
||||
Container mtusi-vasilev-lab6-s3-1 Healthy
|
||||
Container mtusi-vasilev-lab6-grafana-1 Healthy
|
||||
Container mtusi-vasilev-lab6-collector-1 Healthy
|
||||
Container mtusi-vasilev-lab6-broker-1 Healthy
|
||||
Container mtusi-vasilev-lab6-s3-bootstrap-1 Exited
|
||||
Container mtusi-vasilev-lab6-cache-1 Healthy
|
||||
Container mtusi-vasilev-lab6-worker-1 Healthy
|
||||
Container mtusi-vasilev-lab6-tempo-1 Healthy
|
||||
Container mtusi-vasilev-lab6-postgres-1 Healthy
|
||||
Container mtusi-vasilev-lab6-storage-1 Healthy
|
||||
Container mtusi-vasilev-lab6-dispatcher-1 Healthy
|
||||
Container mtusi-vasilev-lab6-loki-1 Healthy
|
||||
Container mtusi-vasilev-lab6-migrate-1 Exited
|
||||
Container mtusi-vasilev-lab6-prometheus-1 Healthy
|
||||
Container mtusi-vasilev-lab6-api-2 Healthy
|
||||
Container mtusi-vasilev-lab6-api-1 Healthy
|
||||
@@ -0,0 +1 @@
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
@@ -0,0 +1 @@
|
||||
{"captured_at" : "2026-11-16T00:16:17.713479+00:00", "users" : 367, "chats" : 140, "messages" : 271, "attachments" : 55, "jobs" : 55}
|
||||
@@ -0,0 +1,61 @@
|
||||
Derived from evidence/incidents.json
|
||||
{
|
||||
"started_at": "2026-11-16T00:11:51.947497+00:00",
|
||||
"finished_at": "2026-11-16T00:15:16.555972+00:00",
|
||||
"passed": true,
|
||||
"scenarios": [
|
||||
{
|
||||
"name": "upload_publish_consume_retry",
|
||||
"passed": true,
|
||||
"trace_id": "1df2ebde21e84372b95437a839d9aa0a",
|
||||
"attachment_id": "2e1aad5f-647d-462b-bf57-705605e29449",
|
||||
"log_streams": 5
|
||||
},
|
||||
{
|
||||
"name": "worker_stop_queue_age",
|
||||
"passed": true,
|
||||
"firing_after_s": 34.739,
|
||||
"firing_observation": {
|
||||
"depth": 3.0,
|
||||
"oldest_age_s": 29.867548,
|
||||
"error_ratio": 0.0
|
||||
},
|
||||
"resolved_after_restore_s": 14.131,
|
||||
"ready_attachment_count": 3
|
||||
},
|
||||
{
|
||||
"name": "database_pause",
|
||||
"passed": true,
|
||||
"firing_after_s": 29.15,
|
||||
"statuses": [
|
||||
503,
|
||||
503,
|
||||
503,
|
||||
503,
|
||||
503,
|
||||
503,
|
||||
503,
|
||||
503,
|
||||
503,
|
||||
503,
|
||||
503,
|
||||
503
|
||||
],
|
||||
"firing_observation": {
|
||||
"depth": 0.0,
|
||||
"oldest_age_s": 0.0,
|
||||
"error_ratio": 0.5328919208698448
|
||||
},
|
||||
"resolved_after_restore_s": 35.22
|
||||
},
|
||||
{
|
||||
"name": "telemetry_backends_outage",
|
||||
"passed": true,
|
||||
"duration_s": 39.125,
|
||||
"business_successes": 200,
|
||||
"failed_span_sends_before": 0.0,
|
||||
"failed_span_sends_after": 118.0,
|
||||
"restored_trace_id": "c2af36cbb59745e0b403a8112b0e21af"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,722 @@
|
||||
{
|
||||
"started_at": "2026-11-16T00:11:51.947497+00:00",
|
||||
"scenarios": [
|
||||
{
|
||||
"name": "upload_publish_consume_retry",
|
||||
"passed": true,
|
||||
"trace_id": "1df2ebde21e84372b95437a839d9aa0a",
|
||||
"attachment_id": "2e1aad5f-647d-462b-bf57-705605e29449",
|
||||
"spans": [
|
||||
{
|
||||
"traceId": "HfLr3iHoQ3K5VDeoOdmqCg==",
|
||||
"spanId": "85PQRG4jEBU=",
|
||||
"parentSpanId": "EjRWeJCrze8=",
|
||||
"flags": 768,
|
||||
"name": "POST /api/v1/chats/{chat_id}/attachments",
|
||||
"kind": "SPAN_KIND_SERVER",
|
||||
"startTimeUnixNano": "1794787924032425043",
|
||||
"endTimeUnixNano": "1794787924232650493",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "http.request.method",
|
||||
"value": {
|
||||
"stringValue": "POST"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "http.route",
|
||||
"value": {
|
||||
"stringValue": "/api/v1/chats/{chat_id}/attachments"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "http.response.status_code",
|
||||
"value": {
|
||||
"intValue": "202"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {}
|
||||
},
|
||||
{
|
||||
"traceId": "HfLr3iHoQ3K5VDeoOdmqCg==",
|
||||
"spanId": "YWdKpwpF9Ds=",
|
||||
"parentSpanId": "85PQRG4jEBU=",
|
||||
"flags": 256,
|
||||
"name": "postgres.transaction",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"startTimeUnixNano": "1794787924032821453",
|
||||
"endTimeUnixNano": "1794787924033245940",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "db.system.name",
|
||||
"value": {
|
||||
"stringValue": "postgresql"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {}
|
||||
},
|
||||
{
|
||||
"traceId": "HfLr3iHoQ3K5VDeoOdmqCg==",
|
||||
"spanId": "9YJKvSiyB5g=",
|
||||
"parentSpanId": "85PQRG4jEBU=",
|
||||
"flags": 256,
|
||||
"name": "postgres.transaction",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"startTimeUnixNano": "1794787924033675481",
|
||||
"endTimeUnixNano": "1794787924034278077",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "db.system.name",
|
||||
"value": {
|
||||
"stringValue": "postgresql"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {}
|
||||
},
|
||||
{
|
||||
"traceId": "HfLr3iHoQ3K5VDeoOdmqCg==",
|
||||
"spanId": "2x/geWQTYmo=",
|
||||
"parentSpanId": "85PQRG4jEBU=",
|
||||
"flags": 256,
|
||||
"name": "postgres.transaction",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"startTimeUnixNano": "1794787924226398258",
|
||||
"endTimeUnixNano": "1794787924227576250",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "db.system.name",
|
||||
"value": {
|
||||
"stringValue": "postgresql"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {}
|
||||
},
|
||||
{
|
||||
"traceId": "HfLr3iHoQ3K5VDeoOdmqCg==",
|
||||
"spanId": "auYsCTJ/YBY=",
|
||||
"parentSpanId": "85PQRG4jEBU=",
|
||||
"flags": 256,
|
||||
"name": "postgres.transaction",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"startTimeUnixNano": "1794787924227618281",
|
||||
"endTimeUnixNano": "1794787924232311322",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "db.system.name",
|
||||
"value": {
|
||||
"stringValue": "postgresql"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {}
|
||||
},
|
||||
{
|
||||
"traceId": "HfLr3iHoQ3K5VDeoOdmqCg==",
|
||||
"spanId": "v9mqJzVyq6s=",
|
||||
"parentSpanId": "auYsCTJ/YBY=",
|
||||
"flags": 256,
|
||||
"name": "s3.request",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"startTimeUnixNano": "1794787924227935058",
|
||||
"endTimeUnixNano": "1794787924231301036",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "rpc.system",
|
||||
"value": {
|
||||
"stringValue": "aws-api"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "rpc.service",
|
||||
"value": {
|
||||
"stringValue": "S3"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {}
|
||||
},
|
||||
{
|
||||
"traceId": "HfLr3iHoQ3K5VDeoOdmqCg==",
|
||||
"spanId": "KNSxMXLkvrw=",
|
||||
"parentSpanId": "auYsCTJ/YBY=",
|
||||
"flags": 768,
|
||||
"name": "attachments publish",
|
||||
"kind": "SPAN_KIND_PRODUCER",
|
||||
"startTimeUnixNano": "1794787924425567005",
|
||||
"endTimeUnixNano": "1794787924427476734",
|
||||
"status": {}
|
||||
},
|
||||
{
|
||||
"traceId": "HfLr3iHoQ3K5VDeoOdmqCg==",
|
||||
"spanId": "iiLJTNZpKyY=",
|
||||
"parentSpanId": "KNSxMXLkvrw=",
|
||||
"flags": 768,
|
||||
"name": "attachments consume",
|
||||
"kind": "SPAN_KIND_CONSUMER",
|
||||
"startTimeUnixNano": "1794787924427715550",
|
||||
"endTimeUnixNano": "1794787924431726697",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "job.type",
|
||||
"value": {
|
||||
"stringValue": "attachment.process.v1"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "job.outcome",
|
||||
"value": {
|
||||
"stringValue": "retry"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {
|
||||
"message": "processing_failed",
|
||||
"code": "STATUS_CODE_ERROR"
|
||||
}
|
||||
},
|
||||
{
|
||||
"traceId": "HfLr3iHoQ3K5VDeoOdmqCg==",
|
||||
"spanId": "3I8sOfMxGUw=",
|
||||
"parentSpanId": "iiLJTNZpKyY=",
|
||||
"flags": 256,
|
||||
"name": "postgres.transaction",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"startTimeUnixNano": "1794787924427780393",
|
||||
"endTimeUnixNano": "1794787924429488867",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "db.system.name",
|
||||
"value": {
|
||||
"stringValue": "postgresql"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {}
|
||||
},
|
||||
{
|
||||
"traceId": "HfLr3iHoQ3K5VDeoOdmqCg==",
|
||||
"spanId": "KENZefPstWA=",
|
||||
"parentSpanId": "iiLJTNZpKyY=",
|
||||
"flags": 256,
|
||||
"name": "postgres.transaction",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"startTimeUnixNano": "1794787924429565242",
|
||||
"endTimeUnixNano": "1794787924430107381",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "db.system.name",
|
||||
"value": {
|
||||
"stringValue": "postgresql"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {}
|
||||
},
|
||||
{
|
||||
"traceId": "HfLr3iHoQ3K5VDeoOdmqCg==",
|
||||
"spanId": "jTmlxkAxFa8=",
|
||||
"parentSpanId": "iiLJTNZpKyY=",
|
||||
"flags": 256,
|
||||
"name": "postgres.transaction",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"startTimeUnixNano": "1794787924430433266",
|
||||
"endTimeUnixNano": "1794787924431559869",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "db.system.name",
|
||||
"value": {
|
||||
"stringValue": "postgresql"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {}
|
||||
},
|
||||
{
|
||||
"traceId": "HfLr3iHoQ3K5VDeoOdmqCg==",
|
||||
"spanId": "+ftjsuipd7I=",
|
||||
"parentSpanId": "auYsCTJ/YBY=",
|
||||
"flags": 768,
|
||||
"name": "attachments publish",
|
||||
"kind": "SPAN_KIND_PRODUCER",
|
||||
"startTimeUnixNano": "1794787925868986427",
|
||||
"endTimeUnixNano": "1794787925870404011",
|
||||
"status": {}
|
||||
},
|
||||
{
|
||||
"traceId": "HfLr3iHoQ3K5VDeoOdmqCg==",
|
||||
"spanId": "j3xUVmra6+M=",
|
||||
"parentSpanId": "+ftjsuipd7I=",
|
||||
"flags": 768,
|
||||
"name": "attachments consume",
|
||||
"kind": "SPAN_KIND_CONSUMER",
|
||||
"startTimeUnixNano": "1794787925870556967",
|
||||
"endTimeUnixNano": "1794787926054299958",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "job.type",
|
||||
"value": {
|
||||
"stringValue": "attachment.process.v1"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "job.outcome",
|
||||
"value": {
|
||||
"stringValue": "success"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {}
|
||||
},
|
||||
{
|
||||
"traceId": "HfLr3iHoQ3K5VDeoOdmqCg==",
|
||||
"spanId": "pzGGkWEL0t0=",
|
||||
"parentSpanId": "j3xUVmra6+M=",
|
||||
"flags": 256,
|
||||
"name": "s3.request",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"startTimeUnixNano": "1794787926043435746",
|
||||
"endTimeUnixNano": "1794787926046214771",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "rpc.system",
|
||||
"value": {
|
||||
"stringValue": "aws-api"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "rpc.service",
|
||||
"value": {
|
||||
"stringValue": "S3"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {}
|
||||
},
|
||||
{
|
||||
"traceId": "HfLr3iHoQ3K5VDeoOdmqCg==",
|
||||
"spanId": "YdH70wxCgds=",
|
||||
"parentSpanId": "j3xUVmra6+M=",
|
||||
"flags": 256,
|
||||
"name": "s3.request",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"startTimeUnixNano": "1794787926046277701",
|
||||
"endTimeUnixNano": "1794787926047537678",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "rpc.system",
|
||||
"value": {
|
||||
"stringValue": "aws-api"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "rpc.service",
|
||||
"value": {
|
||||
"stringValue": "S3"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {}
|
||||
},
|
||||
{
|
||||
"traceId": "HfLr3iHoQ3K5VDeoOdmqCg==",
|
||||
"spanId": "Hl+esmJIzaw=",
|
||||
"parentSpanId": "j3xUVmra6+M=",
|
||||
"flags": 256,
|
||||
"name": "s3.request",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"startTimeUnixNano": "1794787926047602404",
|
||||
"endTimeUnixNano": "1794787926048859375",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "rpc.system",
|
||||
"value": {
|
||||
"stringValue": "aws-api"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "rpc.service",
|
||||
"value": {
|
||||
"stringValue": "S3"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {}
|
||||
},
|
||||
{
|
||||
"traceId": "HfLr3iHoQ3K5VDeoOdmqCg==",
|
||||
"spanId": "ymy5xPLE22s=",
|
||||
"parentSpanId": "j3xUVmra6+M=",
|
||||
"flags": 256,
|
||||
"name": "s3.request",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"startTimeUnixNano": "1794787926048933580",
|
||||
"endTimeUnixNano": "1794787926050261415",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "rpc.system",
|
||||
"value": {
|
||||
"stringValue": "aws-api"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "rpc.service",
|
||||
"value": {
|
||||
"stringValue": "S3"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {}
|
||||
},
|
||||
{
|
||||
"traceId": "HfLr3iHoQ3K5VDeoOdmqCg==",
|
||||
"spanId": "0I0XgttMe7o=",
|
||||
"parentSpanId": "j3xUVmra6+M=",
|
||||
"flags": 256,
|
||||
"name": "postgres.transaction",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"startTimeUnixNano": "1794787926050309688",
|
||||
"endTimeUnixNano": "1794787926050692112",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "db.system.name",
|
||||
"value": {
|
||||
"stringValue": "postgresql"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {}
|
||||
},
|
||||
{
|
||||
"traceId": "HfLr3iHoQ3K5VDeoOdmqCg==",
|
||||
"spanId": "xpBliDqwSJ0=",
|
||||
"parentSpanId": "j3xUVmra6+M=",
|
||||
"flags": 256,
|
||||
"name": "postgres.transaction",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"startTimeUnixNano": "1794787926050724475",
|
||||
"endTimeUnixNano": "1794787926052224647",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "db.system.name",
|
||||
"value": {
|
||||
"stringValue": "postgresql"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {}
|
||||
},
|
||||
{
|
||||
"traceId": "HfLr3iHoQ3K5VDeoOdmqCg==",
|
||||
"spanId": "Nb/KjJuQvYE=",
|
||||
"parentSpanId": "j3xUVmra6+M=",
|
||||
"flags": 256,
|
||||
"name": "postgres.transaction",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"startTimeUnixNano": "1794787926052282315",
|
||||
"endTimeUnixNano": "1794787926052454235",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "db.system.name",
|
||||
"value": {
|
||||
"stringValue": "postgresql"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {}
|
||||
},
|
||||
{
|
||||
"traceId": "HfLr3iHoQ3K5VDeoOdmqCg==",
|
||||
"spanId": "W2UgjhNayzg=",
|
||||
"parentSpanId": "j3xUVmra6+M=",
|
||||
"flags": 256,
|
||||
"name": "postgres.transaction",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"startTimeUnixNano": "1794787926052481587",
|
||||
"endTimeUnixNano": "1794787926054080225",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "db.system.name",
|
||||
"value": {
|
||||
"stringValue": "postgresql"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {}
|
||||
},
|
||||
{
|
||||
"traceId": "HfLr3iHoQ3K5VDeoOdmqCg==",
|
||||
"spanId": "Ir9zAzRDnt8=",
|
||||
"parentSpanId": "W2UgjhNayzg=",
|
||||
"flags": 256,
|
||||
"name": "s3.request",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"startTimeUnixNano": "1794787926052739904",
|
||||
"endTimeUnixNano": "1794787926053461741",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "rpc.system",
|
||||
"value": {
|
||||
"stringValue": "aws-api"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "rpc.service",
|
||||
"value": {
|
||||
"stringValue": "S3"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {}
|
||||
},
|
||||
{
|
||||
"traceId": "HfLr3iHoQ3K5VDeoOdmqCg==",
|
||||
"spanId": "FSMxie8tGuk=",
|
||||
"parentSpanId": "j3xUVmra6+M=",
|
||||
"flags": 256,
|
||||
"name": "postgres.transaction",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"startTimeUnixNano": "1794787925870594559",
|
||||
"endTimeUnixNano": "1794787925871912567",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "db.system.name",
|
||||
"value": {
|
||||
"stringValue": "postgresql"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {}
|
||||
},
|
||||
{
|
||||
"traceId": "HfLr3iHoQ3K5VDeoOdmqCg==",
|
||||
"spanId": "0we4paz46tc=",
|
||||
"parentSpanId": "j3xUVmra6+M=",
|
||||
"flags": 256,
|
||||
"name": "postgres.transaction",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"startTimeUnixNano": "1794787925871962940",
|
||||
"endTimeUnixNano": "1794787925872135026",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "db.system.name",
|
||||
"value": {
|
||||
"stringValue": "postgresql"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {}
|
||||
},
|
||||
{
|
||||
"traceId": "HfLr3iHoQ3K5VDeoOdmqCg==",
|
||||
"spanId": "1ch+xVVRGHo=",
|
||||
"parentSpanId": "j3xUVmra6+M=",
|
||||
"flags": 256,
|
||||
"name": "s3.get",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"startTimeUnixNano": "1794787925872227920",
|
||||
"endTimeUnixNano": "1794787925874040634",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "rpc.system",
|
||||
"value": {
|
||||
"stringValue": "aws-api"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "rpc.service",
|
||||
"value": {
|
||||
"stringValue": "S3"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {}
|
||||
}
|
||||
],
|
||||
"log_streams": 5
|
||||
},
|
||||
{
|
||||
"name": "worker_stop_queue_age",
|
||||
"passed": true,
|
||||
"firing_after_s": 34.739,
|
||||
"firing_observation": {
|
||||
"depth": 3.0,
|
||||
"oldest_age_s": 29.867548,
|
||||
"error_ratio": 0.0
|
||||
},
|
||||
"resolved_after_restore_s": 14.131,
|
||||
"ready_attachment_count": 3
|
||||
},
|
||||
{
|
||||
"name": "database_pause",
|
||||
"passed": true,
|
||||
"firing_after_s": 29.15,
|
||||
"statuses": [
|
||||
503,
|
||||
503,
|
||||
503,
|
||||
503,
|
||||
503,
|
||||
503,
|
||||
503,
|
||||
503,
|
||||
503,
|
||||
503,
|
||||
503,
|
||||
503
|
||||
],
|
||||
"firing_observation": {
|
||||
"depth": 0.0,
|
||||
"oldest_age_s": 0.0,
|
||||
"error_ratio": 0.5328919208698448
|
||||
},
|
||||
"resolved_after_restore_s": 35.22
|
||||
},
|
||||
{
|
||||
"name": "telemetry_backends_outage",
|
||||
"passed": true,
|
||||
"duration_s": 39.125,
|
||||
"business_successes": 200,
|
||||
"failed_span_sends_before": 0.0,
|
||||
"failed_span_sends_after": 118.0,
|
||||
"memory_before": [
|
||||
{
|
||||
"service": "api",
|
||||
"container": "cfafa727059621ed84debff1f81ee3389f245125fe4fa3abb5f94a70fdf75137",
|
||||
"current_bytes": 84967424,
|
||||
"peak_bytes": 151830528,
|
||||
"events": {
|
||||
"low": "0",
|
||||
"high": "0",
|
||||
"max": "0",
|
||||
"oom": "0",
|
||||
"oom_kill": "0",
|
||||
"oom_group_kill": "0"
|
||||
},
|
||||
"limit_bytes": 536870912,
|
||||
"oom_killed": false
|
||||
},
|
||||
{
|
||||
"service": "api",
|
||||
"container": "9f76ed4585708192348539a2ec7faf820544abaab82f9c31242ba5a79b0889e2",
|
||||
"current_bytes": 84553728,
|
||||
"peak_bytes": 158683136,
|
||||
"events": {
|
||||
"low": "0",
|
||||
"high": "0",
|
||||
"max": "0",
|
||||
"oom": "0",
|
||||
"oom_kill": "0",
|
||||
"oom_group_kill": "0"
|
||||
},
|
||||
"limit_bytes": 536870912,
|
||||
"oom_killed": false
|
||||
},
|
||||
{
|
||||
"service": "worker",
|
||||
"container": "2268305d2238f316cf7b5a65d2045a6c1a7a2ec70dd02c59754685fac7d5437f",
|
||||
"current_bytes": 72175616,
|
||||
"peak_bytes": 118439936,
|
||||
"events": {
|
||||
"low": "0",
|
||||
"high": "0",
|
||||
"max": "0",
|
||||
"oom": "0",
|
||||
"oom_kill": "0",
|
||||
"oom_group_kill": "0"
|
||||
},
|
||||
"limit_bytes": 939524096,
|
||||
"oom_killed": false
|
||||
},
|
||||
{
|
||||
"service": "collector",
|
||||
"container": "eb2404f8d1fd88ff2a037004fe07c978fd9e2fb56b5d66fc25aae540333d3938",
|
||||
"current_bytes": 50638848,
|
||||
"peak_bytes": 51539968,
|
||||
"events": {
|
||||
"low": "0",
|
||||
"high": "0",
|
||||
"max": "0",
|
||||
"oom": "0",
|
||||
"oom_kill": "0",
|
||||
"oom_group_kill": "0"
|
||||
},
|
||||
"limit_bytes": 268435456,
|
||||
"oom_killed": false
|
||||
}
|
||||
],
|
||||
"memory_during": [
|
||||
{
|
||||
"service": "api",
|
||||
"container": "cfafa727059621ed84debff1f81ee3389f245125fe4fa3abb5f94a70fdf75137",
|
||||
"current_bytes": 86003712,
|
||||
"peak_bytes": 151830528,
|
||||
"events": {
|
||||
"low": "0",
|
||||
"high": "0",
|
||||
"max": "0",
|
||||
"oom": "0",
|
||||
"oom_kill": "0",
|
||||
"oom_group_kill": "0"
|
||||
},
|
||||
"limit_bytes": 536870912,
|
||||
"oom_killed": false
|
||||
},
|
||||
{
|
||||
"service": "api",
|
||||
"container": "9f76ed4585708192348539a2ec7faf820544abaab82f9c31242ba5a79b0889e2",
|
||||
"current_bytes": 84656128,
|
||||
"peak_bytes": 158683136,
|
||||
"events": {
|
||||
"low": "0",
|
||||
"high": "0",
|
||||
"max": "0",
|
||||
"oom": "0",
|
||||
"oom_kill": "0",
|
||||
"oom_group_kill": "0"
|
||||
},
|
||||
"limit_bytes": 536870912,
|
||||
"oom_killed": false
|
||||
},
|
||||
{
|
||||
"service": "worker",
|
||||
"container": "2268305d2238f316cf7b5a65d2045a6c1a7a2ec70dd02c59754685fac7d5437f",
|
||||
"current_bytes": 72183808,
|
||||
"peak_bytes": 118439936,
|
||||
"events": {
|
||||
"low": "0",
|
||||
"high": "0",
|
||||
"max": "0",
|
||||
"oom": "0",
|
||||
"oom_kill": "0",
|
||||
"oom_group_kill": "0"
|
||||
},
|
||||
"limit_bytes": 939524096,
|
||||
"oom_killed": false
|
||||
},
|
||||
{
|
||||
"service": "collector",
|
||||
"container": "eb2404f8d1fd88ff2a037004fe07c978fd9e2fb56b5d66fc25aae540333d3938",
|
||||
"current_bytes": 52547584,
|
||||
"peak_bytes": 53600256,
|
||||
"events": {
|
||||
"low": "0",
|
||||
"high": "0",
|
||||
"max": "0",
|
||||
"oom": "0",
|
||||
"oom_kill": "0",
|
||||
"oom_group_kill": "0"
|
||||
},
|
||||
"limit_bytes": 268435456,
|
||||
"oom_killed": false
|
||||
}
|
||||
],
|
||||
"restored_trace_id": "c2af36cbb59745e0b403a8112b0e21af"
|
||||
}
|
||||
],
|
||||
"passed": true,
|
||||
"finished_at": "2026-11-16T00:15:16.555972+00:00"
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
All checks passed!
|
||||
@@ -0,0 +1,102 @@
|
||||
{
|
||||
"captured_at": "2026-11-16T09:05:50.509991+00:00",
|
||||
"image_revision": "548855e2c1a34bf8567a6245de0126dac8ed06a0",
|
||||
"roles": [
|
||||
{
|
||||
"role": "api",
|
||||
"container": "cfc128f2198a1b6de57103ce5d3ffe2ce5bd1f18499e778904e5b536b4425f4c",
|
||||
"matches_local": true,
|
||||
"files": {
|
||||
"src/messenger/__init__.py": "9a72a49106367ca514cc6147d0e727e62c4d33fefd61572d209a5c9494551a2c",
|
||||
"src/messenger/__main__.py": "ba9a59ad44ee8884cd65cf88d5ea15cbd7051b10f7c729043ddd44ca2f34defe",
|
||||
"src/messenger/app.py": "e66050123ff895a9fb6da14991457df9c30ed997d0ef983ed17e33fa940f75e1",
|
||||
"src/messenger/attachments.py": "a10f28f9ce7355975c2509e6a360d1086f39d601a563706ad3bcb231da82228e",
|
||||
"src/messenger/config.py": "e4d4aaf674ede8b5abdc361f77c7793ee94d8d7856030c28834eb7c95291b193",
|
||||
"src/messenger/domain.py": "7bf6f177ab5bdeea09b303dc886160335d348f64f47912b9bba45355db378bef",
|
||||
"src/messenger/image_pipeline.py": "3dcca5590704f1f9e8eca71dc28097930eac62701cf479aa8b7e3c71ca9b3551",
|
||||
"src/messenger/image_validation.py": "d9aa8db65edaab88c9a76cf2e5459ad3028ba808579311aeb0d8c4f01f875720",
|
||||
"src/messenger/jobctl.py": "41098fe1088960f6e62a3e3b6bd63e8b7f048b4561d82a994cf64574b939215d",
|
||||
"src/messenger/jobs.py": "9e5c79cd1388f6e065bd68088257e3dc86046922f0317b582a653037ca3f9165",
|
||||
"src/messenger/migrate.py": "eb5cad7ac6abe38dd40b2db307d298b297abe9543b3928c36bab8ef06283664e",
|
||||
"src/messenger/models.py": "dd2c5360e4ed1908fc91e57bbee106dbc5ab5c50a003e6ba0087ed254550452d",
|
||||
"src/messenger/postgres.py": "346cab57cc8020f943da3d12feb9dec8ba8f11798d83754a4ff57bf3dd273143",
|
||||
"src/messenger/storage.py": "d04cfc5d284e1b72437a0b99d3a1c56f358c4ae02d737109a27c3b5ebfa55f9e",
|
||||
"src/messenger/storage_bootstrap.py": "b29a2a1f8864ec3425a7499b1752688f69d8f402f3881957e790ed4522d35589",
|
||||
"src/messenger/telemetry.py": "16cd5f6f98538fd11c30bbe48d8a4ad69745ee1dc331e426855d37904cc439c5",
|
||||
"src/messenger/worker.py": "e4b37b90d918cb0dc8cc4f90b392d82351d22357da75c92b9b895da563eb9aba"
|
||||
}
|
||||
},
|
||||
{
|
||||
"role": "api",
|
||||
"container": "5d7baeaaaf03ec2879eafa26f425508cb82e54de0325d8cb4b364cf199c6d010",
|
||||
"matches_local": true,
|
||||
"files": {
|
||||
"src/messenger/__init__.py": "9a72a49106367ca514cc6147d0e727e62c4d33fefd61572d209a5c9494551a2c",
|
||||
"src/messenger/__main__.py": "ba9a59ad44ee8884cd65cf88d5ea15cbd7051b10f7c729043ddd44ca2f34defe",
|
||||
"src/messenger/app.py": "e66050123ff895a9fb6da14991457df9c30ed997d0ef983ed17e33fa940f75e1",
|
||||
"src/messenger/attachments.py": "a10f28f9ce7355975c2509e6a360d1086f39d601a563706ad3bcb231da82228e",
|
||||
"src/messenger/config.py": "e4d4aaf674ede8b5abdc361f77c7793ee94d8d7856030c28834eb7c95291b193",
|
||||
"src/messenger/domain.py": "7bf6f177ab5bdeea09b303dc886160335d348f64f47912b9bba45355db378bef",
|
||||
"src/messenger/image_pipeline.py": "3dcca5590704f1f9e8eca71dc28097930eac62701cf479aa8b7e3c71ca9b3551",
|
||||
"src/messenger/image_validation.py": "d9aa8db65edaab88c9a76cf2e5459ad3028ba808579311aeb0d8c4f01f875720",
|
||||
"src/messenger/jobctl.py": "41098fe1088960f6e62a3e3b6bd63e8b7f048b4561d82a994cf64574b939215d",
|
||||
"src/messenger/jobs.py": "9e5c79cd1388f6e065bd68088257e3dc86046922f0317b582a653037ca3f9165",
|
||||
"src/messenger/migrate.py": "eb5cad7ac6abe38dd40b2db307d298b297abe9543b3928c36bab8ef06283664e",
|
||||
"src/messenger/models.py": "dd2c5360e4ed1908fc91e57bbee106dbc5ab5c50a003e6ba0087ed254550452d",
|
||||
"src/messenger/postgres.py": "346cab57cc8020f943da3d12feb9dec8ba8f11798d83754a4ff57bf3dd273143",
|
||||
"src/messenger/storage.py": "d04cfc5d284e1b72437a0b99d3a1c56f358c4ae02d737109a27c3b5ebfa55f9e",
|
||||
"src/messenger/storage_bootstrap.py": "b29a2a1f8864ec3425a7499b1752688f69d8f402f3881957e790ed4522d35589",
|
||||
"src/messenger/telemetry.py": "16cd5f6f98538fd11c30bbe48d8a4ad69745ee1dc331e426855d37904cc439c5",
|
||||
"src/messenger/worker.py": "e4b37b90d918cb0dc8cc4f90b392d82351d22357da75c92b9b895da563eb9aba"
|
||||
}
|
||||
},
|
||||
{
|
||||
"role": "worker",
|
||||
"container": "1f0366be1a714aea43fee70af5af1f108d5f2406bceaa9d432c901ed89f4dce7",
|
||||
"matches_local": true,
|
||||
"files": {
|
||||
"src/messenger/__init__.py": "9a72a49106367ca514cc6147d0e727e62c4d33fefd61572d209a5c9494551a2c",
|
||||
"src/messenger/__main__.py": "ba9a59ad44ee8884cd65cf88d5ea15cbd7051b10f7c729043ddd44ca2f34defe",
|
||||
"src/messenger/app.py": "e66050123ff895a9fb6da14991457df9c30ed997d0ef983ed17e33fa940f75e1",
|
||||
"src/messenger/attachments.py": "a10f28f9ce7355975c2509e6a360d1086f39d601a563706ad3bcb231da82228e",
|
||||
"src/messenger/config.py": "e4d4aaf674ede8b5abdc361f77c7793ee94d8d7856030c28834eb7c95291b193",
|
||||
"src/messenger/domain.py": "7bf6f177ab5bdeea09b303dc886160335d348f64f47912b9bba45355db378bef",
|
||||
"src/messenger/image_pipeline.py": "3dcca5590704f1f9e8eca71dc28097930eac62701cf479aa8b7e3c71ca9b3551",
|
||||
"src/messenger/image_validation.py": "d9aa8db65edaab88c9a76cf2e5459ad3028ba808579311aeb0d8c4f01f875720",
|
||||
"src/messenger/jobctl.py": "41098fe1088960f6e62a3e3b6bd63e8b7f048b4561d82a994cf64574b939215d",
|
||||
"src/messenger/jobs.py": "9e5c79cd1388f6e065bd68088257e3dc86046922f0317b582a653037ca3f9165",
|
||||
"src/messenger/migrate.py": "eb5cad7ac6abe38dd40b2db307d298b297abe9543b3928c36bab8ef06283664e",
|
||||
"src/messenger/models.py": "dd2c5360e4ed1908fc91e57bbee106dbc5ab5c50a003e6ba0087ed254550452d",
|
||||
"src/messenger/postgres.py": "346cab57cc8020f943da3d12feb9dec8ba8f11798d83754a4ff57bf3dd273143",
|
||||
"src/messenger/storage.py": "d04cfc5d284e1b72437a0b99d3a1c56f358c4ae02d737109a27c3b5ebfa55f9e",
|
||||
"src/messenger/storage_bootstrap.py": "b29a2a1f8864ec3425a7499b1752688f69d8f402f3881957e790ed4522d35589",
|
||||
"src/messenger/telemetry.py": "16cd5f6f98538fd11c30bbe48d8a4ad69745ee1dc331e426855d37904cc439c5",
|
||||
"src/messenger/worker.py": "e4b37b90d918cb0dc8cc4f90b392d82351d22357da75c92b9b895da563eb9aba"
|
||||
}
|
||||
},
|
||||
{
|
||||
"role": "dispatcher",
|
||||
"container": "39769e4dc12941740226e8e8fb882aa0f05c3e046fc5f4c7a4c887dd4e46c3e4",
|
||||
"matches_local": true,
|
||||
"files": {
|
||||
"src/messenger/__init__.py": "9a72a49106367ca514cc6147d0e727e62c4d33fefd61572d209a5c9494551a2c",
|
||||
"src/messenger/__main__.py": "ba9a59ad44ee8884cd65cf88d5ea15cbd7051b10f7c729043ddd44ca2f34defe",
|
||||
"src/messenger/app.py": "e66050123ff895a9fb6da14991457df9c30ed997d0ef983ed17e33fa940f75e1",
|
||||
"src/messenger/attachments.py": "a10f28f9ce7355975c2509e6a360d1086f39d601a563706ad3bcb231da82228e",
|
||||
"src/messenger/config.py": "e4d4aaf674ede8b5abdc361f77c7793ee94d8d7856030c28834eb7c95291b193",
|
||||
"src/messenger/domain.py": "7bf6f177ab5bdeea09b303dc886160335d348f64f47912b9bba45355db378bef",
|
||||
"src/messenger/image_pipeline.py": "3dcca5590704f1f9e8eca71dc28097930eac62701cf479aa8b7e3c71ca9b3551",
|
||||
"src/messenger/image_validation.py": "d9aa8db65edaab88c9a76cf2e5459ad3028ba808579311aeb0d8c4f01f875720",
|
||||
"src/messenger/jobctl.py": "41098fe1088960f6e62a3e3b6bd63e8b7f048b4561d82a994cf64574b939215d",
|
||||
"src/messenger/jobs.py": "9e5c79cd1388f6e065bd68088257e3dc86046922f0317b582a653037ca3f9165",
|
||||
"src/messenger/migrate.py": "eb5cad7ac6abe38dd40b2db307d298b297abe9543b3928c36bab8ef06283664e",
|
||||
"src/messenger/models.py": "dd2c5360e4ed1908fc91e57bbee106dbc5ab5c50a003e6ba0087ed254550452d",
|
||||
"src/messenger/postgres.py": "346cab57cc8020f943da3d12feb9dec8ba8f11798d83754a4ff57bf3dd273143",
|
||||
"src/messenger/storage.py": "d04cfc5d284e1b72437a0b99d3a1c56f358c4ae02d737109a27c3b5ebfa55f9e",
|
||||
"src/messenger/storage_bootstrap.py": "b29a2a1f8864ec3425a7499b1752688f69d8f402f3881957e790ed4522d35589",
|
||||
"src/messenger/telemetry.py": "16cd5f6f98538fd11c30bbe48d8a4ad69745ee1dc331e426855d37904cc439c5",
|
||||
"src/messenger/worker.py": "e4b37b90d918cb0dc8cc4f90b392d82351d22357da75c92b9b895da563eb9aba"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
{
|
||||
"source_commit": "2b8372427dd8f7980039d694d3821d4e05302b6a",
|
||||
"files": {
|
||||
"migrations/001_initial.sql": "5d88cd12ef9fda008832dc0b886fa0cf5d886738fe9b77180cdc65adac50dfd9",
|
||||
"migrations/002_idempotency.sql": "82b41be828ad55ec5c1d991775528e7fe6766c362fa58a2c4f46690459b2c00d",
|
||||
"migrations/003_unicode_storage.sql": "7282a55da4aa081f7d3d99692e11eeb7a49feab9c9f829a09d3d72dc74009643",
|
||||
"migrations/004_attachments.sql": "ed6b89075160294a2ac0d55da3be65c41be06aaee1fa5ef22b387d3f83841a06",
|
||||
"migrations/005_attachment_only_message.sql": "9c53e55be4306a3d6b3f38ee4ee8013571cd03d1dc49958fdbd15859564db080",
|
||||
"migrations/006_jobs.sql": "dee847c794f0bbbdcf544a69c257979da9f29a4ce11921d7dfcfe3fc71447750",
|
||||
"migrations/007_observability.sql": "70e690806a8f15b9d876c71a63a826c589bb6020d5b425b11c88fdfbf3985f64",
|
||||
"migrations/008_telemetry_acceptance.sql": "b2504c5edae6171fe2e9888dbab4e24ae0a917412ba98fef7e8b89aa31960f59",
|
||||
"src/messenger/__init__.py": "9a72a49106367ca514cc6147d0e727e62c4d33fefd61572d209a5c9494551a2c",
|
||||
"src/messenger/__main__.py": "ba9a59ad44ee8884cd65cf88d5ea15cbd7051b10f7c729043ddd44ca2f34defe",
|
||||
"src/messenger/app.py": "e66050123ff895a9fb6da14991457df9c30ed997d0ef983ed17e33fa940f75e1",
|
||||
"src/messenger/attachments.py": "a10f28f9ce7355975c2509e6a360d1086f39d601a563706ad3bcb231da82228e",
|
||||
"src/messenger/config.py": "e4d4aaf674ede8b5abdc361f77c7793ee94d8d7856030c28834eb7c95291b193",
|
||||
"src/messenger/domain.py": "7bf6f177ab5bdeea09b303dc886160335d348f64f47912b9bba45355db378bef",
|
||||
"src/messenger/image_pipeline.py": "3dcca5590704f1f9e8eca71dc28097930eac62701cf479aa8b7e3c71ca9b3551",
|
||||
"src/messenger/image_validation.py": "d9aa8db65edaab88c9a76cf2e5459ad3028ba808579311aeb0d8c4f01f875720",
|
||||
"src/messenger/jobctl.py": "41098fe1088960f6e62a3e3b6bd63e8b7f048b4561d82a994cf64574b939215d",
|
||||
"src/messenger/jobs.py": "9e5c79cd1388f6e065bd68088257e3dc86046922f0317b582a653037ca3f9165",
|
||||
"src/messenger/migrate.py": "eb5cad7ac6abe38dd40b2db307d298b297abe9543b3928c36bab8ef06283664e",
|
||||
"src/messenger/models.py": "dd2c5360e4ed1908fc91e57bbee106dbc5ab5c50a003e6ba0087ed254550452d",
|
||||
"src/messenger/postgres.py": "346cab57cc8020f943da3d12feb9dec8ba8f11798d83754a4ff57bf3dd273143",
|
||||
"src/messenger/storage.py": "d04cfc5d284e1b72437a0b99d3a1c56f358c4ae02d737109a27c3b5ebfa55f9e",
|
||||
"src/messenger/storage_bootstrap.py": "b29a2a1f8864ec3425a7499b1752688f69d8f402f3881957e790ed4522d35589",
|
||||
"src/messenger/telemetry.py": "16cd5f6f98538fd11c30bbe48d8a4ad69745ee1dc331e426855d37904cc439c5",
|
||||
"src/messenger/worker.py": "e4b37b90d918cb0dc8cc4f90b392d82351d22357da75c92b9b895da563eb9aba"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
sample25 1 PASS 39.78 RPS
|
||||
sample10 1 PASS 39.8 RPS
|
||||
full 2 PASS 39.84 RPS
|
||||
sample25 2 PASS 39.52 RPS
|
||||
sample10 2 PASS 39.84 RPS
|
||||
off 2 PASS 39.85 RPS
|
||||
sample25 3 PASS 39.87 RPS
|
||||
sample10 3 PASS 39.85 RPS
|
||||
off 3 PASS 39.84 RPS
|
||||
full 3 PASS 39.85 RPS
|
||||
{
|
||||
"off": {
|
||||
"median_success_rps": 39.848470847125924,
|
||||
"median_p95_s": 0.03684441700170282,
|
||||
"median_trace_wire_bytes": 0.0,
|
||||
"rare_total": 32,
|
||||
"rare_sampled": 0,
|
||||
"export_failed_spans": 0.0,
|
||||
"queue_lost_spans": 0.0
|
||||
},
|
||||
"full": {
|
||||
"median_success_rps": 39.85229800726845,
|
||||
"median_p95_s": 0.03505475000565639,
|
||||
"median_trace_wire_bytes": 134268.0,
|
||||
"rare_total": 39,
|
||||
"rare_sampled": 39,
|
||||
"export_failed_spans": 0.0,
|
||||
"queue_lost_spans": 0.0
|
||||
},
|
||||
"sample25": {
|
||||
"median_success_rps": 39.77777880417364,
|
||||
"median_p95_s": 0.045096541005477775,
|
||||
"median_trace_wire_bytes": 39431.0,
|
||||
"rare_total": 31,
|
||||
"rare_sampled": 10,
|
||||
"export_failed_spans": 0.0,
|
||||
"queue_lost_spans": 0.0
|
||||
},
|
||||
"sample10": {
|
||||
"median_success_rps": 39.835097210352295,
|
||||
"median_p95_s": 0.030325082996569108,
|
||||
"median_trace_wire_bytes": 20769.0,
|
||||
"rare_total": 32,
|
||||
"rare_sampled": 5,
|
||||
"export_failed_spans": 0.0,
|
||||
"queue_lost_spans": 0.0
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
[
|
||||
{
|
||||
"profile": "off",
|
||||
"runs": 3,
|
||||
"median_success_rps": 39.848470847125924,
|
||||
"median_p50_ms": 5.394749998231418,
|
||||
"median_p95_ms": 36.84441700170282,
|
||||
"median_p99_ms": 288.0857920026756,
|
||||
"mean_total_docker_cpu_percent": 45.94916666666667,
|
||||
"peak_total_docker_memory_mib": 1175.78,
|
||||
"median_traces_wire_bytes": 0.0,
|
||||
"median_logs_wire_bytes": 0.0,
|
||||
"median_metrics_wire_bytes": 0.0,
|
||||
"median_collector_accepted_spans": 0.0,
|
||||
"median_collector_refused_spans": 0.0,
|
||||
"export_failed_spans": 0.0,
|
||||
"queue_lost_spans": 0.0,
|
||||
"not_sent": 0,
|
||||
"rare_total": 32,
|
||||
"rare_found": 0,
|
||||
"errors_found": 0,
|
||||
"slow_found": 0
|
||||
},
|
||||
{
|
||||
"profile": "full",
|
||||
"runs": 3,
|
||||
"median_success_rps": 39.85229800726845,
|
||||
"median_p50_ms": 5.507000001671258,
|
||||
"median_p95_ms": 35.05475000565639,
|
||||
"median_p99_ms": 285.96479200496105,
|
||||
"mean_total_docker_cpu_percent": 52.064166666666665,
|
||||
"peak_total_docker_memory_mib": 983.8240000000001,
|
||||
"median_traces_wire_bytes": 134268.0,
|
||||
"median_logs_wire_bytes": 228873.0,
|
||||
"median_metrics_wire_bytes": 19200.0,
|
||||
"median_collector_accepted_spans": 840.0,
|
||||
"median_collector_refused_spans": 0.0,
|
||||
"export_failed_spans": 0.0,
|
||||
"queue_lost_spans": 0.0,
|
||||
"not_sent": 0,
|
||||
"rare_total": 39,
|
||||
"rare_found": 39,
|
||||
"errors_found": 6,
|
||||
"slow_found": 33
|
||||
},
|
||||
{
|
||||
"profile": "sample25",
|
||||
"runs": 3,
|
||||
"median_success_rps": 39.77777880417364,
|
||||
"median_p50_ms": 5.806125001981854,
|
||||
"median_p95_ms": 45.096541005477775,
|
||||
"median_p99_ms": 272.20208299695514,
|
||||
"mean_total_docker_cpu_percent": 50.21,
|
||||
"peak_total_docker_memory_mib": 1232.86,
|
||||
"median_traces_wire_bytes": 39431.0,
|
||||
"median_logs_wire_bytes": 229445.0,
|
||||
"median_metrics_wire_bytes": 18935.0,
|
||||
"median_collector_accepted_spans": 194.0,
|
||||
"median_collector_refused_spans": 0.0,
|
||||
"export_failed_spans": 0.0,
|
||||
"queue_lost_spans": 0.0,
|
||||
"not_sent": 0,
|
||||
"rare_total": 31,
|
||||
"rare_found": 10,
|
||||
"errors_found": 2,
|
||||
"slow_found": 8
|
||||
},
|
||||
{
|
||||
"profile": "sample10",
|
||||
"runs": 3,
|
||||
"median_success_rps": 39.835097210352295,
|
||||
"median_p50_ms": 6.040542000846472,
|
||||
"median_p95_ms": 30.325082996569108,
|
||||
"median_p99_ms": 286.31024999776855,
|
||||
"mean_total_docker_cpu_percent": 53.4,
|
||||
"peak_total_docker_memory_mib": 1079.92,
|
||||
"median_traces_wire_bytes": 20769.0,
|
||||
"median_logs_wire_bytes": 228854.0,
|
||||
"median_metrics_wire_bytes": 18694.0,
|
||||
"median_collector_accepted_spans": 82.0,
|
||||
"median_collector_refused_spans": 0.0,
|
||||
"export_failed_spans": 0.0,
|
||||
"queue_lost_spans": 0.0,
|
||||
"not_sent": 0,
|
||||
"rare_total": 32,
|
||||
"rare_found": 5,
|
||||
"errors_found": 1,
|
||||
"slow_found": 4
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,90 @@
|
||||
[
|
||||
{
|
||||
"profile": "off",
|
||||
"runs": 3,
|
||||
"median_success_rps": 39.848470847125924,
|
||||
"median_p50_ms": 5.394749998231418,
|
||||
"median_p95_ms": 36.84441700170282,
|
||||
"median_p99_ms": 288.0857920026756,
|
||||
"mean_total_docker_cpu_percent": 45.94916666666667,
|
||||
"peak_total_docker_memory_mib": 1175.78,
|
||||
"median_traces_wire_bytes": 0.0,
|
||||
"median_logs_wire_bytes": 0.0,
|
||||
"median_metrics_wire_bytes": 0.0,
|
||||
"median_collector_accepted_spans": 0.0,
|
||||
"median_collector_refused_spans": 0.0,
|
||||
"export_failed_spans": 0.0,
|
||||
"queue_lost_spans": 0.0,
|
||||
"not_sent": 0,
|
||||
"rare_total": 32,
|
||||
"rare_found": 0,
|
||||
"errors_found": 0,
|
||||
"slow_found": 0
|
||||
},
|
||||
{
|
||||
"profile": "full",
|
||||
"runs": 3,
|
||||
"median_success_rps": 39.85229800726845,
|
||||
"median_p50_ms": 5.507000001671258,
|
||||
"median_p95_ms": 35.05475000565639,
|
||||
"median_p99_ms": 285.96479200496105,
|
||||
"mean_total_docker_cpu_percent": 52.064166666666665,
|
||||
"peak_total_docker_memory_mib": 983.8240000000001,
|
||||
"median_traces_wire_bytes": 134268.0,
|
||||
"median_logs_wire_bytes": 228873.0,
|
||||
"median_metrics_wire_bytes": 19200.0,
|
||||
"median_collector_accepted_spans": 840.0,
|
||||
"median_collector_refused_spans": 0.0,
|
||||
"export_failed_spans": 0.0,
|
||||
"queue_lost_spans": 0.0,
|
||||
"not_sent": 0,
|
||||
"rare_total": 39,
|
||||
"rare_found": 39,
|
||||
"errors_found": 6,
|
||||
"slow_found": 33
|
||||
},
|
||||
{
|
||||
"profile": "sample25",
|
||||
"runs": 3,
|
||||
"median_success_rps": 39.77777880417364,
|
||||
"median_p50_ms": 5.806125001981854,
|
||||
"median_p95_ms": 45.096541005477775,
|
||||
"median_p99_ms": 272.20208299695514,
|
||||
"mean_total_docker_cpu_percent": 50.21,
|
||||
"peak_total_docker_memory_mib": 1232.86,
|
||||
"median_traces_wire_bytes": 39431.0,
|
||||
"median_logs_wire_bytes": 229445.0,
|
||||
"median_metrics_wire_bytes": 18935.0,
|
||||
"median_collector_accepted_spans": 194.0,
|
||||
"median_collector_refused_spans": 0.0,
|
||||
"export_failed_spans": 0.0,
|
||||
"queue_lost_spans": 0.0,
|
||||
"not_sent": 0,
|
||||
"rare_total": 31,
|
||||
"rare_found": 10,
|
||||
"errors_found": 2,
|
||||
"slow_found": 8
|
||||
},
|
||||
{
|
||||
"profile": "sample10",
|
||||
"runs": 3,
|
||||
"median_success_rps": 39.835097210352295,
|
||||
"median_p50_ms": 6.040542000846472,
|
||||
"median_p95_ms": 30.325082996569108,
|
||||
"median_p99_ms": 286.31024999776855,
|
||||
"mean_total_docker_cpu_percent": 53.4,
|
||||
"peak_total_docker_memory_mib": 1079.92,
|
||||
"median_traces_wire_bytes": 20769.0,
|
||||
"median_logs_wire_bytes": 228854.0,
|
||||
"median_metrics_wire_bytes": 18694.0,
|
||||
"median_collector_accepted_spans": 82.0,
|
||||
"median_collector_refused_spans": 0.0,
|
||||
"export_failed_spans": 0.0,
|
||||
"queue_lost_spans": 0.0,
|
||||
"not_sent": 0,
|
||||
"rare_total": 32,
|
||||
"rare_found": 5,
|
||||
"errors_found": 1,
|
||||
"slow_found": 4
|
||||
}
|
||||
]
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1 @@
|
||||
PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.
|
||||
@@ -0,0 +1,133 @@
|
||||
{
|
||||
"captured_at": "2026-11-16T00:14:28.404528+00:00",
|
||||
"source_commit": "a531af3bdbf9059a8e1ef35988010681b156f127",
|
||||
"docker_host": {
|
||||
"NCPU": 4,
|
||||
"MemTotal": 8307167232,
|
||||
"OperatingSystem": "Ubuntu 24.04.4 LTS",
|
||||
"Architecture": "aarch64",
|
||||
"ServerVersion": "29.5.2"
|
||||
},
|
||||
"containers": [
|
||||
{
|
||||
"name": "/mtusi-vasilev-lab6-api-1",
|
||||
"image_ref": "mtusi-vasilev:lab6",
|
||||
"image_id": "sha256:6b173e593253adec07422a3dbac49f92aafba94b03d5e17a3e2b4e32275397ed",
|
||||
"memory_limit": 536870912,
|
||||
"nano_cpus": 500000000,
|
||||
"oom_killed": false
|
||||
},
|
||||
{
|
||||
"name": "/mtusi-vasilev-lab6-api-2",
|
||||
"image_ref": "mtusi-vasilev:lab6",
|
||||
"image_id": "sha256:6b173e593253adec07422a3dbac49f92aafba94b03d5e17a3e2b4e32275397ed",
|
||||
"memory_limit": 536870912,
|
||||
"nano_cpus": 500000000,
|
||||
"oom_killed": false
|
||||
},
|
||||
{
|
||||
"name": "/mtusi-vasilev-lab6-broker-1",
|
||||
"image_ref": "valkey/valkey:8.1.3-alpine",
|
||||
"image_id": "sha256:d827e7f7552cdee40cc7482dbae9da020f42bc47669af6f71182a4ef76a22773",
|
||||
"memory_limit": 201326592,
|
||||
"nano_cpus": 500000000,
|
||||
"oom_killed": false
|
||||
},
|
||||
{
|
||||
"name": "/mtusi-vasilev-lab6-cache-1",
|
||||
"image_ref": "valkey/valkey:8.1.3-alpine",
|
||||
"image_id": "sha256:d827e7f7552cdee40cc7482dbae9da020f42bc47669af6f71182a4ef76a22773",
|
||||
"memory_limit": 134217728,
|
||||
"nano_cpus": 500000000,
|
||||
"oom_killed": false
|
||||
},
|
||||
{
|
||||
"name": "/mtusi-vasilev-lab6-collector-1",
|
||||
"image_ref": "otel/opentelemetry-collector-contrib:0.123.0",
|
||||
"image_id": "sha256:e39311df1f3d941923c00da79ac7ba6269124a870ee87e3c3ad24d60f8aee4d2",
|
||||
"memory_limit": 268435456,
|
||||
"nano_cpus": 500000000,
|
||||
"oom_killed": false
|
||||
},
|
||||
{
|
||||
"name": "/mtusi-vasilev-lab6-dispatcher-1",
|
||||
"image_ref": "mtusi-vasilev:lab6",
|
||||
"image_id": "sha256:6b173e593253adec07422a3dbac49f92aafba94b03d5e17a3e2b4e32275397ed",
|
||||
"memory_limit": 201326592,
|
||||
"nano_cpus": 250000000,
|
||||
"oom_killed": false
|
||||
},
|
||||
{
|
||||
"name": "/mtusi-vasilev-lab6-grafana-1",
|
||||
"image_ref": "grafana/grafana:11.6.0",
|
||||
"image_id": "sha256:62d2b9d20a19714ebfe48d1bb405086081bc602aa053e28cf6d73c7537640dfb",
|
||||
"memory_limit": 268435456,
|
||||
"nano_cpus": 500000000,
|
||||
"oom_killed": false
|
||||
},
|
||||
{
|
||||
"name": "/mtusi-vasilev-lab6-lb-1",
|
||||
"image_ref": "haproxy:3.2-alpine@sha256:96ec3c75a9264ba94e733eeacb44051ed6e7b79bb77a73c2c36f236b6423f1f0",
|
||||
"image_id": "sha256:96ec3c75a9264ba94e733eeacb44051ed6e7b79bb77a73c2c36f236b6423f1f0",
|
||||
"memory_limit": 134217728,
|
||||
"nano_cpus": 500000000,
|
||||
"oom_killed": false
|
||||
},
|
||||
{
|
||||
"name": "/mtusi-vasilev-lab6-loki-1",
|
||||
"image_ref": "grafana/loki:3.4.2",
|
||||
"image_id": "sha256:58a6c186ce78ba04d58bfe2a927eff296ba733a430df09645d56cdc158f3ba08",
|
||||
"memory_limit": 402653184,
|
||||
"nano_cpus": 500000000,
|
||||
"oom_killed": false
|
||||
},
|
||||
{
|
||||
"name": "/mtusi-vasilev-lab6-postgres-1",
|
||||
"image_ref": "postgres:17.10-alpine",
|
||||
"image_id": "sha256:742f40ea20b9ff2ff31db5458d127452988a2164df9e17441e191f3b72252193",
|
||||
"memory_limit": 805306368,
|
||||
"nano_cpus": 1000000000,
|
||||
"oom_killed": false
|
||||
},
|
||||
{
|
||||
"name": "/mtusi-vasilev-lab6-prometheus-1",
|
||||
"image_ref": "prom/prometheus:v3.2.1",
|
||||
"image_id": "sha256:6927e0919a144aa7616fd0137d4816816d42f6b816de3af269ab065250859a62",
|
||||
"memory_limit": 268435456,
|
||||
"nano_cpus": 500000000,
|
||||
"oom_killed": false
|
||||
},
|
||||
{
|
||||
"name": "/mtusi-vasilev-lab6-s3-1",
|
||||
"image_ref": "chrislusf/seaweedfs:4.15@sha256:1593bd2a98964f5d4df83fa7db2b3d4ee7b20190813cc5bf8c3200e0e70d346e",
|
||||
"image_id": "sha256:1593bd2a98964f5d4df83fa7db2b3d4ee7b20190813cc5bf8c3200e0e70d346e",
|
||||
"memory_limit": 268435456,
|
||||
"nano_cpus": 500000000,
|
||||
"oom_killed": false
|
||||
},
|
||||
{
|
||||
"name": "/mtusi-vasilev-lab6-storage-1",
|
||||
"image_ref": "chrislusf/seaweedfs:4.15@sha256:1593bd2a98964f5d4df83fa7db2b3d4ee7b20190813cc5bf8c3200e0e70d346e",
|
||||
"image_id": "sha256:1593bd2a98964f5d4df83fa7db2b3d4ee7b20190813cc5bf8c3200e0e70d346e",
|
||||
"memory_limit": 536870912,
|
||||
"nano_cpus": 1000000000,
|
||||
"oom_killed": false
|
||||
},
|
||||
{
|
||||
"name": "/mtusi-vasilev-lab6-tempo-1",
|
||||
"image_ref": "grafana/tempo:2.7.2",
|
||||
"image_id": "sha256:4b0277a9b572a4b1fa43b01468e58911ec214dd69d033f2cd476596d14de9fe3",
|
||||
"memory_limit": 805306368,
|
||||
"nano_cpus": 500000000,
|
||||
"oom_killed": false
|
||||
},
|
||||
{
|
||||
"name": "/mtusi-vasilev-lab6-worker-1",
|
||||
"image_ref": "mtusi-vasilev:lab6",
|
||||
"image_id": "sha256:6b173e593253adec07422a3dbac49f92aafba94b03d5e17a3e2b4e32275397ed",
|
||||
"memory_limit": 939524096,
|
||||
"nano_cpus": 1000000000,
|
||||
"oom_killed": false
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
python3 tests/smoke.py --base-url "http://127.0.0.1:8086"
|
||||
PASS: lab6 public API scenario; infrastructure and grading evidence remain separate.
|
||||
@@ -0,0 +1,5 @@
|
||||
BASE_URL=http://127.0.0.1:8086 uv run --locked pytest -q
|
||||
........................................................................ [ 39%]
|
||||
........................................................................ [ 79%]
|
||||
...................................... [100%]
|
||||
182 passed in 47.47s
|
||||
@@ -0,0 +1,5 @@
|
||||
BASE_URL=http://127.0.0.1:8086 uv run --locked pytest -q
|
||||
........................................................................ [ 39%]
|
||||
........................................................................ [ 78%]
|
||||
....................................... [100%]
|
||||
183 passed in 58.39s
|
||||
@@ -0,0 +1,3 @@
|
||||
[leveldb2]
|
||||
enabled = true
|
||||
dir = "/data/filerldb2"
|
||||
@@ -0,0 +1,450 @@
|
||||
{
|
||||
"uid": "messenger-overview",
|
||||
"title": "Messenger: API and image pipeline",
|
||||
"schemaVersion": 39,
|
||||
"version": 1,
|
||||
"refresh": "5s",
|
||||
"time": {
|
||||
"from": "now-15m",
|
||||
"to": "now"
|
||||
},
|
||||
"panels": [
|
||||
{
|
||||
"id": 1,
|
||||
"title": "Business RPS",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum(rate(messenger_http_requests_total[1m]))"
|
||||
}
|
||||
],
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 0
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "reqps"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"title": "Business 5xx ratio",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum(rate(messenger_http_requests_total{status_class=\"5xx\"}[1m])) / clamp_min(sum(rate(messenger_http_requests_total[1m])),0.001)"
|
||||
}
|
||||
],
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 0
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percentunit"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"title": "HTTP latency quantiles",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "histogram_quantile(0.5, sum by (le) (rate(messenger_http_duration_seconds_bucket[1m])))",
|
||||
"legendFormat": "p50"
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_http_duration_seconds_bucket[1m])))",
|
||||
"legendFormat": "p95"
|
||||
},
|
||||
{
|
||||
"refId": "C",
|
||||
"expr": "histogram_quantile(0.99, sum by (le) (rate(messenger_http_duration_seconds_bucket[1m])))",
|
||||
"legendFormat": "p99"
|
||||
}
|
||||
],
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 8
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"title": "Job outcomes",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum by (outcome) (rate(messenger_jobs_total[1m]))"
|
||||
}
|
||||
],
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 8
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "ops"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"title": "Job p95 processing",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_job_duration_seconds_bucket[1m])))"
|
||||
}
|
||||
],
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 16
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"title": "End-to-end p95 to ready",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_attachment_ready_duration_seconds_bucket[1m])))"
|
||||
}
|
||||
],
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 16
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 7,
|
||||
"title": "Outstanding jobs",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "max(messenger_queue_depth)"
|
||||
}
|
||||
],
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 24
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 8,
|
||||
"title": "Oldest outstanding age",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "max(messenger_queue_oldest_age_seconds)"
|
||||
}
|
||||
],
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 24
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 9,
|
||||
"title": "DB pool used",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "messenger_db_pool_used"
|
||||
}
|
||||
],
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 32
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 10,
|
||||
"title": "DB pool waiting",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "messenger_db_pool_waiting"
|
||||
}
|
||||
],
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 32
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 11,
|
||||
"title": "Worker busy/capacity",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "messenger_worker_busy / messenger_worker_capacity"
|
||||
}
|
||||
],
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 40
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percentunit"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 12,
|
||||
"title": "Process resident memory",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "messenger_process_memory_bytes"
|
||||
}
|
||||
],
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 40
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "bytes"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 13,
|
||||
"title": "Process CPU (100%=1 core)",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "messenger_process_cpu_percent"
|
||||
}
|
||||
],
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 48
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percent"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 14,
|
||||
"title": "Sampling decisions",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum by (decision) (rate(messenger_http_trace_decisions_total[1m]))"
|
||||
}
|
||||
],
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 48
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "ops"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 15,
|
||||
"title": "Collector failed span sends",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "rate(otelcol_exporter_send_failed_spans_total[1m])"
|
||||
}
|
||||
],
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 56
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "ops"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 99,
|
||||
"title": "Structured application logs (trace_id and request_id)",
|
||||
"type": "logs",
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "loki"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "{service_name=~\"messenger-api|messenger-worker\"}"
|
||||
}
|
||||
],
|
||||
"gridPos": {
|
||||
"h": 9,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 64
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
apiVersion: 1
|
||||
providers:
|
||||
- name: messenger
|
||||
folder: Messenger
|
||||
type: file
|
||||
options:
|
||||
path: /var/lib/grafana/dashboards
|
||||
@@ -0,0 +1,27 @@
|
||||
apiVersion: 1
|
||||
datasources:
|
||||
- name: Prometheus
|
||||
uid: prometheus
|
||||
type: prometheus
|
||||
access: proxy
|
||||
url: http://prometheus:9090
|
||||
isDefault: true
|
||||
editable: false
|
||||
- name: Loki
|
||||
uid: loki
|
||||
type: loki
|
||||
access: proxy
|
||||
url: http://loki:3100
|
||||
editable: false
|
||||
jsonData:
|
||||
derivedFields:
|
||||
- datasourceUid: tempo
|
||||
matcherRegex: '"trace_id": "([a-f0-9]{32})"'
|
||||
name: TraceID
|
||||
url: '$${__value.raw}'
|
||||
- name: Tempo
|
||||
uid: tempo
|
||||
type: tempo
|
||||
access: proxy
|
||||
url: http://tempo:3200
|
||||
editable: false
|
||||
@@ -0,0 +1,37 @@
|
||||
global
|
||||
log stdout format raw local0
|
||||
maxconn 256
|
||||
stats socket /tmp/haproxy.sock mode 600 level admin
|
||||
|
||||
defaults
|
||||
mode http
|
||||
log global
|
||||
timeout connect 1s
|
||||
timeout client 10s
|
||||
timeout server 8s
|
||||
timeout check 1s
|
||||
timeout queue 2s
|
||||
retries 0
|
||||
option httplog
|
||||
|
||||
resolvers docker
|
||||
nameserver dns 127.0.0.11:53
|
||||
resolve_retries 3
|
||||
timeout resolve 1s
|
||||
timeout retry 1s
|
||||
hold valid 1s
|
||||
hold obsolete 1s
|
||||
hold nx 1s
|
||||
|
||||
frontend public
|
||||
bind :8080
|
||||
http-request set-var(txn.lb_request_id) uuid()
|
||||
http-request return status 503 content-type application/json lf-string '{"error":{"code":"unavailable","message":"No ready API","request_id":"%[var(txn.lb_request_id)]"}}' hdr X-Request-Id '%[var(txn.lb_request_id)]' if { nbsrv(apis) eq 0 }
|
||||
default_backend apis
|
||||
|
||||
backend apis
|
||||
balance roundrobin
|
||||
option httpchk GET /health/ready
|
||||
http-check expect status 200
|
||||
default-server inter 1s fastinter 500ms downinter 1s fall 2 rise 2
|
||||
server-template api 1-8 api:8080 check resolvers docker resolve-prefer ipv4 init-addr libc,none
|
||||
@@ -0,0 +1,24 @@
|
||||
groups:
|
||||
- name: messenger-slo
|
||||
rules:
|
||||
- alert: MessengerHighErrorRate
|
||||
expr: sum(rate(messenger_http_requests_total{status_class="5xx"}[1m])) / clamp_min(sum(rate(messenger_http_requests_total[1m])), 0.001) > 0.05
|
||||
for: 10s
|
||||
labels: {severity: warning}
|
||||
annotations:
|
||||
summary: Business HTTP 5xx ratio above 5% over 1 minute
|
||||
runbook_url: file:RUNBOOK.md#api-errors
|
||||
- alert: MessengerQueueLag
|
||||
expr: max(messenger_queue_oldest_age_seconds) > 15 and max(messenger_queue_depth) > 0
|
||||
for: 10s
|
||||
labels: {severity: warning}
|
||||
annotations:
|
||||
summary: Outstanding attachment job age above 15 seconds
|
||||
runbook_url: file:RUNBOOK.md#queue-lag
|
||||
- alert: MessengerTelemetryUnavailable
|
||||
expr: up{job="messenger"} == 0
|
||||
for: 10s
|
||||
labels: {severity: warning}
|
||||
annotations:
|
||||
summary: Collector metrics unavailable
|
||||
runbook_url: file:RUNBOOK.md#telemetry-unavailable
|
||||
@@ -0,0 +1,82 @@
|
||||
extensions:
|
||||
health_check:
|
||||
endpoint: 0.0.0.0:13133
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
grpc:
|
||||
endpoint: 0.0.0.0:4317
|
||||
http:
|
||||
endpoint: 0.0.0.0:4318
|
||||
processors:
|
||||
memory_limiter:
|
||||
check_interval: 1s
|
||||
limit_mib: 160
|
||||
spike_limit_mib: 32
|
||||
batch:
|
||||
timeout: 1s
|
||||
send_batch_size: 128
|
||||
send_batch_max_size: 256
|
||||
exporters:
|
||||
prometheus:
|
||||
endpoint: 0.0.0.0:9464
|
||||
metric_expiration: 15m
|
||||
resource_to_telemetry_conversion:
|
||||
enabled: true
|
||||
otlphttp/tempo:
|
||||
endpoint: http://tempo:4318
|
||||
timeout: 2s
|
||||
sending_queue:
|
||||
queue_size: 128
|
||||
num_consumers: 1
|
||||
retry_on_failure:
|
||||
initial_interval: 1s
|
||||
max_interval: 2s
|
||||
max_elapsed_time: 5s
|
||||
otlphttp/loki:
|
||||
endpoint: http://loki:3100/otlp
|
||||
timeout: 2s
|
||||
sending_queue:
|
||||
queue_size: 128
|
||||
num_consumers: 1
|
||||
retry_on_failure:
|
||||
initial_interval: 1s
|
||||
max_interval: 2s
|
||||
max_elapsed_time: 5s
|
||||
service:
|
||||
extensions:
|
||||
- health_check
|
||||
telemetry:
|
||||
metrics:
|
||||
level: detailed
|
||||
readers:
|
||||
- pull:
|
||||
exporter:
|
||||
prometheus:
|
||||
host: 0.0.0.0
|
||||
port: 8888
|
||||
pipelines:
|
||||
traces:
|
||||
receivers:
|
||||
- otlp
|
||||
processors:
|
||||
- memory_limiter
|
||||
- batch
|
||||
exporters:
|
||||
- otlphttp/tempo
|
||||
metrics:
|
||||
receivers:
|
||||
- otlp
|
||||
processors:
|
||||
- memory_limiter
|
||||
- batch
|
||||
exporters:
|
||||
- prometheus
|
||||
logs:
|
||||
receivers:
|
||||
- otlp
|
||||
processors:
|
||||
- memory_limiter
|
||||
- batch
|
||||
exporters:
|
||||
- otlphttp/loki
|
||||
@@ -0,0 +1,32 @@
|
||||
auth_enabled: false
|
||||
server:
|
||||
http_listen_port: 3100
|
||||
common:
|
||||
instance_addr: 127.0.0.1
|
||||
path_prefix: /loki
|
||||
storage:
|
||||
filesystem:
|
||||
chunks_directory: /loki/chunks
|
||||
rules_directory: /loki/rules
|
||||
replication_factor: 1
|
||||
ring:
|
||||
kvstore:
|
||||
store: inmemory
|
||||
schema_config:
|
||||
configs:
|
||||
- from: 2024-01-01
|
||||
store: tsdb
|
||||
object_store: filesystem
|
||||
schema: v13
|
||||
index:
|
||||
prefix: index_
|
||||
period: 24h
|
||||
limits_config:
|
||||
allow_structured_metadata: true
|
||||
retention_period: 24h
|
||||
compactor:
|
||||
working_directory: /loki/compactor
|
||||
retention_enabled: true
|
||||
delete_request_store: filesystem
|
||||
analytics:
|
||||
reporting_enabled: false
|
||||
@@ -0,0 +1,11 @@
|
||||
global:
|
||||
scrape_interval: 5s
|
||||
evaluation_interval: 5s
|
||||
rule_files: [/etc/prometheus/alerts.yaml]
|
||||
scrape_configs:
|
||||
- job_name: messenger
|
||||
static_configs:
|
||||
- targets: [collector:9464]
|
||||
- job_name: collector
|
||||
static_configs:
|
||||
- targets: [collector:8888]
|
||||
@@ -0,0 +1,24 @@
|
||||
server:
|
||||
http_listen_port: 3200
|
||||
distributor:
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
grpc:
|
||||
endpoint: 0.0.0.0:4317
|
||||
http:
|
||||
endpoint: 0.0.0.0:4318
|
||||
ingester:
|
||||
max_block_duration: 1m
|
||||
compactor:
|
||||
compaction:
|
||||
block_retention: 24h
|
||||
storage:
|
||||
trace:
|
||||
backend: local
|
||||
wal:
|
||||
path: /var/tempo/wal
|
||||
local:
|
||||
path: /var/tempo/blocks
|
||||
usage_report:
|
||||
reporting_enabled: false
|
||||
@@ -0,0 +1,35 @@
|
||||
CREATE TABLE users (
|
||||
id uuid PRIMARY KEY,
|
||||
username text NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'),
|
||||
display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100),
|
||||
password_hash text NOT NULL,
|
||||
created_at timestamptz NOT NULL
|
||||
);
|
||||
CREATE TABLE chats (
|
||||
id uuid PRIMARY KEY,
|
||||
title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100),
|
||||
created_at timestamptz NOT NULL
|
||||
);
|
||||
CREATE TABLE members (
|
||||
chat_id uuid NOT NULL REFERENCES chats(id),
|
||||
user_id uuid NOT NULL REFERENCES users(id),
|
||||
position integer NOT NULL CHECK (position >= 0),
|
||||
PRIMARY KEY(chat_id,user_id),
|
||||
UNIQUE(chat_id,position)
|
||||
);
|
||||
CREATE INDEX members_user_idx ON members(user_id,chat_id);
|
||||
CREATE TABLE messages (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL,
|
||||
sender_id uuid NOT NULL,
|
||||
text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000),
|
||||
created_at timestamptz NOT NULL,
|
||||
FOREIGN KEY(chat_id,sender_id) REFERENCES members(chat_id,user_id)
|
||||
);
|
||||
CREATE INDEX messages_history_idx ON messages(chat_id,created_at,id);
|
||||
CREATE TABLE sessions (
|
||||
token_digest text PRIMARY KEY,
|
||||
user_id uuid NOT NULL REFERENCES users(id),
|
||||
expires_at timestamptz NOT NULL
|
||||
);
|
||||
CREATE INDEX sessions_expiry_idx ON sessions(expires_at);
|
||||
@@ -0,0 +1,10 @@
|
||||
CREATE TABLE message_requests (
|
||||
actor_id uuid NOT NULL REFERENCES users(id),
|
||||
chat_id uuid NOT NULL REFERENCES chats(id),
|
||||
key text NOT NULL,
|
||||
body_digest text NOT NULL,
|
||||
response jsonb NOT NULL,
|
||||
expires_at timestamptz NOT NULL,
|
||||
PRIMARY KEY(actor_id,chat_id,key)
|
||||
);
|
||||
CREATE INDEX message_requests_expiry_idx ON message_requests(expires_at);
|
||||
@@ -0,0 +1,13 @@
|
||||
-- PostgreSQL text rejects U+0000, while the API accepts Unicode code points.
|
||||
-- Preserve UTF-8 losslessly. API validation enforces code point limits; the DB
|
||||
-- additionally bounds byte lengths, without stripping or replacing characters.
|
||||
ALTER TABLE users DROP CONSTRAINT users_display_name_check;
|
||||
ALTER TABLE users ALTER COLUMN display_name TYPE bytea USING convert_to(display_name,'UTF8');
|
||||
ALTER TABLE users ADD CHECK (octet_length(display_name) BETWEEN 1 AND 400);
|
||||
ALTER TABLE chats DROP CONSTRAINT chats_title_check;
|
||||
ALTER TABLE chats ALTER COLUMN title TYPE bytea USING convert_to(title,'UTF8');
|
||||
ALTER TABLE chats ADD CHECK (octet_length(title) BETWEEN 1 AND 400);
|
||||
ALTER TABLE messages DROP CONSTRAINT messages_text_check;
|
||||
ALTER TABLE messages ALTER COLUMN text TYPE bytea USING convert_to(text,'UTF8');
|
||||
ALTER TABLE messages ADD CHECK (octet_length(text) BETWEEN 1 AND 16000);
|
||||
ALTER TABLE message_requests ALTER COLUMN response TYPE bytea USING convert_to(response::text,'UTF8');
|
||||
@@ -0,0 +1,35 @@
|
||||
CREATE TABLE attachments (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL REFERENCES chats(id),
|
||||
owner_id uuid NOT NULL REFERENCES users(id),
|
||||
filename text NOT NULL,
|
||||
content_type text NOT NULL CHECK (content_type IN ('image/png','image/jpeg')),
|
||||
size_bytes bigint NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760),
|
||||
sha256 text NOT NULL,
|
||||
status text NOT NULL CONSTRAINT attachments_status_check CHECK (status IN ('queued','processing','ready','failed')),
|
||||
error_code text,
|
||||
created_at timestamptz NOT NULL,
|
||||
updated_at timestamptz NOT NULL,
|
||||
object_key text NOT NULL UNIQUE,
|
||||
UNIQUE(id,chat_id)
|
||||
);
|
||||
CREATE TABLE message_attachments (
|
||||
message_id uuid NOT NULL REFERENCES messages(id),
|
||||
attachment_id uuid NOT NULL REFERENCES attachments(id),
|
||||
position integer NOT NULL CHECK (position BETWEEN 0 AND 9),
|
||||
PRIMARY KEY(message_id,attachment_id),
|
||||
UNIQUE(message_id,position)
|
||||
);
|
||||
CREATE INDEX message_attachments_attachment_idx ON message_attachments(attachment_id);
|
||||
CREATE TABLE upload_intents (
|
||||
id uuid PRIMARY KEY,
|
||||
object_key text NOT NULL UNIQUE,
|
||||
created_at timestamptz NOT NULL
|
||||
);
|
||||
CREATE INDEX upload_intents_created_idx ON upload_intents(created_at,id);
|
||||
CREATE TABLE attachment_reconcile_audit (
|
||||
id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
|
||||
event text NOT NULL,
|
||||
object_key text NOT NULL,
|
||||
happened_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
@@ -0,0 +1,3 @@
|
||||
-- Empty text is valid only when the API supplies at least one attachment.
|
||||
ALTER TABLE messages DROP CONSTRAINT messages_text_check;
|
||||
ALTER TABLE messages ADD CONSTRAINT messages_text_check CHECK (octet_length(text) <= 16000);
|
||||
@@ -0,0 +1,49 @@
|
||||
ALTER TABLE attachments DROP CONSTRAINT IF EXISTS attachments_status_check;
|
||||
ALTER TABLE attachments ADD CONSTRAINT attachments_status_check
|
||||
CHECK (status IN ('queued','processing','ready','failed'));
|
||||
ALTER TABLE attachments ADD COLUMN pipeline_version integer NOT NULL DEFAULT 1 CHECK (pipeline_version > 0);
|
||||
ALTER TABLE attachments ADD COLUMN variants jsonb NOT NULL DEFAULT '[]';
|
||||
ALTER TABLE attachments ADD COLUMN stage_key text;
|
||||
|
||||
CREATE TABLE attachment_jobs (
|
||||
id uuid PRIMARY KEY,
|
||||
attachment_id uuid NOT NULL REFERENCES attachments(id),
|
||||
pipeline_version integer NOT NULL CHECK (pipeline_version > 0),
|
||||
source_key text,
|
||||
state text NOT NULL CHECK (state IN ('queued','processing','ready','failed','superseded')),
|
||||
attempt integer NOT NULL DEFAULT 0 CHECK (attempt >= 0),
|
||||
attempt_limit integer NOT NULL CHECK (attempt_limit > 0),
|
||||
lease_id uuid,
|
||||
lease_until timestamptz,
|
||||
available_at timestamptz NOT NULL DEFAULT now(),
|
||||
error_code text,
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
updated_at timestamptz NOT NULL DEFAULT now(),
|
||||
UNIQUE(attachment_id,pipeline_version)
|
||||
);
|
||||
CREATE INDEX jobs_recovery_idx ON attachment_jobs(state,lease_until);
|
||||
CREATE TABLE job_outbox (
|
||||
job_id uuid PRIMARY KEY REFERENCES attachment_jobs(id),
|
||||
published_at timestamptz
|
||||
);
|
||||
CREATE TABLE job_audit (
|
||||
id bigserial PRIMARY KEY,
|
||||
job_id uuid NOT NULL REFERENCES attachment_jobs(id),
|
||||
event text NOT NULL,
|
||||
attempt integer NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
CREATE TABLE failed_jobs (
|
||||
job_id uuid PRIMARY KEY REFERENCES attachment_jobs(id),
|
||||
error_code text NOT NULL,
|
||||
failed_at timestamptz NOT NULL DEFAULT now(),
|
||||
replay_count integer NOT NULL DEFAULT 0
|
||||
);
|
||||
CREATE TABLE test_faults (
|
||||
id bigserial PRIMARY KEY,
|
||||
point text NOT NULL,
|
||||
attachment_id uuid,
|
||||
action text NOT NULL CHECK (action IN ('crash','pause','error')),
|
||||
reached_at timestamptz,
|
||||
released boolean NOT NULL DEFAULT false
|
||||
);
|
||||
@@ -0,0 +1,2 @@
|
||||
ALTER TABLE attachment_jobs ADD COLUMN traceparent text;
|
||||
ALTER TABLE attachment_jobs ADD COLUMN tracestate text;
|
||||
@@ -0,0 +1,4 @@
|
||||
ALTER TABLE attachment_jobs ADD COLUMN accepted_at timestamptz;
|
||||
UPDATE attachment_jobs SET accepted_at=created_at;
|
||||
ALTER TABLE attachment_jobs ALTER COLUMN accepted_at SET NOT NULL;
|
||||
ALTER TABLE attachment_jobs ALTER COLUMN accepted_at SET DEFAULT clock_timestamp();
|
||||
@@ -0,0 +1,32 @@
|
||||
[project]
|
||||
name = "mtusi-messenger"
|
||||
version = "1.0.0"
|
||||
requires-python = ">=3.12,<3.13"
|
||||
dependencies = [
|
||||
"fastapi",
|
||||
"uvicorn",
|
||||
"psycopg[binary,pool]",
|
||||
"redis",
|
||||
"argon2-cffi",
|
||||
"boto3>=1.43,<2",
|
||||
"pillow==13.0.0",
|
||||
"opentelemetry-sdk==1.45.2",
|
||||
"opentelemetry-exporter-otlp-proto-http==1.45.2",
|
||||
"psutil>=7,<8",
|
||||
]
|
||||
|
||||
[dependency-groups]
|
||||
dev = ["pytest", "httpx", "ruff", "openapi-spec-validator", "jsonschema"]
|
||||
|
||||
[tool.ruff]
|
||||
target-version = "py312"
|
||||
line-length = 100
|
||||
include = ["src/**/*.py", "student-tests/**/*.py", "research/**/*.py"]
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
testpaths = ["student-tests"]
|
||||
pythonpath = ["src", "tests"]
|
||||
|
||||
[tool.ruff.lint]
|
||||
select = ["E4", "E7", "E9", "F", "I", "UP", "B"]
|
||||
ignore = ["B008"] # FastAPI dependencies are declared in endpoint signatures.
|
||||
@@ -0,0 +1,76 @@
|
||||
# Воспроизводимость исследования ЛР6
|
||||
|
||||
Запускать из корня репозитория после `make bootstrap` и `docker compose up -d --build`.
|
||||
На время исследования не запускать второй стек или внешнюю нагрузку. Секреты остаются
|
||||
в локальном каталоге `secrets/`; скрипты не сохраняют пароли и bearer-токены.
|
||||
|
||||
```sh
|
||||
python3 research/incident_suite.py
|
||||
python3 research/observability_cost.py
|
||||
```
|
||||
|
||||
Первый скрипт проверяет сквозной trace с реальной повторной попыткой, остановку worker,
|
||||
приостановку PostgreSQL и недоступность Loki/Tempo. Результаты сохраняются поэтапно
|
||||
в `evidence/incidents.json`. При недоступности БД/worker/backend восстановление выполняется
|
||||
в `finally`. Нельзя считать частично записанный JSON итоговым PASS: проверяется поле `passed`.
|
||||
|
||||
Второй скрипт сравнивает четыре режима: SDK выключен, head sampling 100%, 25%, 10%.
|
||||
Каждый режим повторяется три раза с вращением порядка. Нагрузка одинакова: 40 запросов/с,
|
||||
10 секунд, два контролируемых HTTP500 после durable COMMIT вложения и одна блокировка
|
||||
таблицы sessions на 350мс. Блокировка создаёт несколько медленных запросов, а не ровно один.
|
||||
Пул генератора ограничен; пропущенные отправки учитываются как `not_sent`, не как ответы.
|
||||
Проверяется, что два ожидаемых HTTP500 действительно получены; любые другие статусы
|
||||
останавливают эксперимент. Данные — синтетические пользователи и один PNG fixture.
|
||||
|
||||
Перед замером выполняются 40 прогревочных запросов, затем 10 секунд тишины. Baseline
|
||||
читается непосредственно с Collector после прогрева. После нагрузки также выдерживаются
|
||||
10 секунд для экспорта. Значения SDK выбираются только для текущих двух `service.instance.id`:
|
||||
исторические серии старых контейнеров не вычитаются из новых. Время нагрузки заканчивается
|
||||
сразу после завершения запросов; ожидание Docker sampler не включается в RPS.
|
||||
|
||||
`telemetry_delta.traces_wire_bytes` — фактические байты protobuf-тел HTTP-запросов SDK,
|
||||
включая повторные отправки. Это не размер Tempo на диске и не полный сетевой трафик:
|
||||
заголовки HTTP/TLS не учитываются. Отдельно записаны байты logs/metrics, accepted/refused
|
||||
spans на Collector, записанные/экспортированные/ошибочные/ожидающие spans SDK. Head sampling
|
||||
не является потерей exporter: несэмплированные spans вообще не записывались. Потери
|
||||
ограниченной очереди вычисляются по балансу после quiescence и отдельно проверены
|
||||
негативным тестом при недоступном exporter.
|
||||
|
||||
CPU и RAM получаются из реального `docker stats`, память/пик/OOM — из cgroup v2 контейнеров.
|
||||
SDK RSS относится к процессу, контейнерные числа включают дочернюю обработку изображений.
|
||||
CPU% Docker может превышать 100% при использовании нескольких ядер. Все контейнеры
|
||||
работают на одном Docker host; короткие замеры показывают свойства этого стенда,
|
||||
а не доказательство производительности продакшена.
|
||||
|
||||
Raw результаты: `evidence/observability-cost.json`. Каждый завершённый профиль сохраняется
|
||||
сразу; итоговое `passed=true` появляется только после всех 12 прогонов. Для редких ошибок
|
||||
и задержек сохраняются trace ID, решение sampling и факт появления в Tempo. Head sampling
|
||||
может отбросить любую ошибку или задержку; меньшая доля не гарантирует их сохранения.
|
||||
|
||||
Остановка telemetry backend проверяет изоляцию бизнес-функции, ограниченную память и
|
||||
наблюдаемую потерю отправок. Доставку без потерь при длительном сбое эта конфигурация
|
||||
не обещает: очередь только в памяти, retry ограничен пятью секундами.
|
||||
|
||||
Выбранный профиль поставляется отдельным override, чтобы при защите по умолчанию было
|
||||
легко показать любой trace (100%). Для включения сбалансированного профиля:
|
||||
|
||||
```sh
|
||||
docker compose -f compose.yaml -f compose.balanced.yaml up -d --no-deps --force-recreate api worker dispatcher
|
||||
```
|
||||
|
||||
Sampling применяется только к новым корневым traces; валидный входящий `traceparent`
|
||||
сохраняет свой sampled-флаг. Для публичного недоверенного traffic это отдельный
|
||||
вопрос trust policy: клиент способен запросить полную трассировку своего запроса.
|
||||
Метрики и логи при уменьшении доли traces не подвергаются head sampling; при сбое exporter их ограниченные очереди также могут терять данные.
|
||||
|
||||
При прерывании запустить `python3 research/observability_cost.py --resume`: сохраняются только
|
||||
ранее законченные пары profile/repeat с той же методикой. Failed/diagnostic runs не добавляются
|
||||
в итоговую выборку. Перед каждым профилем неиспользованные fault hooks этого локального
|
||||
эксперимента очищаются. Контрольным двум upload зарезервирован один слот генератора,
|
||||
чтобы они не исчезали из-за ограничения числа обычных запросов в полёте.
|
||||
|
||||
`docker stats` даёт выборки примерно раз в несколько секунд; максимум их суммы RAM
|
||||
не является гарантированным мгновенным пиком. `memory.peak` cgroup относится ко всей
|
||||
жизни контейнера. API/worker пересоздаются перед профилем, Collector/Tempo сохраняются
|
||||
между профилями, поэтому их cgroup peak может включать предыдущие профили. Отдельные
|
||||
краткие дочерние процессы могут попасть в cgroup peak, но не в редкую выборку stats.
|
||||
@@ -0,0 +1,117 @@
|
||||
"""Synthetic local experiment clients; credentials remain only in memory."""
|
||||
import base64
|
||||
import json
|
||||
import subprocess
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
API = 'http://127.0.0.1:8086'
|
||||
PROM = 'http://127.0.0.1:19096'
|
||||
LOKI = 'http://127.0.0.1:13106'
|
||||
TEMPO = 'http://127.0.0.1:13206'
|
||||
|
||||
|
||||
def get(url):
|
||||
with urllib.request.urlopen(url, timeout=8) as response:
|
||||
return json.load(response)
|
||||
|
||||
|
||||
def compose(*args):
|
||||
result = subprocess.run(['docker', 'compose', *args], cwd=ROOT, capture_output=True, text=True)
|
||||
if result.returncode:
|
||||
raise RuntimeError('Compose action failed: ' + ' '.join(args[:3]))
|
||||
return result.stdout
|
||||
|
||||
|
||||
def query(expression):
|
||||
data = get(PROM + '/api/v1/query?' + urllib.parse.urlencode({'query': expression}))
|
||||
assert data['status'] == 'success', data
|
||||
return data['data']['result']
|
||||
|
||||
|
||||
def scalar(expression):
|
||||
values = query(expression)
|
||||
return float(values[0]['value'][1]) if values else 0.0
|
||||
|
||||
|
||||
def wait_for(check, timeout=90, interval=2):
|
||||
deadline = time.monotonic() + timeout
|
||||
last = None
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
last = check()
|
||||
if last:
|
||||
return last
|
||||
except (urllib.error.URLError, TimeoutError, AssertionError):
|
||||
pass
|
||||
time.sleep(interval)
|
||||
raise AssertionError('Condition did not become true; last=' + str(last)[:100])
|
||||
|
||||
|
||||
def cgroups(services):
|
||||
result = []
|
||||
for service in services:
|
||||
for container in compose('ps', '-q', service).split():
|
||||
inspect = json.loads(subprocess.check_output(['docker', 'inspect', container]))[0]
|
||||
# Read only this container's cgroup through a nonprivileged helper;
|
||||
# Collector's minimal image intentionally has no shell/python.
|
||||
pid = str(inspect['State']['Pid'])
|
||||
script = 'p=$(cut -d: -f3 /proc/' + pid + '/cgroup); cat /sys/fs/cgroup$p/memory.current /sys/fs/cgroup$p/memory.peak; cat /sys/fs/cgroup$p/memory.events'
|
||||
stats = subprocess.check_output(['docker', 'run', '--rm', '--network=none', '--read-only', '--cap-drop=ALL', '--pid=host', '--cgroupns=host', '--entrypoint', 'sh', 'postgres:17.10-alpine', '-c', script], text=True)
|
||||
lines = stats.splitlines()
|
||||
result.append({'service': service, 'container': container,
|
||||
'current_bytes': int(lines[0]), 'peak_bytes': int(lines[1]),
|
||||
'events': dict(line.split() for line in lines[2:]),
|
||||
'limit_bytes': inspect['HostConfig']['Memory'],
|
||||
'oom_killed': inspect['State']['OOMKilled']})
|
||||
return result
|
||||
|
||||
|
||||
class Client:
|
||||
def __init__(self):
|
||||
self.token = None
|
||||
self.username = 'obs_' + uuid4().hex[:20]
|
||||
password = 'Synthetic-' + uuid4().hex
|
||||
status, user, _ = self.request('POST', '/api/v1/users', {'username': self.username, 'display_name': 'Observation fixture', 'password': password})
|
||||
assert status == 201
|
||||
basic = base64.b64encode((self.username + ':' + password).encode()).decode()
|
||||
status, session, _ = self.request('POST', '/api/v1/auth/sessions', headers={'Authorization': 'Basic ' + basic})
|
||||
assert status == 201
|
||||
self.token = session['token']
|
||||
status, chat, _ = self.request('POST', '/api/v1/chats', {'title': 'Observation fixture', 'member_ids': [user['id']]})
|
||||
assert status == 201
|
||||
self.chat = chat['id']
|
||||
|
||||
def request(self, method, path, body=None, headers=None):
|
||||
headers = dict(headers or {})
|
||||
if self.token:
|
||||
headers.setdefault('Authorization', 'Bearer ' + self.token)
|
||||
if isinstance(body, (dict, list)):
|
||||
body = json.dumps(body).encode()
|
||||
headers['Content-Type'] = 'application/json'
|
||||
request = urllib.request.Request(API + path, data=body, headers=headers, method=method)
|
||||
try:
|
||||
response = urllib.request.urlopen(request, timeout=12)
|
||||
except urllib.error.HTTPError as error:
|
||||
response = error
|
||||
with response:
|
||||
raw = response.read()
|
||||
value = json.loads(raw) if raw and 'json' in response.headers.get('Content-Type', '') else raw
|
||||
return response.status, value, dict(response.headers)
|
||||
|
||||
def upload(self, trace_id=None):
|
||||
headers = {'Content-Type': 'image/png', 'X-Filename': 'fixture.png'}
|
||||
if trace_id:
|
||||
headers['traceparent'] = '00-' + trace_id + '-1234567890abcdef-01'
|
||||
status, body, _ = self.request('POST', f'/api/v1/chats/{self.chat}/attachments?filename=fixture.png', (ROOT/'tests/fixtures/sample.png').read_bytes(), headers)
|
||||
assert status == 202, (status, body)
|
||||
return body['id']
|
||||
|
||||
def ready(self, attachment):
|
||||
status, body, _ = self.request('GET', '/api/v1/attachments/' + attachment)
|
||||
return status == 200 and body['status'] == 'ready'
|
||||
@@ -0,0 +1,142 @@
|
||||
"""Real local failures, bounded restoration, query receipts; never persist credentials."""
|
||||
import json
|
||||
import time
|
||||
import urllib.parse
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import UTC, datetime
|
||||
from uuid import uuid4
|
||||
|
||||
from common import LOKI, ROOT, TEMPO, Client, cgroups, compose, get, query, scalar, wait_for
|
||||
|
||||
|
||||
def firing(name):
|
||||
return bool(query('ALERTS{alertname="' + name + '",alertstate="firing"}'))
|
||||
|
||||
|
||||
def trace_spans(trace_id):
|
||||
payload = get(TEMPO + '/api/traces/' + trace_id)
|
||||
return [span for batch in payload.get('batches', []) for scope in batch.get('scopeSpans', []) for span in scope.get('spans', [])]
|
||||
|
||||
|
||||
def snapshot():
|
||||
return {'depth': scalar('max(messenger_queue_depth)'),
|
||||
'oldest_age_s': scalar('max(messenger_queue_oldest_age_seconds)'),
|
||||
'error_ratio': scalar('sum(rate(messenger_http_requests_total{status_class="5xx"}[1m])) / clamp_min(sum(rate(messenger_http_requests_total[1m])),0.001)')}
|
||||
|
||||
|
||||
def main():
|
||||
report = {'started_at': datetime.now(UTC).isoformat(), 'scenarios': []}
|
||||
client = Client()
|
||||
for _ in range(30):
|
||||
assert client.request('GET', '/api/v1/users/me')[0] == 200
|
||||
time.sleep(.1)
|
||||
time.sleep(8)
|
||||
# Incoming W3C context propagates durably through outbox and retry.
|
||||
compose('-f', 'compose.yaml', '-f', 'compose.observability-fault.yaml', 'up', '-d', '--no-deps', 'worker')
|
||||
compose('exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', '-c',
|
||||
"INSERT INTO test_faults(point,action) VALUES ('before_effect','error')")
|
||||
trace_id = uuid4().hex
|
||||
attachment = client.upload(trace_id)
|
||||
wait_for(lambda: client.ready(attachment), 45)
|
||||
def retry_trace():
|
||||
spans = trace_spans(trace_id)
|
||||
consumers = [s for s in spans if s.get('kind') in ('SPAN_KIND_CONSUMER', 5)]
|
||||
errors = [s for s in consumers if s.get('status', {}).get('code') in ('STATUS_CODE_ERROR', 2)]
|
||||
return spans if len(consumers) >= 2 and errors else None
|
||||
spans = wait_for(retry_trace, 30)
|
||||
logs = get(LOKI + '/loki/api/v1/query_range?' + urllib.parse.urlencode({'query': '{service_name=~"messenger-api|messenger-worker"} |= "' + trace_id + '"', 'limit': '100'}))
|
||||
assert logs['data']['result']
|
||||
report['scenarios'].append({'name': 'upload_publish_consume_retry', 'passed': True,
|
||||
'trace_id': trace_id, 'attachment_id': attachment,
|
||||
'spans': spans, 'log_streams': len(logs['data']['result'])})
|
||||
compose('up', '-d', '--no-deps', 'worker')
|
||||
|
||||
(ROOT/'evidence/incidents.json').write_text(json.dumps(report, indent=2))
|
||||
print('trace retry PASS', flush=True)
|
||||
|
||||
# Worker absent: queue gauges must reveal accepted work even with no failures counter.
|
||||
compose('stop', 'worker')
|
||||
started = time.monotonic()
|
||||
try:
|
||||
attachments = [client.upload() for _ in range(3)]
|
||||
wait_for(lambda: firing('MessengerQueueLag'), 100)
|
||||
detected = time.monotonic() - started
|
||||
state = snapshot()
|
||||
finally:
|
||||
compose('start', 'worker')
|
||||
restored = time.monotonic()
|
||||
wait_for(lambda: all(client.ready(a) for a in attachments), 45)
|
||||
wait_for(lambda: not firing('MessengerQueueLag') and scalar('max(messenger_queue_depth)') == 0, 45)
|
||||
report['scenarios'].append({'name': 'worker_stop_queue_age', 'passed': True,
|
||||
'firing_after_s': round(detected, 3), 'firing_observation': state,
|
||||
'resolved_after_restore_s': round(time.monotonic()-restored, 3),
|
||||
'ready_attachment_count': len(attachments)})
|
||||
|
||||
(ROOT/'evidence/incidents.json').write_text(json.dumps(report, indent=2))
|
||||
print('worker stop PASS', flush=True)
|
||||
|
||||
# Database pause: client-side deadline must bound calls; application spans show DB error.
|
||||
compose('pause', 'postgres')
|
||||
started = time.monotonic()
|
||||
statuses = []
|
||||
try:
|
||||
with ThreadPoolExecutor(max_workers=4) as executor:
|
||||
futures = [executor.submit(client.request, 'GET', '/api/v1/users/me') for _ in range(12)]
|
||||
statuses = [future.result()[0] for future in futures]
|
||||
wait_for(lambda: firing('MessengerHighErrorRate'), 60)
|
||||
detected = time.monotonic() - started
|
||||
state = snapshot()
|
||||
finally:
|
||||
compose('unpause', 'postgres')
|
||||
restored = time.monotonic()
|
||||
wait_for(lambda: client.request('GET', '/api/v1/users/me')[0] == 200, 25)
|
||||
for _ in range(60):
|
||||
assert client.request('GET', '/api/v1/users/me')[0] == 200
|
||||
time.sleep(.2)
|
||||
wait_for(lambda: not firing('MessengerHighErrorRate'), 90)
|
||||
report['scenarios'].append({'name': 'database_pause', 'passed': True,
|
||||
'firing_after_s': round(detected, 3), 'statuses': statuses,
|
||||
'firing_observation': state,
|
||||
'resolved_after_restore_s': round(time.monotonic()-restored, 3)})
|
||||
|
||||
(ROOT/'evidence/incidents.json').write_text(json.dumps(report, indent=2))
|
||||
print('database pause PASS', flush=True)
|
||||
|
||||
before = cgroups(['api', 'worker', 'collector'])
|
||||
before_failed = scalar('sum({__name__=~"otelcol_exporter_send_failed_spans(_total)?"})')
|
||||
compose('stop', 'tempo', 'loki')
|
||||
try:
|
||||
started = time.monotonic()
|
||||
statuses = []
|
||||
for _ in range(200):
|
||||
statuses.append(client.request('GET', '/api/v1/users/me')[0])
|
||||
time.sleep(.1)
|
||||
attachment = client.upload()
|
||||
wait_for(lambda: client.ready(attachment), 30)
|
||||
time.sleep(10)
|
||||
during = cgroups(['api', 'worker', 'collector'])
|
||||
after_failed = scalar('sum({__name__=~"otelcol_exporter_send_failed_spans(_total)?"})')
|
||||
assert set(statuses) == {200}
|
||||
assert after_failed > before_failed
|
||||
assert all(not item['oom_killed'] and int(item['events']['oom_kill']) == 0 for item in during)
|
||||
finally:
|
||||
compose('start', 'tempo', 'loki')
|
||||
wait_for(lambda: client.request('GET', '/api/v1/users/me')[0] == 200, 30)
|
||||
new_trace = uuid4().hex
|
||||
new_attachment = client.upload(new_trace)
|
||||
wait_for(lambda: client.ready(new_attachment), 30)
|
||||
wait_for(lambda: len(trace_spans(new_trace)) >= 5, 30)
|
||||
report['scenarios'].append({'name': 'telemetry_backends_outage', 'passed': True,
|
||||
'duration_s': round(time.monotonic()-started, 3),
|
||||
'business_successes': len(statuses),
|
||||
'failed_span_sends_before': before_failed, 'failed_span_sends_after': after_failed,
|
||||
'memory_before': before, 'memory_during': during,
|
||||
'restored_trace_id': new_trace})
|
||||
report['passed'] = all(x['passed'] for x in report['scenarios'])
|
||||
report['finished_at'] = datetime.now(UTC).isoformat()
|
||||
(ROOT/'evidence/incidents.json').write_text(json.dumps(report, indent=2))
|
||||
print(json.dumps({'passed': report['passed'], 'scenarios': len(report['scenarios'])}))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,220 @@
|
||||
"""Track 1: same synthetic offered workload, off/full/25%/10%, 3 repeats."""
|
||||
import concurrent.futures
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import re
|
||||
import statistics
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import urllib.request
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from common import ROOT, TEMPO, Client, cgroups, compose, get, scalar, wait_for
|
||||
|
||||
PROFILES = [('off', '0', '0'), ('full', '1', '1'), ('sample25', '1', '.25'), ('sample10', '1', '.1')]
|
||||
DURATION = 10
|
||||
OFFERED = 40
|
||||
|
||||
|
||||
def percentile(values, fraction):
|
||||
return sorted(values)[min(len(values)-1, math.ceil(len(values)*fraction)-1)]
|
||||
|
||||
|
||||
def observation(name):
|
||||
return scalar('sum(messenger_telemetry_export{service_name="messenger-api",observation="'+name+'"})')
|
||||
|
||||
|
||||
def telemetry_snapshot(instance_ids):
|
||||
# Read Collector endpoints directly after quiescence: no extra Prometheus
|
||||
# scrape delay may move warmup payload into the measurement baseline.
|
||||
with urllib.request.urlopen('http://127.0.0.1:19464/metrics', timeout=5) as response:
|
||||
text = response.read().decode()
|
||||
counts = {name: 0.0 for name in ('spans_recorded', 'spans_exported', 'spans_export_failed',
|
||||
'spans_pending', 'spans_in_flight', 'spans_queue_lost',
|
||||
'traces_wire_bytes', 'logs_wire_bytes', 'metrics_wire_bytes')}
|
||||
for line in text.splitlines():
|
||||
if line.startswith('messenger_telemetry_export{') and any('service_instance_id="'+value+'"' in line for value in instance_ids):
|
||||
match = re.search(r'observation="([^"]+)"', line)
|
||||
if match and match[1] in counts:
|
||||
counts[match[1]] += float(line.rsplit(' ', 1)[1])
|
||||
with urllib.request.urlopen('http://127.0.0.1:18888/metrics', timeout=5) as response:
|
||||
text = response.read().decode()
|
||||
accepted = [float(line.rsplit(' ', 1)[1]) for line in text.splitlines()
|
||||
if line.startswith('otelcol_receiver_accepted_spans') and not line.startswith('#')]
|
||||
counts['collector_accepted_spans'] = sum(accepted)
|
||||
counts['collector_refused_spans'] = sum(float(line.rsplit(' ', 1)[1]) for line in text.splitlines()
|
||||
if line.startswith('otelcol_receiver_refused_spans') and not line.startswith('#'))
|
||||
return counts
|
||||
|
||||
|
||||
def exists(trace_id):
|
||||
if not trace_id or trace_id == '0'*32:
|
||||
return False
|
||||
try:
|
||||
return bool(get(TEMPO + '/api/traces/' + trace_id).get('batches'))
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
class DockerSamples:
|
||||
def __init__(self):
|
||||
self.stop = threading.Event()
|
||||
self.rows = []
|
||||
self.ids = compose('ps', '-q').split()
|
||||
self.thread = threading.Thread(target=self.run, daemon=True)
|
||||
|
||||
def run(self):
|
||||
while not self.stop.is_set():
|
||||
raw = subprocess.check_output(['docker', 'stats', '--no-stream', '--format', '{{json .}}', *self.ids], text=True)
|
||||
self.rows.append({'at': time.monotonic(), 'containers': [json.loads(line) for line in raw.splitlines()]})
|
||||
self.stop.wait(1)
|
||||
|
||||
def __enter__(self):
|
||||
self.thread.start()
|
||||
return self
|
||||
|
||||
def __exit__(self, *args):
|
||||
self.stop.set()
|
||||
self.thread.join(timeout=5)
|
||||
|
||||
|
||||
def run_profile(client, profile, repeat):
|
||||
name, enabled, ratio = profile
|
||||
env = {**os.environ, 'OTEL_ENABLED': enabled, 'OTEL_SAMPLE_RATIO': ratio}
|
||||
command = ['docker','compose','-f','compose.yaml','-f','compose.observability-fault.yaml',
|
||||
'up','-d','--no-deps','--force-recreate','api','worker','dispatcher']
|
||||
subprocess.run(command, env=env, cwd=ROOT, check=True, capture_output=True)
|
||||
wait_for(lambda: client.request('GET','/api/v1/users/me')[0] == 200, 30)
|
||||
# Establish each process/meter and warm the same endpoint before taking baselines.
|
||||
instance_ids = set()
|
||||
for _ in range(40):
|
||||
status, _, headers = client.request('GET','/api/v1/users/me')
|
||||
assert status == 200
|
||||
instance_ids.add({key.lower():value for key,value in headers.items()}['x-instance-id'])
|
||||
time.sleep(.1)
|
||||
assert len(instance_ids) == 2
|
||||
time.sleep(10)
|
||||
before = telemetry_snapshot(instance_ids)
|
||||
memory_before = cgroups(['api','worker','collector','tempo'])
|
||||
compose('exec','-T','postgres','psql','-U','messenger','-d','messenger','-c',
|
||||
"DELETE FROM test_faults WHERE point='after_upload_commit' AND reached_at IS NULL; INSERT INTO test_faults(point,action) VALUES ('after_upload_commit','error'),('after_upload_commit','error')")
|
||||
calls = []
|
||||
dropped = 0
|
||||
lock_future = None
|
||||
started = time.monotonic()
|
||||
|
||||
def request(index):
|
||||
begin = time.monotonic()
|
||||
if index in (80, 240):
|
||||
result = client.request('POST',f'/api/v1/chats/{client.chat}/attachments?filename=fixture.png',
|
||||
(ROOT/'tests/fixtures/sample.png').read_bytes(), {'Content-Type':'image/png'})
|
||||
else:
|
||||
result = client.request('GET','/api/v1/users/me')
|
||||
status, _, headers = result
|
||||
headers = {key.lower():value for key,value in headers.items()}
|
||||
return {'index':index,'status':status,'duration_s':time.monotonic()-begin,
|
||||
'trace_id':headers.get('x-trace-id'), 'sampled':headers.get('x-trace-sampled')=='true',
|
||||
'kind':'controlled_upload_error' if index in (80,240) else 'read'}
|
||||
|
||||
with DockerSamples() as samples, concurrent.futures.ThreadPoolExecutor(max_workers=16) as executor:
|
||||
pending = set()
|
||||
for index in range(DURATION*OFFERED):
|
||||
target = started + index/OFFERED
|
||||
delay = target-time.monotonic()
|
||||
if delay > 0:
|
||||
time.sleep(delay)
|
||||
if index == 160:
|
||||
lock_future = executor.submit(compose,'exec','-T','postgres','psql','-U','messenger','-d','messenger','-c',
|
||||
'BEGIN; LOCK TABLE sessions IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep(0.35); COMMIT;')
|
||||
ready = [future for future in pending if future.done()]
|
||||
for future in ready:
|
||||
calls.append(future.result())
|
||||
pending.remove(future)
|
||||
if len(pending) >= 15 and index not in (80,240):
|
||||
dropped += 1
|
||||
else:
|
||||
pending.add(executor.submit(request,index))
|
||||
for future in pending:
|
||||
calls.append(future.result())
|
||||
if lock_future:
|
||||
lock_future.result()
|
||||
elapsed = time.monotonic()-started
|
||||
if sum(x['status']==500 for x in calls)!=2:
|
||||
(ROOT/'evidence/cost-failed-requests.json').write_text(json.dumps({'profile':name,'repeat':repeat,'not_sent':dropped,'requests':calls},indent=2))
|
||||
raise AssertionError('Expected two controlled upload errors; statuses='+str({status:sum(x['status']==status for x in calls) for status in {x['status'] for x in calls}}))
|
||||
assert all(x['status'] in (200,500) for x in calls), 'Unexpected business failure'
|
||||
time.sleep(10)
|
||||
after = telemetry_snapshot(instance_ids)
|
||||
memory_after = cgroups(['api','worker','collector','tempo'])
|
||||
assert any(row['kind']=='read' and row['duration_s']>=.15 for row in calls), 'Slow read probe did not occur'
|
||||
rare = [row for row in calls if row['status']==500 or (row['kind']=='read' and row['duration_s']>=.15)]
|
||||
for row in rare:
|
||||
row['found_in_tempo'] = exists(row['trace_id'])
|
||||
if enabled=='1' and row['sampled']:
|
||||
assert row['found_in_tempo'], 'Sampled rare trace failed ingestion'
|
||||
latency = [row['duration_s'] for row in calls]
|
||||
deltas = {key: after[key]-before[key] for key in before}
|
||||
return {'profile':name,'instance_ids':sorted(instance_ids),'enabled':enabled,'sampling_ratio':float(ratio),'repeat':repeat,
|
||||
'offered_rps':OFFERED,'duration_s':DURATION,'elapsed_s':round(elapsed,6),
|
||||
'scheduled':DURATION*OFFERED,'not_sent':dropped,'sent':len(calls),
|
||||
'successful':sum(row['status']==200 for row in calls),'controlled_500':2,
|
||||
'successful_rps':sum(row['status']==200 for row in calls)/elapsed,
|
||||
'latency_s':{'p50':percentile(latency,.5),'p95':percentile(latency,.95),'p99':percentile(latency,.99)},
|
||||
'sampled_requests':sum(row['sampled'] for row in calls),
|
||||
'rare_probes':rare,'telemetry_before':before,'telemetry_after':after,'telemetry_delta':deltas,
|
||||
'cgroups_before':memory_before,'cgroups_after':memory_after,
|
||||
'docker_samples':samples.rows,'requests':sorted(calls,key=lambda row:row['index'])}
|
||||
|
||||
|
||||
def main():
|
||||
report = {'started_at':datetime.now(UTC).isoformat(),'track':'observability cost','runs':[],
|
||||
'method':{'profiles':PROFILES,'repeats':3,'offered_rps':OFFERED,'measurement_s':DURATION,
|
||||
'latency_clock':'client perf_counter, includes network and queueing',
|
||||
'rare_slow':'one 350ms exclusive sessions-table lock/run',
|
||||
'rare_errors':'two actual API500 after durable upload COMMIT/run through private local fault hooks',
|
||||
'wire_bytes':'protobuf body length on each SDK HTTP send, retries included; excludes HTTP/TLS headers',
|
||||
'sdk_queue_losses':'conservation after 10s quiescence, separate from sampled-out spans',
|
||||
'baseline':'40 warmup requests at 10/s, then10s silence; direct Collector endpoints avoid Prometheus scrape delay',
|
||||
'post_window':'10s silence before direct Collector counters; bytes include periodic metric exports in observation window',
|
||||
'resources':'same image/CPU/RAM; sequential profiles, all services on one Docker host'}}
|
||||
result_path = ROOT/'evidence/observability-cost.json'
|
||||
if '--resume' in sys.argv and result_path.exists():
|
||||
previous = json.loads(result_path.read_text())
|
||||
assert previous['method']['rare_slow'] == report['method']['rare_slow']
|
||||
report = previous
|
||||
completed = {(row['profile'], row['repeat']) for row in report['runs']}
|
||||
client = Client()
|
||||
try:
|
||||
for repeat in range(1,4):
|
||||
order = PROFILES[repeat-1:]+PROFILES[:repeat-1]
|
||||
for profile in order:
|
||||
if (profile[0],repeat) in completed:
|
||||
continue
|
||||
row = run_profile(client,profile,repeat)
|
||||
report['runs'].append(row)
|
||||
(ROOT/'evidence/observability-cost.json').write_text(json.dumps(report,indent=2))
|
||||
print(profile[0],repeat,'PASS',round(row['successful_rps'],2),'RPS',flush=True)
|
||||
finally:
|
||||
subprocess.run(['docker','compose','up','-d','--no-deps','--force-recreate','api','worker','dispatcher'],cwd=ROOT,check=True,capture_output=True)
|
||||
report['passed'] = len(report['runs'])==12
|
||||
report['finished_at'] = datetime.now(UTC).isoformat()
|
||||
summary={}
|
||||
for name,_,_ in PROFILES:
|
||||
runs=[row for row in report['runs'] if row['profile']==name]
|
||||
summary[name]={'median_success_rps':statistics.median(row['successful_rps'] for row in runs),
|
||||
'median_p95_s':statistics.median(row['latency_s']['p95'] for row in runs),
|
||||
'median_trace_wire_bytes':statistics.median(row['telemetry_delta']['traces_wire_bytes'] for row in runs),
|
||||
'rare_total':sum(len(row['rare_probes']) for row in runs),
|
||||
'rare_sampled':sum(sum(x['sampled'] for x in row['rare_probes']) for row in runs),
|
||||
'export_failed_spans':sum(row['telemetry_delta']['spans_export_failed'] for row in runs),
|
||||
'queue_lost_spans':sum(row['telemetry_delta']['spans_queue_lost'] for row in runs)}
|
||||
report['summary']=summary
|
||||
(ROOT/'evidence/observability-cost.json').write_text(json.dumps(report,indent=2))
|
||||
print(json.dumps(summary,indent=2))
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
main()
|
||||
@@ -0,0 +1,46 @@
|
||||
"""Derive a compact table from authentic raw run receipts; never alter raw data."""
|
||||
import json
|
||||
import re
|
||||
import statistics
|
||||
|
||||
from common import ROOT
|
||||
|
||||
|
||||
def memory_mib(value):
|
||||
number, unit = re.fullmatch(r'([0-9.]+)([A-Za-z]+)', value.split(' / ')[0]).groups()
|
||||
return float(number) * {'B': 1/1048576, 'KiB': 1/1024, 'MiB': 1, 'GiB': 1024}[unit]
|
||||
|
||||
|
||||
def main():
|
||||
data = json.loads((ROOT/'evidence/observability-cost.json').read_text())
|
||||
assert data['passed'] and len(data['runs']) == 12
|
||||
rows = []
|
||||
for profile in ('off', 'full', 'sample25', 'sample10'):
|
||||
runs = [run for run in data['runs'] if run['profile'] == profile]
|
||||
samples = [sample for run in runs for sample in run['docker_samples']]
|
||||
rows.append({
|
||||
'profile': profile, 'runs': len(runs),
|
||||
'median_success_rps': statistics.median(run['successful_rps'] for run in runs),
|
||||
**{'median_'+key+'_ms': statistics.median(run['latency_s'][key]*1000 for run in runs)
|
||||
for key in ('p50', 'p95', 'p99')},
|
||||
'mean_total_docker_cpu_percent': statistics.mean(sum(float(c['CPUPerc'].rstrip('%'))
|
||||
for c in sample['containers']) for sample in samples),
|
||||
'peak_total_docker_memory_mib': max(sum(memory_mib(c['MemUsage']) for c in sample['containers'])
|
||||
for sample in samples),
|
||||
**{'median_'+key: statistics.median(run['telemetry_delta'][key] for run in runs)
|
||||
for key in ('traces_wire_bytes', 'logs_wire_bytes', 'metrics_wire_bytes',
|
||||
'collector_accepted_spans', 'collector_refused_spans')},
|
||||
'export_failed_spans': sum(run['telemetry_delta']['spans_export_failed'] for run in runs),
|
||||
'queue_lost_spans': sum(run['telemetry_delta']['spans_queue_lost'] for run in runs),
|
||||
'not_sent': sum(run['not_sent'] for run in runs),
|
||||
'rare_total': sum(len(run['rare_probes']) for run in runs),
|
||||
'rare_found': sum(sum(probe['found_in_tempo'] for probe in run['rare_probes']) for run in runs),
|
||||
'errors_found': sum(sum(probe['found_in_tempo'] for probe in run['rare_probes'] if probe['status']==500) for run in runs),
|
||||
'slow_found': sum(sum(probe['found_in_tempo'] for probe in run['rare_probes'] if probe['status']!=500) for run in runs),
|
||||
})
|
||||
(ROOT/'evidence/observability-cost-summary.json').write_text(json.dumps(rows, indent=2))
|
||||
print(json.dumps(rows, indent=2))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,50 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Create a private local secret directory once; never rotate existing volumes implicitly."""
|
||||
|
||||
import os
|
||||
import json
|
||||
import secrets
|
||||
from pathlib import Path
|
||||
|
||||
root = Path(__file__).resolve().parents[1]
|
||||
private = root / "secrets"
|
||||
private.mkdir(mode=0o700, exist_ok=True)
|
||||
os.chmod(private, 0o700)
|
||||
password_file = private / "postgres_password"
|
||||
if not password_file.exists():
|
||||
password_file.write_text(secrets.token_urlsafe(32))
|
||||
password = password_file.read_text().strip()
|
||||
broker_password_file = private / "broker_password"
|
||||
if not broker_password_file.exists():
|
||||
broker_password_file.write_text(secrets.token_urlsafe(32))
|
||||
broker_password = broker_password_file.read_text().strip()
|
||||
values = {
|
||||
"database_url": "postgresql://messenger:" + password + "@postgres:5432/messenger",
|
||||
"cache_url": "redis://cache:6379/0",
|
||||
"broker_url": "redis://:" + broker_password + "@broker:6379/0",
|
||||
"broker_config": "bind 0.0.0.0\nprotected-mode yes\nrequirepass " + broker_password + "\nappendonly yes\nappendfsync always\nsave \"\"\ndir /data\nmaxmemory 128mb\nmaxmemory-policy noeviction\n",
|
||||
"s3_access_key": "app-" + secrets.token_hex(10),
|
||||
"s3_secret_key": secrets.token_urlsafe(32),
|
||||
"grafana_password": secrets.token_urlsafe(32),
|
||||
"s3_admin_access_key": "admin-" + secrets.token_hex(10),
|
||||
"s3_admin_secret_key": secrets.token_urlsafe(32),
|
||||
}
|
||||
for name, value in values.items():
|
||||
path = private / name
|
||||
if not path.exists():
|
||||
path.write_text(value)
|
||||
config = private / "s3_config"
|
||||
if not config.exists():
|
||||
config.write_text(json.dumps({"identities": [
|
||||
{"name": "bootstrap", "credentials": [{"accessKey": (private / "s3_admin_access_key").read_text(),
|
||||
"secretKey": (private / "s3_admin_secret_key").read_text()}], "actions": ["Admin", "Read", "Write", "List"]},
|
||||
{"name": "api", "credentials": [{"accessKey": (private / "s3_access_key").read_text(),
|
||||
"secretKey": (private / "s3_secret_key").read_text()}], "actions": ["Read:messenger", "Write:messenger", "List:messenger"]}
|
||||
]}))
|
||||
# The parent directory is private on the host; files must be readable by non-root API
|
||||
# through Compose's file-backed secret mount (uid/mode mapping is not portable).
|
||||
for path in private.iterdir():
|
||||
os.chmod(path, 0o644)
|
||||
if not (root / ".env").exists():
|
||||
(root / ".env").write_text("PUBLIC_PORT=8086\nSESSION_TTL_SECONDS=3600\n")
|
||||
print("Local configuration ready; existing secret values preserved.")
|
||||
Executable
+5
@@ -0,0 +1,5 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
docker compose restart postgres cache broker storage s3 api lb dispatcher worker
|
||||
docker compose up -d --wait --no-recreate
|
||||
Executable
+4
@@ -0,0 +1,4 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
python3 scripts/stop_one.py
|
||||
@@ -0,0 +1,22 @@
|
||||
"""Map a process alias to its real container, then leave exactly it stopped."""
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
wanted = os.environ["TARGET_INSTANCE_ID"]
|
||||
ids = subprocess.check_output(["docker", "compose", "ps", "-q", "api"], text=True).split()
|
||||
for cid in ids:
|
||||
result = subprocess.run(
|
||||
["docker", "exec", cid, "python", "-c",
|
||||
"import urllib.request; r=urllib.request.urlopen('http://127.0.0.1:8080/health/live'); print(r.headers['X-Instance-Id'])"],
|
||||
text=True, capture_output=True, check=False,
|
||||
)
|
||||
if result.returncode == 0 and result.stdout.strip() == wanted:
|
||||
subprocess.run(["docker", "stop", "--time", "8", cid], check=True, stdout=subprocess.DEVNULL)
|
||||
receipt = {"stopped_container": cid, "instance_id": wanted}
|
||||
Path("evidence/last-stopped.json").write_text(json.dumps(receipt))
|
||||
print(json.dumps(receipt))
|
||||
break
|
||||
else:
|
||||
raise SystemExit("Target instance not found among this project's API containers")
|
||||
@@ -0,0 +1 @@
|
||||
"""Учебный REST-мессенджер."""
|
||||
@@ -0,0 +1,12 @@
|
||||
import os
|
||||
|
||||
import uvicorn
|
||||
|
||||
uvicorn.run(
|
||||
"messenger.app:app",
|
||||
host=os.getenv("APP_HOST", "0.0.0.0"),
|
||||
port=int(os.getenv("APP_PORT", "8080")),
|
||||
workers=1,
|
||||
access_log=False,
|
||||
timeout_graceful_shutdown=5,
|
||||
)
|
||||
@@ -0,0 +1,349 @@
|
||||
import base64
|
||||
import binascii
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
from contextlib import asynccontextmanager
|
||||
from threading import BoundedSemaphore
|
||||
from uuid import uuid4
|
||||
|
||||
from fastapi import Depends, FastAPI, Request, Response
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from fastapi.responses import JSONResponse
|
||||
from opentelemetry import propagate
|
||||
from opentelemetry.trace import SpanKind, Status, StatusCode
|
||||
from starlette.exceptions import HTTPException
|
||||
|
||||
from . import telemetry
|
||||
from .attachments import AttachmentService
|
||||
from .domain import Failure, boundary, identifier, page
|
||||
from .models import CreateChat, CreateMessage, CreateUser
|
||||
from .postgres import PostgresStore
|
||||
from .storage import ObjectStorage
|
||||
|
||||
logger = logging.getLogger("messenger")
|
||||
logging.basicConfig(level=os.getenv("LOG_LEVEL", "INFO"), format="%(message)s")
|
||||
INSTANCE_ID = telemetry.INSTANCE_ID
|
||||
|
||||
CODES = {
|
||||
400: "invalid_request",
|
||||
401: "unauthorized",
|
||||
404: "not_found",
|
||||
409: "conflict",
|
||||
413: "payload_too_large",
|
||||
415: "unsupported_media_type",
|
||||
429: "rate_limited",
|
||||
503: "unavailable",
|
||||
500: "internal_error",
|
||||
}
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app):
|
||||
app.state.telemetry = telemetry.setup("messenger-api")
|
||||
app.state.store = PostgresStore()
|
||||
app.state.telemetry.start_background(app.state.store)
|
||||
app.state.storage = ObjectStorage()
|
||||
app.state.attachments = AttachmentService(app.state.store, app.state.storage)
|
||||
app.state.upload_slots = BoundedSemaphore(1)
|
||||
logger.info(json.dumps({"event": "started", "storage": "postgres"}))
|
||||
yield
|
||||
logger.info(json.dumps({"event": "stopped"}))
|
||||
app.state.telemetry.close()
|
||||
app.state.storage.close()
|
||||
app.state.store.close()
|
||||
|
||||
|
||||
app = FastAPI(lifespan=lifespan, docs_url=None, redoc_url=None, openapi_url=None)
|
||||
|
||||
|
||||
def error(request, status, message, headers=None):
|
||||
return JSONResponse(
|
||||
{
|
||||
"error": {
|
||||
"code": CODES.get(status, "invalid_request"),
|
||||
"message": message,
|
||||
"request_id": request.state.request_id,
|
||||
}
|
||||
},
|
||||
status_code=status,
|
||||
headers=headers,
|
||||
)
|
||||
|
||||
|
||||
async def invalid_json_encoding(request: Request):
|
||||
path = request.url.path
|
||||
if request.method != "POST" or not (
|
||||
path in {"/api/v1/users", "/api/v1/chats"}
|
||||
or re.fullmatch(r"/api/v1/chats/[^/]+/messages", path)
|
||||
):
|
||||
return False
|
||||
raw = await request.body()
|
||||
try:
|
||||
raw.decode("utf-8")
|
||||
except UnicodeDecodeError:
|
||||
return True
|
||||
# Literal NUL is invalid JSON; it also prevents json.loads(bytes) from
|
||||
# autodetecting BOM-less UTF-16/32. Escaped JSON \u0000 remains valid.
|
||||
return b"\x00" in raw
|
||||
|
||||
|
||||
@app.middleware("http")
|
||||
async def request_log(request: Request, call_next):
|
||||
request.state.request_id = str(uuid4())
|
||||
start = time.perf_counter()
|
||||
observed = not request.url.path.startswith("/health/")
|
||||
if not observed:
|
||||
response = await call_next(request)
|
||||
response.headers["X-Request-Id"] = request.state.request_id
|
||||
response.headers["X-Instance-Id"] = INSTANCE_ID
|
||||
return response
|
||||
method = request.method if request.method in {"GET", "POST", "DELETE", "PUT", "PATCH", "HEAD", "OPTIONS"} else "OTHER"
|
||||
context = propagate.extract(dict(request.headers))
|
||||
with telemetry.operation("HTTP " + method, context=context, kind=SpanKind.SERVER) as span:
|
||||
try:
|
||||
if (request.method == "POST" and request.url.path == "/api/v1/auth/sessions"
|
||||
and await request.body()):
|
||||
response = error(request, 400, "Вход не принимает тело запроса")
|
||||
elif await invalid_json_encoding(request):
|
||||
response = error(request, 400, "JSON должен быть закодирован в UTF-8")
|
||||
else:
|
||||
response = await call_next(request)
|
||||
except Exception:
|
||||
logger.error(json.dumps({"event": "unexpected_error", "request_id": request.state.request_id}))
|
||||
response = error(request, 500, "Внутренняя ошибка")
|
||||
response.headers["X-Request-Id"] = request.state.request_id
|
||||
response.headers["X-Instance-Id"] = INSTANCE_ID
|
||||
response.headers["X-Trace-Id"] = format(span.get_span_context().trace_id, "032x")
|
||||
response.headers["X-Trace-Sampled"] = "true" if span.is_recording() else "false"
|
||||
route = getattr(request.scope.get("route"), "path", "unknown")
|
||||
elapsed = time.perf_counter() - start
|
||||
attrs = {"method": method, "route": route,
|
||||
"status_class": str(response.status_code // 100) + "xx"}
|
||||
span.update_name(method + " " + route)
|
||||
span.set_attributes({"http.request.method": method, "http.route": route,
|
||||
"http.response.status_code": response.status_code})
|
||||
if response.status_code >= 500:
|
||||
span.set_status(Status(StatusCode.ERROR, "server_error"))
|
||||
t = telemetry.current()
|
||||
if observed and t:
|
||||
t.requests.add(1, attrs)
|
||||
t.http_duration.record(elapsed, attrs)
|
||||
t.sampling.add(1, {"decision": "sampled" if span.is_recording() else "not_sampled"})
|
||||
logger.info(json.dumps({"event": "request", "request_id": request.state.request_id,
|
||||
**attrs, "status": response.status_code,
|
||||
"duration_ms": round(elapsed * 1000, 3)}))
|
||||
return response
|
||||
|
||||
|
||||
@app.exception_handler(Failure)
|
||||
async def domain_error(request, exc):
|
||||
return error(request, exc.status, exc.message, exc.headers)
|
||||
|
||||
|
||||
@app.exception_handler(RequestValidationError)
|
||||
async def validation_error(request, exc):
|
||||
return error(request, 400, "Неверный JSON или поля запроса")
|
||||
|
||||
|
||||
@app.exception_handler(HTTPException)
|
||||
async def http_error(request, exc):
|
||||
return error(request, exc.status_code, "Неверный запрос", exc.headers)
|
||||
|
||||
|
||||
def bearer_failure():
|
||||
return Failure(
|
||||
401, "Требуется действующая сессия", {"WWW-Authenticate": 'Bearer realm="messenger"'}
|
||||
)
|
||||
|
||||
|
||||
def bearer(request):
|
||||
parts = request.headers.get("Authorization", "").split()
|
||||
if len(parts) != 2 or parts[0].lower() != "bearer":
|
||||
raise bearer_failure()
|
||||
return parts[1]
|
||||
|
||||
|
||||
def actor(request: Request):
|
||||
user = request.app.state.store.authenticate(bearer(request))
|
||||
if user is None:
|
||||
raise bearer_failure()
|
||||
return user
|
||||
|
||||
|
||||
@app.post("/api/v1/auth/sessions", status_code=201)
|
||||
def login(request: Request):
|
||||
headers = {"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'}
|
||||
parts = request.headers.get("Authorization", "").split()
|
||||
try:
|
||||
if len(parts) != 2 or parts[0].lower() != "basic" or len(parts[1]) > 2048:
|
||||
raise ValueError
|
||||
decoded = base64.b64decode(parts[1], validate=True).decode("utf-8")
|
||||
username, password = decoded.split(":", 1)
|
||||
if not re.fullmatch(r"[a-z0-9_]{3,32}", username) or not 12 <= len(password) <= 128:
|
||||
raise ValueError
|
||||
except (ValueError, UnicodeError, binascii.Error):
|
||||
raise Failure(401, "Неверные учётные данные", headers) from None
|
||||
session = request.app.state.store.login(username, password)
|
||||
if session is None:
|
||||
raise Failure(401, "Неверные учётные данные", headers)
|
||||
return session
|
||||
|
||||
|
||||
@app.delete("/api/v1/auth/sessions/current", status_code=204)
|
||||
def logout(request: Request, user=Depends(actor)):
|
||||
if not request.app.state.store.logout(bearer(request)):
|
||||
raise bearer_failure()
|
||||
return Response(status_code=204)
|
||||
|
||||
|
||||
def pagination(request, scope):
|
||||
allowed = {"limit", "cursor"}
|
||||
if any(k not in allowed for k in request.query_params):
|
||||
raise Failure(400, "Неизвестный параметр запроса")
|
||||
if any(len(request.query_params.getlist(k)) > 1 for k in allowed):
|
||||
raise Failure(400, "Повторный параметр запроса")
|
||||
value = request.query_params.get("limit", "50")
|
||||
if len(value) > 3 or not re.fullmatch(r"[0-9]+", value) or not 1 <= int(value) <= 100:
|
||||
raise Failure(400, "limit должен быть целым от 1 до 100")
|
||||
return int(value), boundary(scope, request.query_params.get("cursor"))
|
||||
|
||||
|
||||
@app.get("/health/live")
|
||||
def live():
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
@app.get("/health/ready")
|
||||
def ready(request: Request):
|
||||
if not request.app.state.store.ready() or not request.app.state.storage.ready():
|
||||
raise Failure(503, "Хранилище недоступно")
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
@app.post("/api/v1/users", status_code=201)
|
||||
def register(data: CreateUser, request: Request):
|
||||
return request.app.state.store.register(data.model_dump(mode="json"))
|
||||
|
||||
|
||||
@app.get("/api/v1/users/me")
|
||||
def me(user=Depends(actor)):
|
||||
return user
|
||||
|
||||
|
||||
@app.post("/api/v1/chats", status_code=201)
|
||||
def create_chat(data: CreateChat, request: Request, user=Depends(actor)):
|
||||
return request.app.state.store.create_chat(user["id"], data.model_dump(mode="json"))
|
||||
|
||||
|
||||
@app.get("/api/v1/chats")
|
||||
def chats(request: Request, user=Depends(actor)):
|
||||
scope = "chats:" + user["id"]
|
||||
limit, after = pagination(request, scope)
|
||||
return page(request.app.state.store.list_chats(user["id"], after, limit), limit, scope)
|
||||
|
||||
|
||||
@app.get("/api/v1/chats/{chat_id}")
|
||||
def chat(chat_id: str, request: Request, user=Depends(actor)):
|
||||
return request.app.state.store.chat(user["id"], identifier(chat_id))
|
||||
|
||||
|
||||
@app.post("/api/v1/chats/{chat_id}/messages", status_code=201)
|
||||
def send(chat_id: str, data: CreateMessage, request: Request, user=Depends(actor)):
|
||||
key = request.headers.get("Idempotency-Key", "")
|
||||
if not re.fullmatch(r"[A-Za-z0-9._:-]{1,128}", key):
|
||||
raise Failure(400, "Требуется корректный Idempotency-Key")
|
||||
return request.app.state.store.send(
|
||||
user["id"], identifier(chat_id), data.text, key, [str(a) for a in data.attachment_ids]
|
||||
)
|
||||
|
||||
|
||||
@app.get("/api/v1/chats/{chat_id}/messages")
|
||||
def history(chat_id: str, request: Request, user=Depends(actor)):
|
||||
cid = identifier(chat_id)
|
||||
scope = "messages:" + cid
|
||||
limit, after = pagination(request, scope)
|
||||
return page(request.app.state.store.history(user["id"], cid, after, limit), limit, scope)
|
||||
|
||||
|
||||
@app.post("/api/v1/chats/{chat_id}/attachments", status_code=202)
|
||||
async def upload_attachment(chat_id: str, request: Request, user=Depends(actor)):
|
||||
import asyncio
|
||||
import tempfile
|
||||
|
||||
from starlette.concurrency import run_in_threadpool
|
||||
|
||||
from .attachments import MAX_BYTES, filename_valid
|
||||
|
||||
if (
|
||||
set(request.query_params) != {"filename"}
|
||||
or len(request.query_params.getlist("filename")) != 1
|
||||
):
|
||||
raise Failure(400, "Требуется одно имя файла")
|
||||
filename = filename_valid(request.query_params["filename"])
|
||||
cid = identifier(chat_id)
|
||||
await run_in_threadpool(request.app.state.store.chat, user["id"], cid)
|
||||
content_type = request.headers.get("content-type", "").lower()
|
||||
if content_type not in {"image/png", "image/jpeg"}:
|
||||
raise Failure(415, "Ожидается PNG или JPEG")
|
||||
if not request.app.state.upload_slots.acquire(blocking=False):
|
||||
raise Failure(429, "Все места загрузки заняты", {"Retry-After": "1"})
|
||||
try:
|
||||
with tempfile.SpooledTemporaryFile(max_size=256 * 1024) as spool:
|
||||
size = 0
|
||||
try:
|
||||
async with asyncio.timeout(15):
|
||||
async for chunk in request.stream():
|
||||
size += len(chunk)
|
||||
if size > MAX_BYTES:
|
||||
raise Failure(413, "Файл больше 10 MiB")
|
||||
spool.write(chunk)
|
||||
except TimeoutError:
|
||||
raise Failure(400, "Истекло время загрузки") from None
|
||||
spool.seek(0)
|
||||
return await run_in_threadpool(
|
||||
request.app.state.attachments.upload,
|
||||
user["id"],
|
||||
cid,
|
||||
filename,
|
||||
content_type,
|
||||
spool,
|
||||
size,
|
||||
)
|
||||
finally:
|
||||
request.app.state.upload_slots.release()
|
||||
|
||||
|
||||
@app.get("/api/v1/attachments/{attachment_id}")
|
||||
def attachment_metadata(attachment_id: str, request: Request, user=Depends(actor)):
|
||||
return request.app.state.attachments.metadata(user["id"], identifier(attachment_id))
|
||||
|
||||
|
||||
@app.get("/api/v1/attachments/{attachment_id}/content")
|
||||
def attachment_content(attachment_id: str, request: Request, user=Depends(actor)):
|
||||
from fastapi.responses import StreamingResponse
|
||||
|
||||
if (
|
||||
any(k != "variant" for k in request.query_params)
|
||||
or len(request.query_params.getlist("variant")) > 1
|
||||
):
|
||||
raise Failure(400, "Неверный вариант")
|
||||
variant = request.query_params.get("variant", "original")
|
||||
if variant not in {"original", "thumbnail", "crop", "optimized"}:
|
||||
raise Failure(400, "Неверный вариант")
|
||||
obj = request.app.state.attachments.content(user["id"], identifier(attachment_id), variant)
|
||||
|
||||
def chunks():
|
||||
try:
|
||||
while data := obj["Body"].read(64 * 1024):
|
||||
yield data
|
||||
finally:
|
||||
obj["Body"].close()
|
||||
|
||||
return StreamingResponse(
|
||||
chunks(),
|
||||
media_type=obj["ContentType"],
|
||||
headers={"Content-Length": str(obj["ContentLength"])},
|
||||
)
|
||||
@@ -0,0 +1,176 @@
|
||||
"""Durable asynchronous intake with full bounded validation before acceptance."""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import unicodedata
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
from .domain import Failure, timestamp
|
||||
from .jobs import Jobs, fault
|
||||
from .postgres import public
|
||||
|
||||
MAX_BYTES = 10 * 1024 * 1024
|
||||
MAX_PIXELS = 40_000_000
|
||||
PUBLIC_FIELDS = (
|
||||
"id",
|
||||
"chat_id",
|
||||
"owner_id",
|
||||
"filename",
|
||||
"content_type",
|
||||
"size_bytes",
|
||||
"sha256",
|
||||
"status",
|
||||
"error_code",
|
||||
"created_at",
|
||||
"updated_at",
|
||||
)
|
||||
|
||||
|
||||
def filename_valid(name):
|
||||
if not isinstance(name, str) or not 1 <= len(name) <= 255 or not name.strip():
|
||||
raise Failure(400, "Неверное имя файла")
|
||||
if any(c in "/\\" or unicodedata.category(c) == "Cc" for c in name):
|
||||
raise Failure(400, "Неверное имя файла")
|
||||
return name
|
||||
|
||||
|
||||
def validate_image(fileobj, declared_type):
|
||||
if declared_type not in {"image/png", "image/jpeg"}:
|
||||
raise Failure(415, "Ожидается PNG или JPEG")
|
||||
fileobj.seek(0)
|
||||
encoded = fileobj.read(MAX_BYTES + 1)
|
||||
if len(encoded) > MAX_BYTES:
|
||||
raise Failure(413, "Файл больше 10 MiB")
|
||||
try:
|
||||
with subprocess.Popen(
|
||||
[sys.executable, "-m", "messenger.image_validation", declared_type],
|
||||
stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
start_new_session=True,
|
||||
env={
|
||||
"PATH": os.environ.get("PATH", os.defpath),
|
||||
"PYTHONPATH": str(Path(__file__).resolve().parents[1]),
|
||||
"PYTHONDONTWRITEBYTECODE": "1",
|
||||
"LANG": "C.UTF-8",
|
||||
},
|
||||
) as process:
|
||||
try:
|
||||
output, _ = process.communicate(input=encoded, timeout=5)
|
||||
except subprocess.TimeoutExpired:
|
||||
# The validator can itself run djpeg: kill its whole process group
|
||||
# before releasing the upload slot, not only the Python parent.
|
||||
try:
|
||||
os.killpg(process.pid, signal.SIGKILL)
|
||||
except ProcessLookupError:
|
||||
pass # The child may have exited at the timeout boundary.
|
||||
process.communicate()
|
||||
raise Failure(415, "Изображение не прошло ограниченную проверку") from None
|
||||
if process.returncode:
|
||||
try:
|
||||
os.killpg(process.pid, signal.SIGKILL)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
raise Failure(415, "Изображение не прошло ограниченную проверку")
|
||||
except OSError:
|
||||
raise Failure(503, "Проверка изображения временно недоступна") from None
|
||||
finally:
|
||||
del encoded
|
||||
try:
|
||||
outcome = json.loads(output)
|
||||
except (ValueError, UnicodeError):
|
||||
raise Failure(415, "Повреждённое изображение") from None
|
||||
if not isinstance(outcome, dict):
|
||||
raise Failure(415, "Изображение не прошло ограниченную проверку")
|
||||
if "error" in outcome:
|
||||
status = outcome["error"]
|
||||
if type(status) is not int or status not in {413, 415}:
|
||||
status = 415
|
||||
raise Failure(status, "Изображение не прошло ограниченную проверку")
|
||||
values = outcome.get("ok")
|
||||
if not isinstance(values, list) or len(values) != 3:
|
||||
raise Failure(415, "Изображение не прошло ограниченную проверку")
|
||||
fileobj.seek(0)
|
||||
return tuple(values)
|
||||
|
||||
|
||||
def metadata_public(row):
|
||||
return {**public({k: row[k] for k in PUBLIC_FIELDS}), "variants": row["variants"]}
|
||||
|
||||
|
||||
class AttachmentService:
|
||||
def __init__(self, store, storage):
|
||||
self.store = store
|
||||
self.storage = storage
|
||||
self.jobs = Jobs(store)
|
||||
|
||||
def upload(self, actor, cid, filename, content_type, fileobj, size):
|
||||
filename_valid(filename)
|
||||
if not 1 <= size <= MAX_BYTES:
|
||||
raise Failure(413 if size > MAX_BYTES else 415, "Неверный размер изображения")
|
||||
validate_image(fileobj, content_type)
|
||||
digest = hashlib.file_digest(fileobj, "sha256").hexdigest()
|
||||
aid = str(uuid4())
|
||||
key = "staging/" + aid
|
||||
now = timestamp()
|
||||
with self.store.connection() as conn:
|
||||
self.store._chat(conn, actor, cid)
|
||||
conn.execute(
|
||||
"INSERT INTO upload_intents(id,object_key,created_at) VALUES (%s,%s,%s)",
|
||||
(aid, key, now),
|
||||
)
|
||||
# Keep the intent locked throughout PUT/finalization: reconciler skips live
|
||||
# uploads even if a slow request crosses the grace boundary.
|
||||
with self.store.connection() as conn, conn.transaction():
|
||||
intent = conn.execute(
|
||||
"SELECT id FROM upload_intents WHERE id=%s FOR UPDATE", (aid,)
|
||||
).fetchone()
|
||||
if not intent:
|
||||
raise Failure(503, "Загрузка больше не активна")
|
||||
self.storage.put(key, fileobj, size, content_type)
|
||||
fault(self.store, "after_staging", aid)
|
||||
row = conn.execute(
|
||||
"INSERT INTO attachments(id,chat_id,owner_id,filename,content_type,size_bytes,sha256,"
|
||||
"status,error_code,created_at,updated_at,object_key,stage_key) "
|
||||
"VALUES (%s,%s,%s,%s,%s,%s,%s,'queued',NULL,%s,%s,%s,%s) RETURNING *",
|
||||
(aid, cid, actor, filename, content_type, size, digest, now, now, key, key),
|
||||
).fetchone()
|
||||
self.jobs.enqueue(conn, aid, key)
|
||||
conn.execute("DELETE FROM upload_intents WHERE id=%s", (aid,))
|
||||
# Never delete S3 on an ambiguous COMMIT: the transaction may have committed.
|
||||
fault(self.store, "after_upload_commit", aid)
|
||||
return metadata_public(row)
|
||||
|
||||
def metadata(self, actor, aid):
|
||||
with self.store.connection() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT a.* FROM attachments a JOIN members m ON m.chat_id=a.chat_id "
|
||||
"WHERE a.id=%s AND m.user_id=%s",
|
||||
(aid, actor),
|
||||
).fetchone()
|
||||
if not row:
|
||||
raise Failure(404, "Объект не найден")
|
||||
return metadata_public(row)
|
||||
|
||||
def content(self, actor, aid, variant="original"):
|
||||
with self.store.connection() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT a.* FROM attachments a JOIN members m ON m.chat_id=a.chat_id "
|
||||
"WHERE a.id=%s AND m.user_id=%s",
|
||||
(aid, actor),
|
||||
).fetchone()
|
||||
if not row:
|
||||
raise Failure(404, "Объект не найден")
|
||||
if row["status"] != "ready":
|
||||
raise Failure(409, "Вложение не готово")
|
||||
key = row["object_key"]
|
||||
if variant != "original":
|
||||
if variant not in {v["name"] for v in row["variants"]}:
|
||||
raise Failure(404, "Вариант не найден")
|
||||
key = key.rsplit("/", 1)[0] + "/" + variant
|
||||
return self.storage.get(key)
|
||||
@@ -0,0 +1,10 @@
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def secret(name: str) -> str:
|
||||
filename = os.environ.get(name + "_FILE")
|
||||
value = Path(filename).read_text().strip() if filename else os.environ.get(name, "")
|
||||
if not value:
|
||||
raise RuntimeError("Отсутствует настройка " + name)
|
||||
return value
|
||||
@@ -0,0 +1,63 @@
|
||||
import base64
|
||||
import json
|
||||
import re
|
||||
from datetime import UTC, datetime
|
||||
from uuid import UUID
|
||||
|
||||
|
||||
class Failure(Exception):
|
||||
def __init__(self, status: int, message: str, headers: dict | None = None):
|
||||
self.status = status
|
||||
self.message = message
|
||||
self.headers = headers or {}
|
||||
|
||||
|
||||
def timestamp() -> str:
|
||||
return datetime.now(UTC).isoformat(timespec="microseconds").replace("+00:00", "Z")
|
||||
|
||||
|
||||
def identifier(value: str) -> str:
|
||||
try:
|
||||
return str(UUID(value))
|
||||
except (ValueError, TypeError, AttributeError):
|
||||
raise Failure(400, "Некорректный UUID") from None
|
||||
|
||||
|
||||
def boundary(scope: str, cursor: str | None) -> tuple[str, str] | None:
|
||||
if cursor is None:
|
||||
return None
|
||||
try:
|
||||
if not cursor or len(cursor) > 1024 or not re.fullmatch(r"[A-Za-z0-9_-]+", cursor):
|
||||
raise ValueError
|
||||
data = json.loads(base64.urlsafe_b64decode(cursor + "=" * (-len(cursor) % 4)))
|
||||
if (
|
||||
not isinstance(data, list)
|
||||
or len(data) != 4
|
||||
or type(data[0]) is not int
|
||||
or data[0] != 1
|
||||
or data[1] != scope
|
||||
):
|
||||
raise ValueError
|
||||
date, uid = data[2:]
|
||||
if not isinstance(date, str) or not re.fullmatch(
|
||||
r"[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(?:\.[0-9]{1,6})?Z",
|
||||
date,
|
||||
):
|
||||
raise ValueError
|
||||
datetime.fromisoformat(date.replace("Z", "+00:00"))
|
||||
if not isinstance(uid, str) or str(UUID(uid)) != uid:
|
||||
raise ValueError
|
||||
return date, uid
|
||||
except (ValueError, TypeError, KeyError, UnicodeError):
|
||||
raise Failure(400, "Некорректный cursor или область запроса") from None
|
||||
|
||||
|
||||
def page(items: list[dict], limit: int, scope: str) -> dict:
|
||||
more = len(items) > limit
|
||||
selected = items[:limit]
|
||||
cursor = None
|
||||
if more:
|
||||
last = selected[-1]
|
||||
payload = json.dumps([1, scope, last["created_at"], last["id"]], separators=(",", ":"))
|
||||
cursor = base64.urlsafe_b64encode(payload.encode()).decode().rstrip("=")
|
||||
return {"items": selected, "next_cursor": cursor}
|
||||
@@ -0,0 +1,82 @@
|
||||
"""Image transformations, run in a bounded child process by the worker."""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from PIL import Image, ImageOps
|
||||
|
||||
Image.MAX_IMAGE_PIXELS = 40_000_000
|
||||
|
||||
|
||||
def fitted_size(width, height, bound):
|
||||
"""Fit without upscale; integer arithmetic deliberately rounds down."""
|
||||
longest = max(width, height)
|
||||
if longest <= bound:
|
||||
return width, height
|
||||
return max(1, width * bound // longest), max(1, height * bound // longest)
|
||||
|
||||
|
||||
def render(source, destination, version=1):
|
||||
if version not in (1, 2):
|
||||
raise ValueError("unsupported_pipeline")
|
||||
destination = Path(destination)
|
||||
with Image.open(source) as image:
|
||||
if image.width * image.height > 40_000_000:
|
||||
raise ValueError("pixel_limit")
|
||||
image.load()
|
||||
ImageOps.exif_transpose(image, in_place=True)
|
||||
if "A" in image.getbands() or (image.mode == "P" and "transparency" in image.info):
|
||||
rgba = image if image.mode == "RGBA" else image.convert("RGBA")
|
||||
rgb = Image.new("RGB", rgba.size, "white")
|
||||
alpha = rgba.getchannel("A")
|
||||
rgb.paste(rgba, mask=alpha)
|
||||
alpha.close()
|
||||
if rgba is not image:
|
||||
rgba.close()
|
||||
else:
|
||||
rgb = image if image.mode == "RGB" else image.convert("RGB")
|
||||
thumbnail = rgb.resize(fitted_size(*rgb.size, 256), Image.Resampling.LANCZOS)
|
||||
crop = ImageOps.fit(rgb, (128, 128), method=Image.Resampling.LANCZOS, centering=(0.5, 0.5))
|
||||
optimized = rgb.resize(fitted_size(*rgb.size, 1280), Image.Resampling.LANCZOS)
|
||||
variants = []
|
||||
for name, variant, kind, mime, options in (
|
||||
("thumbnail", thumbnail, "JPEG", "image/jpeg", {"quality": 85}),
|
||||
("crop", crop, "JPEG", "image/jpeg", {"quality": 85}),
|
||||
(
|
||||
"optimized",
|
||||
optimized,
|
||||
"WEBP",
|
||||
"image/webp",
|
||||
{"quality": 80 if version == 1 else 75, "method": 4},
|
||||
),
|
||||
):
|
||||
# No EXIF/ICC arguments: outputs do not propagate source metadata.
|
||||
path = destination / name
|
||||
variant.save(path, format=kind, **options)
|
||||
data = path.read_bytes()
|
||||
variants.append(
|
||||
{
|
||||
"name": name,
|
||||
"content_type": mime,
|
||||
"size_bytes": len(data),
|
||||
"width": variant.width,
|
||||
"height": variant.height,
|
||||
"sha256": hashlib.sha256(data).hexdigest(),
|
||||
}
|
||||
)
|
||||
variant.close()
|
||||
if rgb is not image:
|
||||
rgb.close()
|
||||
return variants
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import resource
|
||||
|
||||
resource.setrlimit(resource.RLIMIT_AS, (700 * 1024**2, 700 * 1024**2))
|
||||
resource.setrlimit(resource.RLIMIT_CPU, (15, 15))
|
||||
source, destination, version = sys.argv[1:]
|
||||
result = render(source, destination, int(version))
|
||||
(Path(destination) / "variants.json").write_text(json.dumps(result))
|
||||
@@ -0,0 +1,146 @@
|
||||
"""Short-lived bounded pixel validation; no DB, credentials or application imports."""
|
||||
|
||||
import json
|
||||
import resource
|
||||
import subprocess
|
||||
import sys
|
||||
from io import BytesIO
|
||||
|
||||
from PIL import Image, UnidentifiedImageError
|
||||
|
||||
from .domain import Failure
|
||||
|
||||
MAX_BYTES = 10 * 1024 * 1024
|
||||
MAX_PIXELS = 40_000_000
|
||||
Image.MAX_IMAGE_PIXELS = MAX_PIXELS
|
||||
|
||||
|
||||
def inspect_image(fileobj, declared_type):
|
||||
if declared_type not in {"image/png", "image/jpeg"}:
|
||||
raise Failure(415, "Ожидается PNG или JPEG")
|
||||
try:
|
||||
fileobj.seek(0)
|
||||
with Image.open(fileobj) as header:
|
||||
width, height = header.size
|
||||
if width * height > MAX_PIXELS:
|
||||
raise Failure(413, "Слишком много пикселей")
|
||||
actual = {"PNG": "image/png", "JPEG": "image/jpeg"}.get(header.format)
|
||||
if actual != declared_type:
|
||||
raise Failure(415, "Формат изображения не соответствует MIME")
|
||||
frames = getattr(header, "n_frames", 1)
|
||||
default_image = bool(getattr(header, "default_image", False))
|
||||
if width * height * frames > MAX_PIXELS:
|
||||
raise Failure(413, "Слишком много пикселей во всех кадрах")
|
||||
header.verify()
|
||||
# Drop parsed metadata before the separate strict decoder process starts.
|
||||
del header
|
||||
if actual == "image/png":
|
||||
# verify() checks CRC, but does not compare acTL's frame count with
|
||||
# every fcTL. Do not let a forged count hide a corrupt later frame.
|
||||
fileobj.seek(8)
|
||||
controls = frame_controls = 0
|
||||
declared_frames = None
|
||||
animation_data = False
|
||||
while True:
|
||||
chunk_header = fileobj.read(8)
|
||||
if len(chunk_header) != 8:
|
||||
raise ValueError("Truncated PNG chunk")
|
||||
length = int.from_bytes(chunk_header[:4], "big")
|
||||
kind = chunk_header[4:]
|
||||
controls += kind == b"acTL"
|
||||
frame_controls += kind == b"fcTL"
|
||||
animation_data |= kind == b"fdAT"
|
||||
if kind == b"acTL":
|
||||
if length != 8:
|
||||
raise ValueError("Invalid acTL length")
|
||||
declared_frames = int.from_bytes(fileobj.read(4), "big")
|
||||
fileobj.seek(length, 1) # remaining payload plus CRC
|
||||
else:
|
||||
fileobj.seek(length + 4, 1)
|
||||
if kind == b"IEND":
|
||||
break
|
||||
if controls:
|
||||
if (
|
||||
controls != 1
|
||||
or not declared_frames
|
||||
or frame_controls != declared_frames
|
||||
or declared_frames + int(default_image) != frames
|
||||
):
|
||||
raise ValueError("Invalid APNG frame count")
|
||||
elif frame_controls or animation_data:
|
||||
raise ValueError("Animation chunks without acTL")
|
||||
if actual == "image/jpeg":
|
||||
# Validate scan warnings before allocating Pillow's decoded canvas.
|
||||
# 384 MiB covers coefficient arrays for <=40M pixels and <=4 channels;
|
||||
# keeping both full decoded buffers alive could exceed the API cgroup.
|
||||
fileobj.seek(0)
|
||||
# A bounded compressed copy avoids sharing buffered file-descriptor
|
||||
# offsets with the child, and supports BytesIO callers as well.
|
||||
encoded = fileobj.read(MAX_BYTES + 1)
|
||||
if len(encoded) > MAX_BYTES:
|
||||
raise Failure(413, "Файл больше 10 MiB")
|
||||
try:
|
||||
decoded = subprocess.run(
|
||||
["djpeg", "-strict", "-maxmemory", "384m", "-outfile", "/dev/null"],
|
||||
input=encoded,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.PIPE,
|
||||
timeout=4,
|
||||
check=False,
|
||||
)
|
||||
except subprocess.TimeoutExpired:
|
||||
raise Failure(415, "Изображение не прошло ограниченную проверку") from None
|
||||
finally:
|
||||
del encoded
|
||||
if decoded.returncode:
|
||||
raise Failure(415, "Повреждённое изображение")
|
||||
# djpeg has already decoded every JPEG pixel, streaming output rows.
|
||||
# A second Pillow load would overlap full progressive coefficient
|
||||
# arrays with a 160 MiB CMYK canvas and violate the API's 512 MiB cap.
|
||||
fileobj.seek(0)
|
||||
return actual, width, height
|
||||
# Decode one frame at a time. Pillow may retain a previous APNG canvas
|
||||
# for compositing, but no list/copies of every decoded frame is collected.
|
||||
fileobj.seek(0)
|
||||
with Image.open(fileobj) as image:
|
||||
decoded_pixels = 0
|
||||
for frame in range(frames):
|
||||
if decoded_pixels + width * height > MAX_PIXELS:
|
||||
raise Failure(413, "Слишком много пикселей во всех кадрах")
|
||||
image.seek(frame)
|
||||
decoded_pixels += image.size[0] * image.size[1]
|
||||
if decoded_pixels > MAX_PIXELS:
|
||||
raise Failure(413, "Слишком много пикселей во всех кадрах")
|
||||
image.load()
|
||||
# Image's context manager closes its input handle, not the decoded core.
|
||||
# Releasing the object frees native canvases before hashing/S3 publication.
|
||||
del image
|
||||
fileobj.seek(0)
|
||||
return actual, width, height
|
||||
except Image.DecompressionBombError:
|
||||
raise Failure(413, "Слишком много пикселей") from None
|
||||
except (UnidentifiedImageError, OSError, ValueError, SyntaxError, EOFError):
|
||||
raise Failure(415, "Повреждённое изображение") from None
|
||||
|
||||
|
||||
def main():
|
||||
resource.setrlimit(resource.RLIMIT_CORE, (0, 0))
|
||||
resource.setrlimit(resource.RLIMIT_CPU, (4, 5))
|
||||
if sys.platform == "linux":
|
||||
resource.setrlimit(resource.RLIMIT_AS, (512 * 1024 * 1024, 512 * 1024 * 1024))
|
||||
try:
|
||||
data = sys.stdin.buffer.read(MAX_BYTES + 1)
|
||||
if len(data) > MAX_BYTES:
|
||||
raise Failure(413, "Файл больше 10 MiB")
|
||||
result = inspect_image(BytesIO(data), sys.argv[1])
|
||||
print(json.dumps({"ok": result}))
|
||||
except Failure as exc:
|
||||
print(json.dumps({"error": exc.status}))
|
||||
except MemoryError:
|
||||
print(json.dumps({"error": 415}))
|
||||
except Exception:
|
||||
print(json.dumps({"error": 415}))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,123 @@
|
||||
"""Local operator commands; no public HTTP replay or arbitrary object paths."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
|
||||
from .domain import Failure
|
||||
from .jobs import Jobs, fault
|
||||
from .postgres import PostgresStore
|
||||
from .storage import ObjectStorage
|
||||
from .worker import cleanup_ready
|
||||
|
||||
|
||||
def cleanup(store, storage, failed_hours=168, intent_seconds=3600):
|
||||
if failed_hours < 0 or intent_seconds < 0:
|
||||
raise ValueError("Retention must be nonnegative")
|
||||
counts = {"ready_staging": 0, "failed_sources": 0, "orphan_intents": 0}
|
||||
with store.connection() as conn:
|
||||
ready = conn.execute(
|
||||
"SELECT id FROM attachments WHERE status='ready' AND stage_key IS NOT NULL"
|
||||
).fetchall()
|
||||
for row in ready:
|
||||
cleanup_ready(store, storage, row["id"])
|
||||
counts["ready_staging"] += 1
|
||||
with store.connection() as conn:
|
||||
candidates = conn.execute(
|
||||
"SELECT id FROM attachments WHERE status='failed' AND updated_at<now()-(%s * interval '1 hour')",
|
||||
(failed_hours,),
|
||||
).fetchall()
|
||||
for row in candidates:
|
||||
with store.connection() as conn, conn.transaction():
|
||||
a = conn.execute(
|
||||
"SELECT * FROM attachments WHERE id=%s FOR UPDATE", (row["id"],)
|
||||
).fetchone()
|
||||
# Recheck under the same row lock used by replay and publication.
|
||||
if not a or a["status"] != "failed":
|
||||
continue
|
||||
current = conn.execute(
|
||||
"SELECT * FROM attachment_jobs WHERE attachment_id=%s AND pipeline_version=%s FOR UPDATE",
|
||||
(a["id"], a["pipeline_version"]),
|
||||
).fetchone()
|
||||
if not current or current["state"] != "failed" or current["source_key"] is None:
|
||||
continue
|
||||
old = conn.execute(
|
||||
"SELECT updated_at<now()-(%s * interval '1 hour') AS old FROM attachments WHERE id=%s",
|
||||
(failed_hours, a["id"]),
|
||||
).fetchone()["old"]
|
||||
if not old:
|
||||
continue
|
||||
fault(store, "cleanup_before_delete", a["id"])
|
||||
# If cleanup commits first, later replay refuses source_missing. If replay
|
||||
# commits first, status is queued and cleanup skips it. DELETE stays locked.
|
||||
source = current["source_key"]
|
||||
storage.delete(source)
|
||||
prefix = f"attachments/{a['id']}/v{a['pipeline_version']}"
|
||||
for variant in ("original", "thumbnail", "crop", "optimized"):
|
||||
storage.delete(prefix + "/" + variant)
|
||||
conn.execute("UPDATE attachment_jobs SET source_key=NULL WHERE id=%s", (current["id"],))
|
||||
conn.execute("UPDATE attachments SET stage_key=NULL WHERE id=%s", (a["id"],))
|
||||
Jobs.audit(conn, current["id"], "source_retention_expired", current["attempt"])
|
||||
counts["failed_sources"] += 1
|
||||
with store.connection() as conn:
|
||||
intents = conn.execute(
|
||||
"SELECT id FROM upload_intents WHERE created_at<now()-(%s * interval '1 second')",
|
||||
(intent_seconds,),
|
||||
).fetchall()
|
||||
for row in intents:
|
||||
with store.connection() as conn, conn.transaction():
|
||||
intent = conn.execute(
|
||||
"SELECT * FROM upload_intents WHERE id=%s FOR UPDATE SKIP LOCKED", (row["id"],)
|
||||
).fetchone()
|
||||
if intent is None:
|
||||
continue
|
||||
referenced = conn.execute(
|
||||
"SELECT 1 FROM attachments WHERE id=%s", (row["id"],)
|
||||
).fetchone()
|
||||
if referenced is None:
|
||||
storage.delete(intent["object_key"])
|
||||
counts["orphan_intents"] += 1
|
||||
conn.execute("DELETE FROM upload_intents WHERE id=%s", (row["id"],))
|
||||
return counts
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
sub = parser.add_subparsers(dest="command", required=True)
|
||||
for name in ("replay", "reprocess"):
|
||||
command = sub.add_parser(name)
|
||||
command.add_argument("attachment_id")
|
||||
if name == "reprocess":
|
||||
command.add_argument("--version", type=int, required=True)
|
||||
sub.add_parser("failed")
|
||||
command = sub.add_parser("cleanup")
|
||||
command.add_argument("--failed-hours", type=float, default=168)
|
||||
command.add_argument("--intent-seconds", type=float, default=3600)
|
||||
args = parser.parse_args()
|
||||
store, storage = PostgresStore(), ObjectStorage()
|
||||
try:
|
||||
jobs = Jobs(store)
|
||||
if args.command == "replay":
|
||||
result = {"job_id": jobs.replay(args.attachment_id, storage)}
|
||||
elif args.command == "reprocess":
|
||||
result = {"job_id": jobs.reprocess(args.attachment_id, args.version, storage)}
|
||||
elif args.command == "cleanup":
|
||||
result = cleanup(store, storage, args.failed_hours, args.intent_seconds)
|
||||
else:
|
||||
with store.connection() as conn:
|
||||
result = [
|
||||
dict(row)
|
||||
for row in conn.execute(
|
||||
"SELECT j.id,j.attachment_id,j.attempt,f.error_code,f.replay_count FROM failed_jobs f JOIN attachment_jobs j ON j.id=f.job_id WHERE j.state='failed'"
|
||||
).fetchall()
|
||||
]
|
||||
print(json.dumps(result, default=str))
|
||||
except Failure as exc:
|
||||
print(json.dumps({"status": exc.status, "error": exc.message}))
|
||||
raise SystemExit(1) from None
|
||||
finally:
|
||||
storage.close()
|
||||
store.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,330 @@
|
||||
"""PostgreSQL owns job state; broker deliveries are repeatable notifications."""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import random
|
||||
import time
|
||||
from datetime import UTC, datetime
|
||||
from uuid import uuid4
|
||||
|
||||
import redis
|
||||
from opentelemetry import propagate
|
||||
from opentelemetry.trace import SpanKind
|
||||
|
||||
from . import telemetry
|
||||
from .config import secret
|
||||
from .domain import Failure
|
||||
from .postgres import public
|
||||
|
||||
logger = logging.getLogger("messenger.jobs")
|
||||
STREAM = "attachment-jobs:v1"
|
||||
GROUP = "image-workers"
|
||||
|
||||
|
||||
def broker_client():
|
||||
return redis.Redis.from_url(
|
||||
secret("BROKER_URL"),
|
||||
decode_responses=False,
|
||||
socket_connect_timeout=2,
|
||||
socket_timeout=3,
|
||||
)
|
||||
|
||||
|
||||
def fault(store, point, attachment_id):
|
||||
"""Only enabled in the explicit fault-test overlay; never accepts HTTP controls."""
|
||||
if os.getenv("ENABLE_FAULT_INJECTION") != "1":
|
||||
return
|
||||
with store.connection() as conn:
|
||||
row = conn.execute(
|
||||
"UPDATE test_faults SET reached_at=now() WHERE id=("
|
||||
"SELECT id FROM test_faults WHERE point=%s AND reached_at IS NULL "
|
||||
"AND (attachment_id IS NULL OR attachment_id=%s) ORDER BY id LIMIT 1) RETURNING *",
|
||||
(point, attachment_id),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
return
|
||||
logger.warning(json.dumps({"event": "fault_reached", "point": point, "fault_id": row["id"]}))
|
||||
if row["action"] == "crash":
|
||||
os._exit(86)
|
||||
if row["action"] == "error":
|
||||
raise RuntimeError("injected_transient_failure")
|
||||
deadline = time.monotonic() + 60
|
||||
while time.monotonic() < deadline:
|
||||
with store.connection() as conn:
|
||||
released = conn.execute(
|
||||
"SELECT released FROM test_faults WHERE id=%s", (row["id"],)
|
||||
).fetchone()
|
||||
if released and released["released"]:
|
||||
return
|
||||
time.sleep(0.1)
|
||||
raise RuntimeError("fault_barrier_timeout")
|
||||
|
||||
|
||||
class Jobs:
|
||||
def __init__(self, store):
|
||||
self.store = store
|
||||
self.attempt_limit = int(os.getenv("JOB_MAX_ATTEMPTS", "5"))
|
||||
self.lease_seconds = int(os.getenv("JOB_LEASE_SECONDS", "45"))
|
||||
if not 1 <= self.attempt_limit <= 20 or not 5 <= self.lease_seconds <= 300:
|
||||
raise RuntimeError("Invalid job retry/lease configuration")
|
||||
|
||||
def enqueue(self, conn, attachment_id, source_key, version=1):
|
||||
job_id = str(uuid4())
|
||||
conn.execute(
|
||||
"INSERT INTO attachment_jobs(id,attachment_id,pipeline_version,source_key,state,attempt_limit,traceparent,tracestate,accepted_at) "
|
||||
"VALUES (%s,%s,%s,%s,'queued',%s,%s,%s,clock_timestamp())",
|
||||
(job_id, attachment_id, version, source_key, self.attempt_limit, telemetry.carrier().get("traceparent"), telemetry.carrier().get("tracestate")),
|
||||
)
|
||||
conn.execute("INSERT INTO job_outbox(job_id) VALUES (%s)", (job_id,))
|
||||
self.audit(conn, job_id, "accepted", 0)
|
||||
return job_id
|
||||
|
||||
@staticmethod
|
||||
def audit(conn, job_id, event, attempt):
|
||||
conn.execute(
|
||||
"INSERT INTO job_audit(job_id,event,attempt) VALUES (%s,%s,%s)",
|
||||
(job_id, event, attempt),
|
||||
)
|
||||
|
||||
def publish_one(self, broker):
|
||||
with self.store.connection() as conn, conn.transaction():
|
||||
job = conn.execute(
|
||||
"SELECT j.* FROM job_outbox o JOIN attachment_jobs j ON j.id=o.job_id "
|
||||
"WHERE o.published_at IS NULL AND j.available_at<=now() "
|
||||
"ORDER BY j.available_at LIMIT 1 FOR UPDATE OF o SKIP LOCKED"
|
||||
).fetchone()
|
||||
if job is None:
|
||||
return False
|
||||
if job["state"] == "queued":
|
||||
body = {
|
||||
"schema_version": 1,
|
||||
"job_id": str(job["id"]),
|
||||
"type": "attachment.process.v1",
|
||||
"attachment_id": str(job["attachment_id"]),
|
||||
"pipeline_version": job["pipeline_version"],
|
||||
"attempt": job["attempt"] + 1,
|
||||
"created_at": public({"v": job["created_at"]})["v"],
|
||||
}
|
||||
# XADD response with appendfsync=always is the broker acceptance boundary.
|
||||
parent = {key: job.get(key) for key in ("traceparent", "tracestate") if job.get(key)}
|
||||
with telemetry.operation("attachments publish", context=propagate.extract(parent), kind=SpanKind.PRODUCER):
|
||||
body.update(telemetry.carrier())
|
||||
broker.xadd(STREAM, {"body": json.dumps(body)})
|
||||
fault(self.store, "after_publish", job["attachment_id"])
|
||||
conn.execute("UPDATE job_outbox SET published_at=now() WHERE job_id=%s", (job["id"],))
|
||||
return True
|
||||
|
||||
def claim(self, job_id, attachment_id, version):
|
||||
with self.store.connection() as conn, conn.transaction():
|
||||
attachment = conn.execute(
|
||||
"SELECT * FROM attachments WHERE id=%s FOR UPDATE", (attachment_id,)
|
||||
).fetchone()
|
||||
if attachment is None:
|
||||
return None
|
||||
job = conn.execute(
|
||||
"SELECT * FROM attachment_jobs WHERE id=%s FOR UPDATE", (job_id,)
|
||||
).fetchone()
|
||||
if (
|
||||
job is None
|
||||
or str(job["attachment_id"]) != str(attachment_id)
|
||||
or job["pipeline_version"] != version
|
||||
):
|
||||
return None
|
||||
if attachment["pipeline_version"] != version:
|
||||
conn.execute(
|
||||
"UPDATE attachment_jobs SET state='superseded',updated_at=now() WHERE id=%s AND state NOT IN ('ready','superseded')",
|
||||
(job_id,),
|
||||
)
|
||||
return None
|
||||
if job["state"] != "queued":
|
||||
return None
|
||||
lease_id = str(uuid4())
|
||||
job = conn.execute(
|
||||
"UPDATE attachment_jobs SET state='processing',attempt=attempt+1,lease_id=%s,"
|
||||
"lease_until=now()+(%s * interval '1 second'),updated_at=now() "
|
||||
"WHERE id=%s AND available_at<=now() RETURNING *",
|
||||
(lease_id, self.lease_seconds, job_id),
|
||||
).fetchone()
|
||||
if job is None:
|
||||
return None
|
||||
conn.execute(
|
||||
"UPDATE attachments SET status='processing',updated_at=now() WHERE id=%s",
|
||||
(attachment_id,),
|
||||
)
|
||||
self.audit(conn, job_id, "claimed", job["attempt"])
|
||||
job["source_sha256"] = attachment["sha256"]
|
||||
job["source_size"] = attachment["size_bytes"]
|
||||
job["source_content_type"] = attachment["content_type"]
|
||||
return job
|
||||
|
||||
def complete(self, job, object_key, variants):
|
||||
with self.store.connection() as conn, conn.transaction():
|
||||
attachment = conn.execute(
|
||||
"SELECT * FROM attachments WHERE id=%s FOR UPDATE", (job["attachment_id"],)
|
||||
).fetchone()
|
||||
current = conn.execute(
|
||||
"SELECT * FROM attachment_jobs WHERE id=%s FOR UPDATE", (job["id"],)
|
||||
).fetchone()
|
||||
if (
|
||||
attachment["pipeline_version"] != job["pipeline_version"]
|
||||
or current["state"] != "processing"
|
||||
or current["lease_id"] != job["lease_id"]
|
||||
):
|
||||
self.audit(conn, job["id"], "stale_publish_rejected", job["attempt"])
|
||||
return False
|
||||
conn.execute(
|
||||
"UPDATE attachments SET status='ready',error_code=NULL,object_key=%s,variants=%s::jsonb,updated_at=now() WHERE id=%s",
|
||||
(object_key, json.dumps(variants), job["attachment_id"]),
|
||||
)
|
||||
conn.execute(
|
||||
"UPDATE attachment_jobs SET state='ready',lease_until=NULL,error_code=NULL,updated_at=now() WHERE id=%s",
|
||||
(job["id"],),
|
||||
)
|
||||
self.audit(conn, job["id"], "published", job["attempt"])
|
||||
t = telemetry.current()
|
||||
if t:
|
||||
t.ready_duration.record((datetime.now(UTC) - job["accepted_at"]).total_seconds(), {"job_type": "attachment.process.v1"})
|
||||
return True
|
||||
|
||||
def fail(self, job, code, permanent=False):
|
||||
with self.store.connection() as conn, conn.transaction():
|
||||
attachment = conn.execute(
|
||||
"SELECT * FROM attachments WHERE id=%s FOR UPDATE", (job["attachment_id"],)
|
||||
).fetchone()
|
||||
current = conn.execute(
|
||||
"SELECT * FROM attachment_jobs WHERE id=%s FOR UPDATE", (job["id"],)
|
||||
).fetchone()
|
||||
if (
|
||||
attachment["pipeline_version"] != job["pipeline_version"]
|
||||
or current["lease_id"] != job["lease_id"]
|
||||
or current["state"] != "processing"
|
||||
):
|
||||
return
|
||||
if permanent or current["attempt"] >= current["attempt_limit"]:
|
||||
final_code = code if permanent else "retry_exhausted"
|
||||
self._failed(conn, current, final_code)
|
||||
else:
|
||||
delay = min(30, 2 ** (current["attempt"] - 1)) + random.uniform(0, 0.5)
|
||||
conn.execute(
|
||||
"UPDATE attachment_jobs SET state='queued',error_code=%s,lease_until=NULL,"
|
||||
"available_at=now()+(%s * interval '1 second'),updated_at=now() WHERE id=%s",
|
||||
(code, delay, job["id"]),
|
||||
)
|
||||
conn.execute(
|
||||
"UPDATE job_outbox SET published_at=NULL WHERE job_id=%s", (job["id"],)
|
||||
)
|
||||
conn.execute(
|
||||
"UPDATE attachments SET status='queued',updated_at=now() WHERE id=%s",
|
||||
(job["attachment_id"],),
|
||||
)
|
||||
self.audit(conn, job["id"], "retry_scheduled", job["attempt"])
|
||||
|
||||
def _failed(self, conn, job, code):
|
||||
conn.execute(
|
||||
"UPDATE attachment_jobs SET state='failed',error_code=%s,lease_until=NULL,updated_at=now() WHERE id=%s",
|
||||
(code, job["id"]),
|
||||
)
|
||||
conn.execute(
|
||||
"UPDATE attachments SET status='failed',error_code=%s,updated_at=now() WHERE id=%s",
|
||||
(code, job["attachment_id"]),
|
||||
)
|
||||
conn.execute(
|
||||
"INSERT INTO failed_jobs(job_id,error_code) VALUES (%s,%s) "
|
||||
"ON CONFLICT(job_id) DO UPDATE SET error_code=excluded.error_code,failed_at=now()",
|
||||
(job["id"], code),
|
||||
)
|
||||
self.audit(conn, job["id"], "failed:" + code, job["attempt"])
|
||||
|
||||
def recover_one(self):
|
||||
with self.store.connection() as conn, conn.transaction():
|
||||
candidate = conn.execute(
|
||||
"SELECT a.id FROM attachments a JOIN attachment_jobs j ON j.attachment_id=a.id "
|
||||
"WHERE j.state='processing' AND j.lease_until<now() LIMIT 1 FOR UPDATE OF a SKIP LOCKED"
|
||||
).fetchone()
|
||||
if candidate is None:
|
||||
return False
|
||||
job = conn.execute(
|
||||
"SELECT j.* FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id "
|
||||
"WHERE j.attachment_id=%s AND j.state='processing' AND j.lease_until<now() "
|
||||
"AND j.pipeline_version=a.pipeline_version FOR UPDATE OF j",
|
||||
(candidate["id"],),
|
||||
).fetchone()
|
||||
if job is None:
|
||||
return False
|
||||
if job["attempt"] >= job["attempt_limit"]:
|
||||
self._failed(conn, job, "retry_exhausted")
|
||||
else:
|
||||
conn.execute(
|
||||
"UPDATE attachment_jobs SET state='queued',lease_until=NULL,available_at=now(),updated_at=now() WHERE id=%s",
|
||||
(job["id"],),
|
||||
)
|
||||
conn.execute(
|
||||
"UPDATE attachments SET status='queued',updated_at=now() WHERE id=%s",
|
||||
(job["attachment_id"],),
|
||||
)
|
||||
conn.execute(
|
||||
"UPDATE job_outbox SET published_at=NULL WHERE job_id=%s", (job["id"],)
|
||||
)
|
||||
self.audit(conn, job["id"], "lease_recovered", job["attempt"])
|
||||
return True
|
||||
|
||||
def replay(self, attachment_id, storage):
|
||||
with self.store.connection() as conn, conn.transaction():
|
||||
a = conn.execute(
|
||||
"SELECT * FROM attachments WHERE id=%s FOR UPDATE", (attachment_id,)
|
||||
).fetchone()
|
||||
if not a or a["status"] != "failed":
|
||||
raise Failure(409, "Повтор разрешён только для failed")
|
||||
job = conn.execute(
|
||||
"SELECT * FROM attachment_jobs WHERE attachment_id=%s AND pipeline_version=%s FOR UPDATE",
|
||||
(attachment_id, a["pipeline_version"]),
|
||||
).fetchone()
|
||||
if not job["source_key"] or not storage.exists(job["source_key"]):
|
||||
raise Failure(409, "Источник больше недоступен")
|
||||
fault(self.store, "replay_before_commit", attachment_id)
|
||||
conn.execute(
|
||||
"UPDATE attachment_jobs SET state='queued',attempt_limit=attempt+%s,available_at=now(),error_code=NULL,updated_at=now() WHERE id=%s",
|
||||
(self.attempt_limit, job["id"]),
|
||||
)
|
||||
conn.execute(
|
||||
"UPDATE attachments SET status='queued',error_code=NULL,updated_at=now() WHERE id=%s",
|
||||
(attachment_id,),
|
||||
)
|
||||
conn.execute("UPDATE job_outbox SET published_at=NULL WHERE job_id=%s", (job["id"],))
|
||||
conn.execute(
|
||||
"UPDATE failed_jobs SET replay_count=replay_count+1 WHERE job_id=%s", (job["id"],)
|
||||
)
|
||||
self.audit(conn, job["id"], "operator_replay", job["attempt"])
|
||||
return str(job["id"])
|
||||
|
||||
def reprocess(self, attachment_id, version, storage):
|
||||
if version not in (1, 2):
|
||||
raise Failure(400, "Поддерживаются версии pipeline 1 и 2")
|
||||
with self.store.connection() as conn, conn.transaction():
|
||||
a = conn.execute(
|
||||
"SELECT * FROM attachments WHERE id=%s FOR UPDATE", (attachment_id,)
|
||||
).fetchone()
|
||||
if not a:
|
||||
raise Failure(404, "Вложение отсутствует")
|
||||
if version <= a["pipeline_version"]:
|
||||
raise Failure(409, "Нужна более новая версия pipeline")
|
||||
source = a["object_key"] if a["status"] == "ready" else a["stage_key"]
|
||||
if not source or not storage.exists(source):
|
||||
raise Failure(409, "Источник больше недоступен")
|
||||
if a["status"] == "ready" and a["stage_key"] and a["stage_key"].startswith("staging/"):
|
||||
# A worker can be paused after publishing ready but before its
|
||||
# staging cleanup. Do not orphan that key when choosing final v1
|
||||
# as the new source. Delete stays under the same attachment lock;
|
||||
# a storage failure leaves the ready version unchanged.
|
||||
fault(self.store, "reprocess_before_staging_delete", attachment_id)
|
||||
storage.delete(a["stage_key"])
|
||||
conn.execute(
|
||||
"UPDATE attachment_jobs SET state='superseded',updated_at=now() WHERE attachment_id=%s AND state IN ('queued','processing','failed')",
|
||||
(attachment_id,),
|
||||
)
|
||||
conn.execute(
|
||||
"UPDATE attachments SET pipeline_version=%s,status='queued',variants='[]',error_code=NULL,stage_key=%s,updated_at=now() WHERE id=%s",
|
||||
(version, source, attachment_id),
|
||||
)
|
||||
return self.enqueue(conn, attachment_id, source, version)
|
||||
@@ -0,0 +1,32 @@
|
||||
import hashlib
|
||||
from pathlib import Path
|
||||
|
||||
import psycopg
|
||||
|
||||
from .config import secret
|
||||
|
||||
|
||||
def migrate():
|
||||
with psycopg.connect(secret("DATABASE_URL")) as conn:
|
||||
# Serialize independent migration processes without changing application data.
|
||||
conn.execute("SELECT pg_advisory_xact_lock(2301002)")
|
||||
conn.execute(
|
||||
"CREATE TABLE IF NOT EXISTS schema_migrations "
|
||||
"(name text PRIMARY KEY, checksum text NOT NULL)"
|
||||
)
|
||||
for path in sorted(Path("migrations").glob("*.sql")):
|
||||
checksum = hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
old = conn.execute(
|
||||
"SELECT checksum FROM schema_migrations WHERE name=%s", (path.name,)
|
||||
).fetchone()
|
||||
if old:
|
||||
if old[0] != checksum:
|
||||
raise RuntimeError("Изменена применённая миграция: " + path.name)
|
||||
continue
|
||||
conn.execute(path.read_text())
|
||||
conn.execute("INSERT INTO schema_migrations VALUES (%s,%s)", (path.name, checksum))
|
||||
print("Applied", path.name)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
migrate()
|
||||
@@ -0,0 +1,71 @@
|
||||
import re
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, StrictStr, field_validator, model_validator
|
||||
|
||||
|
||||
class RequestModel(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
@field_validator("*", mode="after")
|
||||
@classmethod
|
||||
def nonblank(cls, value):
|
||||
if isinstance(value, str):
|
||||
value.encode("utf-8")
|
||||
if isinstance(value, str) and not value.strip():
|
||||
raise ValueError("Строка не может состоять из пробелов")
|
||||
return value
|
||||
|
||||
|
||||
class CreateUser(RequestModel):
|
||||
username: StrictStr = Field(pattern=r"^[a-z0-9_]{3,32}$")
|
||||
display_name: StrictStr = Field(min_length=1, max_length=100)
|
||||
password: StrictStr = Field(min_length=12, max_length=128)
|
||||
|
||||
@field_validator("username")
|
||||
@classmethod
|
||||
def full_username(cls, value):
|
||||
if not re.fullmatch(r"[a-z0-9_]{3,32}", value):
|
||||
raise ValueError("Неверное имя пользователя")
|
||||
return value
|
||||
|
||||
|
||||
class CreateChat(RequestModel):
|
||||
title: StrictStr = Field(min_length=1, max_length=100)
|
||||
member_ids: list[UUID] = Field(min_length=1, max_length=100)
|
||||
|
||||
@field_validator("member_ids", mode="before")
|
||||
@classmethod
|
||||
def string_ids(cls, value):
|
||||
if not isinstance(value, list) or any(not isinstance(x, str) for x in value):
|
||||
raise ValueError("Ожидается массив UUID-строк")
|
||||
return value
|
||||
|
||||
@field_validator("member_ids")
|
||||
@classmethod
|
||||
def unique_ids(cls, value):
|
||||
if len(set(value)) != len(value):
|
||||
raise ValueError("Участники не должны повторяться")
|
||||
return value
|
||||
|
||||
|
||||
class CreateMessage(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
text: StrictStr = Field(max_length=4000)
|
||||
attachment_ids: list[UUID] = Field(default_factory=list, max_length=10)
|
||||
|
||||
@field_validator("attachment_ids", mode="before")
|
||||
@classmethod
|
||||
def string_ids(cls, value):
|
||||
if not isinstance(value, list) or any(not isinstance(x, str) for x in value):
|
||||
raise ValueError("Ожидается массив UUID-строк")
|
||||
return value
|
||||
|
||||
@model_validator(mode="after")
|
||||
def content(self):
|
||||
self.text.encode("utf-8")
|
||||
if len(set(self.attachment_ids)) != len(self.attachment_ids):
|
||||
raise ValueError("Вложения не должны повторяться")
|
||||
if not self.text.strip() and not self.attachment_ids:
|
||||
raise ValueError("Пустое сообщение без вложений")
|
||||
return self
|
||||
@@ -0,0 +1,334 @@
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import secrets
|
||||
from contextlib import contextmanager
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from threading import BoundedSemaphore
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
import psycopg
|
||||
import redis
|
||||
from argon2 import PasswordHasher
|
||||
from argon2.exceptions import VerificationError
|
||||
from psycopg.rows import dict_row
|
||||
from psycopg_pool import ConnectionPool, PoolTimeout, TooManyRequests
|
||||
|
||||
from .config import secret
|
||||
from .domain import Failure, timestamp
|
||||
from .telemetry import operation
|
||||
|
||||
logger = logging.getLogger("messenger")
|
||||
KDF_SLOTS = BoundedSemaphore(2)
|
||||
KDF = PasswordHasher(time_cost=2, memory_cost=65536, parallelism=1)
|
||||
DUMMY_HASH = KDF.hash("invalid-user-dummy-password")
|
||||
|
||||
|
||||
class DeadlineConnection(psycopg.Connection):
|
||||
def wait(self, gen, interval=0.1, timeout=None):
|
||||
try:
|
||||
return super().wait(gen, interval=interval, timeout=timeout or 3.0)
|
||||
except psycopg.errors._WaitTimeout:
|
||||
# A paused server cannot complete cancellation. Discard an uncertain
|
||||
# connection instead of returning it to the pool or waiting forever.
|
||||
self.pgconn.finish()
|
||||
raise
|
||||
|
||||
|
||||
def public(row):
|
||||
return {
|
||||
k: (
|
||||
str(v)
|
||||
if isinstance(v, UUID)
|
||||
else v.astimezone(UTC).isoformat(timespec="microseconds").replace("+00:00", "Z")
|
||||
if isinstance(v, datetime)
|
||||
else bytes(v).decode("utf-8")
|
||||
if isinstance(v, (bytes, memoryview))
|
||||
else v
|
||||
)
|
||||
for k, v in row.items()
|
||||
if k not in {"password_hash", "token_digest"}
|
||||
}
|
||||
|
||||
|
||||
class PostgresStore:
|
||||
def __init__(self):
|
||||
self.pool = ConnectionPool(
|
||||
secret("DATABASE_URL"),
|
||||
connection_class=DeadlineConnection,
|
||||
min_size=1,
|
||||
max_size=int(os.getenv("POOL_MAX_SIZE", "8")),
|
||||
timeout=2,
|
||||
max_waiting=32,
|
||||
kwargs={
|
||||
"autocommit": True,
|
||||
"row_factory": dict_row,
|
||||
"connect_timeout": 2,
|
||||
"options": "-c statement_timeout=2500 -c lock_timeout=1500",
|
||||
},
|
||||
check=ConnectionPool.check_connection,
|
||||
)
|
||||
self.pool.wait(timeout=15)
|
||||
self.cache = redis.Redis.from_url(
|
||||
secret("CACHE_URL"),
|
||||
socket_connect_timeout=0.2,
|
||||
socket_timeout=0.2,
|
||||
decode_responses=True,
|
||||
)
|
||||
self.ttl = int(os.getenv("SESSION_TTL_SECONDS", "3600"))
|
||||
if self.ttl < 1:
|
||||
raise RuntimeError("SESSION_TTL_SECONDS должен быть положительным")
|
||||
|
||||
@contextmanager
|
||||
def connection(self):
|
||||
try:
|
||||
with operation("postgres.transaction", {"db.system.name": "postgresql"}):
|
||||
with self.pool.connection() as conn:
|
||||
yield conn
|
||||
except (
|
||||
psycopg.OperationalError,
|
||||
psycopg.InterfaceError,
|
||||
PoolTimeout,
|
||||
TooManyRequests,
|
||||
psycopg.errors.QueryCanceled,
|
||||
psycopg.errors.LockNotAvailable,
|
||||
):
|
||||
raise Failure(503, "Хранилище временно недоступно") from None
|
||||
|
||||
def close(self):
|
||||
self.cache.close()
|
||||
self.pool.close()
|
||||
|
||||
def ready(self):
|
||||
try:
|
||||
with self.connection() as conn:
|
||||
conn.execute(
|
||||
"SELECT u.id,u.password_hash,c.id,m.chat_id,msg.id,s.token_digest,r.key,"
|
||||
"a.id,a.stage_key,a.pipeline_version,a.variants,ma.attachment_id,ui.object_key,"
|
||||
"j.id,j.source_key,j.attempt_limit,j.lease_until,j.traceparent,j.tracestate,j.accepted_at,o.job_id,o.published_at,"
|
||||
"audit.job_id,f.error_code FROM users u,chats c,members m,messages msg,"
|
||||
"sessions s,message_requests r,attachments a,message_attachments ma,"
|
||||
"upload_intents ui,attachment_jobs j,job_outbox o,job_audit audit,failed_jobs f "
|
||||
"WHERE false"
|
||||
)
|
||||
return True
|
||||
except (Failure, psycopg.Error):
|
||||
return False
|
||||
|
||||
def _cached_user(self, uid):
|
||||
key = "user:v1:" + uid
|
||||
try:
|
||||
value = self.cache.get(key)
|
||||
if value:
|
||||
cached = json.loads(value)
|
||||
if (
|
||||
isinstance(cached, dict)
|
||||
and cached.get("id") == uid
|
||||
and set(cached) == {"id", "username", "display_name", "created_at"}
|
||||
):
|
||||
return cached
|
||||
except (redis.RedisError, ValueError, TypeError):
|
||||
logger.warning(json.dumps({"event": "cache_bypass"}))
|
||||
user = self.user(uid)
|
||||
if user:
|
||||
try:
|
||||
self.cache.setex(key, 60, json.dumps(user))
|
||||
except redis.RedisError:
|
||||
logger.warning(json.dumps({"event": "cache_bypass"}))
|
||||
return user
|
||||
|
||||
def user(self, uid):
|
||||
with self.connection() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT id,username,display_name,created_at FROM users WHERE id=%s", (uid,)
|
||||
).fetchone()
|
||||
return public(row) if row else None
|
||||
|
||||
def register(self, data):
|
||||
with KDF_SLOTS:
|
||||
hashed = KDF.hash(data.pop("password"))
|
||||
uid = str(uuid4())
|
||||
try:
|
||||
with self.connection() as conn:
|
||||
row = conn.execute(
|
||||
"INSERT INTO users VALUES (%s,%s,%s,%s,%s) "
|
||||
"RETURNING id,username,display_name,created_at",
|
||||
(
|
||||
uid,
|
||||
data["username"],
|
||||
data["display_name"].encode("utf-8"),
|
||||
hashed,
|
||||
timestamp(),
|
||||
),
|
||||
).fetchone()
|
||||
return public(row)
|
||||
except psycopg.errors.UniqueViolation:
|
||||
raise Failure(409, "Имя пользователя занято") from None
|
||||
|
||||
def login(self, username, password):
|
||||
with self.connection() as conn:
|
||||
row = conn.execute("SELECT * FROM users WHERE username=%s", (username,)).fetchone()
|
||||
try:
|
||||
with KDF_SLOTS:
|
||||
KDF.verify(row["password_hash"] if row else DUMMY_HASH, password)
|
||||
except VerificationError:
|
||||
return None
|
||||
if row is None:
|
||||
return None
|
||||
token = secrets.token_urlsafe(32)
|
||||
digest = hashlib.sha256(token.encode()).hexdigest()
|
||||
expires = datetime.now(UTC) + timedelta(seconds=self.ttl)
|
||||
with self.connection() as conn:
|
||||
conn.execute("INSERT INTO sessions VALUES (%s,%s,%s)", (digest, row["id"], expires))
|
||||
return {
|
||||
"token": token,
|
||||
"token_type": "Bearer",
|
||||
"expires_at": public({"v": expires})["v"],
|
||||
"user": public(row),
|
||||
}
|
||||
|
||||
def authenticate(self, token):
|
||||
if not token or len(token) > 512:
|
||||
return None
|
||||
digest = hashlib.sha256(token.encode()).hexdigest()
|
||||
# Session validity always comes from PostgreSQL, including on a cache hit.
|
||||
with self.connection() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT s.user_id FROM sessions s WHERE s.token_digest=%s AND s.expires_at>now()",
|
||||
(digest,),
|
||||
).fetchone()
|
||||
return self._cached_user(str(row["user_id"])) if row else None
|
||||
|
||||
def logout(self, token):
|
||||
with self.connection() as conn:
|
||||
row = conn.execute(
|
||||
"DELETE FROM sessions WHERE token_digest=%s AND expires_at>now() RETURNING user_id",
|
||||
(hashlib.sha256(token.encode()).hexdigest(),),
|
||||
).fetchone()
|
||||
return row is not None
|
||||
|
||||
def create_chat(self, actor, data):
|
||||
if actor not in data["member_ids"]:
|
||||
raise Failure(400, "Вызывающий должен входить в чат")
|
||||
cid = str(uuid4())
|
||||
try:
|
||||
with self.connection() as conn, conn.transaction():
|
||||
row = conn.execute(
|
||||
"INSERT INTO chats VALUES (%s,%s,%s) RETURNING *",
|
||||
(cid, data["title"].encode("utf-8"), timestamp()),
|
||||
).fetchone()
|
||||
for index, uid in enumerate(data["member_ids"]):
|
||||
conn.execute("INSERT INTO members VALUES (%s,%s,%s)", (cid, uid, index))
|
||||
return {**public(row), "member_ids": data["member_ids"]}
|
||||
except psycopg.errors.ForeignKeyViolation:
|
||||
raise Failure(400, "Неизвестный участник") from None
|
||||
|
||||
@staticmethod
|
||||
def _chat(conn, actor, cid):
|
||||
row = conn.execute(
|
||||
"SELECT c.* FROM chats c JOIN members m ON m.chat_id=c.id "
|
||||
"WHERE c.id=%s AND m.user_id=%s",
|
||||
(cid, actor),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
raise Failure(404, "Объект не найден")
|
||||
members = conn.execute(
|
||||
"SELECT user_id FROM members WHERE chat_id=%s ORDER BY position", (cid,)
|
||||
).fetchall()
|
||||
return {**public(row), "member_ids": [str(m["user_id"]) for m in members]}
|
||||
|
||||
def chat(self, actor, cid):
|
||||
with self.connection() as conn:
|
||||
return self._chat(conn, actor, cid)
|
||||
|
||||
def list_chats(self, actor, after, limit):
|
||||
with self.connection() as conn:
|
||||
sql = "SELECT c.* FROM chats c JOIN members m ON m.chat_id=c.id WHERE m.user_id=%s"
|
||||
params = [actor]
|
||||
if after:
|
||||
sql += " AND (c.created_at,c.id)>(%s,%s)"
|
||||
params.extend(after)
|
||||
sql += " ORDER BY c.created_at,c.id LIMIT %s"
|
||||
params.append(limit + 1)
|
||||
rows = conn.execute(sql, params).fetchall()
|
||||
return [self._chat(conn, actor, str(row["id"])) for row in rows]
|
||||
|
||||
def send(self, actor, cid, text, key, attachment_ids=None):
|
||||
attachment_ids = attachment_ids or []
|
||||
body_digest = hashlib.sha256(
|
||||
json.dumps(
|
||||
{"text": text, "attachment_ids": attachment_ids},
|
||||
sort_keys=True,
|
||||
ensure_ascii=False,
|
||||
separators=(",", ":"),
|
||||
).encode()
|
||||
).hexdigest()
|
||||
with self.connection() as conn, conn.transaction():
|
||||
self._chat(conn, actor, cid)
|
||||
# Serializes exactly this semantic key across all API processes. Hash
|
||||
# collisions can only serialize unrelated requests, never mix results.
|
||||
scope = json.dumps([actor, "POST", cid, key], separators=(",", ":"))
|
||||
lock = int.from_bytes(hashlib.sha256(scope.encode()).digest()[:8], "big", signed=True)
|
||||
conn.execute("SELECT pg_advisory_xact_lock(%s)", (lock,))
|
||||
old = conn.execute(
|
||||
"SELECT body_digest,response FROM message_requests "
|
||||
"WHERE actor_id=%s AND chat_id=%s AND key=%s AND expires_at>now()",
|
||||
(actor, cid, key),
|
||||
).fetchone()
|
||||
if old:
|
||||
if old["body_digest"] != body_digest:
|
||||
raise Failure(409, "Ключ уже использован для другого сообщения")
|
||||
return json.loads(bytes(old["response"]))
|
||||
conn.execute(
|
||||
"DELETE FROM message_requests WHERE actor_id=%s AND chat_id=%s AND key=%s",
|
||||
(actor, cid, key),
|
||||
)
|
||||
for aid in attachment_ids:
|
||||
attachment = conn.execute(
|
||||
"SELECT id,status FROM attachments WHERE id=%s AND chat_id=%s FOR SHARE",
|
||||
(aid, cid),
|
||||
).fetchone()
|
||||
if not attachment:
|
||||
raise Failure(404, "Объект не найден")
|
||||
if attachment["status"] == "failed":
|
||||
raise Failure(409, "Вложение недоступно")
|
||||
row = conn.execute(
|
||||
"INSERT INTO messages VALUES (%s,%s,%s,%s,%s) RETURNING *",
|
||||
(str(uuid4()), cid, actor, text.encode("utf-8"), timestamp()),
|
||||
).fetchone()
|
||||
for position, aid in enumerate(attachment_ids):
|
||||
conn.execute(
|
||||
"INSERT INTO message_attachments VALUES (%s,%s,%s)", (row["id"], aid, position)
|
||||
)
|
||||
response = {**public(row), "attachment_ids": attachment_ids}
|
||||
conn.execute(
|
||||
"INSERT INTO message_requests(actor_id,chat_id,key,body_digest,response,expires_at) "
|
||||
"VALUES (%s,%s,%s,%s,%s,now()+interval '24 hours')",
|
||||
(actor, cid, key, body_digest, json.dumps(response).encode("utf-8")),
|
||||
)
|
||||
return response
|
||||
|
||||
def history(self, actor, cid, after, limit):
|
||||
with self.connection() as conn:
|
||||
self._chat(conn, actor, cid)
|
||||
sql = "SELECT * FROM messages WHERE chat_id=%s"
|
||||
params = [cid]
|
||||
if after:
|
||||
sql += " AND (created_at,id)>(%s,%s)"
|
||||
params.extend(after)
|
||||
sql += " ORDER BY created_at,id LIMIT %s"
|
||||
params.append(limit + 1)
|
||||
result = []
|
||||
for row in conn.execute(sql, params).fetchall():
|
||||
attachments = conn.execute(
|
||||
"SELECT attachment_id FROM message_attachments WHERE message_id=%s ORDER BY position",
|
||||
(row["id"],),
|
||||
).fetchall()
|
||||
result.append(
|
||||
{
|
||||
**public(row),
|
||||
"attachment_ids": [str(a["attachment_id"]) for a in attachments],
|
||||
}
|
||||
)
|
||||
return result
|
||||
@@ -0,0 +1,100 @@
|
||||
"""Private S3 access with bounded network waits; never expose storage keys in API."""
|
||||
|
||||
import os
|
||||
from contextlib import contextmanager
|
||||
|
||||
import boto3
|
||||
from botocore.config import Config
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
|
||||
from .config import secret
|
||||
from .domain import Failure
|
||||
from .telemetry import operation
|
||||
|
||||
|
||||
class ObjectStorage:
|
||||
def __init__(self, admin=False):
|
||||
self.bucket = os.getenv("S3_BUCKET", "messenger")
|
||||
prefix = "S3_ADMIN" if admin else "S3"
|
||||
self.client = boto3.client(
|
||||
"s3",
|
||||
endpoint_url=os.getenv("S3_ENDPOINT", "http://s3:8333"),
|
||||
aws_access_key_id=secret(prefix + "_ACCESS_KEY"),
|
||||
aws_secret_access_key=secret(prefix + "_SECRET_KEY"),
|
||||
region_name="us-east-1",
|
||||
config=Config(
|
||||
connect_timeout=1,
|
||||
read_timeout=2,
|
||||
retries={"total_max_attempts": 1},
|
||||
s3={"addressing_style": "path"},
|
||||
request_checksum_calculation="when_required",
|
||||
response_checksum_validation="when_required",
|
||||
),
|
||||
)
|
||||
|
||||
@contextmanager
|
||||
def boundary(self):
|
||||
try:
|
||||
with operation("s3.request", {"rpc.system": "aws-api", "rpc.service": "S3"}):
|
||||
yield
|
||||
except (BotoCoreError, ClientError):
|
||||
raise Failure(503, "Хранилище файлов временно недоступно") from None
|
||||
|
||||
def put(self, key, fileobj, size, content_type):
|
||||
fileobj.seek(0)
|
||||
with self.boundary():
|
||||
self.client.put_object(
|
||||
Bucket=self.bucket,
|
||||
Key=key,
|
||||
Body=fileobj,
|
||||
ContentLength=size,
|
||||
ContentType=content_type,
|
||||
)
|
||||
|
||||
def get(self, key):
|
||||
try:
|
||||
with operation("s3.get", {"rpc.system": "aws-api", "rpc.service": "S3"}):
|
||||
return self.client.get_object(Bucket=self.bucket, Key=key)
|
||||
except ClientError as exc:
|
||||
if exc.response.get("Error", {}).get("Code") in {"NoSuchKey", "404"}:
|
||||
raise Failure(404, "Объект не найден") from None
|
||||
raise Failure(503, "Хранилище файлов временно недоступно") from None
|
||||
except BotoCoreError:
|
||||
raise Failure(503, "Хранилище файлов временно недоступно") from None
|
||||
|
||||
def head(self, key):
|
||||
try:
|
||||
with operation("s3.head", {"rpc.system": "aws-api", "rpc.service": "S3"}):
|
||||
return self.client.head_object(Bucket=self.bucket, Key=key)
|
||||
except ClientError as exc:
|
||||
if exc.response.get("Error", {}).get("Code") in {"NoSuchKey", "404", "NotFound"}:
|
||||
return None
|
||||
raise Failure(503, "Хранилище файлов временно недоступно") from None
|
||||
except BotoCoreError:
|
||||
raise Failure(503, "Хранилище файлов временно недоступно") from None
|
||||
|
||||
def exists(self, key):
|
||||
return self.head(key) is not None
|
||||
|
||||
def delete(self, key):
|
||||
with self.boundary():
|
||||
self.client.delete_object(Bucket=self.bucket, Key=key)
|
||||
|
||||
def objects(self, page_size=100):
|
||||
with self.boundary():
|
||||
paginator = self.client.get_paginator("list_objects_v2")
|
||||
for page in paginator.paginate(
|
||||
Bucket=self.bucket, PaginationConfig={"PageSize": page_size}
|
||||
):
|
||||
yield from page.get("Contents", [])
|
||||
|
||||
def ready(self):
|
||||
try:
|
||||
with self.boundary():
|
||||
self.client.head_bucket(Bucket=self.bucket)
|
||||
return True
|
||||
except Failure:
|
||||
return False
|
||||
|
||||
def close(self):
|
||||
self.client.close()
|
||||
@@ -0,0 +1,30 @@
|
||||
"""Idempotent private bucket bootstrap; this process alone receives admin keys."""
|
||||
|
||||
import time
|
||||
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
|
||||
from .storage import ObjectStorage
|
||||
|
||||
|
||||
def main():
|
||||
storage = ObjectStorage(admin=True)
|
||||
for attempt in range(30):
|
||||
try:
|
||||
try:
|
||||
storage.client.head_bucket(Bucket=storage.bucket)
|
||||
except ClientError as exc:
|
||||
if exc.response["Error"]["Code"] not in {"404", "NoSuchBucket", "NotFound"}:
|
||||
raise
|
||||
storage.client.create_bucket(Bucket=storage.bucket)
|
||||
print("Private bucket ready; anonymous identity is not configured.")
|
||||
storage.close()
|
||||
return
|
||||
except (ClientError, BotoCoreError):
|
||||
if attempt == 29:
|
||||
raise RuntimeError("S3 bootstrap timeout") from None
|
||||
time.sleep(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,274 @@
|
||||
"""Bounded OTLP telemetry; technical attributes only, no payload/credential capture."""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
from contextlib import contextmanager
|
||||
from datetime import UTC, datetime
|
||||
from uuid import uuid4
|
||||
|
||||
import psutil
|
||||
import requests
|
||||
from opentelemetry import metrics, propagate, trace
|
||||
from opentelemetry._logs import set_logger_provider
|
||||
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
|
||||
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
|
||||
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
|
||||
from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
|
||||
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
|
||||
from opentelemetry.sdk.metrics import MeterProvider
|
||||
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
|
||||
from opentelemetry.sdk.metrics.view import ExplicitBucketHistogramAggregation, View
|
||||
from opentelemetry.sdk.resources import Resource
|
||||
from opentelemetry.sdk.trace import SpanProcessor, TracerProvider
|
||||
from opentelemetry.sdk.trace.export import BatchSpanProcessor, SpanExportResult
|
||||
from opentelemetry.sdk.trace.sampling import ParentBased, TraceIdRatioBased
|
||||
from opentelemetry.trace import SpanKind, Status, StatusCode
|
||||
|
||||
HTTP_BUCKETS = (0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10)
|
||||
JOB_BUCKETS = (0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 20, 45, 60, 120)
|
||||
INSTANCE_ID = str(uuid4())
|
||||
_runtime = None
|
||||
|
||||
|
||||
class Accounting(dict):
|
||||
def __init__(self, keys):
|
||||
super().__init__((key, 0) for key in keys)
|
||||
self.lock = threading.Lock()
|
||||
|
||||
def add(self, key, value):
|
||||
with self.lock:
|
||||
self[key] += value
|
||||
|
||||
def snapshot(self):
|
||||
with self.lock:
|
||||
return dict(self)
|
||||
|
||||
|
||||
class RecordedSpans(SpanProcessor):
|
||||
def __init__(self, stats):
|
||||
self.stats = stats
|
||||
|
||||
def on_start(self, span, parent_context=None):
|
||||
pass
|
||||
|
||||
def on_end(self, span):
|
||||
if span.context.trace_flags.sampled:
|
||||
self.stats.add("spans_recorded", 1)
|
||||
|
||||
|
||||
class CountingSession(requests.Session):
|
||||
"""Count actual protobuf HTTP sends, including transport retries, without retaining data."""
|
||||
def __init__(self, stats, signal):
|
||||
super().__init__()
|
||||
self.stats, self.signal = stats, signal
|
||||
|
||||
def send(self, request, **kwargs):
|
||||
self.stats.add(self.signal + "_wire_bytes", len(request.body or b""))
|
||||
self.stats.add(self.signal + "_http_sends", 1)
|
||||
return super().send(request, **kwargs)
|
||||
|
||||
|
||||
class ObservedSpanExporter(OTLPSpanExporter):
|
||||
def __init__(self, stats, **kwargs):
|
||||
self.stats = stats
|
||||
super().__init__(**kwargs)
|
||||
|
||||
def export(self, spans):
|
||||
self.stats.add("spans_in_flight", len(spans))
|
||||
try:
|
||||
result = super().export(spans)
|
||||
key = "spans_exported" if result == SpanExportResult.SUCCESS else "spans_export_failed"
|
||||
self.stats.add(key, len(spans))
|
||||
return result
|
||||
except Exception:
|
||||
self.stats.add("spans_export_failed", len(spans))
|
||||
raise
|
||||
finally:
|
||||
self.stats.add("spans_in_flight", -len(spans))
|
||||
|
||||
|
||||
class TechnicalLogs(logging.Filter):
|
||||
def filter(self, record):
|
||||
return record.name.startswith("messenger")
|
||||
|
||||
|
||||
class JsonFormatter(logging.Formatter):
|
||||
def format(self, record):
|
||||
try:
|
||||
event = json.loads(record.getMessage())
|
||||
if not isinstance(event, dict):
|
||||
event = {"event": "application_event"}
|
||||
except ValueError:
|
||||
event = {"event": "application_event"}
|
||||
context = trace.get_current_span().get_span_context()
|
||||
return json.dumps({
|
||||
**event,
|
||||
"timestamp": datetime.now(UTC).isoformat(),
|
||||
"severity": record.levelname,
|
||||
"service.name": os.getenv("OTEL_SERVICE_NAME", "messenger-api"),
|
||||
"trace_id": format(context.trace_id, "032x") if context.is_valid else None,
|
||||
"span_id": format(context.span_id, "016x") if context.is_valid else None,
|
||||
})
|
||||
|
||||
|
||||
class Telemetry:
|
||||
def __init__(self, service):
|
||||
self.service = service
|
||||
self.enabled = os.getenv("OTEL_ENABLED", "1") == "1"
|
||||
self.busy = 0
|
||||
self.providers = []
|
||||
self.export_stats = Accounting(("spans_recorded", "spans_in_flight", "spans_exported", "spans_export_failed", "traces_wire_bytes", "traces_http_sends", "metrics_wire_bytes", "metrics_http_sends", "logs_wire_bytes", "logs_http_sends"))
|
||||
self.stop = threading.Event()
|
||||
self.thread = None
|
||||
resource = Resource.create({
|
||||
"service.name": service,
|
||||
"service.version": os.getenv("SERVICE_VERSION", "lab6-v1"),
|
||||
"service.instance.id": INSTANCE_ID,
|
||||
"deployment.environment.name": "lab",
|
||||
})
|
||||
os.environ["OTEL_SERVICE_NAME"] = service
|
||||
root = logging.getLogger()
|
||||
root.setLevel(logging.INFO)
|
||||
for handler in root.handlers:
|
||||
handler.setFormatter(JsonFormatter())
|
||||
if not root.handlers:
|
||||
handler = logging.StreamHandler()
|
||||
handler.setFormatter(JsonFormatter())
|
||||
root.addHandler(handler)
|
||||
if self.enabled:
|
||||
endpoint = os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://collector:4318").rstrip("/")
|
||||
common = {"timeout": 1.0}
|
||||
# Standard OTLP environment variables also support HTTPS/mTLS in lab7.
|
||||
trace_provider = TracerProvider(resource=resource, sampler=ParentBased(
|
||||
TraceIdRatioBased(float(os.getenv("OTEL_SAMPLE_RATIO", "1")))))
|
||||
trace_provider.add_span_processor(RecordedSpans(self.export_stats))
|
||||
self.batch_spans = BatchSpanProcessor(
|
||||
ObservedSpanExporter(self.export_stats, endpoint=endpoint + "/v1/traces", session=CountingSession(self.export_stats, "traces"), **common),
|
||||
max_queue_size=256, max_export_batch_size=64,
|
||||
schedule_delay_millis=500, export_timeout_millis=1500,
|
||||
)
|
||||
trace_provider.add_span_processor(self.batch_spans)
|
||||
trace.set_tracer_provider(trace_provider)
|
||||
reader = PeriodicExportingMetricReader(
|
||||
OTLPMetricExporter(endpoint=endpoint + "/v1/metrics", session=CountingSession(self.export_stats, "metrics"), **common),
|
||||
export_interval_millis=int(os.getenv("OTEL_METRIC_INTERVAL_MS", "5000")),
|
||||
export_timeout_millis=1500,
|
||||
)
|
||||
meter_provider = MeterProvider(resource=resource, metric_readers=[reader], views=[
|
||||
View(instrument_name="messenger.http.duration", aggregation=ExplicitBucketHistogramAggregation(HTTP_BUCKETS)),
|
||||
View(instrument_name="messenger.job.duration", aggregation=ExplicitBucketHistogramAggregation(JOB_BUCKETS)),
|
||||
View(instrument_name="messenger.attachment.ready.duration", aggregation=ExplicitBucketHistogramAggregation(JOB_BUCKETS)),
|
||||
])
|
||||
metrics.set_meter_provider(meter_provider)
|
||||
logger_provider = LoggerProvider(resource=resource)
|
||||
logger_provider.add_log_record_processor(BatchLogRecordProcessor(
|
||||
OTLPLogExporter(endpoint=endpoint + "/v1/logs", session=CountingSession(self.export_stats, "logs"), **common),
|
||||
max_queue_size=256, max_export_batch_size=64,
|
||||
schedule_delay_millis=500, export_timeout_millis=1500,
|
||||
))
|
||||
set_logger_provider(logger_provider)
|
||||
otlp_handler = LoggingHandler(level=logging.INFO, logger_provider=logger_provider)
|
||||
otlp_handler.addFilter(TechnicalLogs())
|
||||
# Keep machine-readable JSON in OTLP log body, preserving request ID.
|
||||
otlp_handler.setFormatter(JsonFormatter())
|
||||
root.addHandler(otlp_handler)
|
||||
self.providers = [trace_provider, meter_provider, logger_provider]
|
||||
self.tracer = trace.get_tracer("messenger", "1.0")
|
||||
meter = metrics.get_meter("messenger", "1.0")
|
||||
self.requests = meter.create_counter("messenger.http.requests", unit="{request}")
|
||||
self.http_duration = meter.create_histogram("messenger.http.duration", unit="s")
|
||||
self.sampling = meter.create_counter("messenger.http.trace.decisions", unit="{request}")
|
||||
self.jobs = meter.create_counter("messenger.jobs", unit="{job}")
|
||||
self.job_duration = meter.create_histogram("messenger.job.duration", unit="s")
|
||||
self.ready_duration = meter.create_histogram("messenger.attachment.ready.duration", unit="s")
|
||||
self.queue_depth = meter.create_gauge("messenger.queue.depth", unit="{job}")
|
||||
self.oldest_age = meter.create_gauge("messenger.queue.oldest.age", unit="s")
|
||||
self.pool_used = meter.create_gauge("messenger.db.pool.used", unit="{connection}")
|
||||
self.pool_waiting = meter.create_gauge("messenger.db.pool.waiting", unit="{request}")
|
||||
self.worker_busy = meter.create_gauge("messenger.worker.busy", unit="{worker}")
|
||||
self.worker_capacity = meter.create_gauge("messenger.worker.capacity", unit="{worker}")
|
||||
self.process_cpu = meter.create_gauge("messenger.process.cpu", unit="%")
|
||||
self.process_memory = meter.create_gauge("messenger.process.memory", unit="By")
|
||||
self.export_observation = meter.create_gauge("messenger.telemetry.export", unit="{item}")
|
||||
|
||||
def start_background(self, store):
|
||||
process = psutil.Process()
|
||||
process.cpu_percent()
|
||||
|
||||
def collect():
|
||||
while not self.stop.wait(2):
|
||||
counts = self.export_stats.snapshot()
|
||||
# Read-only queue observation of pinned SDK 1.45.2. Conservation
|
||||
# is evaluated after quiescence, not during concurrent export.
|
||||
pending = len(self.batch_spans._batch_processor._queue)
|
||||
counts["spans_pending"] = pending
|
||||
counts["span_queue_capacity"] = 256
|
||||
counts["spans_queue_lost"] = max(0, counts["spans_recorded"] - counts["spans_exported"] - counts["spans_export_failed"] - counts["spans_in_flight"] - pending)
|
||||
for name, value in counts.items():
|
||||
self.export_observation.set(value, {"observation": name})
|
||||
stats = store.pool.get_stats()
|
||||
self.pool_used.set(max(0, stats.get("pool_size", 0) - stats.get("pool_available", 0)))
|
||||
self.pool_waiting.set(stats.get("requests_waiting", 0))
|
||||
self.process_cpu.set(process.cpu_percent())
|
||||
self.process_memory.set(process.memory_info().rss)
|
||||
if self.service == "messenger-worker":
|
||||
self.worker_busy.set(self.busy)
|
||||
self.worker_capacity.set(1)
|
||||
if self.service == "messenger-dispatcher":
|
||||
try:
|
||||
with store.connection() as conn:
|
||||
row = conn.execute("SELECT count(*) AS depth, COALESCE(EXTRACT(EPOCH FROM now()-min(created_at)),0) AS age FROM attachment_jobs WHERE state IN ('queued','processing')").fetchone()
|
||||
self.queue_depth.set(row["depth"], {"queue": "attachments"})
|
||||
self.oldest_age.set(float(row["age"]), {"queue": "attachments"})
|
||||
except Exception:
|
||||
logging.getLogger("messenger.telemetry").warning(json.dumps({"event": "queue_observation_unavailable"}))
|
||||
if self.enabled:
|
||||
self.thread = threading.Thread(target=collect, name="telemetry-observations", daemon=True)
|
||||
self.thread.start()
|
||||
|
||||
def close(self):
|
||||
self.stop.set()
|
||||
if self.thread:
|
||||
self.thread.join(timeout=4)
|
||||
# Business shutdown has a fixed budget even if telemetry endpoints are down.
|
||||
def shutdown_providers():
|
||||
for provider in self.providers:
|
||||
provider.shutdown()
|
||||
shutdown = threading.Thread(target=shutdown_providers, name="telemetry-shutdown", daemon=True)
|
||||
shutdown.start()
|
||||
shutdown.join(timeout=2)
|
||||
|
||||
|
||||
def setup(service):
|
||||
global _runtime
|
||||
if _runtime is None:
|
||||
_runtime = Telemetry(service)
|
||||
return _runtime
|
||||
|
||||
|
||||
def current():
|
||||
return _runtime
|
||||
|
||||
|
||||
def carrier():
|
||||
result = {}
|
||||
propagate.inject(result)
|
||||
return {key: result[key] for key in ("traceparent", "tracestate") if key in result}
|
||||
|
||||
|
||||
@contextmanager
|
||||
def operation(name, attributes=None, context=None, kind=SpanKind.CLIENT):
|
||||
if kind == SpanKind.CLIENT and context is None and not trace.get_current_span().get_span_context().is_valid:
|
||||
yield trace.get_current_span()
|
||||
return
|
||||
tracer = trace.get_tracer("messenger", "1.0")
|
||||
with tracer.start_as_current_span(name, attributes=attributes or {}, context=context,
|
||||
kind=kind, record_exception=False,
|
||||
set_status_on_exception=False) as span:
|
||||
try:
|
||||
yield span
|
||||
except BaseException as exc:
|
||||
span.set_status(Status(StatusCode.ERROR, type(exc).__name__))
|
||||
raise
|
||||
@@ -0,0 +1,286 @@
|
||||
"""Separate serial worker (prefetch=1) and transactional-outbox dispatcher."""
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
import redis
|
||||
from opentelemetry import propagate, trace
|
||||
from opentelemetry.trace import SpanKind, Status, StatusCode
|
||||
|
||||
from . import telemetry
|
||||
from .domain import Failure
|
||||
from .jobs import GROUP, STREAM, Jobs, broker_client, fault
|
||||
from .postgres import PostgresStore
|
||||
from .storage import ObjectStorage
|
||||
|
||||
logger = logging.getLogger("messenger.worker")
|
||||
logging.basicConfig(level=logging.INFO, format="%(message)s")
|
||||
stopping = False
|
||||
MAX_JOB_BYTES = 8192
|
||||
|
||||
|
||||
def stop(signum, frame):
|
||||
global stopping
|
||||
stopping = True
|
||||
|
||||
|
||||
def process(job, jobs, storage):
|
||||
fault(jobs.store, "before_effect", job["attachment_id"])
|
||||
with tempfile.TemporaryDirectory(prefix="image-job-") as directory:
|
||||
root = Path(directory)
|
||||
source = root / "source"
|
||||
response = storage.get(job["source_key"])
|
||||
try:
|
||||
if response["ContentLength"] > 10_485_760:
|
||||
raise ValueError("invalid_image")
|
||||
with source.open("wb") as target:
|
||||
while block := response["Body"].read(64 * 1024):
|
||||
target.write(block)
|
||||
if target.tell() > 10_485_760:
|
||||
raise ValueError("invalid_image")
|
||||
finally:
|
||||
response["Body"].close()
|
||||
with source.open("rb") as file:
|
||||
digest = hashlib.file_digest(file, "sha256").hexdigest()
|
||||
if digest != job["source_sha256"] or source.stat().st_size != job["source_size"]:
|
||||
raise ValueError("invalid_image")
|
||||
# Resize executes outside the API and is bounded in wall time, CPU and address space.
|
||||
result = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
"-m",
|
||||
"messenger.image_pipeline",
|
||||
str(source),
|
||||
directory,
|
||||
str(job["pipeline_version"]),
|
||||
],
|
||||
timeout=20,
|
||||
capture_output=True,
|
||||
check=False,
|
||||
)
|
||||
if result.returncode:
|
||||
raise ValueError("invalid_image")
|
||||
variants = json.loads((root / "variants.json").read_text())
|
||||
prefix = f"attachments/{job['attachment_id']}/v{job['pipeline_version']}"
|
||||
with source.open("rb") as file:
|
||||
storage.put(
|
||||
prefix + "/original", file, source.stat().st_size, job["source_content_type"]
|
||||
)
|
||||
for variant in variants:
|
||||
path = root / variant["name"]
|
||||
with path.open("rb") as file:
|
||||
storage.put(
|
||||
prefix + "/" + variant["name"],
|
||||
file,
|
||||
variant["size_bytes"],
|
||||
variant["content_type"],
|
||||
)
|
||||
fault(jobs.store, "after_effect", job["attachment_id"])
|
||||
published = jobs.complete(job, prefix + "/original", variants)
|
||||
if published:
|
||||
fault(jobs.store, "after_result_commit", job["attachment_id"])
|
||||
cleanup_ready(jobs.store, storage, job["attachment_id"])
|
||||
return published
|
||||
|
||||
|
||||
def cleanup_ready(store, storage, attachment_id):
|
||||
# Same row lock as reprocess/replay: cannot delete a source chosen by a new job.
|
||||
with store.connection() as conn, conn.transaction():
|
||||
a = conn.execute(
|
||||
"SELECT * FROM attachments WHERE id=%s FOR UPDATE", (attachment_id,)
|
||||
).fetchone()
|
||||
if (
|
||||
a
|
||||
and a["status"] == "ready"
|
||||
and a["stage_key"]
|
||||
and a["stage_key"].startswith("staging/")
|
||||
):
|
||||
storage.delete(a["stage_key"])
|
||||
conn.execute("UPDATE attachments SET stage_key=NULL WHERE id=%s", (attachment_id,))
|
||||
|
||||
|
||||
def validate_delivery(body):
|
||||
if not isinstance(body, (str, bytes, bytearray)):
|
||||
raise ValueError("invalid_job")
|
||||
encoded = body.encode("utf-8") if isinstance(body, str) else body
|
||||
if len(encoded) > MAX_JOB_BYTES:
|
||||
raise ValueError("invalid_job")
|
||||
try:
|
||||
data = json.loads(encoded)
|
||||
except RecursionError:
|
||||
raise ValueError("invalid_job") from None
|
||||
expected = {
|
||||
"schema_version",
|
||||
"job_id",
|
||||
"type",
|
||||
"attachment_id",
|
||||
"pipeline_version",
|
||||
"attempt",
|
||||
"created_at",
|
||||
}
|
||||
optional = {"traceparent", "tracestate"}
|
||||
if not isinstance(data, dict) or not expected <= set(data) or set(data) - expected - optional:
|
||||
raise ValueError("invalid_job")
|
||||
if (
|
||||
type(data["schema_version"]) is not int
|
||||
or data["schema_version"] != 1
|
||||
or data["type"] != "attachment.process.v1"
|
||||
):
|
||||
raise ValueError("invalid_job")
|
||||
if not isinstance(data["job_id"], str) or not isinstance(data["attachment_id"], str):
|
||||
raise ValueError("invalid_job")
|
||||
for field in ("job_id", "attachment_id"):
|
||||
canonical = str(UUID(data[field]))
|
||||
if data[field].lower() != canonical:
|
||||
raise ValueError("invalid_job")
|
||||
# RFC UUID hex is case-insensitive; PostgreSQL and state comparisons use
|
||||
# the same canonical spelling, so a valid uppercase delivery is not lost.
|
||||
data[field] = canonical
|
||||
if type(data["pipeline_version"]) is not int or data["pipeline_version"] < 1:
|
||||
raise ValueError("invalid_job")
|
||||
if type(data["attempt"]) is not int or data["attempt"] < 1:
|
||||
raise ValueError("invalid_job")
|
||||
created = data["created_at"]
|
||||
if (
|
||||
not isinstance(created, str)
|
||||
or len(created) > 64
|
||||
or not re.fullmatch(
|
||||
r"[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(?:\.[0-9]+)?Z", created
|
||||
)
|
||||
):
|
||||
raise ValueError("invalid_job")
|
||||
datetime.fromisoformat(created[:-1] + "+00:00")
|
||||
if "traceparent" in data and (
|
||||
not isinstance(data["traceparent"], str)
|
||||
or not re.fullmatch(r"00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}", data["traceparent"])
|
||||
):
|
||||
raise ValueError("invalid_job")
|
||||
if "tracestate" in data and (
|
||||
not isinstance(data["tracestate"], str) or len(data["tracestate"]) > 512
|
||||
):
|
||||
raise ValueError("invalid_job")
|
||||
return data
|
||||
|
||||
|
||||
def consume_one(store, jobs, storage, broker, entry):
|
||||
entry_id, fields = entry
|
||||
try:
|
||||
body = validate_delivery(fields.get("body", fields.get(b"body")))
|
||||
except (KeyError, ValueError, TypeError):
|
||||
logger.error(json.dumps({"event": "invalid_delivery"}))
|
||||
broker.xack(STREAM, GROUP, entry_id)
|
||||
broker.xdel(STREAM, entry_id)
|
||||
return
|
||||
with telemetry.operation("attachments consume", context=propagate.extract(body), kind=SpanKind.CONSUMER):
|
||||
consume_job(store, jobs, storage, broker, body)
|
||||
# Durable state owns recovery even when notification acknowledgements repeat.
|
||||
broker.xack(STREAM, GROUP, entry_id)
|
||||
broker.xdel(STREAM, entry_id)
|
||||
|
||||
|
||||
def consume_job(store, jobs, storage, broker, body):
|
||||
job = jobs.claim(body["job_id"], body["attachment_id"], body["pipeline_version"])
|
||||
t = telemetry.current()
|
||||
started = time.perf_counter()
|
||||
outcome = "duplicate"
|
||||
if job:
|
||||
try:
|
||||
if t:
|
||||
t.busy = 1
|
||||
outcome = "success" if process(job, jobs, storage) else "superseded"
|
||||
except ValueError:
|
||||
jobs.fail(job, "invalid_image", permanent=True)
|
||||
outcome = "failed"
|
||||
except (Failure, subprocess.TimeoutExpired, OSError, RuntimeError) as exc:
|
||||
logger.warning(
|
||||
json.dumps(
|
||||
{"event": "job_retry", "job_id": str(job["id"]), "kind": type(exc).__name__}
|
||||
)
|
||||
)
|
||||
jobs.fail(job, "processing_failed")
|
||||
outcome = "retry" if job["attempt"] < job["attempt_limit"] else "failed"
|
||||
finally:
|
||||
if t:
|
||||
t.busy = 0
|
||||
attrs = {"job_type": "attachment.process.v1", "outcome": outcome}
|
||||
t.jobs.add(1, attrs)
|
||||
t.job_duration.record(time.perf_counter() - started, attrs)
|
||||
span = trace.get_current_span()
|
||||
span.set_attributes({"job.type": "attachment.process.v1", "job.outcome": outcome})
|
||||
if outcome in {"failed", "retry"}:
|
||||
span.set_status(Status(StatusCode.ERROR, "processing_failed"))
|
||||
logger.info(json.dumps({"event": "job_finished", "outcome": outcome}))
|
||||
else:
|
||||
trace.get_current_span().set_attributes({"job.type": "attachment.process.v1", "job.outcome": "duplicate"})
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--dispatcher", action="store_true")
|
||||
args = parser.parse_args()
|
||||
signal.signal(signal.SIGTERM, stop)
|
||||
signal.signal(signal.SIGINT, stop)
|
||||
t = telemetry.setup("messenger-dispatcher" if args.dispatcher else "messenger-worker")
|
||||
store = PostgresStore()
|
||||
t.start_background(store)
|
||||
jobs = Jobs(store)
|
||||
broker = broker_client()
|
||||
storage = None if args.dispatcher else ObjectStorage()
|
||||
consumer = "worker-" + str(uuid4())
|
||||
last_claim = 0
|
||||
try:
|
||||
while not stopping:
|
||||
try:
|
||||
if args.dispatcher:
|
||||
jobs.recover_one()
|
||||
if not jobs.publish_one(broker):
|
||||
time.sleep(0.2)
|
||||
continue
|
||||
try:
|
||||
broker.xgroup_create(STREAM, GROUP, id="0", mkstream=True)
|
||||
except redis.ResponseError as exc:
|
||||
if "BUSYGROUP" not in str(exc):
|
||||
raise
|
||||
entries = []
|
||||
if time.monotonic() - last_claim >= 5:
|
||||
claimed = broker.xautoclaim(
|
||||
STREAM,
|
||||
GROUP,
|
||||
consumer,
|
||||
min_idle_time=jobs.lease_seconds * 1000,
|
||||
start_id="0-0",
|
||||
count=1,
|
||||
)
|
||||
entries = claimed[1]
|
||||
last_claim = time.monotonic()
|
||||
if not entries:
|
||||
batches = broker.xreadgroup(GROUP, consumer, {STREAM: ">"}, count=1, block=500)
|
||||
entries = batches[0][1] if batches else []
|
||||
for entry in entries:
|
||||
consume_one(store, jobs, storage, broker, entry)
|
||||
except (redis.RedisError, Failure) as exc:
|
||||
logger.warning(
|
||||
json.dumps({"event": "dependency_retry", "kind": type(exc).__name__})
|
||||
)
|
||||
time.sleep(0.5)
|
||||
finally:
|
||||
broker.close()
|
||||
if storage is not None:
|
||||
storage.close()
|
||||
t.close()
|
||||
store.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,201 @@
|
||||
import base64
|
||||
import os
|
||||
import threading
|
||||
import uuid
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from urllib.parse import quote
|
||||
|
||||
import pytest
|
||||
from client import LAB, Client
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def scenario():
|
||||
c = Client(os.environ.get("BASE_URL", "http://127.0.0.1:8085"))
|
||||
return c, *c.bootstrap()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"body",
|
||||
[
|
||||
{"text": ""},
|
||||
{"text": " \n\t"},
|
||||
{"text": 1},
|
||||
{"text": True},
|
||||
{"text": None},
|
||||
{"text": "x", "sender_id": str(uuid.uuid4())},
|
||||
{"text": "x", "unknown": 1},
|
||||
{"text": "x" * 4001},
|
||||
{},
|
||||
[],
|
||||
],
|
||||
)
|
||||
def test_invalid_message(scenario, body):
|
||||
c, alice, _, _, chat = scenario
|
||||
c.call(
|
||||
"POST",
|
||||
f"/api/v1/chats/{chat['id']}/messages",
|
||||
400,
|
||||
actor=alice,
|
||||
body=body,
|
||||
headers={"Idempotency-Key": uuid.uuid4().hex},
|
||||
)
|
||||
|
||||
|
||||
def test_malformed_json(scenario):
|
||||
c, alice, _, _, chat = scenario
|
||||
c.call(
|
||||
"POST",
|
||||
f"/api/v1/chats/{chat['id']}/messages",
|
||||
400,
|
||||
actor=alice,
|
||||
raw=b'{"text":',
|
||||
headers={"Content-Type": "application/json", "Idempotency-Key": uuid.uuid4().hex},
|
||||
)
|
||||
|
||||
|
||||
def test_acl_and_unknown_chat(scenario):
|
||||
c, alice, _, eve, chat = scenario
|
||||
for cid in [chat["id"], str(uuid.uuid4())]:
|
||||
for suffix in ["", "/messages"]:
|
||||
c.call("GET", f"/api/v1/chats/{cid}{suffix}", 404, actor=eve)
|
||||
c.call(
|
||||
"POST",
|
||||
f"/api/v1/chats/{cid}/messages",
|
||||
404,
|
||||
actor=eve,
|
||||
body={"text": "x"},
|
||||
headers={"Idempotency-Key": uuid.uuid4().hex},
|
||||
)
|
||||
c.call("GET", "/api/v1/chats/not-a-uuid", 400, actor=alice)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", ["ab", "UPPER", "a-b", "a" * 33, "имя", "abc\n"])
|
||||
def test_username_validation(scenario, name):
|
||||
c, alice, *_ = scenario
|
||||
body = {**alice["registration"], "username": name}
|
||||
c.call("POST", "/api/v1/users", 400, body=body)
|
||||
|
||||
|
||||
def test_unicode_and_limits(scenario):
|
||||
c, alice, _, _, chat = scenario
|
||||
text = "Ё👩💻漢字" * 500
|
||||
assert c.message(alice, chat, text)["json"]["text"] == text
|
||||
c.call(
|
||||
"POST",
|
||||
"/api/v1/users",
|
||||
400,
|
||||
body={
|
||||
**alice["registration"],
|
||||
"username": "u_" + uuid.uuid4().hex[:12],
|
||||
"display_name": " ",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value", ["0", "101", "1.5", "true", "-1", "", "1&limit=2"])
|
||||
def test_invalid_limit(scenario, value):
|
||||
c, alice, _, _, chat = scenario
|
||||
c.call("GET", f"/api/v1/chats/{chat['id']}/messages?limit={value}", 400, actor=alice)
|
||||
|
||||
|
||||
def test_pagination_and_cursor_scope(scenario):
|
||||
c, alice, bob, _, chat = scenario
|
||||
expected = [c.message(alice, chat, str(i))["json"] for i in range(13)]
|
||||
path = f"/api/v1/chats/{chat['id']}/messages"
|
||||
result, cursor, first = [], None, None
|
||||
for i in range(20):
|
||||
url = path + f"?limit={2 if i == 0 else 3}"
|
||||
if cursor:
|
||||
url += "&cursor=" + quote(cursor, safe="")
|
||||
data = c.call("GET", url, actor=bob, schema="MessagePage")["json"]
|
||||
result += data["items"]
|
||||
cursor = data["next_cursor"]
|
||||
first = first or cursor
|
||||
if cursor is None:
|
||||
break
|
||||
assert result == sorted(expected, key=lambda m: (m["created_at"], m["id"]))
|
||||
other = c.call(
|
||||
"POST",
|
||||
"/api/v1/chats",
|
||||
201,
|
||||
actor=alice,
|
||||
body={"title": "другой", "member_ids": [alice["id"]]},
|
||||
schema="Chat",
|
||||
)["json"]
|
||||
c.call("GET", f"/api/v1/chats/{other['id']}/messages?cursor={first}", 400, actor=alice)
|
||||
c.call("GET", path + "?cursor=broken", 400, actor=alice)
|
||||
|
||||
|
||||
def test_chat_validation_and_atomicity(scenario):
|
||||
c, alice, bob, _, _ = scenario
|
||||
before = c.call("GET", "/api/v1/chats?limit=100", actor=alice, schema="ChatPage")["json"]
|
||||
for members in [
|
||||
[bob["id"]],
|
||||
[alice["id"], alice["id"]],
|
||||
[alice["id"], str(uuid.uuid4())],
|
||||
[],
|
||||
[123],
|
||||
]:
|
||||
c.call(
|
||||
"POST",
|
||||
"/api/v1/chats",
|
||||
400,
|
||||
actor=alice,
|
||||
body={"title": "не должен появиться", "member_ids": members},
|
||||
)
|
||||
after = c.call("GET", "/api/v1/chats?limit=100", actor=alice, schema="ChatPage")["json"]
|
||||
assert before == after
|
||||
|
||||
|
||||
def test_concurrent_messages(scenario):
|
||||
c, alice, bob, _, chat = scenario
|
||||
barrier = threading.Barrier(10)
|
||||
|
||||
def sender(i):
|
||||
local = Client(c.base)
|
||||
barrier.wait()
|
||||
return [local.message(alice, chat, f"{i}:{j}")["json"] for j in range(10)]
|
||||
|
||||
with ThreadPoolExecutor(max_workers=10) as pool:
|
||||
sent = [m for batch in pool.map(sender, range(10)) for m in batch]
|
||||
data = c.call(
|
||||
"GET", f"/api/v1/chats/{chat['id']}/messages?limit=100", actor=bob, schema="MessagePage"
|
||||
)["json"]
|
||||
assert len(sent) == len({m["id"] for m in sent}) == len(data["items"]) == 100
|
||||
assert {m["id"] for m in sent} == {m["id"] for m in data["items"]}
|
||||
assert data["next_cursor"] is None
|
||||
|
||||
|
||||
def test_registration_race(scenario):
|
||||
c, alice, *_ = scenario
|
||||
body = {**alice["registration"], "username": "race_" + uuid.uuid4().hex[:12]}
|
||||
barrier = threading.Barrier(10)
|
||||
|
||||
def register(_):
|
||||
local = Client(c.base)
|
||||
barrier.wait()
|
||||
return local.call("POST", "/api/v1/users", [201, 409], body=body)["status"]
|
||||
|
||||
with ThreadPoolExecutor(max_workers=10) as pool:
|
||||
statuses = list(pool.map(register, range(10)))
|
||||
assert statuses.count(201) == 1
|
||||
assert statuses.count(409) == 9
|
||||
|
||||
|
||||
@pytest.mark.skipif(LAB < 2, reason="Сессии появляются в ЛР2")
|
||||
def test_sessions(scenario):
|
||||
c, alice, _, eve, _ = scenario
|
||||
credential = base64.b64encode(
|
||||
(alice["registration"]["username"] + ":wrong-password").encode()
|
||||
).decode()
|
||||
c.call("POST", "/api/v1/auth/sessions", 401, headers={"Authorization": "Basic " + credential})
|
||||
c.call("GET", "/api/v1/users/me", 401, headers={"X-User-Id": alice["id"]})
|
||||
assert (
|
||||
c.call(
|
||||
"GET", "/api/v1/users/me", actor=alice, headers={"X-User-Id": eve["id"]}, schema="User"
|
||||
)["json"]
|
||||
== alice["user"]
|
||||
)
|
||||
c.call("DELETE", "/api/v1/auth/sessions/current", 204, actor=alice)
|
||||
c.call("GET", "/api/v1/users/me", 401, actor=alice)
|
||||
@@ -0,0 +1,203 @@
|
||||
"""Black-box regressions for attachment contract boundaries."""
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
import struct
|
||||
import uuid
|
||||
import zlib
|
||||
from io import BytesIO
|
||||
from urllib.parse import quote
|
||||
|
||||
import pytest
|
||||
from client import Client
|
||||
from PIL import Image
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def scenario():
|
||||
c = Client(os.environ.get("BASE_URL", "http://127.0.0.1:8085"))
|
||||
return c, *c.bootstrap()
|
||||
|
||||
|
||||
def image_bytes(fmt="PNG", size=(32, 20)):
|
||||
stream = BytesIO()
|
||||
Image.new("RGB", size, (100, 130, 200)).save(stream, fmt)
|
||||
return stream.getvalue()
|
||||
|
||||
|
||||
def upload(scenario, data=None, name="sample.png", mime="image/png", actor=None, status=202):
|
||||
c, alice, _, _, chat = scenario
|
||||
return c.call(
|
||||
"POST",
|
||||
f"/api/v1/chats/{chat['id']}/attachments?filename={quote(name, safe='')}",
|
||||
status,
|
||||
actor=actor or alice,
|
||||
raw=data if data is not None else image_bytes(),
|
||||
headers={"Content-Type": mime},
|
||||
schema="Attachment" if status == 202 else None,
|
||||
)
|
||||
|
||||
|
||||
def test_roundtrip_acl_and_variant(scenario):
|
||||
c, alice, bob, eve, chat = scenario
|
||||
data = image_bytes()
|
||||
a = upload(scenario, data)["json"]
|
||||
assert a["sha256"] == hashlib.sha256(data).hexdigest()
|
||||
assert a["status"] == "queued"
|
||||
a = c.wait_ready(alice, a)
|
||||
assert {v["name"] for v in a["variants"]} == {"thumbnail", "crop", "optimized"}
|
||||
path = f"/api/v1/attachments/{a['id']}"
|
||||
assert c.call("GET", path + "/content", 200, actor=bob)["bytes"] == data
|
||||
c.call("GET", path, 404, actor=eve)
|
||||
c.call("GET", path + "/content", 404, actor=eve)
|
||||
c.call("GET", path + "/content?variant=thumbnail", 200, actor=alice)
|
||||
c.call("GET", path + "/content?variant=unknown", 400, actor=alice)
|
||||
message = c.call(
|
||||
"POST",
|
||||
f"/api/v1/chats/{chat['id']}/messages",
|
||||
201,
|
||||
actor=alice,
|
||||
body={"text": "", "attachment_ids": [a["id"]]},
|
||||
headers={"Idempotency-Key": uuid.uuid4().hex},
|
||||
schema="Message",
|
||||
)["json"]
|
||||
assert message["text"] == "" and message["attachment_ids"] == [a["id"]]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"name", [" ", "../x.png", "a\\b.png", "a\0.png", "a\x7f.png", "a\x85.png", "a\n.png"]
|
||||
)
|
||||
def test_bad_filename(scenario, name):
|
||||
upload(scenario, name=name, status=400)
|
||||
|
||||
|
||||
def chunk(kind, data):
|
||||
return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data))
|
||||
|
||||
|
||||
def png_with_data(data, width=1, height=1, color=2, palette=b""):
|
||||
return (
|
||||
b"\x89PNG\r\n\x1a\n"
|
||||
+ chunk(b"IHDR", struct.pack(">IIBBBBB", width, height, 8, color, 0, 0, 0))
|
||||
+ (chunk(b"PLTE", palette) if palette else b"")
|
||||
+ chunk(b"IDAT", data)
|
||||
+ chunk(b"IEND", b"")
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"data,mime",
|
||||
[
|
||||
(image_bytes()[:33], "image/png"),
|
||||
(image_bytes()[:-15], "image/png"),
|
||||
(png_with_data(b"broken-zlib"), "image/png"),
|
||||
(png_with_data(zlib.compress(b"\x05\0\0\0")), "image/png"),
|
||||
(png_with_data(zlib.compress(b"\0\0"), color=3, palette=bytes(771)), "image/png"),
|
||||
(bytes.fromhex("ffd8ffc0000b080001000101011100ffda0008010100003f00ffd9"), "image/jpeg"),
|
||||
(bytes.fromhex("ffd8ffc00008080001000101ffda0002ffd9"), "image/jpeg"),
|
||||
(image_bytes(), "image/jpeg"),
|
||||
(b"not-an-image", "image/png"),
|
||||
],
|
||||
)
|
||||
def test_corrupt_or_spoofed_image(scenario, data, mime):
|
||||
upload(scenario, data=data, mime=mime, status=415)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("size", [(8000, 5001), (20000, 10000)])
|
||||
def test_pixel_limit_before_decode(scenario, size):
|
||||
data = png_with_data(b"", width=size[0], height=size[1])
|
||||
upload(scenario, data=data, status=413)
|
||||
|
||||
|
||||
def test_byte_limit_and_exact_boundary(scenario):
|
||||
data = image_bytes()
|
||||
exact = data + bytes(10 * 1024 * 1024 - len(data))
|
||||
a = upload(scenario, data=exact)["json"]
|
||||
assert a["size_bytes"] == len(exact)
|
||||
upload(scenario, data=exact + b"x", status=413)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"body",
|
||||
[
|
||||
{"attachment_ids": []},
|
||||
{"text": None},
|
||||
{"text": "x", "attachment_ids": None},
|
||||
{"text": "x", "attachment_ids": [123]},
|
||||
],
|
||||
)
|
||||
def test_strict_message_fields(scenario, body):
|
||||
c, alice, _, _, chat = scenario
|
||||
c.call(
|
||||
"POST",
|
||||
f"/api/v1/chats/{chat['id']}/messages",
|
||||
400,
|
||||
actor=alice,
|
||||
body=body,
|
||||
headers={"Idempotency-Key": uuid.uuid4().hex},
|
||||
)
|
||||
|
||||
|
||||
def test_foreign_unknown_and_ordered_attachments(scenario):
|
||||
c, alice, _, _, chat = scenario
|
||||
first, second = upload(scenario)["json"], upload(scenario)["json"]
|
||||
other = c.call(
|
||||
"POST",
|
||||
"/api/v1/chats",
|
||||
201,
|
||||
actor=alice,
|
||||
body={"title": "other", "member_ids": [alice["id"]]},
|
||||
schema="Chat",
|
||||
)["json"]
|
||||
for aid in [first["id"], str(uuid.uuid4())]:
|
||||
c.call(
|
||||
"POST",
|
||||
f"/api/v1/chats/{other['id']}/messages",
|
||||
404,
|
||||
actor=alice,
|
||||
body={"text": "", "attachment_ids": [aid]},
|
||||
headers={"Idempotency-Key": uuid.uuid4().hex},
|
||||
)
|
||||
key = uuid.uuid4().hex
|
||||
body = {"text": "with files", "attachment_ids": [second["id"], first["id"]]}
|
||||
p = f"/api/v1/chats/{chat['id']}/messages"
|
||||
sent = c.call(
|
||||
"POST", p, 201, actor=alice, body=body, headers={"Idempotency-Key": key}, schema="Message"
|
||||
)["json"]
|
||||
repeat = c.call(
|
||||
"POST", p, 201, actor=alice, body=body, headers={"Idempotency-Key": key}, schema="Message"
|
||||
)["json"]
|
||||
assert repeat == sent
|
||||
c.call(
|
||||
"POST",
|
||||
p,
|
||||
409,
|
||||
actor=alice,
|
||||
body={**body, "attachment_ids": list(reversed(body["attachment_ids"]))},
|
||||
headers={"Idempotency-Key": key},
|
||||
)
|
||||
history = c.call("GET", p, 200, actor=alice, schema="MessagePage")["json"]["items"]
|
||||
assert next(m for m in history if m["id"] == sent["id"]) == sent
|
||||
|
||||
|
||||
def test_nul_message_roundtrip(scenario):
|
||||
c, alice, _, _, chat = scenario
|
||||
assert c.message(alice, chat, "abc\0def")["json"]["text"] == "abc\0def"
|
||||
|
||||
|
||||
def test_truncated_jpeg_with_forged_eoi(scenario):
|
||||
import random
|
||||
|
||||
stream = BytesIO()
|
||||
Image.frombytes("RGB", (512, 512), random.Random(42).randbytes(512 * 512 * 3)).save(
|
||||
stream, "JPEG", quality=95
|
||||
)
|
||||
original = stream.getvalue()
|
||||
upload(scenario, data=original, name="real.jpg", mime="image/jpeg")
|
||||
upload(
|
||||
scenario,
|
||||
data=original[: len(original) // 2] + b"\xff\xd9",
|
||||
name="broken.jpg",
|
||||
mime="image/jpeg",
|
||||
status=415,
|
||||
)
|
||||
@@ -0,0 +1,39 @@
|
||||
import base64
|
||||
import json
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
|
||||
from messenger.domain import Failure, boundary
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"version,date",
|
||||
[
|
||||
(True, "2026-01-01T00:00:00Z"),
|
||||
(1.0, "2026-01-01T00:00:00Z"),
|
||||
(1, "2026-W01-1T00:00:00Z"),
|
||||
(1, "20260101T000000Z"),
|
||||
(1, "2026-01-01 00:00:00Z"),
|
||||
(1, "2026-01-01X00:00:00Z"),
|
||||
(1, "2026-02-30T00:00:00Z"),
|
||||
(1, "2026-01-01T00:00:00.1234567Z"),
|
||||
],
|
||||
)
|
||||
def test_boundary_rejects_python_iso_extensions(version, date):
|
||||
payload = [version, "scope", date, str(uuid4())]
|
||||
cursor = base64.urlsafe_b64encode(json.dumps(payload).encode()).decode().rstrip("=")
|
||||
with pytest.raises(Failure) as error:
|
||||
boundary("scope", cursor)
|
||||
assert error.value.status == 400
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"date", ["2026-01-01T00:00:00Z", "2026-01-01T00:00:00.1Z", "2026-01-01T00:00:00.123456Z"]
|
||||
)
|
||||
def test_boundary_preserves_generated_format(date):
|
||||
uid = str(uuid4())
|
||||
cursor = (
|
||||
base64.urlsafe_b64encode(json.dumps([1, "scope", date, uid]).encode()).decode().rstrip("=")
|
||||
)
|
||||
assert boundary("scope", cursor) == (date, uid)
|
||||
@@ -0,0 +1,49 @@
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import uuid
|
||||
|
||||
import pytest
|
||||
from client import Client
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def shared():
|
||||
client = Client(os.getenv("BASE_URL", "http://127.0.0.1:8085"))
|
||||
return client, *client.bootstrap()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"date",
|
||||
[
|
||||
"2026-W01-1T00:00:00Z",
|
||||
"20260101T000000Z",
|
||||
"2026-01-01 00:00:00Z",
|
||||
"2026-01-01X00:00:00Z",
|
||||
"2026-01-01T00:00:00+00:00",
|
||||
"2026-02-30T00:00:00Z",
|
||||
"2026-01-01T24:00:00Z",
|
||||
"2026-01-01T00:00:00.1234567Z",
|
||||
],
|
||||
)
|
||||
def test_cursor_rejects_noncanonical_dates_over_http(shared, date):
|
||||
c, alice, _, _, chat = shared
|
||||
for path, scope in [
|
||||
("/api/v1/chats", "chats:" + alice["id"]),
|
||||
(f"/api/v1/chats/{chat['id']}/messages", "messages:" + chat["id"]),
|
||||
]:
|
||||
payload = [1, scope, date, str(uuid.uuid4())]
|
||||
cursor = base64.urlsafe_b64encode(json.dumps(payload).encode()).decode().rstrip("=")
|
||||
c.call("GET", path + "?cursor=" + cursor, 400, actor=alice)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("version", [True, 1.0])
|
||||
def test_cursor_rejects_noninteger_version_over_http(shared, version):
|
||||
c, alice, _, _, chat = shared
|
||||
for path, scope in [
|
||||
("/api/v1/chats", "chats:" + alice["id"]),
|
||||
(f"/api/v1/chats/{chat['id']}/messages", "messages:" + chat["id"]),
|
||||
]:
|
||||
payload = [version, scope, "2026-01-01T00:00:00Z", str(uuid.uuid4())]
|
||||
cursor = base64.urlsafe_b64encode(json.dumps(payload).encode()).decode().rstrip("=")
|
||||
c.call("GET", path + "?cursor=" + cursor, 400, actor=alice)
|
||||
@@ -0,0 +1,99 @@
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import uuid
|
||||
from urllib.parse import quote
|
||||
|
||||
import pytest
|
||||
from client import LAB, Client
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def edge_scenario():
|
||||
client = Client(os.environ.get("BASE_URL", "http://127.0.0.1:8085"))
|
||||
return client, *client.bootstrap()
|
||||
|
||||
|
||||
def encoded_cursor(value):
|
||||
return base64.urlsafe_b64encode(json.dumps(value).encode()).decode().rstrip("=")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("identity", ["missing", "malformed", "unknown"])
|
||||
def test_invalid_identity(edge_scenario, identity):
|
||||
client, _, _, _, chat = edge_scenario
|
||||
headers = {}
|
||||
if identity == "malformed":
|
||||
headers["X-User-Id"] = "not-a-uuid"
|
||||
elif identity == "unknown":
|
||||
headers["X-User-Id"] = str(uuid.uuid4())
|
||||
for path in ["/api/v1/users/me", "/api/v1/chats", f"/api/v1/chats/{chat['id']}/messages"]:
|
||||
client.call("GET", path, 401, headers=headers)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("uid", [None, [], {}, 123, True, "not-a-uuid"])
|
||||
def test_cursor_rejects_non_uuid_boundary(edge_scenario, uid):
|
||||
client, alice, _, _, chat = edge_scenario
|
||||
payload = [1, "messages:" + chat["id"], "2026-10-30T12:00:00.000000Z", uid]
|
||||
cursor = quote(encoded_cursor(payload), safe="")
|
||||
client.call("GET", f"/api/v1/chats/{chat['id']}/messages?cursor={cursor}", 400, actor=alice)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"payload", [None, {}, [], [1], [1, "wrong-scope", "2026-10-30T12:00:00Z", str(uuid.uuid4())]]
|
||||
)
|
||||
def test_cursor_rejects_wrong_shape(edge_scenario, payload):
|
||||
client, alice, _, _, chat = edge_scenario
|
||||
cursor = quote(encoded_cursor(payload), safe="")
|
||||
client.call("GET", f"/api/v1/chats/{chat['id']}/messages?cursor={cursor}", 400, actor=alice)
|
||||
|
||||
|
||||
def test_request_id_is_present_and_not_reused(edge_scenario):
|
||||
client, alice, _, _, _ = edge_scenario
|
||||
first = client.call("GET", "/health/live", schema="Health")
|
||||
second = client.call("GET", "/health/ready", schema="Health")
|
||||
error = client.call("GET", "/api/v1/chats/malformed", 400, actor=alice)
|
||||
values = [r["headers"]["X-Request-Id"] for r in [first, second, error]]
|
||||
assert len(set(values)) == 3
|
||||
assert error["json"]["error"]["request_id"] == values[-1]
|
||||
|
||||
|
||||
def test_unicode_codepoint_boundaries(edge_scenario):
|
||||
client, alice, _, _, chat = edge_scenario
|
||||
username = "unicode_" + uuid.uuid4().hex[:12]
|
||||
registration = {"username": username, "display_name": "😀" * 100}
|
||||
if LAB >= 2:
|
||||
registration["password"] = "unicode-test-" + uuid.uuid4().hex
|
||||
user = client.call(
|
||||
"POST",
|
||||
"/api/v1/users",
|
||||
201,
|
||||
body=registration,
|
||||
schema="User",
|
||||
)["json"]
|
||||
assert user["display_name"] == "😀" * 100
|
||||
client.call(
|
||||
"POST", "/api/v1/users", 400, body={"username": username + "x", "display_name": "😀" * 101}
|
||||
)
|
||||
message = client.message(alice, chat, "😀" * 4000)["json"]
|
||||
assert message["text"] == "😀" * 4000
|
||||
client.call(
|
||||
"POST", f"/api/v1/chats/{chat['id']}/messages", 400, actor=alice, body={"text": "😀" * 4001}
|
||||
)
|
||||
|
||||
|
||||
def test_empty_history_and_cursor_exhaustion(edge_scenario):
|
||||
client, alice, _, _, chat = edge_scenario
|
||||
path = f"/api/v1/chats/{chat['id']}/messages"
|
||||
empty = client.call("GET", path, actor=alice, schema="MessagePage")["json"]
|
||||
assert empty == {"items": [], "next_cursor": None}
|
||||
message = client.message(alice, chat, "одна запись")["json"]
|
||||
data = client.call("GET", path + "?limit=1", actor=alice, schema="MessagePage")["json"]
|
||||
assert data == {"items": [message], "next_cursor": None}
|
||||
|
||||
|
||||
def test_missing_chat_is_hidden_like_foreign_chat(edge_scenario):
|
||||
client, _, _, eve, chat = edge_scenario
|
||||
foreign = client.call("GET", f"/api/v1/chats/{chat['id']}", 404, actor=eve)["json"]
|
||||
missing = client.call("GET", f"/api/v1/chats/{uuid.uuid4()}", 404, actor=eve)["json"]
|
||||
assert foreign["error"]["code"] == missing["error"]["code"] == "not_found"
|
||||
assert foreign["error"]["message"] == missing["error"]["message"]
|
||||
@@ -0,0 +1,229 @@
|
||||
"""Full-range JPEG and bounded multi-frame PNG regressions."""
|
||||
|
||||
import os
|
||||
import struct
|
||||
import zlib
|
||||
from io import BytesIO
|
||||
|
||||
import pytest
|
||||
from client import Client
|
||||
from PIL import Image
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def scenario():
|
||||
c = Client(os.getenv("BASE_URL", "http://127.0.0.1:8085"))
|
||||
return c, *c.bootstrap()
|
||||
|
||||
|
||||
def upload(scenario, data, mime, expected):
|
||||
c, alice, _, _, chat = scenario
|
||||
result = c.call(
|
||||
"POST",
|
||||
f"/api/v1/chats/{chat['id']}/attachments?filename=edge.png",
|
||||
202 if expected == 201 else expected,
|
||||
actor=alice,
|
||||
raw=data,
|
||||
headers={"Content-Type": mime},
|
||||
schema="Attachment" if expected == 201 else None,
|
||||
)
|
||||
if expected == 201:
|
||||
result["json"] = c.wait_ready(alice, result["json"])
|
||||
return result
|
||||
|
||||
|
||||
def progressive_jpeg():
|
||||
output = BytesIO()
|
||||
image = Image.new("RGB", (6000, 6000), "blue")
|
||||
try:
|
||||
image.save(output, "JPEG", quality=90, progressive=True, subsampling=0)
|
||||
finally:
|
||||
image.close()
|
||||
return output.getvalue()
|
||||
|
||||
|
||||
def apng(size=(16, 16), default_image=False):
|
||||
output = BytesIO()
|
||||
first, second = Image.new("RGB", size, "red"), Image.new("RGB", size, "blue")
|
||||
try:
|
||||
first.save(
|
||||
output,
|
||||
"PNG",
|
||||
save_all=True,
|
||||
append_images=[second],
|
||||
duration=100,
|
||||
loop=0,
|
||||
default_image=default_image,
|
||||
)
|
||||
finally:
|
||||
first.close()
|
||||
second.close()
|
||||
return output.getvalue()
|
||||
|
||||
|
||||
def broken_later_frame(data):
|
||||
result, position = bytearray(data[:8]), 8
|
||||
while position < len(data):
|
||||
length = struct.unpack(">I", data[position : position + 4])[0]
|
||||
kind = data[position + 4 : position + 8]
|
||||
payload = data[position + 8 : position + 8 + length]
|
||||
if kind == b"fdAT":
|
||||
payload = payload[:4] + b"not-a-valid-deflate-stream"
|
||||
result += (
|
||||
struct.pack(">I", len(payload))
|
||||
+ kind
|
||||
+ payload
|
||||
+ struct.pack(">I", zlib.crc32(kind + payload))
|
||||
)
|
||||
position += length + 12
|
||||
return bytes(result)
|
||||
|
||||
|
||||
def test_36m_progressive_jpeg_is_valid_and_unchanged(scenario):
|
||||
data = progressive_jpeg()
|
||||
attachment = upload(scenario, data, "image/jpeg", 201)["json"]
|
||||
c, _, bob, _, _ = scenario
|
||||
assert (
|
||||
c.call("GET", f"/api/v1/attachments/{attachment['id']}/content", actor=bob)["bytes"] == data
|
||||
)
|
||||
|
||||
|
||||
def test_apng_corrupt_later_frame_is_rejected(scenario):
|
||||
upload(scenario, broken_later_frame(apng()), "image/png", 415)
|
||||
|
||||
|
||||
def test_valid_small_apng_is_accepted_and_unchanged(scenario):
|
||||
data = apng()
|
||||
attachment = upload(scenario, data, "image/png", 201)["json"]
|
||||
c, _, bob, _, _ = scenario
|
||||
assert (
|
||||
c.call("GET", f"/api/v1/attachments/{attachment['id']}/content", actor=bob)["bytes"] == data
|
||||
)
|
||||
|
||||
|
||||
def test_apng_cumulative_canvas_limit(scenario):
|
||||
# Two 25M-pixel canvases exceed the total 40M decode budget.
|
||||
upload(scenario, apng((5000, 5000)), "image/png", 413)
|
||||
|
||||
|
||||
def test_apng_forged_frame_count_cannot_hide_bad_frame(scenario):
|
||||
data = bytearray(broken_later_frame(apng()))
|
||||
position = 8
|
||||
while position < len(data):
|
||||
length = struct.unpack(">I", data[position : position + 4])[0]
|
||||
kind = bytes(data[position + 4 : position + 8])
|
||||
if kind == b"acTL":
|
||||
data[position + 8 : position + 12] = struct.pack(">I", 1)
|
||||
payload = bytes(data[position + 8 : position + 8 + length])
|
||||
data[position + 8 + length : position + 12 + length] = struct.pack(
|
||||
">I", zlib.crc32(kind + payload)
|
||||
)
|
||||
break
|
||||
position += length + 12
|
||||
upload(scenario, bytes(data), "image/png", 415)
|
||||
|
||||
|
||||
def test_apng_zero_count_with_static_fallback_is_rejected(scenario):
|
||||
# Pillow warns and falls back to one valid static PNG frame. The raw acTL
|
||||
# count must still be rejected rather than trusting Pillow's n_frames=1.
|
||||
data = apng()
|
||||
result, position, controls = bytearray(data[:8]), 8, 0
|
||||
while position < len(data):
|
||||
length = int.from_bytes(data[position : position + 4], "big")
|
||||
kind = data[position + 4 : position + 8]
|
||||
payload = data[position + 8 : position + 8 + length]
|
||||
position += length + 12
|
||||
if kind == b"acTL":
|
||||
payload = bytes(4) + payload[4:]
|
||||
if kind == b"fcTL":
|
||||
controls += 1
|
||||
if controls > 1:
|
||||
continue
|
||||
if kind == b"fdAT":
|
||||
continue
|
||||
result += (
|
||||
struct.pack(">I", len(payload))
|
||||
+ kind
|
||||
+ payload
|
||||
+ struct.pack(">I", zlib.crc32(kind + payload))
|
||||
)
|
||||
upload(scenario, bytes(result), "image/png", 415)
|
||||
|
||||
|
||||
def test_apng_exact_cumulative_limit_is_accepted(scenario):
|
||||
upload(scenario, apng((4000, 5000)), "image/png", 201)
|
||||
|
||||
|
||||
def test_40m_cmyk_progressive_jpeg_is_accepted(scenario):
|
||||
output = BytesIO()
|
||||
image = Image.new("CMYK", (8000, 5000), (0, 128, 128, 0))
|
||||
try:
|
||||
image.save(output, "JPEG", quality=90, progressive=True, subsampling=0)
|
||||
finally:
|
||||
image.close()
|
||||
data = output.getvalue()
|
||||
attachment = upload(scenario, data, "image/jpeg", 201)["json"]
|
||||
c, _, bob, _, _ = scenario
|
||||
assert (
|
||||
c.call("GET", f"/api/v1/attachments/{attachment['id']}/content", actor=bob)["bytes"] == data
|
||||
)
|
||||
|
||||
|
||||
def test_repeated_large_image_sequence_stays_available(scenario):
|
||||
# Formerly retained APNG canvases could overlap the next CMYK decoder.
|
||||
c, _, bob, _, _ = scenario
|
||||
data = apng((4000, 5000))
|
||||
attachment = upload(scenario, data, "image/png", 201)["json"]
|
||||
assert (
|
||||
c.call("GET", f"/api/v1/attachments/{attachment['id']}/content", actor=bob)["bytes"] == data
|
||||
)
|
||||
del data
|
||||
test_40m_cmyk_progressive_jpeg_is_accepted(scenario)
|
||||
test_36m_progressive_jpeg_is_valid_and_unchanged(scenario)
|
||||
data = BytesIO()
|
||||
image = Image.new("RGB", (16, 16), "green")
|
||||
image.save(data, "PNG")
|
||||
image.close()
|
||||
attachment = upload(scenario, data.getvalue(), "image/png", 201)["json"]
|
||||
assert (
|
||||
c.call("GET", f"/api/v1/attachments/{attachment['id']}/content", actor=bob)["bytes"]
|
||||
== data.getvalue()
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("progressive", [False, True])
|
||||
def test_small_cmyk_jpeg(progressive, scenario):
|
||||
output = BytesIO()
|
||||
image = Image.new("CMYK", (24, 18), (0, 128, 128, 0))
|
||||
image.save(output, "JPEG", progressive=progressive)
|
||||
image.close()
|
||||
upload(scenario, output.getvalue(), "image/jpeg", 201)
|
||||
|
||||
|
||||
def test_invalid_zero_frame_count_is_rejected(scenario):
|
||||
data = apng()
|
||||
result, position, controls = bytearray(data[:8]), 8, 0
|
||||
while position < len(data):
|
||||
length = struct.unpack(">I", data[position : position + 4])[0]
|
||||
kind = data[position + 4 : position + 8]
|
||||
payload = data[position + 8 : position + 8 + length]
|
||||
position += length + 12
|
||||
if kind == b"fcTL":
|
||||
controls += 1
|
||||
if controls > 1:
|
||||
continue
|
||||
if kind == b"fdAT":
|
||||
continue
|
||||
if kind == b"acTL":
|
||||
payload = bytes(4) + payload[4:]
|
||||
result += (
|
||||
struct.pack(">I", len(payload))
|
||||
+ kind
|
||||
+ payload
|
||||
+ struct.pack(">I", zlib.crc32(kind + payload))
|
||||
)
|
||||
upload(scenario, bytes(result), "image/png", 415)
|
||||
|
||||
|
||||
def test_valid_apng_default_image(scenario):
|
||||
upload(scenario, apng(default_image=True), "image/png", 201)
|
||||
@@ -0,0 +1,72 @@
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from messenger.worker import validate_delivery
|
||||
|
||||
VALID = {
|
||||
"schema_version": 1,
|
||||
"job_id": "11111111-1111-4111-8111-111111111111",
|
||||
"type": "attachment.process.v1",
|
||||
"attachment_id": "22222222-2222-4222-8222-222222222222",
|
||||
"pipeline_version": 1,
|
||||
"attempt": 1,
|
||||
"created_at": "2026-11-06T00:00:00Z",
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"key,value",
|
||||
[
|
||||
("job_id", 123),
|
||||
("job_id", []),
|
||||
("attachment_id", {}),
|
||||
("schema_version", True),
|
||||
("pipeline_version", True),
|
||||
("attempt", 0),
|
||||
("created_at", 123),
|
||||
("created_at", "2026-11-06"),
|
||||
("created_at", "2026-11-06T12Z"),
|
||||
("created_at", "2026-99-99T00:00:00Z"),
|
||||
("traceparent", 123),
|
||||
("tracestate", []),
|
||||
],
|
||||
)
|
||||
def test_invalid_envelopes_are_bounded_validation_errors(key, value):
|
||||
with pytest.raises(ValueError):
|
||||
validate_delivery(json.dumps({**VALID, key: value}))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("body", [b"\xff", "null", "[]", "{}", "not json"])
|
||||
def test_invalid_json(body):
|
||||
with pytest.raises(ValueError):
|
||||
validate_delivery(body)
|
||||
|
||||
|
||||
def test_valid_trace_context():
|
||||
body = {
|
||||
**VALID,
|
||||
"traceparent": "00-" + "a" * 32 + "-" + "b" * 16 + "-01",
|
||||
"tracestate": "test=value",
|
||||
}
|
||||
assert validate_delivery(json.dumps(body)) == body
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"body", ["[" * 2000 + "]" * 2000, "[" * 20000 + "]" * 20000, b" " * 8193 + b"{}"]
|
||||
)
|
||||
def test_deep_and_oversize_envelopes_reject_safely(body):
|
||||
with pytest.raises(ValueError):
|
||||
validate_delivery(body)
|
||||
|
||||
|
||||
def test_uuid_uppercase_is_normalized_before_storage():
|
||||
value = "AAAAAAAA-AAAA-4AAA-8AAA-AAAAAAAAAAAA"
|
||||
parsed = validate_delivery(json.dumps({**VALID, "job_id": value, "attachment_id": value}))
|
||||
assert parsed["job_id"] == parsed["attachment_id"] == value.lower()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("field", ["job_id", "attachment_id"])
|
||||
def test_uuid_urn_is_rejected_before_storage(field):
|
||||
with pytest.raises(ValueError):
|
||||
validate_delivery(json.dumps({**VALID, field: "urn:uuid:" + VALID[field]}))
|
||||
@@ -0,0 +1,169 @@
|
||||
import base64
|
||||
import codecs
|
||||
import json
|
||||
import os
|
||||
import uuid
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from client import Client
|
||||
|
||||
ENCODINGS = [
|
||||
"utf-16",
|
||||
"utf-16-le",
|
||||
"utf-16-be",
|
||||
"utf-16-be-bom",
|
||||
"utf-32",
|
||||
"utf-32-le",
|
||||
"utf-32-be",
|
||||
"utf-32-be-bom",
|
||||
]
|
||||
|
||||
|
||||
def encoded(body, encoding):
|
||||
text = json.dumps(body, ensure_ascii=False)
|
||||
if encoding == "utf-16-be-bom":
|
||||
return codecs.BOM_UTF16_BE + text.encode("utf-16-be")
|
||||
if encoding == "utf-32-be-bom":
|
||||
return codecs.BOM_UTF32_BE + text.encode("utf-32-be")
|
||||
return text.encode(encoding)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def encoding_scenario():
|
||||
c = Client(os.environ["BASE_URL"])
|
||||
return c, *c.bootstrap()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("encoding", ENCODINGS)
|
||||
def test_non_utf8_registration_has_no_write(encoding_scenario, encoding):
|
||||
c, *_ = encoding_scenario
|
||||
body = {
|
||||
"username": "enc_" + uuid.uuid4().hex[:14],
|
||||
"display_name": "Никита\u0000имя",
|
||||
"password": "Пароль:тестовый-12345",
|
||||
}
|
||||
c.call(
|
||||
"POST",
|
||||
"/api/v1/users",
|
||||
400,
|
||||
raw=encoded(body, encoding),
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
# UNIQUE username proves the rejected request did not insert a user.
|
||||
created = c.call("POST", "/api/v1/users", 201, body=body, schema="User")["json"]
|
||||
assert created["display_name"] == body["display_name"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("encoding", ENCODINGS)
|
||||
def test_non_utf8_chat_has_no_write(encoding_scenario, encoding):
|
||||
c, alice, *_ = encoding_scenario
|
||||
before = c.call("GET", "/api/v1/chats?limit=100", actor=alice, schema="ChatPage")["json"]
|
||||
body = {"title": "Чат\u0000" + uuid.uuid4().hex, "member_ids": [alice["id"]]}
|
||||
c.call(
|
||||
"POST",
|
||||
"/api/v1/chats",
|
||||
400,
|
||||
actor=alice,
|
||||
raw=encoded(body, encoding),
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
after = c.call("GET", "/api/v1/chats?limit=100", actor=alice, schema="ChatPage")["json"]
|
||||
assert before == after
|
||||
assert (
|
||||
c.call("POST", "/api/v1/chats", 201, actor=alice, body=body, schema="Chat")["json"]["title"]
|
||||
== body["title"]
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("encoding", ENCODINGS)
|
||||
def test_non_utf8_message_has_no_write(encoding_scenario, encoding):
|
||||
c, alice, _, _, chat = encoding_scenario
|
||||
path = f"/api/v1/chats/{chat['id']}/messages"
|
||||
key = uuid.uuid4().hex
|
||||
body = {"text": "Сообщение\u0000" + key}
|
||||
c.call(
|
||||
"POST",
|
||||
path,
|
||||
400,
|
||||
actor=alice,
|
||||
raw=encoded(body, encoding),
|
||||
headers={"Content-Type": "application/json", "Idempotency-Key": key},
|
||||
)
|
||||
history = c.call("GET", path + "?limit=100", actor=alice, schema="MessagePage")["json"]["items"]
|
||||
assert all(item["text"] != body["text"] for item in history)
|
||||
good = c.call(
|
||||
"POST",
|
||||
path,
|
||||
201,
|
||||
actor=alice,
|
||||
body=body,
|
||||
headers={"Idempotency-Key": key},
|
||||
schema="Message",
|
||||
)["json"]
|
||||
repeat = c.call(
|
||||
"POST",
|
||||
path,
|
||||
201,
|
||||
actor=alice,
|
||||
body=body,
|
||||
headers={"Idempotency-Key": key},
|
||||
schema="Message",
|
||||
)["json"]
|
||||
assert good == repeat and good["text"] == body["text"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("encoding", ["utf-8", "utf-8-sig"])
|
||||
def test_valid_utf8_registration_and_login(encoding_scenario, encoding):
|
||||
c, *_ = encoding_scenario
|
||||
body = {
|
||||
"username": "utf8_" + uuid.uuid4().hex[:14],
|
||||
"display_name": "Юникод\u0000😀",
|
||||
"password": "Пароль:тестовый-12345",
|
||||
}
|
||||
user = c.call(
|
||||
"POST",
|
||||
"/api/v1/users",
|
||||
201,
|
||||
raw=encoded(body, encoding),
|
||||
headers={"Content-Type": "application/json"},
|
||||
schema="User",
|
||||
)["json"]
|
||||
assert user["display_name"] == body["display_name"]
|
||||
credential = base64.b64encode((body["username"] + ":" + body["password"]).encode()).decode()
|
||||
session = c.call(
|
||||
"POST",
|
||||
"/api/v1/auth/sessions",
|
||||
201,
|
||||
headers={"Authorization": "Basic " + credential},
|
||||
schema="Session",
|
||||
)["json"]
|
||||
assert session["user"] == user
|
||||
|
||||
|
||||
def requests_for(alice, chat):
|
||||
return [
|
||||
(
|
||||
"/api/v1/users",
|
||||
{
|
||||
"username": "utf_" + uuid4().hex[:12],
|
||||
"display_name": "Имя\0",
|
||||
"password": "synthetic-password-123",
|
||||
},
|
||||
),
|
||||
("/api/v1/chats", {"title": "Чат\0", "member_ids": [alice["id"]]}),
|
||||
(f"/api/v1/chats/{chat['id']}/messages", {"text": "Сообщение\0"}),
|
||||
]
|
||||
|
||||
|
||||
def test_utf8_and_escaped_nul_remain_valid(encoding_scenario):
|
||||
c, alice, _, _, chat = encoding_scenario
|
||||
for path, body in requests_for(alice, chat):
|
||||
c.call(
|
||||
"POST",
|
||||
path,
|
||||
201,
|
||||
actor=alice,
|
||||
raw=json.dumps(body, ensure_ascii=False).encode("utf-8"),
|
||||
headers={"Content-Type": "application/json", "Idempotency-Key": uuid4().hex},
|
||||
)
|
||||
@@ -0,0 +1,46 @@
|
||||
from io import BytesIO
|
||||
|
||||
import pytest
|
||||
from PIL import Image
|
||||
|
||||
from messenger.image_pipeline import fitted_size, render
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"source,expected",
|
||||
[((319, 200), (256, 160)), ((320, 200), (256, 160)), ((10, 4), (10, 4)), ((1, 1000), (1, 256))],
|
||||
)
|
||||
def test_fits_contract(source, expected):
|
||||
assert fitted_size(*source, 256) == expected
|
||||
|
||||
|
||||
def test_variants_alpha_and_metadata(tmp_path):
|
||||
source = BytesIO()
|
||||
Image.new("RGBA", (319, 200), (255, 0, 0, 0)).save(source, "PNG")
|
||||
source.seek(0)
|
||||
variants = render(source, tmp_path)
|
||||
assert {v["name"] for v in variants} == {"thumbnail", "crop", "optimized"}
|
||||
with Image.open(tmp_path / "thumbnail") as thumb:
|
||||
assert thumb.format == "JPEG"
|
||||
assert thumb.size == (256, 160)
|
||||
assert all(channel > 245 for channel in thumb.getpixel((0, 0)))
|
||||
assert not thumb.getexif()
|
||||
with Image.open(tmp_path / "crop") as crop:
|
||||
assert crop.size == (128, 128)
|
||||
with Image.open(tmp_path / "optimized") as optimized:
|
||||
assert optimized.format == "WEBP"
|
||||
assert optimized.size == (319, 200)
|
||||
|
||||
|
||||
def test_exif_orientation_without_upscale(tmp_path):
|
||||
source = BytesIO()
|
||||
exif = Image.Exif()
|
||||
exif[274] = 6
|
||||
Image.new("RGB", (80, 40), "blue").save(source, "JPEG", exif=exif)
|
||||
original = source.getvalue()
|
||||
source.seek(0)
|
||||
render(source, tmp_path)
|
||||
assert source.getvalue() == original
|
||||
with Image.open(tmp_path / "thumbnail") as thumbnail:
|
||||
assert thumbnail.size == (40, 80)
|
||||
assert not thumbnail.getexif()
|
||||
@@ -0,0 +1,62 @@
|
||||
import base64
|
||||
import uuid
|
||||
|
||||
import pytest
|
||||
from client import Client
|
||||
|
||||
|
||||
@pytest.mark.parametrize("password", ["x" * 11, "x" * 129, "😀" * 129, 123, None, True])
|
||||
def test_password_contract(password):
|
||||
import os
|
||||
|
||||
c = Client(os.environ["BASE_URL"])
|
||||
c.call(
|
||||
"POST",
|
||||
"/api/v1/users",
|
||||
400,
|
||||
body={
|
||||
"username": "pwd_" + uuid.uuid4().hex[:12],
|
||||
"display_name": "Тест",
|
||||
"password": password,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def test_full_unicode_password_and_colon():
|
||||
import os
|
||||
|
||||
c = Client(os.environ["BASE_URL"])
|
||||
username = "pwd_" + uuid.uuid4().hex[:12]
|
||||
password = "😀" * 126 + ":Я"
|
||||
user = c.call(
|
||||
"POST",
|
||||
"/api/v1/users",
|
||||
201,
|
||||
body={"username": username, "display_name": "Пароль128", "password": password},
|
||||
schema="User",
|
||||
)["json"]
|
||||
good = base64.b64encode((username + ":" + password).encode()).decode()
|
||||
session = c.call(
|
||||
"POST",
|
||||
"/api/v1/auth/sessions",
|
||||
201,
|
||||
headers={"Authorization": "bAsIc " + good},
|
||||
schema="Session",
|
||||
)["json"]
|
||||
actor = {"id": user["id"], "token": session["token"]}
|
||||
assert session["user"] == user
|
||||
bad = base64.b64encode((username + ":" + password[:-1] + "Б").encode()).decode()
|
||||
c.call("POST", "/api/v1/auth/sessions", 401, headers={"Authorization": "Basic " + bad})
|
||||
c.call("GET", "/api/v1/users/me", 401, headers={"Authorization": "Basic " + good})
|
||||
c.call("GET", "/api/v1/users/me", actor=actor, schema="User")
|
||||
c.call("DELETE", "/api/v1/auth/sessions/current", 204, actor=actor)
|
||||
c.call("GET", "/api/v1/users/me", 401, actor=actor)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("credential", ["Basic !invalid", "Basic", "Bearer abc", "", "Basic YTpi"])
|
||||
def test_invalid_basic(credential):
|
||||
import os
|
||||
|
||||
Client(os.environ["BASE_URL"]).call(
|
||||
"POST", "/api/v1/auth/sessions", 401, headers={"Authorization": credential}
|
||||
)
|
||||
@@ -0,0 +1,38 @@
|
||||
"""Real exporter failure must remain bounded and account for lost sampled spans."""
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def test_span_queue_loss_conservation_when_collector_absent():
|
||||
code = '''
|
||||
import json, time
|
||||
from messenger.telemetry import setup
|
||||
class Pool:
|
||||
def get_stats(self): return {}
|
||||
class Store:
|
||||
pool=Pool()
|
||||
t=setup("messenger-api")
|
||||
t.start_background(Store())
|
||||
for index in range(1200):
|
||||
with t.tracer.start_as_current_span("synthetic-safe-span"):
|
||||
pass
|
||||
time.sleep(7)
|
||||
counts=t.export_stats.snapshot()
|
||||
pending=len(t.batch_spans._batch_processor._queue)
|
||||
loss=counts["spans_recorded"]-counts["spans_exported"]-counts["spans_export_failed"]-counts["spans_in_flight"]-pending
|
||||
assert counts["spans_recorded"]==1200
|
||||
assert 0<=pending<=256
|
||||
assert loss>0
|
||||
assert counts["spans_export_failed"]>0
|
||||
print(json.dumps({"recorded":1200,"pending":pending,"queue_lost":loss,"export_failed":counts["spans_export_failed"]}))
|
||||
t.close()
|
||||
'''
|
||||
env = {**os.environ, "PYTHONPATH": str(Path(__file__).resolve().parents[1]/"src"),
|
||||
"OTEL_ENABLED": "1", "OTEL_EXPORTER_OTLP_ENDPOINT": "http://127.0.0.1:1"}
|
||||
result = subprocess.run([sys.executable, "-c", code], env=env, capture_output=True, text=True, timeout=15)
|
||||
assert result.returncode == 0, result.stderr[-1000:]
|
||||
payload = json.loads(next(line for line in result.stdout.splitlines() if line.startswith('{')))
|
||||
assert payload["queue_lost"] > 0
|
||||
@@ -0,0 +1,19 @@
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from messenger.models import CreateChat, CreateMessage, CreateUser
|
||||
|
||||
|
||||
@pytest.mark.parametrize('scalar',['\ud800','\udfff'])
|
||||
def test_unpaired_surrogates_rejected(scalar):
|
||||
for factory in (
|
||||
lambda: CreateUser(username='valid_user',display_name=scalar,password='valid-pass-123'),
|
||||
lambda: CreateUser(username='valid_user',display_name='Valid',password='valid-pass-'+scalar),
|
||||
lambda: CreateChat(title=scalar,member_ids=['00000000-0000-4000-8000-000000000001']),
|
||||
lambda: CreateMessage(text=scalar),
|
||||
):
|
||||
with pytest.raises(ValidationError):
|
||||
factory()
|
||||
|
||||
def test_valid_unicode_and_nul():
|
||||
assert CreateMessage(text='😀\x00').text=='😀\x00'
|
||||
Reference in New Issue
Block a user