Author SHA1 Message Date
Roflyan 9ebc39886c Обновить REPORT.md 2026-11-19 19:30:00 +03:00
R0flyan 6a9c4175a4 LR6: monitoring and SLO 2026-11-19 19:24:45 +03:00
75 changed files with 8734 additions and 42 deletions
+10 -6
View File
@@ -7,7 +7,7 @@ SESSION_TTL_SECONDS=3600
# Choose credentials locally. File paths contain no secret values.
DATABASE_URL_FILE=/run/secrets/database_url
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
SESSION_STORE=postgres
SESSION_STORE=valkey
CACHE_URL_FILE=/run/secrets/cache_url
API_REPLICAS=2
IDEMPOTENCY_TTL_SECONDS=86400
@@ -19,12 +19,16 @@ S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
MAX_UPLOAD_BYTES=10485760
MAX_IMAGE_PIXELS=40000000
BROKER_URL_FILE=/run/secrets/broker_url
WORKER_CONCURRENCY=2
WORKER_CONCURRENCY=1
JOB_MAX_ATTEMPTS=5
JOB_TIMEOUT_SECONDS=60
JOB_TIMEOUT_SECONDS=30
STAGING_RETENTION_SECONDS=86400
OTEL_SERVICE_NAME=messenger-api
# Worker must set a distinct service.name.
OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318
# The application uses http/protobuf exporters.
OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab
OTEL_SDK_DISABLED=false
# Optional host ports for investigating a local, synthetic test stack.
GRAFANA_PORT=3000
PROMETHEUS_PORT=9090
LOKI_PORT=3100
TEMPO_PORT=3200
+4
View File
@@ -0,0 +1,4 @@
/scripts/restart.sh text eol=lf
/scripts/stop-one.sh text eol=lf
/infra/s3-init.sh text eol=lf
/migrations/*.sql text eol=lf
+40 -14
View File
@@ -1,18 +1,44 @@
## Лабораторная 6
Автор / группа: TODO
Целевая оценка (3/4/5): TODO
Трек 5, если заявлен: TODO
- Автор и группа — Ситосенко, БВТ2303.
- Целевая оценка — **5**, трек 2 «SLO и обнаружение инцидентов».
Что реализовано и что перенесено: TODO
Команды запуска из чистого clone: TODO
Результаты проверок и ссылка на REPORT.md: TODO
Ограничения / известные ошибки: TODO
Источники и использование ИИ: TODO
### Что реализовано
- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
- [ ] Контракт и публичные тесты не ослаблены.
- [ ] make check и make test выполнены; приложен реальный вывод.
- [ ] Сценарии отказов/безопасности своего уровня воспроизведены.
- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
- Перенесена основа Lab5. HAProxy, две API-реплики, PostgreSQL, Valkey для сессий и брокера, приватный MinIO, транзакционный outbox и отдельный worker сохраняют бизнес-API.
- API, dispatcher и worker отправляют метрики, логи и трассы по OTLP. Добавлены Collector, Prometheus, Loki, Tempo, Grafana provisioning и дашборд с 18 панелями.
- W3C Trace Context проходит через PostgreSQL и Valkey Stream. По одному trace ID видны upload, публикация, ошибка обработки, retry, запись в S3 и PostgreSQL.
- Для API и ready заданы SLO с окном 30 минут, две пары окон burn rate и отдельный alert по возрасту очереди. В [runbook](../RUNBOOK.md) описана диагностика.
### Запуск из чистого clone
```sh
python3 scripts/bootstrap.py
docker compose --env-file .env config --quiet
docker compose --env-file .env up -d --build --wait --wait-timeout 180
make check
make test BASE_URL=http://127.0.0.1:8080
```
Grafana доступна на порту 3000, правила Prometheus — на 9090. Команды прогонов и исходные замеры приведены в [REPORT.md](../REPORT.md).
### Результаты
- Публичные `make check`, `make validate` и `make test` прошли. Проверены форматы изображений, правила Prometheus, три источника Grafana и связь логов и трасс.
- Persistence и failover прошли для обеих API-реплик. Задание очереди и сообщение с queued-вложением сохранились после рестарта API и брокера.
- Короткая блокировка таблицы сообщений дала 6 ответов 503. Firing отсутствовал. При длительной блокировке сработали два API-алерта. В trace найден ошибочный SQL span длительностью около 1 с.
- При остановке worker backlog достиг 60 заданий. После восстановления все 77 загрузок из окна отказа и запуска дошли до ready. Очередь опустела, алерты вернулись в inactive.
- При отключении Tempo проверены запись сообщений, появление ready для вложения, конечные потери spans и восстановление экспорта. Замеры и ограничения приведены в отчёте.
### Ограничения и источники
Один учебный хост и синтетический профиль. API SLI покрывает ответы самого приложения на известных маршрутах. Буферы telemetry в памяти ограничены, поэтому во время длительного отказа backend возможна потеря сигналов.
Для реализации и проверки использованы документация [OpenTelemetry](https://opentelemetry.io/docs/), [Prometheus](https://prometheus.io/docs/), [Grafana](https://grafana.com/docs/) и [Google SRE Workbook](https://sre.google/workbook/alerting-on-slos/). Codex использован для переноса, инструментирования, подготовки сценариев, проверки и оформления материалов.
- [x] Обязательный минимум текущей и предыдущих работ сохранён.
- [x] Публичные тесты и контракт сохранены.
- [x] Вывод `make check` и `make test` приложен.
- [x] Воспроизведены сценарии отказов для заявленного уровня.
- [x] Секреты локальные и исключены из Git. В отчёте нет токенов.
- [x] README задания сохранён. Инструкция запуска и результаты находятся в REPORT.md.
+34
View File
@@ -0,0 +1,34 @@
name: Lab 6 smoke
on:
push:
pull_request:
permissions:
contents: read
jobs:
compose-smoke:
runs-on: ubuntu-latest
timeout-minutes: 20
steps:
- uses: actions/checkout@v4
- name: Build and start the Lab 6 stand
run: |
python3 scripts/bootstrap.py
docker compose --env-file .env config --quiet
docker compose --env-file .env up -d --build --wait --wait-timeout 180
- name: Check the contract and run the public smoke
run: |
make check
make test BASE_URL=http://127.0.0.1:8080
- name: Check rules and provisioned telemetry
run: |
docker compose exec -T prometheus promtool check config /etc/prometheus/prometheus.yml
python3 student-tests/observability.py correlation
- name: Show service state on failure
if: failure()
run: docker compose ps --all
- name: Stop containers
if: always()
run: docker compose down
+23
View File
@@ -0,0 +1,23 @@
FROM golang:1.27.3-bookworm@sha256:d9b6a1beb9474adc2a1ce842573639b44b2a0dd49b6669f51d3821baf9ec37c9 AS build
WORKDIR /src
COPY go.mod go.sum ./
RUN go mod download
COPY *.go ./
COPY migrations ./migrations
RUN CGO_ENABLED=0 go build -trimpath -ldflags="-s -w" -o /messenger .
FROM python:3.13.18-slim-bookworm AS worker
RUN pip install --no-cache-dir --only-binary=:all: Pillow==13.0.0
COPY --from=build /messenger /messenger
COPY pipeline.py /pipeline.py
USER 65532:65532
STOPSIGNAL SIGTERM
ENTRYPOINT ["/messenger"]
FROM scratch AS api
COPY --from=build /messenger /messenger
USER 65532:65532
ENV APP_HOST=0.0.0.0 APP_PORT=8080 GOMEMLIMIT=256MiB
EXPOSE 8080
HEALTHCHECK --interval=5s --timeout=3s --start-period=10s --retries=3 CMD ["/messenger","healthcheck"]
STOPSIGNAL SIGTERM
ENTRYPOINT ["/messenger"]
+15
View File
@@ -0,0 +1,15 @@
# Build official sources: upstream registry images are no longer public.
FROM golang:1.27.3-bookworm@sha256:d9b6a1beb9474adc2a1ce842573639b44b2a0dd49b6669f51d3821baf9ec37c9 AS build
ENV GOTOOLCHAIN=local CGO_ENABLED=0
RUN go install -p 4 github.com/minio/minio@v0.0.0-20251015172955-9e49d5e7a648
RUN go install -p 4 github.com/minio/mc@v0.0.0-20250813083541-7394ce0dd2a8
COPY infra/s3-health.go /health.go
RUN go build -trimpath -o /healthcheck /health.go
FROM alpine:3.23.3@sha256:25109184c71bdad752c8312a8623239686a9a2071e8825f20acb8f2198c3f659 AS tools
COPY --from=build /go/bin/mc /usr/bin/mc
ENTRYPOINT ["/usr/bin/mc"]
FROM scratch AS server
COPY --from=build /go/bin/minio /usr/bin/minio
COPY --from=build /healthcheck /healthcheck
COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
ENTRYPOINT ["/usr/bin/minio"]
+133 -22
View File
@@ -1,37 +1,148 @@
# Отчёт — лабораторная 6
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
- Автор и группа — Ситосенко, БВТ2303.
- Целевая оценка — **5**, трек 2 «SLO и обнаружение инцидентов».
- Ветка — `solution/bvt2303-sitosenko`. PR: [https://gitea.privcloud.tech/mtusi2026/mtusi2026lab6/pulls/47].
- [Инструментирование OpenTelemetry Go](https://opentelemetry.io/docs/languages/go/instrumentation/) · [Collector configuration](https://opentelemetry.io/docs/collector/configuration/) · [W3C Trace Context](https://www.w3.org/TR/trace-context/).
[OTLP в Prometheus](https://prometheus.io/docs/guides/opentelemetry/) · [Гистограммы Prometheus](https://prometheus.io/docs/practices/histograms/) · [OTLP в Loki](https://grafana.com/docs/loki/latest/send-data/otel/) · [Связь Tempo и Loki в Grafana](https://grafana.com/docs/grafana/latest/datasources/tempo/configure-trace-to-logs/).
[Google SRE Workbook — Alerting on SLOs](https://sre.google/workbook/alerting-on-slos/).
Codex использован для переноса из lab5, реализации, подготовки сценариев, анализа результатов и отчета.
## Запуск из чистого clone
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
Нужны Docker с Linux-контейнерами, Compose v2+, Python 3.9+ и make. Для панелей CPU/RAM нужен cgroup v2. Сборка обращается к реестру образов, Go modules и PyPI. Bootstrap создаёт локальные `.env` и `secrets/`, сохраняя существующие значения. Миграции 001–006 выполняет одноразовый сервис `migrate`, приватный бакет и отдельные права API и worker создаёт `s3-init`. Успешный выход этих двух сервисов является частью запуска.
## Архитектура и решения
```sh
python3 scripts/bootstrap.py
docker compose --env-file .env config --quiet
docker compose --env-file .env up -d --build --wait --wait-timeout 180
make check
make test BASE_URL=http://127.0.0.1:8080
```
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
В PowerShell можно задать `$env:PYTHONUTF8 = '1'` и использовать `python`. Для Makefile с Windows Python передайте `PYTHON=python`. При изменении `APP_PORT` передайте соответствующий `BASE_URL`. При изменении портов наблюдаемости экспортируйте для тестовых скриптов также `GRAFANA_PORT`, `PROMETHEUS_PORT`, `LOKI_PORT`, `TEMPO_PORT`.
## Доказательства критериев
- API через HAProxy — `http://127.0.0.1:8080`, готовность — `/health/ready`.
- [Grafana](http://127.0.0.1:3000/d/messenger-lab6) автоматически загружает дашборд и три источника. Для локального просмотра выделена анонимная роль Viewer.
- [Prometheus Alerts](http://127.0.0.1:9090/alerts) показывает статус правил. HTTP-порты проверки Loki и Tempo — `127.0.0.1:3100` и `127.0.0.1:3200`. OTLP Collector доступен во внутренней сети Compose.
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
`docker compose down` останавливает проект с сохранением volumes. `docker compose down --volumes` намеренно очищает все данные **этого учебного проекта**, включая историю сигналов.
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
### Версии и настройки
## Эксперименты и отказы
| Компонент | Версия / роль |
| --- | --- |
| API, dispatcher, worker | Go toolchain 1.27.3, pgx 5.9.2. Python 3.13.18 и Pillow 13.0.0 в worker |
| Бизнес-хранилища | PostgreSQL 17.6, Valkey 8.1.3 для сессий и отдельный Valkey 8.1.3 для Streams, MinIO CE из коммита `9e49d5e7a648` |
| Балансировщик | HAProxy 3.2.6, две API-реплики по умолчанию |
| OTel Go | SDK traces/metrics 1.38.0, logs 0.14.0. Имена HTTP и DB attributes по Semantic Conventions 1.37.0, метрики проекта перечислены ниже |
| Хранилища наблюдаемости | Collector contrib 0.133.0, Prometheus 3.5.0, Loki 3.5.3, Tempo 2.8.2, Grafana 12.1.1 |
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
`APP_HOST` и `APP_PORT` задают bind API внутри контейнера и внешний порт HAProxy соответственно. `APP_ENV` становится атрибутом ресурса. `DATABASE_URL_FILE`, `CACHE_URL_FILE`, `BROKER_URL_FILE` и S3 `*_FILE` указывают на файлы в `/run/secrets`. Compose фиксирует изолированные S3 endpoint, region и bucket. `MAX_UPLOAD_BYTES` и `MAX_IMAGE_PIXELS` отражают контрактные константы в коде. `WORKER_CONCURRENCY=1`, `JOB_MAX_ATTEMPTS=5`, `JOB_TIMEOUT_SECONDS=30` и `STAGING_RETENTION_SECONDS=86400` задают ограничения pipeline. `OTEL_EXPORTER_OTLP_ENDPOINT` задаёт адрес Collector. Транспорт SDK — HTTP/protobuf. `OTEL_SDK_DISABLED=true` отключает три сигнала, по умолчанию SDK включён.
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
## Архитектура
## Ограничения и следующие шаги
Перенесены исходники собственного Go API, Dockerfile, миграции 001–005, pipeline изображений и нужные конфиги HAProxy, PostgreSQL, Valkey и MinIO. Сохранены общие сессии, проверка членства, идемпотентность сообщений, приватный S3, `202 queued`, outbox и worker с обработкой изображений и повторами. Публичные тесты и контракт взяты из skeleton Lab6. Workflow сборки отдельный.
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
При upload сохранение оригинала в staging предшествует commit строки вложения и `image_jobs`. PostgreSQL определяет состояние задания, версию и возможность публикации. Dispatcher отправляет JSON job в Valkey через XADD. Фиксация SQL-признака публикации отдельная, поэтому при сбое возможна повторная доставка. Worker берёт lease, пишет оригинал, thumbnail, crop и WebP с условным PUT, фиксирует `ready` транзакцией и подтверждает очередь. Worker ограничен 1 vCPU, 640 MiB и 32 MiB tmpfs, API-реплики — 1 vCPU и 768 MiB каждая.
При открытии server span приложение назначает `request_id` для HTTP и логов. Миграция 006 хранит часть W3C Trace Context в outbox и время фиксации `ready`. JSON job остаётся формата Lab5, его метаданные W3C передаются отдельными полями Streams. Producer spans создаются из контекста upload. Consumer извлекает контекст конкретного producer. При retry создаётся новая пара spans в том же trace. Трассы S3 и SQL входят в consumer. Старые строки с пустым контекстом обрабатываются самостоятельно, прежняя семантика доставки сохраняется.
SDK отправляет spans, логи и метрики асинхронно в общий OTLP HTTP receiver. Collector обрабатывает каждый поток через `memory_limiter` и `batch`. Prometheus принимает OTLP-метрики и скрапит сам Collector, Loki принимает OTLP-логи с индексным label `service_name` и structured metadata, Tempo принимает OTLP spans. История хранится в volumes `metrics`, `logs` и `traces` до 7 дней, в Prometheus действует также лимит 512 MB. Grafana сопоставляет логи и трассы из этих источников.
### Сигналы и единицы
| Сигнал | Имя или запрос | Единица и смысл |
| --- | --- | --- |
| HTTP RPS и ошибки | `sum(rate(messenger_http_requests_total[30s]))` и серия с `status_class="5xx"` | запросы/с по `method`, `route`, `status_class` |
| HTTP p50/p95/p99 | `histogram_quantile(0.95, sum by (le) (rate(messenger_http_duration_seconds_bucket[1m])))` и аналоги 0.50, 0.99 | секунды, bucket агрегируются по репликам |
| Попытки worker | `messenger_job_attempts_total`, `messenger_job_duration_seconds_bucket` | число попыток и секунды с разбивкой по `job_type`, `outcome`. `queued` после ошибки означает retry |
| Очередь | `messenger_queue_depth`, `messenger_queue_oldest_age_seconds` | число ожидающих строк SQL и возраст старейшей от создания, включая непубликованные задания и backoff |
| Пул PostgreSQL | `messenger_db_pool_used`, `messenger_db_pool_capacity`, `messenger_db_pool_acquiring`, `messenger_db_pool_waits_total` | занятые соединения, лимит, текущие Acquire и кумулятивные запросы получения при пустом пуле |
| Worker и ресурсы | `messenger_worker_busy / messenger_worker_capacity`, `rate(messenger_container_cpu_seconds_total[30s])`, `messenger_container_memory_bytes` | доля занятых слотов, ядра CPU и байты cgroup v2, включая Pillow |
| Время до ready | `messenger_attachment_time_to_ready_seconds_bucket` | секунды между записями SQL о принятии и публикации первой версии, только для завершённых задач |
| SLI готовности | `messenger_attachment_slo_jobs_total{outcome="good"\|"missed"}` | счётчик созревших событий, включая просроченные, failed и незавершённые |
| Логи и трассы | События `http.request`, `image.published`, `image.attempt` и spans HTTP, очереди, S3, PostgreSQL | лог содержит timestamp, severity и correlation IDs. Имя сервиса, версия, экземпляр и окружение находятся в OTel resource |
HTTP routes нормализованы до шаблонов. Нестандартные методы попадают в `OTHER`, неизвестные пути — в `unmatched`. ID запросов и задания используются для поиска событий и трасс. Отбор полей SDK исключает raw URL, SQL-параметры, логин, пароли, Authorization, токены, текст сообщений и байты изображений. Клиентский `traceparent` с публичной границы не импортируется. SDK использует полную выборку трасс.
## SLO
Цель API — **99% ответов без 5xx** на известных маршрутах приложения за скользящие 30 минут. Числитель плохих событий — 5xx, знаменатель — все ответы на этих маршрутах, включая 4xx. Health и ответы, сформированные до попадания в приложение, исключены из измерения.
Цель вложений — **99% первых загрузок становятся ready за ≤5 секунд** за скользящие 30 минут. Метрика учитывает событие после дедлайна по данным PostgreSQL. Загрузка с ошибкой или без ready к сроку расходует бюджет. SQL-времена фиксируются внутри транзакций рядом с commit. Гистограмма строится только после успешной фиксации. Цели и срок выбраны для этого учебного профиля и регулярной проверки пользовательского ожидания.
Burn rate равен доле плохих событий, делённой на 0,01. Быстрая пара использует 30 и 10 с, порог 14,4 и `for: 2s`. Медленная пара использует 75 и 25 с, порог 6 и `for: 12s`. При равномерном трафике каждый порог за длинное окно расходует примерно четверть бюджета 30 минут (24% и 25%). Обе части пары должны превышать порог, а в каждом окне требуется минимальное число событий. Окна сжаты для воспроизведения в аудитории. Отдельный `QueueLagHigh` срабатывает, если возраст >5 секунд и в очереди есть задания не менее 4 секунд. Шаг оценки Prometheus — 2 с. Оповещения локальные. [Правила](infra/otel/alerts.yml) · [Runbook](RUNBOOK.md)
## Проверки и эксперименты
Два предполагаемых результата. При остановке worker API продолжает принимать загрузки, но растут backlog и расход бюджета ready. При блокировке таблицы сообщений 5xx видны на одном маршруте, а трассы указывают на PostgreSQL. Краткий всплеск наблюдаем по 503, но недостаточен для firing при стабильно прогретом потоке. Проверка отказа Tempo должна показать ограниченную потерю spans при доступном бизнес-пути.
Среда — Docker Desktop 29.3.0 и Compose 5.1.0 на Windows, Linux amd64 в Docker VM, 12 доступных логических CPU и 3,68 GiB RAM. В исследовании SLO после 82 секунд прогрева генератор планирует 4 POST сообщения/с и 1 PNG upload/с, одновременно до 12 запросов. Тест блокирует только таблицу `messages` на 2,5 и 65 секунд, затем отдельно останавливает worker примерно на минуту. До и после каждой длительной деградации генератор продолжает отправлять трафик. Инциденты ограничены локальным проектом. Отдельные прогоны на одном стенде проводят последовательно.
```sh
export BASE_URL=http://127.0.0.1:8080
python3 student-tests/image_rules.py
python3 student-tests/observability.py correlation
python3 student-tests/slo_experiment.py
python3 student-tests/observability.py outage
python3 student-tests/observability.py queue-restart
python3 tests/load.py --duration 30 --concurrency 4 --output evidence/load.json
docker compose exec -T prometheus promtool check config /etc/prometheus/prometheus.yml
make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL="$BASE_URL"
make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL="$BASE_URL"
docker compose start api
```
Скрипты fault/restart выполняются только на выделенном стенде. Публичные resilience hooks требуют POSIX shell. При необходимости выполните `chmod +x scripts/restart.sh scripts/stop-one.sh`.
### Доказательства по уровням
| Уровень и критерий | Проверка и наблюдение | Доказательство |
| --- | --- | --- |
| 3. Compose, миграции, источники сигналов | `docker compose config --quiet`, `up -d --build --wait`, два контейнера API, отдельный worker, три backend и Grafana | [Сервисы](evidence/services.txt), [Compose](compose.yaml) |
| 3. Публичный контракт и поведение приложения | `make check`, `make validate`, `make test`, абонентский сценарий Lab6 с включённым SDK. Проверены форматы и размеры вариантов | [Контракт](evidence/contract.txt), [OpenAPI](evidence/openapi.txt), [Smoke](evidence/smoke.txt), [Обработка изображений](evidence/image-rules.txt) |
| 3. Очередь и сохранность Lab5 | При выключенном worker сообщение с queued-вложением прошло перезапуск двух API и broker. Та же запись Streams сохранилась, затем получены ready и JPEG | [Перезапуск очереди](evidence/queue-restart.json) |
| 3. Логи, traces, безопасные labels | Найдены 45 spans одного upload и три связанных API/worker лога. Пробные секреты, имя файла и текст отсутствовали в проверенных событиях, проверено 818 серий метрик | [Корреляция](evidence/correlation.json) |
| 3. Дашборд и сигналы | Grafana API показал 18 панелей и три provisioned источника. Запросы панелей выполняются на Prometheus | [Дашборд](infra/otel/grafana/dashboards/messenger.json), [Проверка запросов](evidence/dashboard.json) |
| 4. Контекст и retry | Для попыток 1 и 2 проверены родительские span ID на переходах upload → producer → consumer. Результат первой попытки — `queued`, второй — `ready` с четырьмя S3 PUT и SQL | [Трасса и логи](evidence/correlation.json) |
| 4. SLI, алерты и два инцидента | `promtool` принял пять правил. При блокировке таблицы и при остановке worker зафиксированы firing и возврат в inactive | [Правила](evidence/promtool.txt), [Хронология](evidence/slo-experiment.json), [Runbook](RUNBOOK.md) |
| 4. Отказ telemetry backend | Отдельный прогон отключения Tempo с непрерывной записью сообщений и обработкой загрузки | [Отказ Tempo](evidence/backend-outage.json) |
| 4. Наблюдаемое насыщение | В запросах Grafana видны занятые соединения и ожидание пула, рабочие слоты и cgroup CPU/RAM. Возраст и глубина выходят из PostgreSQL | [Панели](infra/otel/grafana/dashboards/messenger.json), [Наблюдения](evidence/slo-experiment.json) |
| 5. Трек SLO | Краткий всплеск и две длительные деградации при постоянном профиле. Измерены задержки срабатывания и восстановления | [Результаты и снимки](evidence/slo-experiment.json), [Проверка измерений](evidence/slo-validation.txt) |
### Что показал прогон 19 ноября 2026 года
В прогретом окне подтверждено 328 сообщений и 82 загрузки. Скорость составила 4,00 успешных сообщения/с и 1,00 принятую загрузку/с. Клиентские p50/p95/p99 для сообщений — 29,9 / 177,7 / 415,0 мс, для загрузок — 62,7 / 368,5 / 730,6 мс. Пик выборочных значений CPU после первых 20 с — 0,28 ядра на две API-реплики и 0,57 ядра на worker. Пик свежих показаний cgroup RAM — около 370 MiB у двух API-реплик и 29 MiB у worker. Генератор пропустил два запуска из-за лимита одновременных запросов. Это менее 0,2% от числа событий в четырёх оцениваемых фазах. В длительном DB-сценарии отдельно получена одна транспортная или контрактная ошибка клиента. [Клиентские измерения](evidence/slo-experiment.json) · [Запросы к истории Prometheus](evidence/study-promql.json)
| Воздействие | Наблюдение из сохранённых сигналов | Результат |
| --- | --- | --- |
| Фон и короткая блокировка таблицы `messages` на 2,5 с | В фоне 410 успешных ответов. Во время блокировки получено 6 ответов 503 | В фоне и после короткого всплеска зарегистрировано 0 firing. Короткая деградация видна в логах и счётчиках |
| Длительная блокировка на 65 с | Клиент получил 253 ответа 503 на POST сообщений. По `X-Request-Id` найден HTTP-лог. В trace `8fbfa739f88710aef39a7c9f068b44b1` ошибочный span `postgresql query` длился 1001,3 мс | Задержка локализована в SQL. HTTP восстановился после снятия блокировки. В восстановлении отдельно наблюдался ReadySlowBurn, когда Prometheus оценил около 13 просрочек из 27 созревших загрузок за 25 с |
| Остановка worker | Поток сообщений сохранился на 4,00 успешных/с, upload — около 1,00/с. Наблюдаемый backlog достиг 60, возраст — 59,5 с | После запуска worker очередь опустела. Все 77 загрузок из интервала воздействия и запуска дошли до `ready` |
| Алерт | Первое firing после начала воздействия | Первое inactive после восстановления |
| --- | ---: | ---: |
| APIFastBurn | 12,0 с | 10,0 с |
| APISlowBurn | 22,0 с | 24,0 с |
| QueueLagHigh | 12,0 с | 22,0 с |
| ReadyFastBurn | 14,0 с | 28,0 с |
| ReadySlowBurn | 22,0 с | 44,0 с |
Алерты оценивались и опрашивались каждые 2 с. Для БД моменты отсчитывались от получения маркера захвата блокировки и выхода `psql` после commit. Для worker использовались моменты завершения команд Compose. Длительность шага запуска worker и остатки ошибок в скользящем окне влияют на наблюдаемые задержки. Числа описывают один прогон всех трёх фаз. [Замеры](evidence/slo-experiment.json)
При отключении Tempo на 36,5 с приложение подтвердило 274 сообщения, одна синтетическая загрузка дошла до ready. Во время отказа очередь экспортёра достигла лимита 16. По наблюдаемым дельтам Collector 270 spans получены им, но исчерпали попытки отправки, ещё 1180 не поставлены в очередь экспортёра. Пик наблюдаемых значений RSS Collector — 75,3 MiB при лимите контейнера 180 MiB. Количество рестартов осталось 0, OOMKilled — false. После команды запуска Tempo восстановление подтверждено новой трассой через 19,9 с. [Наблюдения Collector](evidence/backend-outage.json)
Публичный `tests/load.py` за 30,1 с с четырьмя параллельными клиентами подтвердил 10 833 сообщения без ошибок. В этом closed-loop прогоне 359,9 успешных ответов/с, p50/p95/p99 успешных запросов — 8,1 / 28,3 / 44,3 мс. Выбранная нагрузка и coordinated omission ограничивают интерпретацию. [Отдельный нагрузочный прогон](evidence/load.json)
Публичная проверка persistence подтвердила сохранение данных и сессии. Хук сначала перезапускает хранилища и ждёт healthcheck, затем перезапускает API, LB и фоновые процессы. Одновременный рестарт в первом прогоне вызывал гонку старта API и PostgreSQL. Порядок исправлен и результат подтверждён повторным прогоном. По одному историческому моменту открыт прежний набор из 56 рядов HTTP, восстановились 45 spans того же trace ID и связанная запись Loki. [Persistence](evidence/persistence.txt) · [До рестарта](evidence/restart-before.txt) · [После рестарта](evidence/restart-after.txt) · [История сигналов](evidence/history-after-restart.json)
Публичный failover пройден для каждой API-реплики. В двух прогонах получено 122 и 121 успешная запись при 3 и 2 переходных ошибках. Первая успешная запись после запуска хука произошла через 3,765 и 4,008 с. На Windows публичный тест вызвал те же POSIX-хуки через локальные адаптеры Git Bash. [Реплика 1](evidence/failover.txt) · [Реплика 2](evidence/failover-second.txt) · [Состояние первой реплики](evidence/stopped-after.txt) · [Состояние второй реплики](evidence/stopped-second-after.txt)
Для проверки минимума Lab5 при остановленном worker записано сообщение с queued-вложением. После рестарта двух API и broker проверены новые `StartedAt`, тот же идентификатор записи Valkey Streams и доступность прежнего сообщения по действующей сессии. После запуска worker получены `ready`, побайтно совпадающий оригинал и thumbnail JPEG. [Очередь при рестарте](evidence/queue-restart.json)
## Ограничения
Один локальный Compose-хост, одна DB, один брокер и один worker. Результаты относятся к синтетической нагрузке, размерам изображений и ограничениям выше. Они не задают максимальную пропускную способность или эксплуатационные цели. Скользящие окна чувствительны к объёму трафика. SQL-агрегат SLI читает историю заданий каждые 2 секунды, поэтому при росте таблицы потребуется отдельная агрегация и политика хранения. Момент готовности берётся из SQL перед завершением транзакции, задержка самого commit в гистограмму входит приблизительно. Показатели насыщения снимаются периодически и могут пропускать субсекундную работу. Бизнес-запросы, не дошедшие до API, исключены из его SLI.
При недоступном backend допускается потеря сигналов. SDK использует буферы на 1024 spans и 1024 лога, экспорт с таймаутом 2 с, асинхронный сброс и отключённые повторы SDK. Collector ограничивает объём очереди каждым exporter до 16 элементов, число одновременных отправок — до одной, время повторов — до 8 с и память контейнера — до 180 MiB. `memory_limiter` имеет лимит 125 MiB и резерв для пика 25 MiB. Счётчики Collector охватывают его собственные потери. Потери до приёма Collector и незасэмплированные сигналы из них не выводятся.
+34
View File
@@ -0,0 +1,34 @@
# Диагностика мессенджера
Откройте [дашборд Lab 6 · Messenger](http://127.0.0.1:3000/d/messenger-lab6) и [Prometheus Alerts](http://127.0.0.1:9090/alerts). Выберите период, который начинается за минуту до первого сигнала. Состояния `pending` и `firing` видны в Prometheus. После восстановления правило снова `inactive`, а история `ALERTS` на графике обрывается. Доставка уведомлений наружу для этого стенда отключена.
## API
1. Сверьте «Запросы API», «Ошибки API» и «Задержка API». Если выросли только клиентские 4xx, проверьте запросы клиента. Если выросли 5xx, откройте «Соединения PostgreSQL» и «Получение соединения при пустом пуле».
2. В Grafana Explore выберите Loki. Для поиска используйте:
```logql
{service_name="messenger-api"} | event="http.request" | status="503"
```
По конкретному ответу можно искать `{service_name="messenger-api"} | request_id="<X-Request-Id>"`. В деталях лога есть route, status, trace_id и span_id. Откройте Trace, или вставьте trace_id в поиск по ID в Tempo.
3. Раскройте server span. При деградации БД тяжёлая часть времени приходится на ошибочный `postgresql query`. Сравните со здоровой трассой того же маршрута. Если задержка находится вне SQL, изучите другие child spans и время внутри самого handler. Наличие длинного SQL span локализует задержку на пути запроса к БД. В контролируемом прогоне интервал совпадает с удержанием блокировки таблицы `messages`.
4. После восстановления проверьте 201 на обычной записи, исчезновение 5xx и возврат обоих API-алертов в inactive. Их окна продолжают учитывать ошибки ещё некоторое время после первой успешной записи.
## Ready
1. Сверьте «Глубина очереди», «Возраст старейшего задания», «Попытки worker» и «Занятые слоты worker». При остановке worker API продолжает принимать uploads, пополнение очереди видно в SQL backlog, счётчики попыток перестают расти, а SLI ready падает. Откройте диапазон начиная с первых 202.
2. Найдите upload по `request_id` в Loki и откройте Tempo. Если видна публикация и отсутствует consumer, доставка ждёт обработки. Если consumer присутствует, найдите исход `image.attempt` и просмотрите его S3 и SQL spans. Чтобы найти все записи этой цепочки:
```logql
{service_name=~"messenger-api|messenger-dispatcher|messenger-worker"} | trace_id="<trace id>"
```
В повторах сравните `attempt` и `outcome`. Каждый publish — дочерний для upload, каждая обработка — дочерняя для соответствующего publish.
3. После `docker compose start worker` дождитесь обработки накопившихся задач и убедитесь, что очередь опустела, метаданные отвечают `ready`, а `ReadyFastBurn`, `ReadySlowBurn` и `QueueLagHigh` вернулись в inactive. Гистограмма ready в это время показывает длинный хвост из-за завершения старых задач.
## Потеря телеметрии
Если HTTP и worker продолжают работать, а один источник пуст, проверьте его readiness и ошибки экспортёра Collector. В случае Tempo ищите `otelcol_exporter_send_failed_spans{exporter="otlphttp/tempo"}`, размер очереди и RSS Collector. При восстановлении найдите новую трассу по свежему HTTP-логу. Старые span, для которых истёк срок повторов, уже потеряны. Лимиты и наблюдения прогона описаны в отчёте.
Для самостоятельного повтора используйте команды из [REPORT.md](REPORT.md). Сначала запустите рабочий стенд без посторонней нагрузки.
+324
View File
@@ -0,0 +1,324 @@
package main
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"image"
_ "image/jpeg"
_ "image/png"
"io"
"mime"
"net"
"net/http"
"net/url"
"os"
"strconv"
"strings"
"time"
"unicode"
"unicode/utf8"
"github.com/jackc/pgx/v5"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
)
const maxUploadBytes int64 = 10 * 1024 * 1024
const maxImagePixels int64 = 40000000
type Variant struct {
Name string `json:"name"`
ContentType string `json:"content_type"`
SizeBytes int64 `json:"size_bytes"`
Width int `json:"width"`
Height int `json:"height"`
SHA256 string `json:"sha256"`
}
type Attachment struct {
ID string `json:"id"`
ChatID string `json:"chat_id"`
OwnerID string `json:"owner_id"`
Filename string `json:"filename"`
ContentType string `json:"content_type"`
SizeBytes int64 `json:"size_bytes"`
SHA256 string `json:"sha256"`
Status string `json:"status"`
Variants []Variant `json:"variants"`
ErrorCode *string `json:"error_code"`
CreatedAt string `json:"created_at"`
UpdatedAt string `json:"updated_at"`
resultVersion int
}
func s3Client() (*minio.Client, string, error) {
endpoint, err := url.Parse(env("S3_ENDPOINT", "http://s3:9000"))
if err != nil || (endpoint.Scheme != "http" && endpoint.Scheme != "https") || endpoint.Host == "" || endpoint.User != nil || endpoint.RawQuery != "" || (endpoint.Path != "" && endpoint.Path != "/") {
return nil, "", errors.New("invalid S3 endpoint")
}
access, err := secretFile("S3_ACCESS_KEY_ID_FILE")
if err != nil {
return nil, "", err
}
secret, err := secretFile("S3_SECRET_ACCESS_KEY_FILE")
if err != nil {
return nil, "", err
}
transport := &http.Transport{Proxy: http.ProxyFromEnvironment, DialContext: (&net.Dialer{Timeout: 2 * time.Second, KeepAlive: 30 * time.Second}).DialContext, MaxIdleConns: 16, MaxIdleConnsPerHost: 8, MaxConnsPerHost: 8, ResponseHeaderTimeout: 5 * time.Second, TLSHandshakeTimeout: 3 * time.Second, IdleConnTimeout: 30 * time.Second, DisableCompression: true}
client, err := minio.New(endpoint.Host, &minio.Options{Creds: credentials.NewStaticV4(access, secret, ""), Secure: endpoint.Scheme == "https", Region: env("S3_REGION", "us-east-1"), Transport: transport, MaxRetries: 1, BucketLookup: minio.BucketLookupPath})
return client, env("S3_BUCKET", "messenger-attachments"), err
}
func validFilename(s string) bool {
if !utf8.ValidString(s) || strings.TrimSpace(s) == "" || utf8.RuneCountInString(s) > 255 {
return false
}
for _, c := range s {
if c == '/' || c == '\\' || unicode.IsControl(c) {
return false
}
}
return true
}
func messageFields(body map[string]json.RawMessage) (string, []string, error) {
var text string
if string(body["text"]) == "null" || json.Unmarshal(body["text"], &text) != nil || utf8.RuneCountInString(text) > 4000 {
return "", nil, invalid()
}
ids := []string{}
if raw, ok := body["attachment_ids"]; ok {
if string(raw) == "null" || json.Unmarshal(raw, &ids) != nil || len(ids) > 10 {
return "", nil, invalid()
}
}
seen := map[string]bool{}
for i, id := range ids {
normalized, err := normalizeID(id)
if err != nil || seen[normalized] {
return "", nil, invalid()
}
ids[i] = normalized
seen[normalized] = true
}
if strings.TrimSpace(text) == "" && len(ids) == 0 {
return "", nil, invalid()
}
return text, ids, nil
}
func (a *App) s3Ready(ctx context.Context) bool {
check, cancel := context.WithTimeout(ctx, time.Second)
defer cancel()
ok, err := a.s3.BucketExists(check, a.bucket)
return err == nil && ok
}
func mediaError() error {
return apiError{415, "unsupported_media_type", "Expected a valid PNG or JPEG matching Content-Type"}
}
func tooLarge() error { return apiError{413, "payload_too_large", "Image exceeds byte or pixel limit"} }
// The encoded file lives only in a bounded tmpfs during validation/PUT. At most
// five uploads and one decoded image per process; no bytes enter PostgreSQL.
func (a *App) upload(ctx context.Context, w http.ResponseWriter, r *http.Request, chat, user string) (Attachment, error) {
result := Attachment{}
query, err := url.ParseQuery(r.URL.RawQuery)
if err != nil || len(query) != 1 || len(query["filename"]) != 1 || !validFilename(query.Get("filename")) {
return result, invalid()
}
contentType, params, err := mime.ParseMediaType(r.Header.Get("Content-Type"))
if err != nil || len(params) != 0 || (contentType != "image/png" && contentType != "image/jpeg") {
return result, mediaError()
}
if r.ContentLength > maxUploadBytes {
return result, tooLarge()
}
select {
case a.uploadSlots <- struct{}{}:
defer func() { <-a.uploadSlots }()
default:
return result, unavailable()
}
f, err := os.CreateTemp("", "upload-*")
if err != nil {
return result, unavailable()
}
defer os.Remove(f.Name())
defer f.Close()
digest := sha256.New()
size, err := io.Copy(io.MultiWriter(f, digest), http.MaxBytesReader(w, r.Body, maxUploadBytes))
if err != nil {
var big *http.MaxBytesError
if errors.As(err, &big) {
return result, tooLarge()
}
return result, invalid()
}
if size == 0 {
return result, mediaError()
}
if _, err = f.Seek(0, io.SeekStart); err != nil {
return result, unavailable()
}
cfg, format, err := image.DecodeConfig(f)
if err != nil || (format != "png" && format != "jpeg") || "image/"+format != contentType {
return result, mediaError()
}
if cfg.Width <= 0 || cfg.Height <= 0 || int64(cfg.Width) > maxImagePixels/int64(cfg.Height) {
return result, tooLarge()
}
select {
case a.decodeSlot <- struct{}{}:
case <-ctx.Done():
return result, unavailable()
}
validation := func() error {
defer func() { <-a.decodeSlot }()
if _, e := f.Seek(0, io.SeekStart); e != nil {
return unavailable()
}
decoded, actual, e := image.Decode(f)
if e != nil || actual != format || decoded.Bounds().Dx() != cfg.Width || decoded.Bounds().Dy() != cfg.Height {
return mediaError()
}
return nil
}
if err = validation(); err != nil {
return result, err
}
if ctx.Err() != nil {
return result, unavailable()
}
if _, err = f.Seek(0, io.SeekStart); err != nil {
return result, unavailable()
}
result = Attachment{ID: uuid(), ChatID: chat, OwnerID: user, Filename: query.Get("filename"), ContentType: contentType, SizeBytes: size, SHA256: hex.EncodeToString(digest.Sum(nil)), Status: "queued", Variants: []Variant{}}
tx, err := a.db.Begin(ctx)
if err != nil {
return result, unavailable()
}
defer rollbackAttachment(tx)
if err = attachmentLock(ctx, tx, result.ID); err != nil {
return result, unavailable()
}
key := "original/" + result.ID
staging, s3Span := startSpan(ctx, "s3 put staging")
_, err = a.s3.PutObject(staging, a.bucket, stagingKey(result.ID), f, size, minio.PutObjectOptions{ContentType: contentType, DisableMultipart: true, UserMetadata: map[string]string{"sha256": result.SHA256}})
endSpan(s3Span, err)
if err != nil {
return result, unavailable()
}
var created, updated time.Time
err = tx.QueryRow(ctx, `INSERT INTO attachments(id,chat_id,owner_id,object_key,filename,content_type,size_bytes,sha256,status,active_version)
VALUES($1,$2,$3,$4,$5,$6,$7,$8,'queued',1) RETURNING created_at,updated_at`, result.ID, chat, user, key, result.Filename, contentType, size, result.SHA256).Scan(&created, &updated)
if err != nil {
return result, unavailable()
}
if err = enqueueImage(ctx, tx, result.ID, 1); err != nil {
return result, unavailable()
}
// COMMIT may have succeeded even when its acknowledgement is lost. Never
// blindly delete the blob here: reconciler checks durable metadata + grace.
if err = tx.Commit(ctx); err != nil {
return result, unavailable()
}
result.CreatedAt = stamp(created)
result.UpdatedAt = stamp(updated)
return result, nil
}
func attachmentLock(ctx context.Context, tx pgx.Tx, id string) error {
_, err := tx.Exec(ctx, "SELECT pg_advisory_xact_lock(hashtextextended($1,0))", "attachment:"+id)
return err
}
func rollbackAttachment(tx pgx.Tx) {
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
defer cancel()
_ = tx.Rollback(ctx)
}
func (a *App) attachment(ctx context.Context, id, user string) (Attachment, error) {
x := Attachment{}
var created, updated time.Time
var missingSince *time.Time
err := a.db.QueryRow(ctx, `SELECT a.id::text,a.chat_id::text,a.owner_id::text,a.filename,a.content_type,a.size_bytes,a.sha256,a.created_at,a.updated_at,a.missing_since,a.status,a.variants,a.error_code,a.result_version
FROM attachments a JOIN chat_members m ON m.chat_id=a.chat_id AND m.user_id=$2 WHERE a.id=$1`, id, user).Scan(&x.ID, &x.ChatID, &x.OwnerID, &x.Filename, &x.ContentType, &x.SizeBytes, &x.SHA256, &created, &updated, &missingSince, &x.Status, &x.Variants, &x.ErrorCode, &x.resultVersion)
if err != nil {
return x, dbError(err)
}
if missingSince != nil {
return x, unavailable()
}
x.CreatedAt = stamp(created)
x.UpdatedAt = stamp(updated)
return x, nil
}
type binaryContent struct {
attachment Attachment
object *minio.Object
}
func (a *App) content(ctx context.Context, r *http.Request, x Attachment) (binaryContent, error) {
q, err := url.ParseQuery(r.URL.RawQuery)
if err != nil || len(q) > 1 || len(q["variant"]) > 1 {
return binaryContent{}, invalid()
}
for k := range q {
if k != "variant" {
return binaryContent{}, invalid()
}
}
name := "original"
if values, ok := q["variant"]; ok {
name = values[0]
switch name {
case "original", "thumbnail", "crop", "optimized":
default:
return binaryContent{}, invalid()
}
}
if x.Status != "ready" {
return binaryContent{}, apiError{409, "conflict", "Attachment is not ready"}
}
if name != "original" {
found := false
for _, variant := range x.Variants {
if variant.Name == name {
x.ContentType, x.SizeBytes = variant.ContentType, variant.SizeBytes
found = true
break
}
}
if !found {
return binaryContent{}, missing()
}
}
object, err := a.s3.GetObject(ctx, a.bucket, resultKey(x.ID, x.resultVersion, name), minio.GetObjectOptions{})
if err != nil {
return binaryContent{}, unavailable()
}
info, err := object.Stat()
if err != nil || info.Size != x.SizeBytes {
object.Close()
return binaryContent{}, unavailable()
}
return binaryContent{x, object}, nil
}
func writeContent(w http.ResponseWriter, b binaryContent) {
defer b.object.Close()
w.Header().Set("Content-Type", b.attachment.ContentType)
w.Header().Set("Content-Length", strconv.FormatInt(b.attachment.SizeBytes, 10))
w.Header().Set("Content-Disposition", mime.FormatMediaType("attachment", map[string]string{"filename": b.attachment.Filename}))
w.Header().Set("X-Content-Type-Options", "nosniff")
w.Header().Set("Cache-Control", "private, no-store")
w.WriteHeader(200)
if _, err := io.CopyN(w, b.object, b.attachment.SizeBytes); err != nil {
panic(http.ErrAbortHandler)
}
}
+147
View File
@@ -0,0 +1,147 @@
package main
import (
"context"
"crypto/rand"
"crypto/sha256"
"crypto/subtle"
"encoding/base64"
"encoding/hex"
"encoding/json"
"errors"
"net/http"
"strings"
"time"
"unicode/utf8"
"github.com/jackc/pgx/v5"
"github.com/redis/go-redis/v9"
"golang.org/x/crypto/argon2"
)
func password(body map[string]json.RawMessage) (string, error) {
var p string
if json.Unmarshal(body["password"], &p) != nil || utf8.RuneCountInString(p) < 12 || utf8.RuneCountInString(p) > 128 {
return "", invalid()
}
return p, nil
}
func hashPassword(p string) string {
salt := make([]byte, 16)
if _, err := rand.Read(salt); err != nil {
panic(err)
}
key := argon2.IDKey([]byte(p), salt, 3, 64*1024, 1, 32)
return "$argon2id$v=19$m=65536,t=3,p=1$" + base64.RawStdEncoding.EncodeToString(salt) + "$" + base64.RawStdEncoding.EncodeToString(key)
}
func verifyPassword(hash, p string) bool {
parts := strings.Split(hash, "$")
if len(parts) != 6 || parts[1] != "argon2id" || parts[2] != "v=19" || parts[3] != "m=65536,t=3,p=1" {
return false
}
salt, e := base64.RawStdEncoding.DecodeString(parts[4])
if e != nil || len(salt) != 16 {
return false
}
want, e := base64.RawStdEncoding.DecodeString(parts[5])
if e != nil || len(want) != 32 {
return false
}
actual := argon2.IDKey([]byte(p), salt, 3, 64*1024, 1, 32)
return subtle.ConstantTimeCompare(want, actual) == 1
}
func (a *App) kdfSlot(ctx context.Context) error {
select {
case a.kdf <- struct{}{}:
return nil
case <-ctx.Done():
return unavailable()
}
}
type storedSession struct {
UserID string `json:"user_id"`
ExpiresAt time.Time `json:"expires_at"`
}
func sessionKey(token string) string {
h := sha256.Sum256([]byte(token))
return "sess:" + hex.EncodeToString(h[:])
}
func (a *App) authenticate(ctx context.Context, r *http.Request) (string, string, error) {
values := r.Header.Values("Authorization")
if len(values) != 1 {
return "", "", unauthorized()
}
parts := strings.Fields(values[0])
if len(parts) != 2 || !strings.EqualFold(parts[0], "Bearer") {
return "", "", unauthorized()
}
token, e := base64.RawURLEncoding.DecodeString(parts[1])
if e != nil || len(token) != 32 {
return "", "", unauthorized()
}
key := sessionKey(parts[1])
raw, err := a.sessions.Get(ctx, key).Bytes()
if errors.Is(err, redis.Nil) {
return "", "", unauthorized()
}
if err != nil {
return "", "", unavailable()
}
var session storedSession
if json.Unmarshal(raw, &session) != nil {
return "", "", unavailable()
}
if !time.Now().Before(session.ExpiresAt) {
return "", "", unauthorized()
}
if _, err = a.user(ctx, session.UserID); err != nil {
var ae apiError
if errors.As(err, &ae) && ae.status == 404 {
return "", "", unauthorized()
}
return "", "", err
}
return session.UserID, key, nil
}
func (a *App) login(ctx context.Context, r *http.Request) (int, any, error) {
if r.ContentLength > 0 || len(r.TransferEncoding) > 0 {
return 0, nil, invalid()
}
username, p, ok := r.BasicAuth()
if len(r.Header.Values("Authorization")) != 1 || !ok || !usernamePattern.MatchString(username) || !utf8.ValidString(p) || utf8.RuneCountInString(p) < 12 || utf8.RuneCountInString(p) > 128 {
return 0, nil, unauthorized()
}
var hash string
var u User
var created time.Time
err := a.db.QueryRow(ctx, "SELECT id::text,username,display_name,created_at,password_hash FROM users WHERE username=$1", username).Scan(&u.ID, &u.Username, utf8String{&u.DisplayName}, &created, &hash)
absent := errors.Is(err, pgx.ErrNoRows)
if err != nil && !absent {
return 0, nil, unavailable()
}
if absent {
hash = a.dummyHash
}
if err = a.kdfSlot(ctx); err != nil {
return 0, nil, err
}
valid := verifyPassword(hash, p)
<-a.kdf
if !valid || absent {
return 0, nil, unauthorized()
}
u.CreatedAt = stamp(created)
tokenBytes := make([]byte, 32)
if _, err = rand.Read(tokenBytes); err != nil {
return 0, nil, unavailable()
}
token := base64.RawURLEncoding.EncodeToString(tokenBytes)
expires := time.Now().UTC().Truncate(time.Second).Add(a.ttl)
raw, _ := json.Marshal(storedSession{u.ID, expires})
if err = a.sessions.SetArgs(ctx, sessionKey(token), raw, redis.SetArgs{Mode: "NX", ExpireAt: expires}).Err(); err != nil {
return 0, nil, unavailable()
}
return 201, map[string]any{"token": token, "token_type": "Bearer", "expires_at": expires.Format(time.RFC3339), "user": u}, nil
}
+295
View File
@@ -0,0 +1,295 @@
name: mtusi-lab6-local
x-telemetry: &telemetry
OTEL_SDK_DISABLED: ${OTEL_SDK_DISABLED:-false}
OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT:-http://otel-collector:4318}
x-image-service: &image-service
restart: unless-stopped
environment:
<<: *telemetry
APP_ENV: ${APP_ENV:-lab}
DATABASE_URL_FILE: /run/secrets/database_url
DB_POOL_MAX: 4
S3_ENDPOINT: http://s3:9000
S3_REGION: us-east-1
S3_BUCKET: messenger-attachments
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_worker_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_worker_secret_access_key
BROKER_URL_FILE: ${BROKER_URL_FILE:-/run/secrets/broker_url}
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-30}
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-1}
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
GOMEMLIMIT: 96MiB
secrets: [database_url, s3_worker_access_key_id, s3_worker_secret_access_key, broker_url]
depends_on:
migrate: {condition: service_completed_successfully}
s3-init: {condition: service_completed_successfully}
broker: {condition: service_started}
read_only: true
cap_drop: [ALL]
security_opt: [no-new-privileges:true]
healthcheck: {disable: true}
logging:
options: {max-size: 10m, max-file: "2"}
services:
db:
image: postgres:17.6-bookworm@sha256:f3bd19c606e442c3d7bdfa8002e03fe260a1023351e0ea4598032022b68dd6e3
environment:
POSTGRES_USER: messenger
POSTGRES_DB: messenger
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
secrets: [postgres_password]
volumes: [pgdata:/var/lib/postgresql/data]
healthcheck:
test: [CMD-SHELL, "pg_isready -h 127.0.0.1 -U messenger -d messenger"]
interval: 2s
timeout: 2s
retries: 30
mem_limit: 512m
cpus: ${DB_CPUS:-1.0}
valkey:
image: valkey/valkey:8.1.3-bookworm@sha256:fea8b3e67b15729d4bb70589eb03367bab9ad1ee89c876f54327fc7c6e618571
command: [valkey-server, /run/secrets/valkey_config]
secrets: [valkey_config, valkey_password]
volumes: [sessions:/data]
healthcheck:
test: [CMD-SHELL, 'REDISCLI_AUTH="$$(cat /run/secrets/valkey_password)" valkey-cli ping | grep -q PONG']
interval: 2s
timeout: 2s
retries: 30
mem_limit: 256m
cpus: 0.5
broker:
image: valkey/valkey:8.1.3-bookworm@sha256:fea8b3e67b15729d4bb70589eb03367bab9ad1ee89c876f54327fc7c6e618571
command: [valkey-server, /run/secrets/broker_config]
secrets: [broker_config, broker_password]
volumes: [brokerdata:/data]
healthcheck:
test: [CMD-SHELL, 'REDISCLI_AUTH="$$(cat /run/secrets/broker_password)" valkey-cli ping | grep -q PONG']
interval: 2s
timeout: 2s
retries: 30
mem_limit: 256m
cpus: 0.5
migrate:
image: mtusi-messenger:lab6
build: .
command: [migrate]
environment:
DATABASE_URL_FILE: /run/secrets/database_url
secrets: [database_url]
depends_on:
db: {condition: service_healthy}
mem_limit: 256m
cpus: 0.5
read_only: true
cap_drop: [ALL]
security_opt: [no-new-privileges:true]
api:
image: mtusi-messenger:lab6
build: .
deploy:
replicas: ${API_REPLICAS:-2}
environment:
<<: *telemetry
APP_HOST: ${APP_HOST:-0.0.0.0}
APP_PORT: 8080
APP_ENV: ${APP_ENV:-lab}
SESSION_STORE: ${SESSION_STORE:-valkey}
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
DB_POOL_MAX: ${DB_POOL_MAX:-8}
MAX_INFLIGHT: ${MAX_INFLIGHT:-32}
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
GOMEMLIMIT: ${GOMEMLIMIT:-512MiB}
GODEBUG: madvdontneed=0
POOL_METRICS: ${POOL_METRICS:-0}
REQUEST_LOG: ${REQUEST_LOG:-1}
DATABASE_URL_FILE: ${DATABASE_URL_FILE:-/run/secrets/database_url}
CACHE_URL_FILE: ${CACHE_URL_FILE:-/run/secrets/cache_url}
CURSOR_SECRET_FILE: ${CURSOR_SECRET_FILE:-/run/secrets/cursor_secret}
S3_ENDPOINT: http://s3:9000
S3_REGION: us-east-1
S3_BUCKET: messenger-attachments
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
secrets: [database_url, cache_url, cursor_secret, s3_access_key_id, s3_secret_access_key]
tmpfs: ["/tmp:size=64m,mode=1777,noexec,nosuid,nodev"]
depends_on:
db: {condition: service_healthy}
valkey: {condition: service_healthy}
migrate: {condition: service_completed_successfully}
s3-init: {condition: service_completed_successfully}
read_only: true
cap_drop: [ALL]
security_opt: [no-new-privileges:true]
mem_limit: ${API_MEMORY:-768m}
cpus: ${API_CPUS:-1.0}
pids_limit: 128
logging:
options: {max-size: 10m, max-file: "2"}
stop_grace_period: 10s
s3:
image: mtusi-minio:RELEASE.2025-10-15T17-29-55Z
build:
context: .
dockerfile: Dockerfile.minio
command: [server, /data, --console-address, "127.0.0.1:9001"]
environment:
MINIO_ROOT_USER_FILE: /run/secrets/s3_root_user
MINIO_ROOT_PASSWORD_FILE: /run/secrets/s3_root_password
MINIO_BROWSER: "off"
secrets: [s3_root_user, s3_root_password]
volumes: [objects:/data]
networks:
object_store:
aliases: [minio-backend]
cap_drop: [ALL]
security_opt: [no-new-privileges:true]
mem_limit: 1g
cpus: 1.0
healthcheck:
test: [CMD, /healthcheck]
interval: 2s
timeout: 2s
retries: 60
s3-init:
image: mtusi-minio-tools:RELEASE.2025-08-13T08-35-41Z
build:
context: .
dockerfile: Dockerfile.minio
target: tools
entrypoint: [/bin/sh, /bootstrap/s3-init.sh]
secrets: [s3_root_user, s3_root_password, s3_access_key_id, s3_secret_access_key, s3_worker_access_key_id, s3_worker_secret_access_key]
volumes: [./infra/s3-init.sh:/bootstrap/s3-init.sh:ro, ./infra/s3-policy.json:/bootstrap/policy.json:ro, ./infra/s3-worker-policy.json:/bootstrap/worker-policy.json:ro]
depends_on:
s3: {condition: service_healthy}
lb: {condition: service_started}
tmpfs: ["/tmp:size=16m,mode=1777,noexec,nosuid,nodev"]
environment: {MC_CONFIG_DIR: /tmp/mc}
read_only: true
cap_drop: [ALL]
security_opt: [no-new-privileges:true]
mem_limit: 128m
cpus: 0.5
dispatcher:
<<: *image-service
image: mtusi-messenger:lab6
build: .
command: [dispatcher]
mem_limit: 192m
cpus: 0.5
worker:
<<: *image-service
image: mtusi-messenger-worker:lab6
build: {context: ., target: worker}
command: [worker]
tmpfs: ["/tmp:size=32m,mode=1777,noexec,nosuid,nodev"]
mem_limit: 640m
cpus: 1.0
pids_limit: 64
lb:
image: haproxy:3.2.6-alpine
ports: ["127.0.0.1:${APP_PORT:-8080}:8080"]
volumes: [./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro]
depends_on:
s3: {condition: service_healthy}
networks:
default:
aliases: [s3]
object_store: {}
read_only: true
cap_drop: [ALL]
security_opt: [no-new-privileges:true]
mem_limit: 128m
cpus: 0.5
stop_grace_period: 10s
logging:
options: {max-size: 10m, max-file: "2"}
healthcheck:
test: [CMD-SHELL, "wget -q -O /dev/null http://127.0.0.1:8080/health/ready"]
interval: 2s
timeout: 2s
retries: 30
otel-collector:
image: otel/opentelemetry-collector-contrib:0.133.0
command: [--config=/etc/otelcol/config.yaml]
volumes: [./infra/otel/collector.yaml:/etc/otelcol/config.yaml:ro]
environment: {GOMEMLIMIT: 110MiB}
mem_limit: 180m
cpus: 0.5
restart: unless-stopped
prometheus:
image: prom/prometheus:v3.5.0
command: [--config.file=/etc/prometheus/prometheus.yml, --storage.tsdb.path=/prometheus,
--storage.tsdb.retention.time=7d, --storage.tsdb.retention.size=512MB, --web.enable-otlp-receiver]
volumes:
- ./infra/otel/prometheus.yml:/etc/prometheus/prometheus.yml:ro
- ./infra/otel/alerts.yml:/etc/prometheus/alerts.yml:ro
- metrics:/prometheus
ports: ["127.0.0.1:${PROMETHEUS_PORT:-9090}:9090"]
mem_limit: 384m
cpus: 0.75
restart: unless-stopped
loki:
image: grafana/loki:3.5.3
command: [-config.file=/etc/loki/lab.yml]
volumes: [./infra/otel/loki.yml:/etc/loki/lab.yml:ro, logs:/loki]
ports: ["127.0.0.1:${LOKI_PORT:-3100}:3100"]
mem_limit: 384m
cpus: 0.75
restart: unless-stopped
tempo:
image: grafana/tempo:2.8.2
command: [-config.file=/etc/tempo/lab.yml]
volumes: [./infra/otel/tempo.yml:/etc/tempo/lab.yml:ro, traces:/var/tempo]
ports: ["127.0.0.1:${TEMPO_PORT:-3200}:3200"]
mem_limit: 384m
cpus: 0.75
restart: unless-stopped
grafana:
image: grafana/grafana:12.1.1
environment:
GF_AUTH_ANONYMOUS_ENABLED: "true"
GF_AUTH_ANONYMOUS_ORG_ROLE: Viewer
GF_AUTH_DISABLE_LOGIN_FORM: "true"
GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION: "true"
GF_ANALYTICS_REPORTING_ENABLED: "false"
GF_ANALYTICS_CHECK_FOR_UPDATES: "false"
GF_ANALYTICS_CHECK_FOR_PLUGIN_UPDATES: "false"
volumes:
- ./infra/otel/grafana/provisioning:/etc/grafana/provisioning:ro
- ./infra/otel/grafana/dashboards:/var/lib/grafana/dashboards:ro
- dashboards:/var/lib/grafana
ports: ["127.0.0.1:${GRAFANA_PORT:-3000}:3000"]
mem_limit: 256m
cpus: 0.5
restart: unless-stopped
networks:
object_store:
internal: true
volumes:
pgdata:
sessions:
brokerdata:
objects:
metrics:
logs:
traces:
dashboards:
secrets:
postgres_password: {file: ./secrets/postgres_password}
database_url: {file: ./secrets/database_url}
valkey_password: {file: ./secrets/valkey_password}
valkey_config: {file: ./secrets/valkey.conf}
cache_url: {file: ./secrets/cache_url}
cursor_secret: {file: ./secrets/cursor_secret}
s3_root_user: {file: ./secrets/s3_root_user}
s3_root_password: {file: ./secrets/s3_root_password}
s3_access_key_id: {file: ./secrets/s3_access_key_id}
s3_secret_access_key: {file: ./secrets/s3_secret_access_key}
broker_url: {file: ./secrets/broker_url}
broker_password: {file: ./secrets/broker_password}
broker_config: {file: ./secrets/broker.conf}
s3_worker_access_key_id: {file: ./secrets/s3_worker_access_key_id}
s3_worker_secret_access_key: {file: ./secrets/s3_worker_secret_access_key}
+170
View File
@@ -0,0 +1,170 @@
{
"fault": "tempo stopped",
"baseline": {
"send_failed": 465.0,
"enqueue_failed": 1315.0,
"queue": 0.0,
"queue_capacity": 16.0,
"collector_rss_bytes": 76185600.0
},
"samples": [
{
"elapsed_s": 0.081,
"send_failed": 465.0,
"enqueue_failed": 1315.0,
"queue": 0.0,
"queue_capacity": 16.0,
"collector_rss_bytes": 76845056.0
},
{
"elapsed_s": 2.033,
"send_failed": 465.0,
"enqueue_failed": 1315.0,
"queue": 1.0,
"queue_capacity": 16.0,
"collector_rss_bytes": 76910592.0
},
{
"elapsed_s": 4.09,
"send_failed": 465.0,
"enqueue_failed": 1315.0,
"queue": 3.0,
"queue_capacity": 16.0,
"collector_rss_bytes": 76910592.0
},
{
"elapsed_s": 6.089,
"send_failed": 465.0,
"enqueue_failed": 1315.0,
"queue": 5.0,
"queue_capacity": 16.0,
"collector_rss_bytes": 76910592.0
},
{
"elapsed_s": 8.104,
"send_failed": 465.0,
"enqueue_failed": 1315.0,
"queue": 7.0,
"queue_capacity": 16.0,
"collector_rss_bytes": 76914688.0
},
{
"elapsed_s": 10.013,
"send_failed": 515.0,
"enqueue_failed": 1315.0,
"queue": 8.0,
"queue_capacity": 16.0,
"collector_rss_bytes": 76918784.0
},
{
"elapsed_s": 12.031,
"send_failed": 515.0,
"enqueue_failed": 1315.0,
"queue": 10.0,
"queue_capacity": 16.0,
"collector_rss_bytes": 77066240.0
},
{
"elapsed_s": 14.104,
"send_failed": 515.0,
"enqueue_failed": 1315.0,
"queue": 12.0,
"queue_capacity": 16.0,
"collector_rss_bytes": 77168640.0
},
{
"elapsed_s": 16.06,
"send_failed": 515.0,
"enqueue_failed": 1315.0,
"queue": 14.0,
"queue_capacity": 16.0,
"collector_rss_bytes": 77508608.0
},
{
"elapsed_s": 18.019,
"send_failed": 585.0,
"enqueue_failed": 1315.0,
"queue": 15.0,
"queue_capacity": 16.0,
"collector_rss_bytes": 77586432.0
},
{
"elapsed_s": 20.053,
"send_failed": 585.0,
"enqueue_failed": 1395.0,
"queue": 16.0,
"queue_capacity": 16.0,
"collector_rss_bytes": 78282752.0
},
{
"elapsed_s": 22.003,
"send_failed": 585.0,
"enqueue_failed": 1555.0,
"queue": 16.0,
"queue_capacity": 16.0,
"collector_rss_bytes": 78946304.0
},
{
"elapsed_s": 24.033,
"send_failed": 665.0,
"enqueue_failed": 1635.0,
"queue": 16.0,
"queue_capacity": 16.0,
"collector_rss_bytes": 78962688.0
},
{
"elapsed_s": 26.034,
"send_failed": 665.0,
"enqueue_failed": 1785.0,
"queue": 16.0,
"queue_capacity": 16.0,
"collector_rss_bytes": 78962688.0
},
{
"elapsed_s": 28.096,
"send_failed": 665.0,
"enqueue_failed": 1935.0,
"queue": 16.0,
"queue_capacity": 16.0,
"collector_rss_bytes": 78979072.0
},
{
"elapsed_s": 30.054,
"send_failed": 665.0,
"enqueue_failed": 2105.0,
"queue": 16.0,
"queue_capacity": 16.0,
"collector_rss_bytes": 78979072.0
},
{
"elapsed_s": 32.008,
"send_failed": 735.0,
"enqueue_failed": 2185.0,
"queue": 16.0,
"queue_capacity": 16.0,
"collector_rss_bytes": 78983168.0
},
{
"elapsed_s": 34.085,
"send_failed": 735.0,
"enqueue_failed": 2335.0,
"queue": 16.0,
"queue_capacity": 16.0,
"collector_rss_bytes": 78991360.0
},
{
"elapsed_s": 36.352,
"send_failed": 735.0,
"enqueue_failed": 2495.0,
"queue": 16.0,
"queue_capacity": 16.0,
"collector_rss_bytes": 78995456.0
}
],
"successful_message_writes": 274,
"upload_ready_while_tempo_stopped": true,
"collector_started_at_restarts_oom_limit_bytes": "2026-11-19T14:55:59.148905711Z 0 false 188743680",
"recovery_trace_id": "2ef768fb95857988ffbfd9cb8bff706f",
"fault_duration_s": 36.468,
"new_trace_after_restart_s": 19.85
}
+2
View File
@@ -0,0 +1,2 @@
python scripts/check_contract.py
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
+722
View File
@@ -0,0 +1,722 @@
{
"trace_id": "5a6a7480a7d39cd8872e69cda2a7db49",
"request_id": "a22d1cec-9360-49ae-bedf-061c31f8934c",
"logs": [
{
"deployment_environment_name": "lab",
"detected_level": "INFO",
"duration_seconds": "0.113465624",
"event": "http.request",
"flags": "1",
"method": "POST",
"observed_timestamp": "1795101125224693239",
"request_id": "a22d1cec-9360-49ae-bedf-061c31f8934c",
"route": "/api/v1/chats/{chat_id}/attachments",
"scope_name": "mtusi/messenger",
"service_instance_id": "b3490044297c",
"service_name": "messenger-api",
"service_version": "6.0.0",
"severity_number": "9",
"severity_text": "INFO",
"span_id": "cb82f12c320bdec9",
"status": "202",
"trace_id": "5a6a7480a7d39cd8872e69cda2a7db49",
"timestamp_ns": "1795101125224687981",
"body": "http.request"
},
{
"attempt": "1",
"deployment_environment_name": "lab",
"detected_level": "ERROR",
"duration_seconds": "0.007419824",
"event": "image.attempt",
"flags": "1",
"job_id": "32d3a0c1-5025-4e89-98d3-0a04a613b2c1",
"observed_timestamp": "1795101126170042959",
"outcome": "queued",
"scope_name": "mtusi/messenger",
"service_instance_id": "c7ce8573ebfd",
"service_name": "messenger-worker",
"service_version": "6.0.0",
"severity_number": "17",
"severity_text": "ERROR",
"span_id": "b7078277bfa428ae",
"trace_id": "5a6a7480a7d39cd8872e69cda2a7db49",
"timestamp_ns": "1795101126170001997",
"body": "image.attempt"
},
{
"attempt": "2",
"deployment_environment_name": "lab",
"detected_level": "INFO",
"duration_seconds": "0.494099263",
"event": "image.attempt",
"flags": "1",
"job_id": "32d3a0c1-5025-4e89-98d3-0a04a613b2c1",
"observed_timestamp": "1795101128086496153",
"outcome": "ready",
"scope_name": "mtusi/messenger",
"service_instance_id": "c7ce8573ebfd",
"service_name": "messenger-worker",
"service_version": "6.0.0",
"severity_number": "9",
"severity_text": "INFO",
"span_id": "85d8bed17c2a983b",
"trace_id": "5a6a7480a7d39cd8872e69cda2a7db49",
"timestamp_ns": "1795101128086482410",
"body": "image.attempt"
}
],
"spans": [
{
"name": "POST /api/v1/chats/{chat_id}/attachments",
"service": "messenger-api",
"service_version": "6.0.0",
"environment": "lab",
"id": "cb82f12c320bdec9",
"parent": null,
"kind": "SPAN_KIND_SERVER",
"status": {},
"duration_ms": 113.472,
"attributes": {
"http.request.method": "POST",
"http.route": "/api/v1/chats/{chat_id}/attachments",
"request.id": "a22d1cec-9360-49ae-bedf-061c31f8934c",
"http.response.status_code": "202"
}
},
{
"name": "postgresql query",
"service": "messenger-api",
"service_version": "6.0.0",
"environment": "lab",
"id": "bfad0321d52b658d",
"parent": "cb82f12c320bdec9",
"kind": "SPAN_KIND_CLIENT",
"status": {},
"duration_ms": 0.42,
"attributes": {
"db.system.name": "postgresql"
}
},
{
"name": "postgresql query",
"service": "messenger-api",
"service_version": "6.0.0",
"environment": "lab",
"id": "c33a717587da4131",
"parent": "cb82f12c320bdec9",
"kind": "SPAN_KIND_CLIENT",
"status": {},
"duration_ms": 0.723,
"attributes": {
"db.system.name": "postgresql"
}
},
{
"name": "postgresql query",
"service": "messenger-api",
"service_version": "6.0.0",
"environment": "lab",
"id": "772e07958b4cd5c5",
"parent": "cb82f12c320bdec9",
"kind": "SPAN_KIND_CLIENT",
"status": {},
"duration_ms": 1.546,
"attributes": {
"db.system.name": "postgresql"
}
},
{
"name": "postgresql query",
"service": "messenger-api",
"service_version": "6.0.0",
"environment": "lab",
"id": "cbfb8136b1705709",
"parent": "cb82f12c320bdec9",
"kind": "SPAN_KIND_CLIENT",
"status": {},
"duration_ms": 0.316,
"attributes": {
"db.system.name": "postgresql"
}
},
{
"name": "s3 put staging",
"service": "messenger-api",
"service_version": "6.0.0",
"environment": "lab",
"id": "6ed1178255aca64d",
"parent": "cb82f12c320bdec9",
"kind": "SPAN_KIND_INTERNAL",
"status": {},
"duration_ms": 27.349,
"attributes": {}
},
{
"name": "postgresql query",
"service": "messenger-api",
"service_version": "6.0.0",
"environment": "lab",
"id": "55ba79021d84459f",
"parent": "cb82f12c320bdec9",
"kind": "SPAN_KIND_CLIENT",
"status": {},
"duration_ms": 2.848,
"attributes": {
"db.system.name": "postgresql"
}
},
{
"name": "postgresql query",
"service": "messenger-api",
"service_version": "6.0.0",
"environment": "lab",
"id": "0b3bd94a696a7315",
"parent": "cb82f12c320bdec9",
"kind": "SPAN_KIND_CLIENT",
"status": {},
"duration_ms": 3.973,
"attributes": {
"db.system.name": "postgresql"
}
},
{
"name": "postgresql query",
"service": "messenger-api",
"service_version": "6.0.0",
"environment": "lab",
"id": "b7778f2677fb2938",
"parent": "cb82f12c320bdec9",
"kind": "SPAN_KIND_CLIENT",
"status": {},
"duration_ms": 2.195,
"attributes": {
"db.system.name": "postgresql"
}
},
{
"name": "attachment-jobs publish",
"service": "messenger-dispatcher",
"service_version": "6.0.0",
"environment": "lab",
"id": "729bce6d32aad08c",
"parent": "cb82f12c320bdec9",
"kind": "SPAN_KIND_PRODUCER",
"status": {},
"duration_ms": 10.783,
"attributes": {
"messaging.system": "valkey",
"messaging.destination.name": "attachment-jobs",
"messaging.message.id": "32d3a0c1-5025-4e89-98d3-0a04a613b2c1",
"job.attempt": "1"
}
},
{
"name": "postgresql query",
"service": "messenger-dispatcher",
"service_version": "6.0.0",
"environment": "lab",
"id": "692e866a70efd5dc",
"parent": "729bce6d32aad08c",
"kind": "SPAN_KIND_CLIENT",
"status": {},
"duration_ms": 2.66,
"attributes": {
"db.system.name": "postgresql"
}
},
{
"name": "postgresql query",
"service": "messenger-dispatcher",
"service_version": "6.0.0",
"environment": "lab",
"id": "c358146a81bb947a",
"parent": "729bce6d32aad08c",
"kind": "SPAN_KIND_CLIENT",
"status": {},
"duration_ms": 2.319,
"attributes": {
"db.system.name": "postgresql"
}
},
{
"name": "attachment-jobs process",
"service": "messenger-worker",
"service_version": "6.0.0",
"environment": "lab",
"id": "b7078277bfa428ae",
"parent": "729bce6d32aad08c",
"kind": "SPAN_KIND_CONSUMER",
"status": {
"message": "queued",
"code": "STATUS_CODE_ERROR"
},
"duration_ms": 24.826,
"attributes": {
"messaging.system": "valkey",
"messaging.destination.name": "attachment-jobs",
"messaging.message.id": "32d3a0c1-5025-4e89-98d3-0a04a613b2c1",
"job.attempt": "1",
"error.type": "processing_failed"
}
},
{
"name": "postgresql query",
"service": "messenger-worker",
"service_version": "6.0.0",
"environment": "lab",
"id": "44e5045ef5cdb271",
"parent": "b7078277bfa428ae",
"kind": "SPAN_KIND_CLIENT",
"status": {},
"duration_ms": 0.46,
"attributes": {
"db.system.name": "postgresql"
}
},
{
"name": "postgresql query",
"service": "messenger-worker",
"service_version": "6.0.0",
"environment": "lab",
"id": "67a030bcf008a4fc",
"parent": "b7078277bfa428ae",
"kind": "SPAN_KIND_CLIENT",
"status": {},
"duration_ms": 3.87,
"attributes": {
"db.system.name": "postgresql"
}
},
{
"name": "postgresql query",
"service": "messenger-worker",
"service_version": "6.0.0",
"environment": "lab",
"id": "70174e976f198166",
"parent": "b7078277bfa428ae",
"kind": "SPAN_KIND_CLIENT",
"status": {},
"duration_ms": 7.427,
"attributes": {
"db.system.name": "postgresql"
}
},
{
"name": "postgresql query",
"service": "messenger-worker",
"service_version": "6.0.0",
"environment": "lab",
"id": "2858bb2caca7f251",
"parent": "b7078277bfa428ae",
"kind": "SPAN_KIND_CLIENT",
"status": {},
"duration_ms": 1.437,
"attributes": {
"db.system.name": "postgresql"
}
},
{
"name": "postgresql query",
"service": "messenger-worker",
"service_version": "6.0.0",
"environment": "lab",
"id": "15e52515d8815a40",
"parent": "b7078277bfa428ae",
"kind": "SPAN_KIND_CLIENT",
"status": {},
"duration_ms": 0.906,
"attributes": {
"db.system.name": "postgresql"
}
},
{
"name": "postgresql query",
"service": "messenger-worker",
"service_version": "6.0.0",
"environment": "lab",
"id": "6b3e8a3493f6ab44",
"parent": "b7078277bfa428ae",
"kind": "SPAN_KIND_CLIENT",
"status": {},
"duration_ms": 2.088,
"attributes": {
"db.system.name": "postgresql"
}
},
{
"name": "postgresql query",
"service": "messenger-worker",
"service_version": "6.0.0",
"environment": "lab",
"id": "01c279a5420a3b3d",
"parent": "b7078277bfa428ae",
"kind": "SPAN_KIND_CLIENT",
"status": {},
"duration_ms": 0.297,
"attributes": {
"db.system.name": "postgresql"
}
},
{
"name": "postgresql query",
"service": "messenger-worker",
"service_version": "6.0.0",
"environment": "lab",
"id": "c9e5516f9a438875",
"parent": "b7078277bfa428ae",
"kind": "SPAN_KIND_CLIENT",
"status": {},
"duration_ms": 0.353,
"attributes": {
"db.system.name": "postgresql"
}
},
{
"name": "postgresql query",
"service": "messenger-worker",
"service_version": "6.0.0",
"environment": "lab",
"id": "6e1c4f53744d512b",
"parent": "b7078277bfa428ae",
"kind": "SPAN_KIND_CLIENT",
"status": {},
"duration_ms": 1.583,
"attributes": {
"db.system.name": "postgresql"
}
},
{
"name": "postgresql query",
"service": "messenger-worker",
"service_version": "6.0.0",
"environment": "lab",
"id": "3511005f73405b11",
"parent": "b7078277bfa428ae",
"kind": "SPAN_KIND_CLIENT",
"status": {},
"duration_ms": 0.977,
"attributes": {
"db.system.name": "postgresql"
}
},
{
"name": "postgresql query",
"service": "messenger-worker",
"service_version": "6.0.0",
"environment": "lab",
"id": "3697447557c7174b",
"parent": "b7078277bfa428ae",
"kind": "SPAN_KIND_CLIENT",
"status": {},
"duration_ms": 1.648,
"attributes": {
"db.system.name": "postgresql"
}
},
{
"name": "postgresql query",
"service": "messenger-worker",
"service_version": "6.0.0",
"environment": "lab",
"id": "90236fe11694b341",
"parent": "b7078277bfa428ae",
"kind": "SPAN_KIND_CLIENT",
"status": {},
"duration_ms": 1.722,
"attributes": {
"db.system.name": "postgresql"
}
},
{
"name": "attachment-jobs publish",
"service": "messenger-dispatcher",
"service_version": "6.0.0",
"environment": "lab",
"id": "6e33ffcb8ab839b8",
"parent": "cb82f12c320bdec9",
"kind": "SPAN_KIND_PRODUCER",
"status": {},
"duration_ms": 9.795,
"attributes": {
"messaging.system": "valkey",
"messaging.destination.name": "attachment-jobs",
"messaging.message.id": "32d3a0c1-5025-4e89-98d3-0a04a613b2c1",
"job.attempt": "2"
}
},
{
"name": "postgresql query",
"service": "messenger-dispatcher",
"service_version": "6.0.0",
"environment": "lab",
"id": "109b4f1901365727",
"parent": "6e33ffcb8ab839b8",
"kind": "SPAN_KIND_CLIENT",
"status": {},
"duration_ms": 2.09,
"attributes": {
"db.system.name": "postgresql"
}
},
{
"name": "postgresql query",
"service": "messenger-dispatcher",
"service_version": "6.0.0",
"environment": "lab",
"id": "dd1eea380140a5f6",
"parent": "6e33ffcb8ab839b8",
"kind": "SPAN_KIND_CLIENT",
"status": {},
"duration_ms": 1.681,
"attributes": {
"db.system.name": "postgresql"
}
},
{
"name": "attachment-jobs process",
"service": "messenger-worker",
"service_version": "6.0.0",
"environment": "lab",
"id": "85d8bed17c2a983b",
"parent": "6e33ffcb8ab839b8",
"kind": "SPAN_KIND_CONSUMER",
"status": {},
"duration_ms": 505.465,
"attributes": {
"messaging.system": "valkey",
"messaging.destination.name": "attachment-jobs",
"messaging.message.id": "32d3a0c1-5025-4e89-98d3-0a04a613b2c1",
"job.attempt": "2"
}
},
{
"name": "postgresql query",
"service": "messenger-worker",
"service_version": "6.0.0",
"environment": "lab",
"id": "2c9d90656209ec8a",
"parent": "85d8bed17c2a983b",
"kind": "SPAN_KIND_CLIENT",
"status": {},
"duration_ms": 0.239,
"attributes": {
"db.system.name": "postgresql"
}
},
{
"name": "postgresql query",
"service": "messenger-worker",
"service_version": "6.0.0",
"environment": "lab",
"id": "f1220d5fb103764f",
"parent": "85d8bed17c2a983b",
"kind": "SPAN_KIND_CLIENT",
"status": {},
"duration_ms": 1.062,
"attributes": {
"db.system.name": "postgresql"
}
},
{
"name": "postgresql query",
"service": "messenger-worker",
"service_version": "6.0.0",
"environment": "lab",
"id": "1b2b0b7c986a4cfd",
"parent": "85d8bed17c2a983b",
"kind": "SPAN_KIND_CLIENT",
"status": {},
"duration_ms": 1.586,
"attributes": {
"db.system.name": "postgresql"
}
},
{
"name": "postgresql query",
"service": "messenger-worker",
"service_version": "6.0.0",
"environment": "lab",
"id": "f56614767d35353e",
"parent": "85d8bed17c2a983b",
"kind": "SPAN_KIND_CLIENT",
"status": {},
"duration_ms": 0.655,
"attributes": {
"db.system.name": "postgresql"
}
},
{
"name": "postgresql query",
"service": "messenger-worker",
"service_version": "6.0.0",
"environment": "lab",
"id": "2ff7311f6904be09",
"parent": "85d8bed17c2a983b",
"kind": "SPAN_KIND_CLIENT",
"status": {},
"duration_ms": 0.377,
"attributes": {
"db.system.name": "postgresql"
}
},
{
"name": "postgresql query",
"service": "messenger-worker",
"service_version": "6.0.0",
"environment": "lab",
"id": "c1a82b84da10c3ea",
"parent": "85d8bed17c2a983b",
"kind": "SPAN_KIND_CLIENT",
"status": {},
"duration_ms": 5.553,
"attributes": {
"db.system.name": "postgresql"
}
},
{
"name": "s3 get source",
"service": "messenger-worker",
"service_version": "6.0.0",
"environment": "lab",
"id": "71c81ad580104e19",
"parent": "85d8bed17c2a983b",
"kind": "SPAN_KIND_INTERNAL",
"status": {},
"duration_ms": 21.795,
"attributes": {}
},
{
"name": "s3 put result",
"service": "messenger-worker",
"service_version": "6.0.0",
"environment": "lab",
"id": "12ec4340343aba83",
"parent": "85d8bed17c2a983b",
"kind": "SPAN_KIND_INTERNAL",
"status": {},
"duration_ms": 17.09,
"attributes": {}
},
{
"name": "s3 put result",
"service": "messenger-worker",
"service_version": "6.0.0",
"environment": "lab",
"id": "46417edf36ab8178",
"parent": "85d8bed17c2a983b",
"kind": "SPAN_KIND_INTERNAL",
"status": {},
"duration_ms": 9.113,
"attributes": {}
},
{
"name": "s3 put result",
"service": "messenger-worker",
"service_version": "6.0.0",
"environment": "lab",
"id": "7f8ea743f7a47625",
"parent": "85d8bed17c2a983b",
"kind": "SPAN_KIND_INTERNAL",
"status": {},
"duration_ms": 10.203,
"attributes": {}
},
{
"name": "s3 put result",
"service": "messenger-worker",
"service_version": "6.0.0",
"environment": "lab",
"id": "70e158035b438fde",
"parent": "85d8bed17c2a983b",
"kind": "SPAN_KIND_INTERNAL",
"status": {},
"duration_ms": 9.493,
"attributes": {}
},
{
"name": "postgresql query",
"service": "messenger-worker",
"service_version": "6.0.0",
"environment": "lab",
"id": "aef9af71f9ae1713",
"parent": "85d8bed17c2a983b",
"kind": "SPAN_KIND_CLIENT",
"status": {},
"duration_ms": 0.212,
"attributes": {
"db.system.name": "postgresql"
}
},
{
"name": "postgresql query",
"service": "messenger-worker",
"service_version": "6.0.0",
"environment": "lab",
"id": "ca951d1add2c301e",
"parent": "85d8bed17c2a983b",
"kind": "SPAN_KIND_CLIENT",
"status": {},
"duration_ms": 0.174,
"attributes": {
"db.system.name": "postgresql"
}
},
{
"name": "postgresql query",
"service": "messenger-worker",
"service_version": "6.0.0",
"environment": "lab",
"id": "a954de7157fa8da0",
"parent": "85d8bed17c2a983b",
"kind": "SPAN_KIND_CLIENT",
"status": {},
"duration_ms": 5.575,
"attributes": {
"db.system.name": "postgresql"
}
},
{
"name": "postgresql query",
"service": "messenger-worker",
"service_version": "6.0.0",
"environment": "lab",
"id": "ad1356ed8a750091",
"parent": "85d8bed17c2a983b",
"kind": "SPAN_KIND_CLIENT",
"status": {},
"duration_ms": 2.231,
"attributes": {
"db.system.name": "postgresql"
}
},
{
"name": "postgresql query",
"service": "messenger-worker",
"service_version": "6.0.0",
"environment": "lab",
"id": "010c5601f972fe44",
"parent": "85d8bed17c2a983b",
"kind": "SPAN_KIND_CLIENT",
"status": {},
"duration_ms": 1.35,
"attributes": {
"db.system.name": "postgresql"
}
}
],
"privacy": {
"synthetic_canaries_absent": true,
"metric_series_checked": 818
},
"grafana": {
"dashboard_uid": "messenger-lab6",
"panels": 18,
"datasources": [
"lab-loki",
"lab-prom",
"lab-tempo"
]
}
}
+45
View File
@@ -0,0 +1,45 @@
{
"dashboard": "messenger-lab6",
"panels": 18,
"query_result_series_counts": {
"1/A": 1,
"1/B": 1,
"2/A": 1,
"2/B": 1,
"3/A": 1,
"3/B": 1,
"3/C": 1,
"4/A": 1,
"5/A": 1,
"5/B": 1,
"5/C": 1,
"6/A": 1,
"6/B": 1,
"6/C": 1,
"7/A": 1,
"8/A": 1,
"8/B": 1,
"9/A": 4,
"9/B": 4,
"9/C": 4,
"10/A": 4,
"11/A": 1,
"11/B": 1,
"12/A": 4,
"13/A": 4,
"14/A": 1,
"14/B": 1,
"14/C": 1,
"15/A": 1,
"15/B": 1,
"15/C": 1,
"15/D": 1,
"15/E": 1,
"15/F": 1,
"15/G": 1,
"15/H": 1,
"16/A": 0,
"17/A": 3,
"17/B": 1
}
}
+3
View File
@@ -0,0 +1,3 @@
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
{"observed_instances_before": 2, "successful_writes": 121, "failed_attempts": 2, "first_recovery_seconds": 4.008}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
+3
View File
@@ -0,0 +1,3 @@
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
{"observed_instances_before": 2, "successful_writes": 122, "failed_attempts": 3, "first_recovery_seconds": 3.765}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
+4
View File
@@ -0,0 +1,4 @@
? mtusi-messenger [no test files]
? mtusi-messenger/infra [no test files]
go vet ./...
PASS
+6
View File
@@ -0,0 +1,6 @@
{
"timestamp_utc_unix": 1795101704.0,
"historical_metric_series": 56,
"trace_spans": 45,
"loki_has_same_trace": true
}
+3
View File
@@ -0,0 +1,3 @@
image/jpeg attachment=a33cfbff-85c7-4cc1-ad05-06a9ea68f307 dimensions=[(85, 256), (128, 128), (100, 300)]
image/png attachment=cc7875e5-4567-4a53-9a25-96b7759c46f6 dimensions=[(8, 4), (128, 128), (8, 4)]
PASS EXIF, fit, crop, WebP, transparency, metadata removal, byte/pixel/format limits
+17
View File
@@ -0,0 +1,17 @@
{
"lab": 6,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 30.0,
"elapsed_seconds": 30.099,
"concurrency": 4,
"successful_requests": 10833,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 359.916,
"successful_latency_ms": {
"p50": 8.063,
"p95": 28.328,
"p99": 44.278
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
}
+2
View File
@@ -0,0 +1,2 @@
python scripts/validate_openapi.py
PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.
+2
View File
@@ -0,0 +1,2 @@
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
+7
View File
@@ -0,0 +1,7 @@
Checking /etc/prometheus/prometheus.yml
SUCCESS: 1 rule files found
SUCCESS: /etc/prometheus/prometheus.yml is valid prometheus config file syntax
Checking /etc/prometheus/alerts.yml
SUCCESS: 5 rules found
+19
View File
@@ -0,0 +1,19 @@
{
"api_and_broker_started_at_before": {
"b3490044297c032e64169009292602013a135297f81c1c1fb0325ed407e89265": "2026-11-19T15:22:04.660528916Z",
"a3593069ffcbb27029199a623467b3e03588be267276d05dfa1d3b3905cbf1ba": "2026-11-19T15:22:41.877466473Z",
"81e2248f40861fff6b040e03b21e094e440f026c5fd38b57405e67e6da955dc8": "2026-11-19T15:21:57.545000764Z"
},
"api_and_broker_started_at_after": {
"b3490044297c032e64169009292602013a135297f81c1c1fb0325ed407e89265": "2026-11-19T15:22:48.941842698Z",
"a3593069ffcbb27029199a623467b3e03588be267276d05dfa1d3b3905cbf1ba": "2026-11-19T15:22:48.919052841Z",
"81e2248f40861fff6b040e03b21e094e440f026c5fd38b57405e67e6da955dc8": "2026-11-19T15:22:47.299849344Z"
},
"stream_entry_before_and_after": "1795101766315-0",
"worker_stopped_during_restart": true,
"attachment_status_before": "queued",
"attachment_status_after": "ready",
"message_preserved": true,
"original_equal": true,
"thumbnail_jpeg": true
}
+14
View File
@@ -0,0 +1,14 @@
b3490044297c032e64169009292602013a135297f81c1c1fb0325ed407e89265 api 2026-11-19T15:22:04.660528916Z running
a3593069ffcbb27029199a623467b3e03588be267276d05dfa1d3b3905cbf1ba api 2026-11-19T15:22:04.718705863Z running
81e2248f40861fff6b040e03b21e094e440f026c5fd38b57405e67e6da955dc8 broker 2026-11-19T15:21:57.545000764Z running
8e8caf1c0b92fe8a396d8d17ba0f434d333a593b9f3a8ca3d51c9644f0454dff db 2026-11-19T15:21:57.443508985Z running
e7fec4458e180dbfe1e9090e5f3c965c4969f946bea8b2c4935efb096f771c62 dispatcher 2026-11-19T15:22:03.216342897Z running
c9fc21f8e01b4b5e7494a441f21dfe5c99419b7be13671f4e46fc334570b5bef grafana 2026-11-19T15:21:56.881592788Z running
24f436d4c181a0a4dee586d51bbdc414db164a72ee26e5d5aa3c372d4e309e27 lb 2026-11-19T15:22:03.717366023Z running
ca3218b7de15da333652bac0acd52cc1970618a16f8ff20aa5c5dc374669782e loki 2026-11-19T15:21:57.633609518Z running
49242f770bc3942d2d2b48f1e10fe4fbe9e0a6c22eb32e8de0b7ec204166abfd otel-collector 2026-11-19T15:21:57.350753091Z running
b86cec226e5ee13e688914c0cbe02d3f2a9582fdbe24e08aa9c2f156cbed453e prometheus 2026-11-19T15:21:57.233000032Z running
f422c361d3b73c21192d55f210e16f4a92e41d01255ab3b253a6e72401927262 s3 2026-11-19T15:21:56.455639643Z running
d0a1fdadae9bcbb9884a26f130cc5794c804a64b76b16e8f247de44233618d88 tempo 2026-11-19T15:21:57.915001534Z running
13375cdf7532c9d8196e0c13f69304b20d91b4902e3aeed68b3f5cf2324a5282 valkey 2026-11-19T15:21:56.506879436Z running
d203da2fd37a6083c37792df0876a29a355066c413c81c6f2e99c3e3d55d5885 worker 2026-11-19T15:22:03.639788826Z running
+14
View File
@@ -0,0 +1,14 @@
b3490044297c032e64169009292602013a135297f81c1c1fb0325ed407e89265 api 2026-11-19T15:20:56.217827116Z running
a3593069ffcbb27029199a623467b3e03588be267276d05dfa1d3b3905cbf1ba api 2026-11-19T15:20:55.945663535Z running
81e2248f40861fff6b040e03b21e094e440f026c5fd38b57405e67e6da955dc8 broker 2026-11-19T15:17:35.003012286Z running
8e8caf1c0b92fe8a396d8d17ba0f434d333a593b9f3a8ca3d51c9644f0454dff db 2026-11-19T15:17:36.53753761Z running
e7fec4458e180dbfe1e9090e5f3c965c4969f946bea8b2c4935efb096f771c62 dispatcher 2026-11-19T15:17:35.043378833Z running
c9fc21f8e01b4b5e7494a441f21dfe5c99419b7be13671f4e46fc334570b5bef grafana 2026-11-19T15:17:35.613855763Z running
24f436d4c181a0a4dee586d51bbdc414db164a72ee26e5d5aa3c372d4e309e27 lb 2026-11-19T15:17:36.965294838Z running
ca3218b7de15da333652bac0acd52cc1970618a16f8ff20aa5c5dc374669782e loki 2026-11-19T15:17:36.424441654Z running
49242f770bc3942d2d2b48f1e10fe4fbe9e0a6c22eb32e8de0b7ec204166abfd otel-collector 2026-11-19T15:17:36.857405626Z running
b86cec226e5ee13e688914c0cbe02d3f2a9582fdbe24e08aa9c2f156cbed453e prometheus 2026-11-19T15:17:35.846347534Z running
f422c361d3b73c21192d55f210e16f4a92e41d01255ab3b253a6e72401927262 s3 2026-11-19T15:17:35.377275893Z running
d0a1fdadae9bcbb9884a26f130cc5794c804a64b76b16e8f247de44233618d88 tempo 2026-11-19T15:17:36.774750656Z running
13375cdf7532c9d8196e0c13f69304b20d91b4902e3aeed68b3f5cf2324a5282 valkey 2026-11-19T15:17:35.42001053Z running
d203da2fd37a6083c37792df0876a29a355066c413c81c6f2e99c3e3d55d5885 worker 2026-11-19T15:17:35.244593186Z running
+16
View File
@@ -0,0 +1,16 @@
api mtusi-messenger:lab6 Up About a minute (healthy)
api mtusi-messenger:lab6 Up About a minute (healthy)
broker valkey/valkey:8.1.3-bookworm@sha256:fea8b3e67b15729d4bb70589eb03367bab9ad1ee89c876f54327fc7c6e618571 Up About a minute (healthy)
db postgres:17.6-bookworm@sha256:f3bd19c606e442c3d7bdfa8002e03fe260a1023351e0ea4598032022b68dd6e3 Up About a minute (healthy)
dispatcher mtusi-messenger:lab6 Up About a minute
grafana grafana/grafana:12.1.1 Up 6 minutes
lb haproxy:3.2.6-alpine Up About a minute (healthy)
loki grafana/loki:3.5.3 Up 6 minutes
migrate mtusi-messenger:lab6 Exited (0) About a minute ago
otel-collector otel/opentelemetry-collector-contrib:0.133.0 Up 6 minutes
prometheus prom/prometheus:v3.5.0 Up 6 minutes
s3 mtusi-minio:RELEASE.2025-10-15T17-29-55Z Up About a minute (healthy)
s3-init mtusi-minio-tools:RELEASE.2025-08-13T08-35-41Z Exited (0) About a minute ago
tempo grafana/tempo:2.8.2 Up 6 minutes
valkey valkey/valkey:8.1.3-bookworm@sha256:fea8b3e67b15729d4bb70589eb03367bab9ad1ee89c876f54327fc7c6e618571 Up About a minute (healthy)
worker mtusi-messenger-worker:lab6 Up About a minute
File diff suppressed because it is too large Load Diff
+2
View File
@@ -0,0 +1,2 @@
PASS: captured fault measurements and recovered states meet study criteria
{'baseline': [], 'short_db_spike': [], 'database_lock': ['APIFastBurn', 'APISlowBurn'], 'worker_stopped': ['QueueLagHigh', 'ReadyFastBurn', 'ReadySlowBurn']}
+2
View File
@@ -0,0 +1,2 @@
python tests/smoke.py --base-url "http://127.0.0.1:8080"
PASS: lab6 public API scenario; infrastructure and grading evidence remain separate.
+1
View File
@@ -0,0 +1 @@
a3593069ffcbb27029199a623467b3e03588be267276d05dfa1d3b3905cbf1ba exited 2026-11-19T15:22:18.560680121Z 0
+1
View File
@@ -0,0 +1 @@
a3593069ffcbb27029199a623467b3e03588be267276d05dfa1d3b3905cbf1ba running 2026-11-19T15:22:04.718705863Z
+1
View File
@@ -0,0 +1 @@
b3490044297c032e64169009292602013a135297f81c1c1fb0325ed407e89265 exited 2026-11-19T15:24:04.56002428Z 0
+1
View File
@@ -0,0 +1 @@
b3490044297c032e64169009292602013a135297f81c1c1fb0325ed407e89265 running 2026-11-19T15:22:48.941842698Z
+19
View File
@@ -0,0 +1,19 @@
{
"baseline_fresh_ram_peaks_bytes": {
"api": 388411392.0,
"worker": 30007296.0
},
"db_recovery_cross_alert": {
"timestamp": 1795100942.6297588,
"observed_upload_outcomes_25s_estimate": [
{
"outcome": "missed",
"value": 12.971290210999655
},
{
"outcome": "good",
"value": 14.052231061916292
}
]
}
}
+58
View File
@@ -0,0 +1,58 @@
module mtusi-messenger
go 1.26.0
toolchain go1.27.3
require (
github.com/jackc/pgx/v5 v5.9.2
github.com/minio/minio-go/v7 v7.4.1
github.com/redis/go-redis/v9 v9.12.1
go.opentelemetry.io/otel v1.38.0
go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.14.0
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.38.0
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0
go.opentelemetry.io/otel/log v0.14.0
go.opentelemetry.io/otel/metric v1.38.0
go.opentelemetry.io/otel/sdk v1.38.0
go.opentelemetry.io/otel/sdk/log v0.14.0
go.opentelemetry.io/otel/sdk/metric v1.38.0
go.opentelemetry.io/otel/trace v1.38.0
golang.org/x/crypto v0.57.0
)
require (
github.com/cenkalti/backoff/v5 v5.0.3 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/go-logr/logr v1.4.3 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
github.com/jackc/puddle/v2 v2.2.2 // indirect
github.com/klauspost/compress v1.19.2 // indirect
github.com/klauspost/cpuid/v2 v2.4.0 // indirect
github.com/klauspost/crc32 v1.3.0 // indirect
github.com/minio/crc64nvme v1.1.1 // indirect
github.com/minio/md5-simd v1.1.2 // indirect
github.com/philhofer/fwd v1.2.0 // indirect
github.com/rs/xid v1.6.0 // indirect
github.com/tinylib/msgp v1.6.4 // indirect
github.com/zeebo/xxh3 v1.1.0 // indirect
go.opentelemetry.io/auto/sdk v1.1.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 // indirect
go.opentelemetry.io/proto/otlp v1.7.1 // indirect
go.yaml.in/yaml/v3 v3.0.5 // indirect
golang.org/x/net v0.60.0 // indirect
golang.org/x/sync v0.23.0 // indirect
golang.org/x/sys v0.48.0 // indirect
golang.org/x/text v0.42.0 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5 // indirect
google.golang.org/grpc v1.75.0 // indirect
google.golang.org/protobuf v1.36.12 // indirect
gopkg.in/ini.v1 v1.67.3 // indirect
)
+130
View File
@@ -0,0 +1,130 @@
github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM=
github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 h1:8Tjv8EJ+pM1xP8mK6egEbD1OgnVTyacbefKhmbLhIhU=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2/go.mod h1:pkJQ2tZHJ0aFOVEEot6oZmaVEZcRme73eIFmhiVuRWs=
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
github.com/jackc/pgx/v5 v5.9.2 h1:3ZhOzMWnR4yJ+RW1XImIPsD1aNSz4T4fyP7zlQb56hw=
github.com/jackc/pgx/v5 v5.9.2/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4=
github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8=
github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw=
github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU=
github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM=
github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw=
github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI=
github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg=
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
github.com/minio/minio-go/v7 v7.4.1 h1:PtU/zIOfQRWZ+QydXAZysc+xZYHdG+BBR4Z6TUvy73w=
github.com/minio/minio-go/v7 v7.4.1/go.mod h1:nQnQi7zTfXR4lREtYXZQWqo5aw0Rqv+ZqCGrTORygRk=
github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM=
github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/redis/go-redis/v9 v9.12.1 h1:k5iquqv27aBtnTm2tIkROUDp8JBXhXZIVu1InSgvovg=
github.com/redis/go-redis/v9 v9.12.1/go.mod h1:huWgSWd8mW6+m0VPhJjSSQ+d6Nh1VICQ6Q5lHuCH/Iw=
github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE=
github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg=
github.com/tinylib/msgp v1.6.4 h1:mOwYbyYDLPj35mkA2BjjYejgJk9BuHxDdvRnb6v2ZcQ=
github.com/tinylib/msgp v1.6.4/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA=
github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs=
github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s=
go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA=
go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A=
go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8=
go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM=
go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.14.0 h1:QQqYw3lkrzwVsoEX0w//EhH/TCnpRdEenKBOOEIMjWc=
go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.14.0/go.mod h1:gSVQcr17jk2ig4jqJ2DX30IdWH251JcNAecvrqTxH1s=
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.38.0 h1:Oe2z/BCg5q7k4iXC3cqJxKYg0ieRiOqF0cecFYdPTwk=
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.38.0/go.mod h1:ZQM5lAJpOsKnYagGg/zV2krVqTtaVdYdDkhMoX6Oalg=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 h1:GqRJVj7UmLjCVyVJ3ZFLdPRmhDUp2zFmQe3RHIOsw24=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0/go.mod h1:ri3aaHSmCTVYu2AWv44YMauwAQc0aqI9gHKIcSbI1pU=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0 h1:aTL7F04bJHUlztTsNGJ2l+6he8c+y/b//eR0jjjemT4=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0/go.mod h1:kldtb7jDTeol0l3ewcmd8SDvx3EmIE7lyvqbasU3QC4=
go.opentelemetry.io/otel/log v0.14.0 h1:2rzJ+pOAZ8qmZ3DDHg73NEKzSZkhkGIua9gXtxNGgrM=
go.opentelemetry.io/otel/log v0.14.0/go.mod h1:5jRG92fEAgx0SU/vFPxmJvhIuDU9E1SUnEQrMlJpOno=
go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA=
go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI=
go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E=
go.opentelemetry.io/otel/sdk v1.38.0/go.mod h1:ghmNdGlVemJI3+ZB5iDEuk4bWA3GkTpW+DOoZMYBVVg=
go.opentelemetry.io/otel/sdk/log v0.14.0 h1:JU/U3O7N6fsAXj0+CXz21Czg532dW2V4gG1HE/e8Zrg=
go.opentelemetry.io/otel/sdk/log v0.14.0/go.mod h1:imQvII+0ZylXfKU7/wtOND8Hn4OpT3YUoIgqJVksUkM=
go.opentelemetry.io/otel/sdk/log/logtest v0.14.0 h1:Ijbtz+JKXl8T2MngiwqBlPaHqc4YCaP/i13Qrow6gAM=
go.opentelemetry.io/otel/sdk/log/logtest v0.14.0/go.mod h1:dCU8aEL6q+L9cYTqcVOk8rM9Tp8WdnHOPLiBgp0SGOA=
go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6qT5wthqPoM=
go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA=
go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE=
go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs=
go.opentelemetry.io/proto/otlp v1.7.1 h1:gTOMpGDb0WTBOP8JaO72iL3auEZhVmAQg4ipjOVAtj4=
go.opentelemetry.io/proto/otlp v1.7.1/go.mod h1:b2rVh6rfI/s2pHWNlB7ILJcRALpcNDzKhACevjI+ZnE=
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
golang.org/x/net v0.60.0 h1:79p50tfZlm0J9YfoDsSi639qSXNGVwEzOPLCxM2FsYU=
golang.org/x/net v0.60.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg=
golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk=
golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI=
golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E=
gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E=
google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5 h1:BIRfGDEjiHRrk0QKZe3Xv2ieMhtgRGeLcZQ0mIVn4EY=
google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5/go.mod h1:j3QtIyytwqGr1JUDtYXwtMXWPKsEa5LtzIFN1Wn5WvE=
google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5 h1:eaY8u2EuxbRv7c3NiGK0/NedzVsCcV6hDuU5qPX5EGE=
google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5/go.mod h1:M4/wBTSeyLxupu3W3tJtOgB14jILAS/XWPSSa3TAlJc=
google.golang.org/grpc v1.75.0 h1:+TW+dqTd2Biwe6KKfhE5JpiYIBWq865PhKGSXiivqt4=
google.golang.org/grpc v1.75.0/go.mod h1:JtPAzKiq4v1xcAB2hydNlWI2RnF85XXcV0mhKXr2ecQ=
google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc=
google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/ini.v1 v1.67.3 h1:iM9Lhz5MRSGhHVGGwCuzG9KO8PoirCXj/m/qTmOJJQw=
gopkg.in/ini.v1 v1.67.3/go.mod h1:x/cyOwCgZqOkJoDIJ3c1KNHMo10+nLGAhh+kn3Zizss=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+81
View File
@@ -0,0 +1,81 @@
package main
import (
"context"
"errors"
"slices"
"sort"
"time"
"github.com/jackc/pgx/v5"
)
// The advisory lock serializes this key across processes. Message and result
// commit together; an interrupted/rolled-back transaction reserves nothing.
func (a *App) createMessage(ctx context.Context, chat, user, key, text string, ids []string) (Message, error) {
m := Message{ChatID: chat, SenderID: user, Text: text, AttachmentIDs: ids}
tx, err := a.db.Begin(ctx)
if err != nil {
return m, unavailable()
}
defer tx.Rollback(ctx)
// POST and the canonical chat path are implicit in this dedicated table.
if _, err = tx.Exec(ctx, "SELECT pg_advisory_xact_lock(hashtextextended($1,0))", user+":"+chat+":"+key); err != nil {
return m, unavailable()
}
var previous Message
var created time.Time
err = tx.QueryRow(ctx, `SELECT m.id::text,m.chat_id::text,m.sender_id::text,m.text,m.created_at,ARRAY(SELECT unnest(m.attachment_ids)::text)
FROM message_keys k JOIN messages m ON m.id=k.message_id
WHERE k.user_id=$1 AND k.chat_id=$2 AND k.key=$3 AND k.expires_at>clock_timestamp()`, user, chat, key).Scan(&previous.ID, &previous.ChatID, &previous.SenderID, utf8String{&previous.Text}, &created, &previous.AttachmentIDs)
if err == nil {
if previous.Text != text || !slices.Equal(previous.AttachmentIDs, ids) {
return m, apiError{409, "conflict", "Idempotency key has a different message"}
}
previous.CreatedAt = stamp(created)
return previous, nil
}
if !errors.Is(err, pgx.ErrNoRows) {
return m, unavailable()
}
if _, err = tx.Exec(ctx, "DELETE FROM message_keys WHERE user_id=$1 AND chat_id=$2 AND key=$3", user, chat, key); err != nil {
return m, unavailable()
}
// Serialize with attachment state transitions; stable ordering prevents cross-upload deadlocks.
locked := slices.Clone(ids)
sort.Strings(locked)
for _, id := range locked {
if err = attachmentLock(ctx, tx, id); err != nil {
return m, unavailable()
}
var missingSince *time.Time
var status string
err = tx.QueryRow(ctx, "SELECT missing_since,status FROM attachments WHERE id=$1 AND chat_id=$2", id, chat).Scan(&missingSince, &status)
if err != nil {
return m, dbError(err)
}
if status == "failed" {
return m, apiError{409, "conflict", "Attachment processing failed"}
}
if missingSince != nil {
return m, unavailable()
}
}
m.ID = uuid()
if err = tx.QueryRow(ctx, "INSERT INTO messages(id,chat_id,sender_id,text,attachment_ids) VALUES($1,$2,$3,$4,$5) RETURNING created_at", m.ID, chat, user, []byte(text), ids).Scan(&created); err != nil {
return m, dbError(err)
}
if _, err = tx.Exec(ctx, "INSERT INTO message_keys(user_id,chat_id,key,message_id,expires_at) VALUES($1,$2,$3,$4,clock_timestamp()+$5::bigint*interval '1 second')", user, chat, key, m.ID, int64(a.idempotencyTTL.Seconds())); err != nil {
return m, unavailable()
}
for _, id := range ids {
if _, err = tx.Exec(ctx, "INSERT INTO message_attachments(message_id,chat_id,attachment_id) VALUES($1,$2,$3)", m.ID, chat, id); err != nil {
return m, dbError(err)
}
}
if err = tx.Commit(ctx); err != nil {
return m, unavailable()
}
m.CreatedAt = stamp(created)
return m, nil
}
+54
View File
@@ -0,0 +1,54 @@
package main
import (
"context"
"errors"
"strings"
"time"
"github.com/jackc/pgx/v5"
"github.com/minio/minio-go/v7"
)
func (a *App) cleanStaging(ctx context.Context, retention int) error {
listCtx, cancel := context.WithCancel(ctx)
defer cancel()
for entry := range a.s3.ListObjects(listCtx, a.bucket, minio.ListObjectsOptions{Prefix: "staging/", Recursive: true}) {
if entry.Err != nil {
return entry.Err
}
id := strings.TrimPrefix(entry.Key, "staging/")
if normalized, err := normalizeID(id); err != nil || normalized != id {
continue
}
if err := a.cleanStagedImage(ctx, id, entry.LastModified, retention); err != nil {
return err
}
}
return ctx.Err()
}
func (a *App) cleanStagedImage(ctx context.Context, id string, modified time.Time, retention int) error {
tx, err := a.db.Begin(ctx)
if err != nil {
return err
}
defer rollbackAttachment(tx)
var locked bool
if err = tx.QueryRow(ctx, "SELECT pg_try_advisory_xact_lock(hashtextextended($1,0))", "attachment:"+id).Scan(&locked); err != nil || !locked {
return err
}
var removable bool
err = tx.QueryRow(ctx, `SELECT result_version>0 OR
(status='failed' AND updated_at<=clock_timestamp()-$2::bigint*interval '1 second')
FROM attachments WHERE id=$1`, id, retention).Scan(&removable)
if errors.Is(err, pgx.ErrNoRows) {
removable = time.Since(modified) > time.Hour
} else if err != nil {
return err
}
if removable {
return a.s3.RemoveObject(ctx, a.bucket, stagingKey(id), minio.RemoveObjectOptions{})
}
return nil
}
+136
View File
@@ -0,0 +1,136 @@
package main
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"io"
"os"
"os/exec"
"path/filepath"
"strconv"
"github.com/minio/minio-go/v7"
)
type imageFailure string
func (e imageFailure) Error() string { return string(e) }
func (a *App) generateImage(ctx context.Context, j imageJob, src imageSource) ([]Variant, error) {
dir, err := os.MkdirTemp("", "image-job-")
if err != nil {
return nil, err
}
defer os.RemoveAll(dir)
key := stagingKey(j.AttachmentID)
if j.Version == 2 {
key = resultKey(j.AttachmentID, 1, "original")
}
original := filepath.Join(dir, "input")
if err = a.fetchImage(ctx, src, key, original); err != nil {
return nil, err
}
command := exec.CommandContext(ctx, "/usr/local/bin/python3", "-I", "/pipeline.py", dir, src.contentType, strconv.Itoa(j.Version))
command.Env = []string{"LC_ALL=C.UTF-8"}
manifest, err := command.Output()
if err != nil {
var exit *exec.ExitError
if errors.As(err, &exit) && exit.ExitCode() == 2 {
return nil, imageFailure("invalid_image")
}
return nil, err
}
var variants []Variant
if err = json.Unmarshal(manifest, &variants); err != nil || len(variants) != 3 {
return nil, errors.New("invalid encoder manifest")
}
if err = a.putImage(ctx, resultKey(j.AttachmentID, j.Version, "original"), original, src); err != nil {
return nil, err
}
injectImageFault("after_effect", j.AttachmentID)
for i, name := range []string{"thumbnail", "crop", "optimized"} {
v := &variants[i]
v.Name = name
v.ContentType = "image/jpeg"
if name == "optimized" {
v.ContentType = "image/webp"
}
path := filepath.Join(dir, name)
v.SizeBytes, v.SHA256, err = fileDigest(path)
if err != nil {
return nil, err
}
if err = a.putImage(ctx, resultKey(j.AttachmentID, j.Version, name), path, imageSource{v.ContentType, v.SizeBytes, v.SHA256}); err != nil {
return nil, err
}
}
return variants, nil
}
func (a *App) fetchImage(ctx context.Context, src imageSource, key, path string) (result error) {
ctx, span := startSpan(ctx, "s3 get source")
defer func() { endSpan(span, result) }()
object, err := a.s3.GetObject(ctx, a.bucket, key, minio.GetObjectOptions{})
if err != nil {
return err
}
defer object.Close()
file, err := os.Create(path)
if err != nil {
return err
}
digest := sha256.New()
size, err := io.Copy(io.MultiWriter(file, digest), io.LimitReader(object, maxUploadBytes+1))
closeErr := file.Close()
if err != nil {
return err
}
if closeErr != nil {
return closeErr
}
if size != src.size || hex.EncodeToString(digest.Sum(nil)) != src.sha256 {
return imageFailure("source_integrity")
}
return nil
}
func fileDigest(path string) (int64, string, error) {
file, err := os.Open(path)
if err != nil {
return 0, "", err
}
defer file.Close()
digest := sha256.New()
size, err := io.Copy(digest, file)
return size, hex.EncodeToString(digest.Sum(nil)), err
}
func (a *App) putImage(ctx context.Context, key, path string, src imageSource) (result error) {
ctx, span := startSpan(ctx, "s3 put result")
defer func() { endSpan(span, result) }()
file, err := os.Open(path)
if err != nil {
return err
}
defer file.Close()
opts := minio.PutObjectOptions{ContentType: src.contentType, DisableMultipart: true,
UserMetadata: map[string]string{"sha256": src.sha256}}
opts.SetMatchETagExcept("*")
_, err = a.s3.PutObject(ctx, a.bucket, key, file, src.size, opts)
if minio.ToErrorResponse(err).Code != "PreconditionFailed" {
return err
}
info, err := a.s3.StatObject(ctx, a.bucket, key, minio.StatObjectOptions{})
if err != nil {
return err
}
if info.Size != src.size || info.ContentType != src.contentType || info.Metadata.Get("X-Amz-Meta-Sha256") != src.sha256 {
return imageFailure("result_integrity")
}
return nil
}
+263
View File
@@ -0,0 +1,263 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"math/rand/v2"
"os"
"time"
"github.com/jackc/pgx/v5"
"go.opentelemetry.io/otel/propagation"
)
type imageJob struct {
SchemaVersion int `json:"schema_version"`
JobID string `json:"job_id"`
Type string `json:"type"`
AttachmentID string `json:"attachment_id"`
Version int `json:"pipeline_version"`
Attempt int `json:"attempt"`
CreatedAt string `json:"created_at"`
}
type imageSource struct {
contentType string
size int64
sha256 string
}
func enqueueImage(ctx context.Context, tx pgx.Tx, attachment string, version int) error {
carrier := propagation.MapCarrier{}
traceContext.Inject(ctx, carrier)
_, err := tx.Exec(ctx, `INSERT INTO image_jobs(id,attachment_id,version,traceparent,tracestate)
VALUES($1,$2,$3,$4,$5)`, uuid(), attachment, version, carrier.Get("traceparent"), carrier.Get("tracestate"))
return err
}
func stagingKey(id string) string { return "staging/" + id }
func resultKey(id string, version int, variant string) string {
if version == 0 {
return "original/" + id
}
return fmt.Sprintf("%s/%s/v%d", variant, id, version)
}
func (a *App) claimImage(ctx context.Context, j imageJob, lease time.Duration) (imageSource, bool, error) {
var src imageSource
tx, err := a.db.Begin(ctx)
if err != nil {
return src, false, err
}
defer rollbackAttachment(tx)
if err = attachmentLock(ctx, tx, j.AttachmentID); err != nil {
return src, false, err
}
tag, err := tx.Exec(ctx, `UPDATE image_jobs j SET state='processing',
lease_until=clock_timestamp()+$5::bigint*interval '1 second', updated_at=clock_timestamp()
FROM attachments a WHERE a.id=j.attachment_id AND a.active_version=j.version
AND j.id=$1 AND j.attachment_id=$2 AND j.version=$3 AND j.attempt=$4
AND j.state='queued' AND j.next_at<=clock_timestamp()`, j.JobID, j.AttachmentID, j.Version, j.Attempt, int64(lease.Seconds()))
if err != nil || tag.RowsAffected() == 0 {
return src, false, err
}
if _, err = tx.Exec(ctx, `UPDATE attachments SET status='processing',updated_at=clock_timestamp()
WHERE id=$1 AND result_version=0`, j.AttachmentID); err != nil {
return src, false, err
}
err = tx.QueryRow(ctx, "SELECT content_type,size_bytes,sha256 FROM attachments WHERE id=$1", j.AttachmentID).
Scan(&src.contentType, &src.size, &src.sha256)
if err != nil {
return src, false, err
}
return src, true, tx.Commit(ctx)
}
func (a *App) completeImage(ctx context.Context, j imageJob, variants []Variant) (float64, bool, error) {
tx, err := a.db.Begin(ctx)
if err != nil {
return 0, false, err
}
defer rollbackAttachment(tx)
if err = attachmentLock(ctx, tx, j.AttachmentID); err != nil {
return 0, false, err
}
var seconds float64
err = tx.QueryRow(ctx, `UPDATE image_jobs SET state='ready',lease_until=NULL,error_code=NULL,
updated_at=clock_timestamp(),ready_at=clock_timestamp()
WHERE id=$1 AND attempt=$2 AND state='processing' AND lease_until>clock_timestamp()
RETURNING extract(epoch FROM (ready_at-created_at))::float8`, j.JobID, j.Attempt).Scan(&seconds)
if errors.Is(err, pgx.ErrNoRows) {
return 0, false, nil // A lost lease is recovered by the dispatcher.
}
if err != nil {
return 0, false, err
}
tag, err := tx.Exec(ctx, `UPDATE attachments SET status='ready',error_code=NULL,result_version=$2,
variants=$3,updated_at=clock_timestamp() WHERE id=$1 AND active_version=$2 AND result_version<$2`,
j.AttachmentID, j.Version, variants)
if err != nil {
return 0, false, err
}
if tag.RowsAffected() != 1 {
return 0, false, errors.New("image result fence rejected")
}
if err = tx.Commit(ctx); err != nil {
return 0, false, err
}
return seconds, true, nil
}
func (a *App) failImage(ctx context.Context, j imageJob, code string, permanent, expired bool, maxAttempts int) (string, error) {
tx, err := a.db.Begin(ctx)
if err != nil {
return "", err
}
defer rollbackAttachment(tx)
if err = attachmentLock(ctx, tx, j.AttachmentID); err != nil {
return "", err
}
var cycleStart int
err = tx.QueryRow(ctx, `SELECT cycle_start FROM image_jobs WHERE id=$1 AND attempt=$2
AND state='processing' AND (lease_until<=clock_timestamp())=$3 FOR UPDATE`, j.JobID, j.Attempt, expired).Scan(&cycleStart)
if errors.Is(err, pgx.ErrNoRows) {
return "lease_lost", nil
}
if err != nil {
return "", err
}
used := j.Attempt - cycleStart + 1
state := "failed"
if !permanent {
if used >= maxAttempts {
code = "retry_exhausted"
} else {
state = "queued"
}
}
attempt := j.Attempt
var delay int64
if state == "queued" {
attempt++
base := int64(1000) << min(used-1, 5)
delay = base + rand.Int64N(base)
}
_, err = tx.Exec(ctx, `UPDATE image_jobs SET state=$2,error_code=$3,lease_until=NULL,attempt=$4,
published=false,next_at=clock_timestamp()+$5::bigint*interval '1 millisecond',updated_at=clock_timestamp() WHERE id=$1`,
j.JobID, state, code, attempt, delay)
if err != nil {
return "", err
}
var publicCode *string
if state == "failed" {
publicCode = &code
}
if _, err = tx.Exec(ctx, `UPDATE attachments SET status=$2,error_code=$3,updated_at=clock_timestamp()
WHERE id=$1 AND active_version=$4 AND result_version=0`, j.AttachmentID, state, publicCode, j.Version); err != nil {
return "", err
}
return state, tx.Commit(ctx)
}
func (a *App) recoverImage(ctx context.Context, maxAttempts int) error {
var j imageJob
err := a.db.QueryRow(ctx, `SELECT id::text,attachment_id::text,version,attempt FROM image_jobs
WHERE state='processing' AND lease_until<=clock_timestamp() ORDER BY lease_until LIMIT 1`).
Scan(&j.JobID, &j.AttachmentID, &j.Version, &j.Attempt)
if errors.Is(err, pgx.ErrNoRows) {
return nil
}
if err != nil {
return err
}
_, err = a.failImage(ctx, j, "lease_expired", false, true, maxAttempts)
return err
}
func (a *App) upgradeImage(ctx context.Context, id string) error {
tx, err := a.db.Begin(ctx)
if err != nil {
return err
}
defer rollbackAttachment(tx)
if err = attachmentLock(ctx, tx, id); err != nil {
return err
}
tag, err := tx.Exec(ctx, `UPDATE attachments SET active_version=2
WHERE id=$1 AND active_version=1 AND result_version=1 AND status='ready'`, id)
if err != nil {
return err
}
if tag.RowsAffected() != 1 {
return errors.New("attachment is not eligible for upgrade")
}
if err = enqueueImage(ctx, tx, id, 2); err != nil {
return err
}
return tx.Commit(ctx)
}
func (a *App) replayImage(ctx context.Context, id string, retention int) error {
var attachment string
if err := a.db.QueryRow(ctx, "SELECT attachment_id::text FROM image_jobs WHERE id=$1", id).Scan(&attachment); err != nil {
return errors.New("job lookup failed")
}
tx, err := a.db.Begin(ctx)
if err != nil {
return err
}
defer rollbackAttachment(tx)
if err = attachmentLock(ctx, tx, attachment); err != nil {
return err
}
var attempt int
var code string
err = tx.QueryRow(ctx, `SELECT j.attempt,j.error_code FROM image_jobs j JOIN attachments a ON a.id=j.attachment_id
WHERE j.id=$1 AND j.state='failed' AND a.active_version=j.version AND a.result_version<j.version
AND (j.version=2 OR j.updated_at>clock_timestamp()-$2::bigint*interval '1 second')`, id, retention).Scan(&attempt, &code)
if errors.Is(err, pgx.ErrNoRows) {
return errors.New("job is not eligible for replay")
}
if err != nil {
return err
}
attempt++
if _, err = tx.Exec(ctx, "INSERT INTO image_job_replays(job_id,attempt,previous_error) VALUES($1,$2,$3)", id, attempt, code); err != nil {
return err
}
if _, err = tx.Exec(ctx, `UPDATE image_jobs SET state='queued',attempt=$2,cycle_start=$2,error_code=NULL,
published=false,next_at=clock_timestamp(),updated_at=clock_timestamp() WHERE id=$1`, id, attempt); err != nil {
return err
}
if _, err = tx.Exec(ctx, `UPDATE attachments SET status='queued',error_code=NULL,updated_at=clock_timestamp()
WHERE id=$1 AND result_version=0`, attachment); err != nil {
return err
}
return tx.Commit(ctx)
}
func (a *App) listFailedImages(ctx context.Context) error {
rows, err := a.db.Query(ctx, `SELECT id::text,attachment_id::text,version,attempt,error_code,updated_at FROM image_jobs
WHERE state='failed' ORDER BY updated_at DESC,id LIMIT 100`)
if err != nil {
return err
}
defer rows.Close()
out := json.NewEncoder(os.Stdout)
for rows.Next() {
var j imageJob
var code string
var updated time.Time
if err = rows.Scan(&j.JobID, &j.AttachmentID, &j.Version, &j.Attempt, &code, &updated); err != nil {
return err
}
if err = out.Encode(map[string]any{"job_id": j.JobID, "attachment_id": j.AttachmentID,
"pipeline_version": j.Version, "attempt": j.Attempt, "error_code": code, "updated_at": stamp(updated)}); err != nil {
return err
}
}
return rows.Err()
}
+316
View File
@@ -0,0 +1,316 @@
package main
import (
"context"
"encoding/json"
"errors"
"log/slog"
"os"
"os/signal"
"strings"
"syscall"
"time"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool"
"github.com/redis/go-redis/v9"
"go.opentelemetry.io/otel/attribute"
otellog "go.opentelemetry.io/otel/log"
"go.opentelemetry.io/otel/metric"
"go.opentelemetry.io/otel/propagation"
)
const imageStream = "attachment-jobs"
const imageGroup = "image-workers"
var ackImage = redis.NewScript(`
if redis.call('XACK', KEYS[1], ARGV[1], ARGV[2]) == 1 then
redis.call('XDEL', KEYS[1], ARGV[2])
end
return 1
`)
func imageCommand(db *pgxpool.Pool, dbTrace *databaseTracer, args []string) error {
a := &App{db: db}
retention := positive("STAGING_RETENTION_SECONDS", 86400)
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
if args[0] == "jobs" {
ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
if len(args) == 2 && args[1] == "failed" {
return a.listFailedImages(ctx)
}
if len(args) != 3 {
return errors.New("usage: jobs failed | jobs replay <job-id> | jobs upgrade <attachment-id>")
}
id, err := normalizeID(args[2])
if err != nil {
return errors.New("invalid job or attachment ID")
}
switch args[1] {
case "replay":
return a.replayImage(ctx, id, retention)
case "upgrade":
return a.upgradeImage(ctx, id)
default:
return errors.New("unknown jobs command")
}
}
if len(args) != 1 || (args[0] != "dispatcher" && args[0] != "worker") {
return errors.New("unknown command")
}
attempts := positive("JOB_MAX_ATTEMPTS", 5)
timeoutSeconds := positive("JOB_TIMEOUT_SECONDS", 30)
if attempts > 20 || timeoutSeconds > 120 || positive("WORKER_CONCURRENCY", 1) != 1 {
return errors.New("invalid image job limits")
}
timeout := time.Duration(timeoutSeconds) * time.Second
s3, bucket, err := s3Client()
if err != nil {
return errors.New("invalid image storage configuration")
}
a.s3, a.bucket = s3, bucket
brokerURL, err := secretFile("BROKER_URL_FILE")
if err != nil {
return err
}
opts, err := redis.ParseURL(brokerURL)
if err != nil {
return errors.New("invalid broker configuration")
}
opts.MaxRetries = -1
opts.PoolSize = 2
opts.DialTimeout = time.Second
opts.ReadTimeout = 2 * time.Second
opts.WriteTimeout = time.Second
opts.ContextTimeoutEnabled = true
broker := redis.NewClient(opts)
defer broker.Close()
instruments, shutdown, err := startTelemetry(args[0], db, dbTrace)
if err != nil {
return err
}
defer shutdown()
a.telemetry = instruments
if args[0] == "dispatcher" {
a.dispatchImages(ctx, broker, attempts, retention)
} else {
a.consumeImages(ctx, broker, attempts, timeout)
}
return nil
}
func injectImageFault(phase, attachment string) {
if os.Getenv("LAB5_FAULT_ATTACHMENT") == attachment && os.Getenv("LAB5_FAULT") == phase {
slog.Info("test crash", "phase", phase, "attachment_id", attachment)
os.Exit(86)
}
}
func (a *App) publishImage(ctx context.Context, broker *redis.Client) (published bool, result error) {
tx, err := a.db.Begin(ctx)
if err != nil {
return false, err
}
defer rollbackAttachment(tx)
j := imageJob{SchemaVersion: 1, Type: "attachment.process.v1"}
var created time.Time
var parent, state string
err = tx.QueryRow(ctx, `SELECT id::text,attachment_id::text,version,attempt,created_at,traceparent,tracestate FROM image_jobs
WHERE state='queued' AND NOT published AND next_at<=clock_timestamp()
ORDER BY next_at,id LIMIT 1 FOR UPDATE SKIP LOCKED`).Scan(&j.JobID, &j.AttachmentID, &j.Version, &j.Attempt, &created, &parent, &state)
if errors.Is(err, pgx.ErrNoRows) {
return false, nil
}
if err != nil {
return false, err
}
j.CreatedAt = stamp(created)
payload, err := json.Marshal(j)
if err != nil {
return false, err
}
ctx, span := a.telemetry.startJob(ctx, propagation.MapCarrier{"traceparent": parent, "tracestate": state}, j, true)
defer func() { endSpan(span, result) }()
carrier := propagation.MapCarrier{}
traceContext.Inject(ctx, carrier)
values := map[string]any{"job": string(payload)}
for key, value := range carrier {
values[key] = value
}
injectImageFault("before_publish", j.AttachmentID)
// XADD replies after the broker's AOF durability boundary. A lost reply
// or crash before the SQL commit causes another delivery of the same job.
err = broker.XAdd(ctx, &redis.XAddArgs{Stream: imageStream, Values: values}).Err()
if err != nil {
return false, err
}
injectImageFault("after_publish", j.AttachmentID)
if _, err = tx.Exec(ctx, "UPDATE image_jobs SET published=true WHERE id=$1", j.JobID); err != nil {
return false, err
}
err = tx.Commit(ctx)
if err == nil {
a.telemetry.event(ctx, "image.published", otellog.SeverityInfo,
otellog.String("job_id", j.JobID), otellog.Int("attempt", j.Attempt))
}
return true, err
}
func (a *App) dispatchImages(ctx context.Context, broker *redis.Client, attempts, retention int) {
cleanup := time.Now()
for ctx.Err() == nil {
step, cancel := context.WithTimeout(ctx, 5*time.Second)
recoverErr := a.recoverImage(step, attempts)
published, publishErr := a.publishImage(step, broker)
cancel()
if recoverErr != nil || publishErr != nil {
slog.Warn("image outbox unavailable", "recover_failed", recoverErr != nil, "publish_failed", publishErr != nil)
}
if time.Now().After(cleanup) {
step, cancel = context.WithTimeout(ctx, 5*time.Second)
if err := a.cleanStaging(step, retention); err != nil && ctx.Err() == nil {
slog.Warn("staging cleanup incomplete")
}
cancel()
cleanup = time.Now().Add(15 * time.Second)
}
if !published || recoverErr != nil {
waitImage(ctx, 300*time.Millisecond)
}
}
}
func waitImage(ctx context.Context, delay time.Duration) {
timer := time.NewTimer(delay)
defer timer.Stop()
select {
case <-ctx.Done():
case <-timer.C:
}
}
func (a *App) consumeImages(ctx context.Context, broker *redis.Client, attempts int, timeout time.Duration) {
cursor := "0-0"
groupReady := false
for ctx.Err() == nil {
if !groupReady {
err := broker.XGroupCreateMkStream(ctx, imageStream, imageGroup, "0").Err()
if err != nil && !strings.HasPrefix(err.Error(), "BUSYGROUP") {
waitImage(ctx, time.Second)
continue
}
groupReady = true
}
messages, next, err := broker.XAutoClaim(ctx, &redis.XAutoClaimArgs{
Stream: imageStream, Group: imageGroup, Consumer: "worker", MinIdle: timeout + 10*time.Second, Start: cursor, Count: 1,
}).Result()
if err == nil {
cursor = next
if len(messages) == 0 {
var streams []redis.XStream
streams, err = broker.XReadGroup(ctx, &redis.XReadGroupArgs{
Group: imageGroup, Consumer: "worker", Streams: []string{imageStream, ">"}, Count: 1, Block: time.Second,
}).Result()
if err == nil && len(streams) > 0 {
messages = streams[0].Messages
}
}
}
if err != nil && !errors.Is(err, redis.Nil) {
groupReady = false
waitImage(ctx, time.Second)
continue
}
for _, message := range messages {
if err = a.handleImage(ctx, message, attempts, timeout); err != nil {
slog.Warn("image delivery left pending", "delivery_id", message.ID)
continue
}
if err = ackImage.Run(ctx, broker, []string{imageStream}, imageGroup, message.ID).Err(); err != nil {
slog.Warn("image acknowledgement unavailable", "delivery_id", message.ID)
}
}
}
}
func (a *App) handleImage(ctx context.Context, message redis.XMessage, attempts int, timeout time.Duration) (result error) {
var j imageJob
payload, ok := message.Values["job"].(string)
if !ok || len(payload) > 2048 || json.Unmarshal([]byte(payload), &j) != nil ||
j.SchemaVersion != 1 || j.Type != "attachment.process.v1" || j.Attempt < 1 || (j.Version != 1 && j.Version != 2) {
slog.Warn("invalid image delivery", "delivery_id", message.ID)
return nil
}
for _, id := range []string{j.JobID, j.AttachmentID} {
if normalized, err := normalizeID(id); err != nil || normalized != id {
slog.Warn("invalid image delivery ID", "delivery_id", message.ID)
return nil
}
}
parent, _ := message.Values["traceparent"].(string)
state, _ := message.Values["tracestate"].(string)
ctx, span := a.telemetry.startJob(ctx, propagation.MapCarrier{"traceparent": parent, "tracestate": state}, j, false)
defer func() { endSpan(span, result) }()
claimCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
src, claimed, err := a.claimImage(claimCtx, j, timeout+5*time.Second)
cancel()
if err != nil || !claimed {
if err != nil {
a.telemetry.event(ctx, "image.claim_failed", otellog.SeverityError, otellog.String("job_id", j.JobID))
}
return err // Stale/duplicate attempts are no-ops. A processing row has a recoverable lease.
}
outcome := "error"
started := time.Now()
if a.telemetry != nil {
a.telemetry.workerBusy.Add(1)
defer a.telemetry.workerBusy.Add(-1)
}
defer func() { a.telemetry.finishJob(ctx, j, outcome, started) }()
injectImageFault("before_effect", j.AttachmentID)
processCtx, cancel := context.WithTimeout(ctx, timeout)
var variants []Variant
switch {
case os.Getenv("LAB5_FAULT_ATTACHMENT") == j.AttachmentID && os.Getenv("LAB5_FAULT") == "invalid_image":
err = imageFailure("invalid_image")
case os.Getenv("LAB5_FAULT_ATTACHMENT") == j.AttachmentID &&
(os.Getenv("LAB5_FAULT") == "temporary" || (os.Getenv("LAB5_FAULT") == "temporary_once" && j.Attempt == 1)):
err = errors.New("test transient failure")
default:
variants, err = a.generateImage(processCtx, j, src)
}
cancel()
finishCtx, cancel := context.WithTimeout(ctx, 4*time.Second)
defer cancel()
if err != nil {
code := "processing_failed"
var permanent imageFailure
if errors.As(err, &permanent) {
code = string(permanent)
}
next, saveErr := a.failImage(finishCtx, j, code, permanent != "", false, attempts)
if saveErr == nil {
outcome = next
}
span.SetAttributes(attribute.String("error.type", code))
return saveErr
}
seconds, committed, err := a.completeImage(finishCtx, j, variants)
if err != nil {
return err
}
if !committed {
outcome = "lease_lost"
return nil
}
outcome = "ready"
if a.telemetry != nil && j.Version == 1 {
a.telemetry.readyTime.Record(ctx, seconds, metric.WithAttributes(attribute.String("job_type", j.Type)))
}
injectImageFault("after_commit", j.AttachmentID)
return nil
}
+63
View File
@@ -0,0 +1,63 @@
global
log stdout format raw local0
maxconn 2048
defaults
log global
mode http
option httplog
timeout connect 500ms
timeout client 40s
timeout server 40s
timeout http-request 3s
timeout http-keep-alive 5s
timeout check 500ms
retries 0
resolvers docker
nameserver dns 127.0.0.11:53
accepted_payload_size 8192
resolve_retries 2
timeout resolve 1s
timeout retry 1s
hold valid 1s
hold obsolete 1s
hold nx 1s
frontend public
bind :8080
log-format "status=%ST duration_ms=%Ta method=%HM backend=%b server=%s"
default_backend api
backend api
balance roundrobin
option httpchk
http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host api
http-check expect status 200
server-template replica 1-4 api:8080 check inter 500ms fall 2 rise 2 resolvers docker resolve-prefer ipv4 init-addr none
listen stats
bind :8404
stats enable
stats uri /stats
# MinIO is reachable only through this internal listener. Public port 9000 is
# never published. Block unsupported vulnerable features in the archived CE.
frontend private_s3
bind :9000
no log
acl unsigned_trailer req.hdr(x-amz-content-sha256) -i STREAMING-UNSIGNED-PAYLOAD-TRAILER
http-request deny deny_status 403 if unsigned_trailer
acl internal_storage path,url_dec -m beg -i /minio/storage /minio/peer /minio/grid /minio/bootstrap
http-request deny deny_status 403 if internal_storage
acl s3_select query,url_dec -m reg -i '(^|&)select([=&]|$)'
http-request deny deny_status 403 if s3_select
acl replication_header req.hdr(x-minio-source-replication-request) -m found
http-request deny deny_status 403 if replication_header
default_backend minio_private
backend minio_private
option httpchk
http-check send meth GET uri /minio/health/ready ver HTTP/1.1 hdr Host minio-backend
http-check expect status 200
server minio minio-backend:9000 check inter 1s resolvers docker init-addr none
+54
View File
@@ -0,0 +1,54 @@
groups:
- name: messenger
rules:
- alert: APIFastBurn
expr: |
((sum(rate(messenger_http_requests_total{route!="unmatched",status_class="5xx"}[30s])) / sum(rate(messenger_http_requests_total{route!="unmatched"}[30s]))) / 0.01 > 14.4)
and ((sum(rate(messenger_http_requests_total{route!="unmatched",status_class="5xx"}[10s])) / sum(rate(messenger_http_requests_total{route!="unmatched"}[10s]))) / 0.01 > 14.4)
and (sum(increase(messenger_http_requests_total{route!="unmatched"}[30s])) >= 8)
and (sum(increase(messenger_http_requests_total{route!="unmatched"}[10s])) >= 3)
for: 2s
labels: {severity: page, slo: api}
annotations:
summary: "Доля 5xx на маршрутах API. Burn rate выше 14.4 в окнах 30s и 10s."
runbook: "RUNBOOK.md#api"
- alert: APISlowBurn
expr: |
((sum(rate(messenger_http_requests_total{route!="unmatched",status_class="5xx"}[75s])) / sum(rate(messenger_http_requests_total{route!="unmatched"}[75s]))) / 0.01 > 6)
and ((sum(rate(messenger_http_requests_total{route!="unmatched",status_class="5xx"}[25s])) / sum(rate(messenger_http_requests_total{route!="unmatched"}[25s]))) / 0.01 > 6)
and (sum(increase(messenger_http_requests_total{route!="unmatched"}[75s])) >= 15)
and (sum(increase(messenger_http_requests_total{route!="unmatched"}[25s])) >= 5)
for: 12s
labels: {severity: ticket, slo: api}
annotations:
summary: "Доля 5xx на маршрутах API. Burn rate выше 6 в окнах 75s и 25s."
runbook: "RUNBOOK.md#api"
- alert: ReadyFastBurn
expr: |
((sum(rate(messenger_attachment_slo_jobs_total{outcome="missed"}[30s])) / sum(rate(messenger_attachment_slo_jobs_total[30s]))) / 0.01 > 14.4)
and ((sum(rate(messenger_attachment_slo_jobs_total{outcome="missed"}[10s])) / sum(rate(messenger_attachment_slo_jobs_total[10s]))) / 0.01 > 14.4)
and (sum(increase(messenger_attachment_slo_jobs_total[30s])) >= 8)
and (sum(increase(messenger_attachment_slo_jobs_total[10s])) >= 3)
for: 2s
labels: {severity: page, slo: ready}
annotations:
summary: "Доля загрузок, не готовых за 5 секунд. Burn rate выше 14.4 в окнах 30s и 10s."
runbook: "RUNBOOK.md#ready"
- alert: ReadySlowBurn
expr: |
((sum(rate(messenger_attachment_slo_jobs_total{outcome="missed"}[75s])) / sum(rate(messenger_attachment_slo_jobs_total[75s]))) / 0.01 > 6)
and ((sum(rate(messenger_attachment_slo_jobs_total{outcome="missed"}[25s])) / sum(rate(messenger_attachment_slo_jobs_total[25s]))) / 0.01 > 6)
and (sum(increase(messenger_attachment_slo_jobs_total[75s])) >= 15)
and (sum(increase(messenger_attachment_slo_jobs_total[25s])) >= 5)
for: 12s
labels: {severity: ticket, slo: ready}
annotations:
summary: "Доля загрузок, не готовых за 5 секунд. Burn rate выше 6 в окнах 75s и 25s."
runbook: "RUNBOOK.md#ready"
- alert: QueueLagHigh
expr: messenger_queue_oldest_age_seconds > 5 and messenger_queue_depth > 0
for: 4s
labels: {severity: ticket}
annotations:
summary: "Старейшее ожидающее задание старше 5 секунд."
runbook: "RUNBOOK.md#ready"
+62
View File
@@ -0,0 +1,62 @@
receivers:
otlp:
protocols:
http:
endpoint: 0.0.0.0:4318
processors:
memory_limiter:
check_interval: 1s
limit_mib: 125
spike_limit_mib: 25
batch:
timeout: 1s
send_batch_size: 128
send_batch_max_size: 256
exporters:
otlphttp/prometheus:
metrics_endpoint: http://prometheus:9090/api/v1/otlp/v1/metrics
timeout: 2s
sending_queue: &bounded_queue
enabled: true
num_consumers: 1
queue_size: 16
retry_on_failure: &bounded_retry
enabled: true
initial_interval: 1s
max_interval: 2s
max_elapsed_time: 8s
otlphttp/loki:
endpoint: http://loki:3100/otlp
timeout: 2s
sending_queue: *bounded_queue
retry_on_failure: *bounded_retry
otlphttp/tempo:
endpoint: http://tempo:4318
timeout: 2s
sending_queue: *bounded_queue
retry_on_failure: *bounded_retry
service:
telemetry:
metrics:
readers:
- pull:
exporter:
prometheus:
host: 0.0.0.0
port: 8888
pipelines:
metrics:
receivers: [otlp]
processors: [memory_limiter, batch]
exporters: [otlphttp/prometheus]
logs:
receivers: [otlp]
processors: [memory_limiter, batch]
exporters: [otlphttp/loki]
traces:
receivers: [otlp]
processors: [memory_limiter, batch]
exporters: [otlphttp/tempo]
@@ -0,0 +1,810 @@
{
"uid": "messenger-lab6",
"title": "Lab 6 · Messenger",
"description": "OpenTelemetry API и очереди изображений",
"tags": [
"messenger",
"opentelemetry"
],
"schemaVersion": 39,
"refresh": "5s",
"time": {
"from": "now-15m",
"to": "now"
},
"timezone": "browser",
"panels": [
{
"id": 1,
"title": "Запросы API",
"description": "Интенсивность по двум API-репликам. Readiness и liveness выключены на этапе инструментации.",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "lab-prom"
},
"gridPos": {
"x": 0,
"y": 0,
"w": 12,
"h": 8
},
"fieldConfig": {
"defaults": {
"unit": "reqps"
},
"overrides": []
},
"targets": [
{
"refId": "A",
"expr": "sum(rate(messenger_http_requests_total{route!=\"unmatched\"}[30s]))",
"legendFormat": "известные маршруты",
"datasource": {
"type": "prometheus",
"uid": "lab-prom"
}
},
{
"refId": "B",
"expr": "sum(rate(messenger_http_requests_total{route!=\"unmatched\",status_class=\"5xx\"}[30s]))",
"legendFormat": "5xx",
"datasource": {
"type": "prometheus",
"uid": "lab-prom"
}
}
]
},
{
"id": 2,
"title": "Ошибки API",
"description": "5xx из приложения на известных маршрутах. 4xx входят в знаменатель.",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "lab-prom"
},
"gridPos": {
"x": 12,
"y": 0,
"w": 12,
"h": 8
},
"fieldConfig": {
"defaults": {
"unit": "percentunit"
},
"overrides": []
},
"targets": [
{
"refId": "A",
"expr": "(sum(rate(messenger_http_requests_total{route!=\"unmatched\",status_class=\"5xx\"}[30s])) / sum(rate(messenger_http_requests_total{route!=\"unmatched\"}[30s])))",
"legendFormat": "5xx / все",
"datasource": {
"type": "prometheus",
"uid": "lab-prom"
}
},
{
"refId": "B",
"expr": "vector(0.01)",
"legendFormat": "допустимая доля",
"datasource": {
"type": "prometheus",
"uid": "lab-prom"
}
}
]
},
{
"id": 3,
"title": "Задержка API",
"description": "Перцентили из общих bucket всех реплик. Первая минута и низкий трафик дают шумную оценку.",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "lab-prom"
},
"gridPos": {
"x": 0,
"y": 8,
"w": 12,
"h": 8
},
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
},
"targets": [
{
"refId": "A",
"expr": "histogram_quantile(0.50, sum by (le) (rate(messenger_http_duration_seconds_bucket[1m])))",
"legendFormat": "p50",
"datasource": {
"type": "prometheus",
"uid": "lab-prom"
}
},
{
"refId": "B",
"expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_http_duration_seconds_bucket[1m])))",
"legendFormat": "p95",
"datasource": {
"type": "prometheus",
"uid": "lab-prom"
}
},
{
"refId": "C",
"expr": "histogram_quantile(0.99, sum by (le) (rate(messenger_http_duration_seconds_bucket[1m])))",
"legendFormat": "p99",
"datasource": {
"type": "prometheus",
"uid": "lab-prom"
}
}
]
},
{
"id": 4,
"title": "Попытки worker",
"description": "Считаются обработанные попытки по исходу. Повторная попытка учитывается отдельно.",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "lab-prom"
},
"gridPos": {
"x": 12,
"y": 8,
"w": 12,
"h": 8
},
"fieldConfig": {
"defaults": {
"unit": "ops"
},
"overrides": []
},
"targets": [
{
"refId": "A",
"expr": "sum by (outcome) (rate(messenger_job_attempts_total[30s]))",
"legendFormat": "{{outcome}}",
"datasource": {
"type": "prometheus",
"uid": "lab-prom"
}
}
]
},
{
"id": 5,
"title": "Задержка попытки worker",
"description": "От захвата lease до записи итога попытки или решения о повторе.",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "lab-prom"
},
"gridPos": {
"x": 0,
"y": 16,
"w": 12,
"h": 8
},
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
},
"targets": [
{
"refId": "A",
"expr": "histogram_quantile(0.50, sum by (le) (rate(messenger_job_duration_seconds_bucket[1m])))",
"legendFormat": "p50",
"datasource": {
"type": "prometheus",
"uid": "lab-prom"
}
},
{
"refId": "B",
"expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_job_duration_seconds_bucket[1m])))",
"legendFormat": "p95",
"datasource": {
"type": "prometheus",
"uid": "lab-prom"
}
},
{
"refId": "C",
"expr": "histogram_quantile(0.99, sum by (le) (rate(messenger_job_duration_seconds_bucket[1m])))",
"legendFormat": "p99",
"datasource": {
"type": "prometheus",
"uid": "lab-prom"
}
}
]
},
{
"id": 6,
"title": "Время до ready",
"description": "SQL-время от принятия загрузки до записи ready. Гистограмма включает завершившиеся первые публикации.",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "lab-prom"
},
"gridPos": {
"x": 12,
"y": 16,
"w": 12,
"h": 8
},
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
},
"targets": [
{
"refId": "A",
"expr": "histogram_quantile(0.50, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket[1m])))",
"legendFormat": "p50",
"datasource": {
"type": "prometheus",
"uid": "lab-prom"
}
},
{
"refId": "B",
"expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket[1m])))",
"legendFormat": "p95",
"datasource": {
"type": "prometheus",
"uid": "lab-prom"
}
},
{
"refId": "C",
"expr": "histogram_quantile(0.99, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket[1m])))",
"legendFormat": "p99",
"datasource": {
"type": "prometheus",
"uid": "lab-prom"
}
}
]
},
{
"id": 7,
"title": "Глубина очереди",
"description": "Строки outbox в queued, в том числе ожидающие XADD и backoff.",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "lab-prom"
},
"gridPos": {
"x": 0,
"y": 24,
"w": 12,
"h": 8
},
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"targets": [
{
"refId": "A",
"expr": "messenger_queue_depth",
"legendFormat": "queued",
"datasource": {
"type": "prometheus",
"uid": "lab-prom"
}
}
]
},
{
"id": 8,
"title": "Возраст старейшего задания",
"description": "Возраст от создания самой старой строки outbox в queued.",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "lab-prom"
},
"gridPos": {
"x": 12,
"y": 24,
"w": 12,
"h": 8
},
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
},
"targets": [
{
"refId": "A",
"expr": "messenger_queue_oldest_age_seconds",
"legendFormat": "oldest",
"datasource": {
"type": "prometheus",
"uid": "lab-prom"
}
},
{
"refId": "B",
"expr": "vector(5)",
"legendFormat": "порог",
"datasource": {
"type": "prometheus",
"uid": "lab-prom"
}
}
]
},
{
"id": 9,
"title": "Соединения PostgreSQL",
"description": "В процессе Acquire учитываются и короткие вызовы. Смотрите динамику при устойчивом ожидании.",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "lab-prom"
},
"gridPos": {
"x": 0,
"y": 32,
"w": 12,
"h": 8
},
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"targets": [
{
"refId": "A",
"expr": "messenger_db_pool_used",
"legendFormat": "{{job}} {{instance}} / занято",
"datasource": {
"type": "prometheus",
"uid": "lab-prom"
}
},
{
"refId": "B",
"expr": "messenger_db_pool_capacity",
"legendFormat": "{{job}} {{instance}} / лимит",
"datasource": {
"type": "prometheus",
"uid": "lab-prom"
}
},
{
"refId": "C",
"expr": "messenger_db_pool_acquiring",
"legendFormat": "{{job}} {{instance}} / получение",
"datasource": {
"type": "prometheus",
"uid": "lab-prom"
}
}
]
},
{
"id": 10,
"title": "Получение соединения при пустом пуле",
"description": "Производная EmptyAcquireCount pgxpool. Включает ожидание создания соединения.",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "lab-prom"
},
"gridPos": {
"x": 12,
"y": 32,
"w": 12,
"h": 8
},
"fieldConfig": {
"defaults": {
"unit": "ops"
},
"overrides": []
},
"targets": [
{
"refId": "A",
"expr": "rate(messenger_db_pool_waits_total[30s])",
"legendFormat": "{{job}} {{instance}}",
"datasource": {
"type": "prometheus",
"uid": "lab-prom"
}
}
]
},
{
"id": 11,
"title": "Занятые слоты worker",
"description": "Только свежие показания. Снимки с шагом 2 с могут пропускать очень короткие задания.",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "lab-prom"
},
"gridPos": {
"x": 0,
"y": 40,
"w": 12,
"h": 8
},
"fieldConfig": {
"defaults": {
"unit": "percentunit"
},
"overrides": []
},
"targets": [
{
"refId": "A",
"expr": "sum(messenger_worker_busy and (time() - timestamp(messenger_worker_busy) < 8)) / sum(messenger_worker_capacity and (time() - timestamp(messenger_worker_capacity) < 8))",
"legendFormat": "busy / capacity",
"datasource": {
"type": "prometheus",
"uid": "lab-prom"
}
},
{
"refId": "B",
"expr": "avg_over_time((sum(messenger_worker_busy and (time() - timestamp(messenger_worker_busy) < 8)) / sum(messenger_worker_capacity and (time() - timestamp(messenger_worker_capacity) < 8)))[30s:2s])",
"legendFormat": "среднее за 30 с",
"datasource": {
"type": "prometheus",
"uid": "lab-prom"
}
}
]
},
{
"id": 12,
"title": "CPU контейнеров приложения",
"description": "CPU-seconds / wall-second по cgroup v2, включая дочерний Pillow в worker.",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "lab-prom"
},
"gridPos": {
"x": 12,
"y": 40,
"w": 12,
"h": 8
},
"fieldConfig": {
"defaults": {
"unit": "cores"
},
"overrides": []
},
"targets": [
{
"refId": "A",
"expr": "rate(messenger_container_cpu_seconds_total[30s])",
"legendFormat": "{{job}} {{instance}}",
"datasource": {
"type": "prometheus",
"uid": "lab-prom"
}
}
]
},
{
"id": 13,
"title": "RAM контейнеров приложения",
"description": "Текущий memory.current cgroup v2.",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "lab-prom"
},
"gridPos": {
"x": 0,
"y": 48,
"w": 12,
"h": 8
},
"fieldConfig": {
"defaults": {
"unit": "bytes"
},
"overrides": []
},
"targets": [
{
"refId": "A",
"expr": "messenger_container_memory_bytes",
"legendFormat": "{{job}} {{instance}}",
"datasource": {
"type": "prometheus",
"uid": "lab-prom"
}
}
]
},
{
"id": 14,
"title": "SLI за 30 минут",
"description": "Каждый upload попадает в когорту по истечении 5 с. Без трафика SLI отсутствует.",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "lab-prom"
},
"gridPos": {
"x": 12,
"y": 48,
"w": 12,
"h": 8
},
"fieldConfig": {
"defaults": {
"unit": "percentunit"
},
"overrides": []
},
"targets": [
{
"refId": "A",
"expr": "1 - (sum(rate(messenger_http_requests_total{route!=\"unmatched\",status_class=\"5xx\"}[30m])) / sum(rate(messenger_http_requests_total{route!=\"unmatched\"}[30m])))",
"legendFormat": "API",
"datasource": {
"type": "prometheus",
"uid": "lab-prom"
}
},
{
"refId": "B",
"expr": "1 - (sum(rate(messenger_attachment_slo_jobs_total{outcome=\"missed\"}[30m])) / sum(rate(messenger_attachment_slo_jobs_total[30m])))",
"legendFormat": "ready ≤ 5 с",
"datasource": {
"type": "prometheus",
"uid": "lab-prom"
}
},
{
"refId": "C",
"expr": "vector(0.99)",
"legendFormat": "цель",
"datasource": {
"type": "prometheus",
"uid": "lab-prom"
}
}
]
},
{
"id": 15,
"title": "Burn rate",
"description": "Burn rate 1 соответствует расходу бюджета с нормативной скоростью.",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "lab-prom"
},
"gridPos": {
"x": 0,
"y": 56,
"w": 12,
"h": 8
},
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"targets": [
{
"refId": "A",
"expr": "(sum(rate(messenger_http_requests_total{route!=\"unmatched\",status_class=\"5xx\"}[10s])) / sum(rate(messenger_http_requests_total{route!=\"unmatched\"}[10s]))) / 0.01",
"legendFormat": "API 10s",
"datasource": {
"type": "prometheus",
"uid": "lab-prom"
}
},
{
"refId": "B",
"expr": "(sum(rate(messenger_http_requests_total{route!=\"unmatched\",status_class=\"5xx\"}[30s])) / sum(rate(messenger_http_requests_total{route!=\"unmatched\"}[30s]))) / 0.01",
"legendFormat": "API 30s",
"datasource": {
"type": "prometheus",
"uid": "lab-prom"
}
},
{
"refId": "C",
"expr": "(sum(rate(messenger_http_requests_total{route!=\"unmatched\",status_class=\"5xx\"}[25s])) / sum(rate(messenger_http_requests_total{route!=\"unmatched\"}[25s]))) / 0.01",
"legendFormat": "API 25s",
"datasource": {
"type": "prometheus",
"uid": "lab-prom"
}
},
{
"refId": "D",
"expr": "(sum(rate(messenger_http_requests_total{route!=\"unmatched\",status_class=\"5xx\"}[75s])) / sum(rate(messenger_http_requests_total{route!=\"unmatched\"}[75s]))) / 0.01",
"legendFormat": "API 75s",
"datasource": {
"type": "prometheus",
"uid": "lab-prom"
}
},
{
"refId": "E",
"expr": "(sum(rate(messenger_attachment_slo_jobs_total{outcome=\"missed\"}[10s])) / sum(rate(messenger_attachment_slo_jobs_total[10s]))) / 0.01",
"legendFormat": "Ready 10s",
"datasource": {
"type": "prometheus",
"uid": "lab-prom"
}
},
{
"refId": "F",
"expr": "(sum(rate(messenger_attachment_slo_jobs_total{outcome=\"missed\"}[30s])) / sum(rate(messenger_attachment_slo_jobs_total[30s]))) / 0.01",
"legendFormat": "Ready 30s",
"datasource": {
"type": "prometheus",
"uid": "lab-prom"
}
},
{
"refId": "G",
"expr": "(sum(rate(messenger_attachment_slo_jobs_total{outcome=\"missed\"}[25s])) / sum(rate(messenger_attachment_slo_jobs_total[25s]))) / 0.01",
"legendFormat": "Ready 25s",
"datasource": {
"type": "prometheus",
"uid": "lab-prom"
}
},
{
"refId": "H",
"expr": "(sum(rate(messenger_attachment_slo_jobs_total{outcome=\"missed\"}[75s])) / sum(rate(messenger_attachment_slo_jobs_total[75s]))) / 0.01",
"legendFormat": "Ready 75s",
"datasource": {
"type": "prometheus",
"uid": "lab-prom"
}
}
]
},
{
"id": 16,
"title": "Alerts в Prometheus",
"description": "Локальная история pending и firing за выбранный период.",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "lab-prom"
},
"gridPos": {
"x": 12,
"y": 56,
"w": 12,
"h": 8
},
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"targets": [
{
"refId": "A",
"expr": "ALERTS{alertname=~\"APIFastBurn|APISlowBurn|ReadyFastBurn|ReadySlowBurn|QueueLagHigh\"}",
"legendFormat": "{{alertname}} / {{alertstate}}",
"datasource": {
"type": "prometheus",
"uid": "lab-prom"
}
}
]
},
{
"id": 17,
"title": "Потери Collector",
"description": "Число элементов в терминально неотправленных или не поставленных в очередь батчах за минуту.",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "lab-prom"
},
"gridPos": {
"x": 0,
"y": 64,
"w": 12,
"h": 8
},
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"targets": [
{
"refId": "A",
"expr": "sum by (exporter) (increase({__name__=~\"otelcol_exporter_send_failed_(spans|log_records|metric_points)(_total)?\"}[1m]))",
"legendFormat": "{{exporter}} / send failed",
"datasource": {
"type": "prometheus",
"uid": "lab-prom"
}
},
{
"refId": "B",
"expr": "sum by (exporter) (increase({__name__=~\"otelcol_exporter_enqueue_failed_(spans|log_records|metric_points)(_total)?\"}[1m]))",
"legendFormat": "{{exporter}} / enqueue failed",
"datasource": {
"type": "prometheus",
"uid": "lab-prom"
}
}
]
},
{
"id": 18,
"title": "Логи API и worker",
"description": "В деталях записи есть trace_id и span_id. Ссылка Trace открывает Tempo.",
"type": "logs",
"datasource": {
"type": "loki",
"uid": "lab-loki"
},
"gridPos": {
"x": 12,
"y": 64,
"w": 12,
"h": 8
},
"fieldConfig": {
"defaults": {
"unit": "none"
},
"overrides": []
},
"targets": [
{
"refId": "A",
"expr": "{service_name=~\"messenger-api|messenger-worker\"}",
"legendFormat": "",
"datasource": {
"type": "loki",
"uid": "lab-loki"
}
}
]
}
]
}
@@ -0,0 +1,8 @@
apiVersion: 1
providers:
- name: Lab 6
folder: Messenger
type: file
allowUiUpdates: false
options:
path: /var/lib/grafana/dashboards
@@ -0,0 +1,41 @@
apiVersion: 1
datasources:
- name: Prometheus
uid: lab-prom
type: prometheus
url: http://prometheus:9090
access: proxy
isDefault: true
editable: false
jsonData:
timeInterval: 2s
prometheusType: Prometheus
prometheusVersion: 3.5.0
- name: Loki
uid: lab-loki
type: loki
url: http://loki:3100
access: proxy
editable: false
jsonData:
derivedFields:
- name: Trace
matcherType: label
matcherRegex: trace_id
datasourceUid: lab-tempo
url: '$${__value.raw}'
- name: Tempo
uid: lab-tempo
type: tempo
url: http://tempo:3200
access: proxy
editable: false
jsonData:
tracesToLogsV2:
datasourceUid: lab-loki
tags:
- {key: service.name, value: service_name}
spanStartTimeShift: "-30s"
spanEndTimeShift: "30s"
customQuery: true
query: '{service_name=~"messenger-api|messenger-dispatcher|messenger-worker"} | trace_id="$${__span.traceId}" | span_id="$${__span.spanId}"'
+38
View File
@@ -0,0 +1,38 @@
auth_enabled: false
server:
http_listen_port: 3100
common:
instance_addr: 127.0.0.1
path_prefix: /loki
storage:
filesystem:
chunks_directory: /loki/chunks
rules_directory: /loki/rules
replication_factor: 1
ring:
kvstore:
store: inmemory
schema_config:
configs:
- from: 2024-01-01
store: tsdb
object_store: filesystem
schema: v13
index:
prefix: index_
period: 24h
limits_config:
allow_structured_metadata: true
retention_period: 168h
otlp_config:
resource_attributes:
ignore_defaults: true
attributes_config:
- attributes: [service.name]
action: index_label
compactor:
working_directory: /loki/compactor
retention_enabled: true
delete_request_store: filesystem
analytics:
reporting_enabled: false
+16
View File
@@ -0,0 +1,16 @@
global:
scrape_interval: 2s
evaluation_interval: 2s
storage:
tsdb:
out_of_order_time_window: 30s
otlp:
promote_resource_attributes: [service.name, service.version, deployment.environment.name]
translation_strategy: UnderscoreEscapingWithSuffixes
rule_files: [/etc/prometheus/alerts.yml]
scrape_configs:
- job_name: otel-collector
static_configs:
- targets: [otel-collector:8888]
+22
View File
@@ -0,0 +1,22 @@
server:
http_listen_port: 3200
distributor:
receivers:
otlp:
protocols:
http:
endpoint: 0.0.0.0:4318
ingester:
max_block_duration: 1m
compactor:
compaction:
block_retention: 168h
storage:
trace:
backend: local
wal:
path: /var/tempo/wal
local:
path: /var/tempo/blocks
usage_report:
reporting_enabled: false
+20
View File
@@ -0,0 +1,20 @@
// Standalone health probe for the minimal MinIO container.
package main
import (
"net/http"
"os"
"time"
)
func main() {
client := http.Client{Timeout: time.Second}
response, err := client.Get("http://127.0.0.1:9000/minio/health/ready")
if err != nil {
os.Exit(1)
}
response.Body.Close()
if response.StatusCode != 200 {
os.Exit(1)
}
}
+17
View File
@@ -0,0 +1,17 @@
#!/bin/sh
set -eu
# No shell tracing and no credentials in persistent config or output.
for attempt in $(seq 1 30); do
if wget -q -O /dev/null http://s3:9000/minio/health/ready; then break; fi
sleep 1
done
mc alias set local http://s3:9000 "$(cat /run/secrets/s3_root_user)" "$(cat /run/secrets/s3_root_password)" >/dev/null
mc mb --ignore-existing local/messenger-attachments >/dev/null
mc anonymous set none local/messenger-attachments >/dev/null
mc admin user add local "$(cat /run/secrets/s3_access_key_id)" "$(cat /run/secrets/s3_secret_access_key)" >/dev/null
mc admin policy create local messenger-api /bootstrap/policy.json >/dev/null
mc admin policy attach local messenger-api --user "$(cat /run/secrets/s3_access_key_id)" >/dev/null
mc admin user add local "$(cat /run/secrets/s3_worker_access_key_id)" "$(cat /run/secrets/s3_worker_secret_access_key)" >/dev/null
mc admin policy create local messenger-worker /bootstrap/worker-policy.json >/dev/null
mc admin policy attach local messenger-worker --user "$(cat /run/secrets/s3_worker_access_key_id)" >/dev/null
echo 'Private bucket and restricted service accounts ready.'
+8
View File
@@ -0,0 +1,8 @@
{
"Version": "2012-10-17",
"Statement": [
{"Effect": "Allow", "Action": ["s3:GetBucketLocation", "s3:ListBucket"], "Resource": ["arn:aws:s3:::messenger-attachments"]},
{"Effect": "Allow", "Action": ["s3:PutObject"], "Resource": ["arn:aws:s3:::messenger-attachments/staging/*"]},
{"Effect": "Allow", "Action": ["s3:GetObject"], "Resource": ["arn:aws:s3:::messenger-attachments/original/*", "arn:aws:s3:::messenger-attachments/thumbnail/*", "arn:aws:s3:::messenger-attachments/crop/*", "arn:aws:s3:::messenger-attachments/optimized/*"]}
]
}
+8
View File
@@ -0,0 +1,8 @@
{
"Version": "2012-10-17",
"Statement": [
{"Effect": "Allow", "Action": ["s3:GetBucketLocation", "s3:ListBucket"], "Resource": ["arn:aws:s3:::messenger-attachments"]},
{"Effect": "Allow", "Action": ["s3:GetObject", "s3:DeleteObject"], "Resource": ["arn:aws:s3:::messenger-attachments/staging/*"]},
{"Effect": "Allow", "Action": ["s3:PutObject", "s3:GetObject"], "Resource": ["arn:aws:s3:::messenger-attachments/original/*", "arn:aws:s3:::messenger-attachments/thumbnail/*", "arn:aws:s3:::messenger-attachments/crop/*", "arn:aws:s3:::messenger-attachments/optimized/*"]}
]
}
+465
View File
@@ -0,0 +1,465 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"log/slog"
"net"
"net/http"
"os"
"os/signal"
"strconv"
"strings"
"sync/atomic"
"syscall"
"time"
"github.com/jackc/pgx/v5/pgxpool"
"github.com/minio/minio-go/v7"
"github.com/redis/go-redis/v9"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/trace"
)
type App struct {
db *pgxpool.Pool
sessions *redis.Client
secret []byte
ttl time.Duration
slots, kdf chan struct{}
stopping atomic.Bool
dummyHash string
instance string
idempotencyTTL time.Duration
s3 *minio.Client
bucket string
uploadSlots, decodeSlot chan struct{}
telemetry *telemetry
}
func env(key, value string) string {
if v := os.Getenv(key); v != "" {
return v
}
return value
}
func positive(key string, fallback int) int {
n, e := strconv.Atoi(env(key, strconv.Itoa(fallback)))
if e != nil || n <= 0 {
panic("invalid positive configuration: " + key)
}
return n
}
func secretFile(key string) (string, error) {
b, e := os.ReadFile(os.Getenv(key))
if e != nil {
return "", fmt.Errorf("cannot read %s", key)
}
s := strings.TrimSpace(string(b))
if s == "" {
return "", fmt.Errorf("empty %s", key)
}
return s, nil
}
func (a *App) dispatch(ctx context.Context, w http.ResponseWriter, r *http.Request) (int, any, error) {
path := r.URL.Path
if r.Method == "GET" && (path == "/health/live" || path == "/health/ready") {
if path == "/health/ready" && (a.stopping.Load() || a.db.Ping(ctx) != nil || a.sessions.Ping(ctx).Err() != nil || !a.s3Ready(ctx)) {
return 0, nil, unavailable()
}
return 200, map[string]string{"status": "ok"}, nil
}
if a.stopping.Load() {
return 0, nil, unavailable()
}
if r.Method == "POST" && path == "/api/v1/auth/sessions" {
return a.login(ctx, r)
}
if r.Method == "POST" && path == "/api/v1/users" {
body, err := readObject(w, r, 128*1024, []string{"username", "display_name", "password"})
if err != nil {
return 0, nil, err
}
name, err := textField(body, "username", 32)
if err != nil || !usernamePattern.MatchString(name) {
return 0, nil, invalid()
}
display, err := textField(body, "display_name", 100)
if err != nil {
return 0, nil, err
}
p, err := password(body)
if err != nil {
return 0, nil, err
}
if err = a.kdfSlot(ctx); err != nil {
return 0, nil, err
}
hash := hashPassword(p)
<-a.kdf
u, err := scanUser(a.db.QueryRow(ctx, "INSERT INTO users(id,username,display_name,password_hash) VALUES($1,$2,$3,$4) RETURNING id::text,username,display_name,created_at", uuid(), name, []byte(display), hash))
return 201, u, err
}
user, key, err := a.authenticate(ctx, r)
if err != nil {
return 0, nil, err
}
if r.Method == "DELETE" && path == "/api/v1/auth/sessions/current" {
count, e := a.sessions.Del(ctx, key).Result()
if e != nil {
return 0, nil, unavailable()
}
if count == 0 {
return 0, nil, unauthorized()
}
return 204, nil, nil
}
if r.Method == "GET" && path == "/api/v1/users/me" {
u, e := a.user(ctx, user)
return 200, u, e
}
if path == "/api/v1/chats" {
if r.Method == "GET" {
scope := "chats:" + user
limit, c, e := a.pagination(r, scope)
if e != nil {
return 0, nil, e
}
page, e := a.listChats(ctx, user, limit, c, scope)
return 200, page, e
}
if r.Method == "POST" {
body, e := readObject(w, r, 128*1024, []string{"title", "member_ids"})
if e != nil {
return 0, nil, e
}
title, e := textField(body, "title", 100)
if e != nil {
return 0, nil, e
}
var ids []string
if json.Unmarshal(body["member_ids"], &ids) != nil || len(ids) < 1 || len(ids) > 100 {
return 0, nil, invalid()
}
seen := map[string]bool{}
for i, id := range ids {
normalized, e := normalizeID(id)
if e != nil || seen[normalized] {
return 0, nil, invalid()
}
ids[i] = normalized
seen[normalized] = true
}
if !seen[user] {
return 0, nil, invalid()
}
chat, e := a.createChat(ctx, title, ids)
return 201, chat, e
}
}
if strings.HasPrefix(path, "/api/v1/attachments/") {
parts := strings.Split(strings.TrimPrefix(path, "/api/v1/attachments/"), "/")
id, e := normalizeID(parts[0])
if e != nil {
return 0, nil, e
}
if r.Method != "GET" || len(parts) > 2 || (len(parts) == 2 && parts[1] != "content") {
return 0, nil, missing()
}
x, e := a.attachment(ctx, id, user)
if e != nil {
return 0, nil, e
}
if len(parts) == 1 {
return 200, x, nil
}
b, e := a.content(ctx, r, x)
return 200, b, e
}
if strings.HasPrefix(path, "/api/v1/chats/") {
parts := strings.Split(strings.TrimPrefix(path, "/api/v1/chats/"), "/")
id, e := normalizeID(parts[0])
if e != nil {
return 0, nil, e
}
if len(parts) > 2 || (len(parts) == 2 && parts[1] != "messages" && parts[1] != "attachments") {
return 0, nil, missing()
}
chat, e := a.chat(ctx, id, user)
if e != nil {
return 0, nil, e
}
if len(parts) == 1 && r.Method == "GET" {
return 200, chat, nil
}
if len(parts) == 2 && parts[1] == "attachments" {
if r.Method != "POST" {
return 0, nil, missing()
}
x, e := a.upload(ctx, w, r, id, user)
return 202, x, e
}
if len(parts) == 2 {
if r.Method == "GET" {
scope := "messages:" + id
limit, c, e := a.pagination(r, scope)
if e != nil {
return 0, nil, e
}
page, e := a.history(ctx, id, limit, c, scope)
return 200, page, e
}
if r.Method == "POST" {
keys := r.Header.Values("Idempotency-Key")
if len(keys) != 1 || !idempotencyPattern.MatchString(keys[0]) {
return 0, nil, invalid()
}
body, e := readObject(w, r, 128*1024, []string{"text", "attachment_ids"})
if e != nil {
return 0, nil, e
}
text, ids, e := messageFields(body)
if e != nil {
return 0, nil, e
}
m, e := a.createMessage(ctx, id, user, keys[0], text, ids)
return 201, m, e
}
}
}
return 0, nil, missing()
}
func (a *App) ServeHTTP(w http.ResponseWriter, r *http.Request) {
start := time.Now()
requestID := uuid()
status := 500
ctx := r.Context()
health := r.Method == "GET" && (r.URL.Path == "/health/live" || r.URL.Path == "/health/ready")
method, route := safeHTTPRoute(r.Method, r.URL.Path)
if !health && a.telemetry != nil {
var span trace.Span
ctx, span = startSpan(ctx, method+" "+route, trace.WithSpanKind(trace.SpanKindServer),
trace.WithAttributes(attribute.String("http.request.method", method), attribute.String("http.route", route),
attribute.String("request.id", requestID)))
defer span.End()
}
w.Header().Set("Content-Type", "application/json")
w.Header().Set("X-Request-Id", requestID)
w.Header().Set("X-Instance-Id", a.instance)
fail := func(err error) {
var e apiError
if !errors.As(err, &e) {
e = unavailable()
}
status = e.status
if status == 401 {
scheme := `Bearer realm="messenger"`
if r.URL.Path == "/api/v1/auth/sessions" {
scheme = `Basic realm="messenger", charset="UTF-8"`
}
w.Header().Set("WWW-Authenticate", scheme)
}
if status == 503 {
w.Header().Set("Retry-After", "1")
}
w.WriteHeader(status)
json.NewEncoder(w).Encode(map[string]any{"error": map[string]string{"code": e.code, "message": e.message, "request_id": requestID}})
}
defer func() {
if recovered := recover(); recovered != nil {
if recovered == http.ErrAbortHandler {
panic(recovered)
}
fail(apiError{500, "internal_error", "Internal server error"})
}
if !health {
a.telemetry.finishHTTP(ctx, route, method, requestID, status, start)
if a.telemetry == nil && env("REQUEST_LOG", "1") == "1" {
slog.Info("request", "method", method, "route", route, "status", status,
"request_id", requestID, "duration_ms", time.Since(start).Milliseconds())
}
}
}()
work, cancel := context.WithTimeout(ctx, 30*time.Second)
defer cancel()
if !health {
select {
case a.slots <- struct{}{}:
defer func() { <-a.slots }()
default:
fail(unavailable())
return
}
}
code, value, err := a.dispatch(work, w, r)
if err != nil {
fail(err)
return
}
status = code
if b, ok := value.(binaryContent); ok {
writeContent(w, b)
return
}
w.WriteHeader(code)
if code != 204 {
json.NewEncoder(w).Encode(value)
}
}
func run() error {
if len(os.Args) > 1 && os.Args[1] == "memory-snapshot" {
return memorySnapshot()
}
if env("SESSION_STORE", "valkey") != "valkey" {
return errors.New("SESSION_STORE must be valkey")
}
address := net.JoinHostPort(env("APP_HOST", "0.0.0.0"), env("APP_PORT", "8080"))
if len(os.Args) > 1 && os.Args[1] == "healthcheck" {
client := http.Client{Timeout: 2 * time.Second}
res, e := client.Get("http://127.0.0.1:" + env("APP_PORT", "8080") + "/health/ready")
if e != nil {
return errors.New("not ready")
}
defer res.Body.Close()
if res.StatusCode != 200 {
return errors.New("not ready")
}
return nil
}
dbURL, err := secretFile("DATABASE_URL_FILE")
if err != nil {
return err
}
cfg, err := pgxpool.ParseConfig(dbURL)
if err != nil {
return errors.New("invalid database configuration")
}
cfg.MaxConns = int32(positive("DB_POOL_MAX", 8))
dbTrace := &databaseTracer{}
cfg.ConnConfig.Tracer = dbTrace
cfg.MinConns = 0
cfg.ConnConfig.ConnectTimeout = 2 * time.Second
cfg.ConnConfig.RuntimeParams["statement_timeout"] = "3000"
cfg.ConnConfig.RuntimeParams["lock_timeout"] = "1000"
cfg.ConnConfig.RuntimeParams["synchronous_commit"] = "on"
startup, cancel := context.WithTimeout(context.Background(), 45*time.Second)
defer cancel()
db, err := pgxpool.NewWithConfig(startup, cfg)
if err != nil {
return errors.New("database setup failed")
}
defer db.Close()
if len(os.Args) > 1 && os.Args[1] == "migrate" {
if err = migrate(startup, db); err != nil {
return fmt.Errorf("migration failed: %w", err)
}
slog.Info("migrations complete")
return nil
}
var schemaReady bool
if err = db.QueryRow(startup, "SELECT EXISTS(SELECT 1 FROM schema_migrations WHERE version='006_observability.sql')").Scan(&schemaReady); err != nil || !schemaReady {
return errors.New("run the migrate service before starting API")
}
if len(os.Args) > 1 {
return imageCommand(db, dbTrace, os.Args[1:])
}
cacheURL, err := secretFile("CACHE_URL_FILE")
if err != nil {
return err
}
signing, err := secretFile("CURSOR_SECRET_FILE")
if err != nil {
return err
}
if len(signing) < 32 {
return errors.New("cursor secret too short")
}
options, err := redis.ParseURL(cacheURL)
if err != nil {
return errors.New("invalid session store configuration")
}
options.PoolSize = 8
options.MaxActiveConns = 8
options.DialTimeout = time.Second
options.ReadTimeout = time.Second
options.WriteTimeout = time.Second
options.PoolTimeout = time.Second
options.MaxRetries = -1
options.ContextTimeoutEnabled = true
sessions := redis.NewClient(options)
defer sessions.Close()
hostname, err := os.Hostname()
if err != nil {
return errors.New("cannot identify instance")
}
s3, bucket, err := s3Client()
if err != nil {
return err
}
a := &App{s3: s3, bucket: bucket, uploadSlots: make(chan struct{}, 5), decodeSlot: make(chan struct{}, 1), db: db, sessions: sessions, secret: []byte(signing), ttl: time.Duration(positive("SESSION_TTL_SECONDS", 3600)) * time.Second, slots: make(chan struct{}, positive("MAX_INFLIGHT", 32)), kdf: make(chan struct{}, 2), dummyHash: hashPassword("unknown-user-dummy-password"), instance: hostname, idempotencyTTL: time.Duration(positive("IDEMPOTENCY_TTL_SECONDS", 86400)) * time.Second}
if a.idempotencyTTL < 24*time.Hour {
return errors.New("idempotency retention must be at least 24 hours")
}
instruments, cancelTelemetry, err := startTelemetry("api", db, dbTrace)
if err != nil {
return err
}
defer cancelTelemetry()
a.telemetry = instruments
server := &http.Server{Addr: address, Handler: a, ReadHeaderTimeout: 2 * time.Second, ReadTimeout: 30 * time.Second, WriteTimeout: 35 * time.Second, IdleTimeout: 5 * time.Second, MaxHeaderBytes: 16 * 1024}
listener, err := net.Listen("tcp", address)
if err != nil {
return errors.New("listen failed")
}
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
if env("POOL_METRICS", "0") == "1" {
go func() {
ticker := time.NewTicker(time.Second)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
s := db.Stat()
slog.Info("pool", "instance", a.instance, "acquired", s.AcquiredConns(), "max", s.MaxConns(), "empty_acquires", s.EmptyAcquireCount(), "acquire_ns", s.AcquireDuration().Nanoseconds())
}
}
}()
}
done := make(chan struct{})
go func() {
<-ctx.Done()
a.stopping.Store(true)
slog.Info("shutdown started")
// Allow two LB readiness checks to drain this backend before closing it.
time.Sleep(1500 * time.Millisecond)
shutdown, cancel := context.WithTimeout(context.Background(), 6*time.Second)
defer cancel()
err := server.Shutdown(shutdown)
if err != nil {
server.Close()
}
slog.Info("shutdown complete", "forced", err != nil)
close(done)
}()
slog.Info("listening", "address", address, "environment", env("APP_ENV", "lab"), "session_store", "valkey", "session_ttl_seconds", int(a.ttl.Seconds()))
if err = server.Serve(listener); err != nil && !errors.Is(err, http.ErrServerClosed) {
return errors.New("serve failed")
}
<-done
return nil
}
func main() {
slog.SetDefault(slog.New(slog.NewJSONHandler(os.Stdout, nil)))
if err := run(); err != nil {
slog.Error("startup failed", "reason", err.Error())
os.Exit(1)
}
}
+27
View File
@@ -0,0 +1,27 @@
package main
import (
"encoding/json"
"fmt"
"os"
"strconv"
"strings"
)
// Local operator command, never an HTTP endpoint. cgroup v2 records peaks
// between Docker samples too; the experiment recreates API cgroups per case.
func memorySnapshot() error {
result := map[string]uint64{}
for _, name := range []string{"memory.current", "memory.peak"} {
raw, err := os.ReadFile("/sys/fs/cgroup/" + name)
if err != nil {
return fmt.Errorf("cgroup v2 %s unavailable", name)
}
value, err := strconv.ParseUint(strings.TrimSpace(string(raw)), 10, 64)
if err != nil {
return fmt.Errorf("invalid cgroup %s", name)
}
result[name] = value
}
return json.NewEncoder(os.Stdout).Encode(result)
}
+28
View File
@@ -0,0 +1,28 @@
CREATE TABLE users (
id uuid PRIMARY KEY,
username text NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'),
display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100),
password_hash text NOT NULL,
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
);
CREATE TABLE chats (
id uuid PRIMARY KEY,
title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100),
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
);
CREATE TABLE chat_members (
chat_id uuid NOT NULL REFERENCES chats(id),
user_id uuid NOT NULL REFERENCES users(id),
PRIMARY KEY (chat_id, user_id)
);
CREATE INDEX chat_members_user_idx ON chat_members(user_id, chat_id);
CREATE INDEX chats_page_idx ON chats(created_at, id);
CREATE TABLE messages (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats(id),
sender_id uuid NOT NULL,
text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000),
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
FOREIGN KEY (chat_id, sender_id) REFERENCES chat_members(chat_id, user_id)
);
CREATE INDEX messages_history_idx ON messages(chat_id, created_at, id);
+12
View File
@@ -0,0 +1,12 @@
-- PostgreSQL text cannot contain U+0000, although it is a valid JSON string
-- code point. Keep exact UTF-8 bytes for free-form API strings. The API checks
-- Unicode length/whitespace; these checks also bound stored byte lengths.
ALTER TABLE users DROP CONSTRAINT users_display_name_check;
ALTER TABLE users ALTER COLUMN display_name TYPE bytea USING convert_to(display_name,'UTF8');
ALTER TABLE users ADD CHECK (octet_length(display_name) BETWEEN 1 AND 400);
ALTER TABLE chats DROP CONSTRAINT chats_title_check;
ALTER TABLE chats ALTER COLUMN title TYPE bytea USING convert_to(title,'UTF8');
ALTER TABLE chats ADD CHECK (octet_length(title) BETWEEN 1 AND 400);
ALTER TABLE messages DROP CONSTRAINT messages_text_check;
ALTER TABLE messages ALTER COLUMN text TYPE bytea USING convert_to(text,'UTF8');
ALTER TABLE messages ADD CHECK (octet_length(text) BETWEEN 1 AND 16000);
+8
View File
@@ -0,0 +1,8 @@
CREATE TABLE message_keys (
user_id uuid NOT NULL REFERENCES users(id),
chat_id uuid NOT NULL REFERENCES chats(id),
key text NOT NULL CHECK (key ~ '^[A-Za-z0-9._:-]{1,128}$'),
message_id uuid NOT NULL REFERENCES messages(id),
expires_at timestamptz NOT NULL,
PRIMARY KEY (user_id, chat_id, key)
);
+27
View File
@@ -0,0 +1,27 @@
CREATE TABLE attachments (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats(id),
owner_id uuid NOT NULL,
object_key text NOT NULL UNIQUE CHECK (object_key = 'original/' || id::text),
filename text NOT NULL CHECK (char_length(filename) BETWEEN 1 AND 255),
content_type text NOT NULL CHECK (content_type IN ('image/png','image/jpeg')),
size_bytes bigint NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760),
sha256 text NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'),
missing_since timestamptz,
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
updated_at timestamptz NOT NULL DEFAULT clock_timestamp(),
UNIQUE (chat_id,id),
FOREIGN KEY (chat_id,owner_id) REFERENCES chat_members(chat_id,user_id)
);
CREATE INDEX attachments_page_idx ON attachments(id);
ALTER TABLE messages DROP CONSTRAINT messages_text_check;
ALTER TABLE messages ADD CHECK (octet_length(text) BETWEEN 0 AND 16000);
ALTER TABLE messages ADD COLUMN attachment_ids uuid[] NOT NULL DEFAULT '{}';
ALTER TABLE messages ADD CHECK (cardinality(attachment_ids) <= 10);
CREATE TABLE message_attachments (
message_id uuid NOT NULL REFERENCES messages(id),
chat_id uuid NOT NULL,
attachment_id uuid NOT NULL,
PRIMARY KEY (message_id,attachment_id),
FOREIGN KEY (chat_id,attachment_id) REFERENCES attachments(chat_id,id)
);
+37
View File
@@ -0,0 +1,37 @@
ALTER TABLE attachments
ADD COLUMN status text NOT NULL DEFAULT 'ready' CHECK (status IN ('queued','processing','ready','failed')),
ADD COLUMN error_code text,
ADD COLUMN active_version integer NOT NULL DEFAULT 0,
ADD COLUMN result_version integer NOT NULL DEFAULT 0,
ADD COLUMN variants jsonb NOT NULL DEFAULT '[]',
ADD CHECK ((status='failed') = (error_code IS NOT NULL)),
ADD CHECK (active_version >= result_version AND result_version >= 0);
-- A queued row is also the transactional outbox entry. Keep completed rows as
-- the deduplication ledger and failed rows as the operator's dead-letter registry.
CREATE TABLE image_jobs (
id uuid PRIMARY KEY,
attachment_id uuid NOT NULL REFERENCES attachments(id),
version integer NOT NULL CHECK (version IN (1,2)),
attempt integer NOT NULL DEFAULT 1,
cycle_start integer NOT NULL DEFAULT 1 CHECK (cycle_start BETWEEN 1 AND attempt),
state text NOT NULL DEFAULT 'queued' CHECK (state IN ('queued','processing','ready','failed')),
published boolean NOT NULL DEFAULT false,
next_at timestamptz NOT NULL DEFAULT clock_timestamp(),
lease_until timestamptz,
error_code text,
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
updated_at timestamptz NOT NULL DEFAULT clock_timestamp(),
UNIQUE (attachment_id,version),
CHECK ((state='processing') = (lease_until IS NOT NULL))
);
CREATE INDEX image_jobs_outbox ON image_jobs(next_at,id) WHERE state='queued' AND NOT published;
CREATE INDEX image_jobs_lease ON image_jobs(lease_until,id) WHERE state='processing';
CREATE TABLE image_job_replays (
job_id uuid NOT NULL REFERENCES image_jobs(id),
attempt integer NOT NULL,
previous_error text NOT NULL,
replayed_at timestamptz NOT NULL DEFAULT clock_timestamp(),
PRIMARY KEY (job_id,attempt)
);
+7
View File
@@ -0,0 +1,7 @@
ALTER TABLE image_jobs
ADD COLUMN traceparent text NOT NULL DEFAULT '',
ADD COLUMN tracestate text NOT NULL DEFAULT '',
ADD COLUMN ready_at timestamptz;
-- Use the historical ready timestamp for lab5 rows that already completed.
UPDATE image_jobs SET ready_at=updated_at WHERE state='ready';
+65
View File
@@ -0,0 +1,65 @@
"""Decode a bounded local input and write metadata-free variants for one job."""
import json
from pathlib import Path
import sys
import warnings
from PIL import Image, ImageOps, UnidentifiedImageError
Image.MAX_IMAGE_PIXELS = 40_000_000
warnings.simplefilter("error", Image.DecompressionBombWarning)
def fit(image, edge):
width, height = image.size
longest = max(width, height)
if longest <= edge:
return image
size = (max(1, width * edge // longest), max(1, height * edge // longest))
return image.resize(size, Image.Resampling.LANCZOS)
def white_rgb(image):
if image.mode == "RGBA":
background = Image.new("RGB", image.size, "white")
background.paste(image, mask=image.getchannel("A"))
return background
return image.convert("RGB")
def render(directory, mime, version):
try:
with Image.open(directory / "input", formats=("JPEG", "PNG")) as source:
if Image.MIME[source.format] != mime or source.width * source.height > Image.MAX_IMAGE_PIXELS:
return 2
source.seek(0)
# PNG may carry transparency in a tRNS chunk or in its alpha channel.
mode = "RGBA" if "A" in source.getbands() or "transparency" in source.info else "RGB"
ImageOps.exif_transpose(source, in_place=True)
image = source.convert(mode)
except (UnidentifiedImageError, OSError, ValueError, SyntaxError,
Image.DecompressionBombError, Image.DecompressionBombWarning):
return 2
square = min(image.size)
left = (image.width - square) // 2
top = (image.height - square) // 2
outputs = [
("thumbnail", fit(image, 256), "JPEG", 85),
("crop", image.resize((128, 128), Image.Resampling.LANCZOS,
box=(left, top, left + square, top + square)), "JPEG", 85),
("optimized", fit(image, 1280), "WEBP", 80 if version == "1" else 90),
]
variants = []
for name, pixels, format_name, quality in outputs:
clean = white_rgb(pixels)
clean.info.clear()
clean.save(directory / name, format=format_name, quality=quality)
variants.append({"width": clean.width, "height": clean.height})
print(json.dumps(variants))
return 0
if __name__ == "__main__":
sys.exit(render(Path(sys.argv[1]), sys.argv[2], sys.argv[3]))
+239
View File
@@ -0,0 +1,239 @@
package main
import (
"crypto/hmac"
"crypto/rand"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"encoding/json"
"errors"
"io"
"mime"
"net/http"
"net/url"
"regexp"
"strconv"
"strings"
"unicode/utf8"
)
type User struct {
ID string `json:"id"`
Username string `json:"username"`
DisplayName string `json:"display_name"`
CreatedAt string `json:"created_at"`
}
type Chat struct {
ID string `json:"id"`
Title string `json:"title"`
MemberIDs []string `json:"member_ids"`
CreatedAt string `json:"created_at"`
}
type Message struct {
AttachmentIDs []string `json:"attachment_ids"`
ID string `json:"id"`
ChatID string `json:"chat_id"`
SenderID string `json:"sender_id"`
Text string `json:"text"`
CreatedAt string `json:"created_at"`
}
type Page[T any] struct {
Items []T `json:"items"`
NextCursor *string `json:"next_cursor"`
}
type apiError struct {
status int
code, message string
}
func (e apiError) Error() string { return e.code }
func invalid() apiError { return apiError{400, "invalid_request", "Invalid request"} }
func unavailable() apiError { return apiError{503, "unavailable", "Dependency unavailable"} }
func missing() apiError { return apiError{404, "not_found", "Object not found"} }
func unauthorized() apiError { return apiError{401, "unauthorized", "Valid Bearer session required"} }
func uuid() string {
b := make([]byte, 16)
if _, err := rand.Read(b); err != nil {
panic(err)
}
b[6] = (b[6] & 15) | 64
b[8] = (b[8] & 63) | 128
s := hex.EncodeToString(b)
return s[:8] + "-" + s[8:12] + "-" + s[12:16] + "-" + s[16:20] + "-" + s[20:]
}
var uuidPattern = regexp.MustCompile(`^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$`)
var usernamePattern = regexp.MustCompile(`^[a-z0-9_]{3,32}$`)
var idempotencyPattern = regexp.MustCompile(`^[A-Za-z0-9._:-]{1,128}$`)
func normalizeID(s string) (string, error) {
if !uuidPattern.MatchString(s) {
return "", invalid()
}
return strings.ToLower(s), nil
}
func textField(m map[string]json.RawMessage, key string, max int) (string, error) {
var s string
b, ok := m[key]
if !ok || string(b) == "null" || json.Unmarshal(b, &s) != nil || strings.TrimSpace(s) == "" || utf8.RuneCountInString(s) > max {
return "", invalid()
}
return s, nil
}
// Reject duplicate keys, invalid UTF-8, and unpaired UTF-16 escapes rather than
// letting encoding/json silently replace malformed Unicode with U+FFFD.
func validUnicode(b []byte) bool {
if !utf8.Valid(b) {
return false
}
for i := 0; i < len(b); i++ {
if b[i] != '\\' {
continue
}
i++
if i >= len(b) {
return false
}
if b[i] != 'u' {
continue
}
if i+4 >= len(b) {
return false
}
n, e := strconv.ParseUint(string(b[i+1:i+5]), 16, 16)
if e != nil {
return false
}
i += 4
if n >= 0xDC00 && n <= 0xDFFF {
return false
}
if n >= 0xD800 && n <= 0xDBFF {
if i+6 >= len(b) || string(b[i+1:i+3]) != `\u` {
return false
}
low, e := strconv.ParseUint(string(b[i+3:i+7]), 16, 16)
if e != nil || low < 0xDC00 || low > 0xDFFF {
return false
}
i += 6
}
}
return true
}
func readObject(w http.ResponseWriter, r *http.Request, max int, fields []string) (map[string]json.RawMessage, error) {
media, params, err := mime.ParseMediaType(r.Header.Get("Content-Type"))
if err != nil || media != "application/json" || (params["charset"] != "" && !strings.EqualFold(params["charset"], "utf-8")) {
return nil, apiError{415, "unsupported_media_type", "Expected application/json UTF-8"}
}
b, err := io.ReadAll(http.MaxBytesReader(w, r.Body, int64(max)))
if err != nil {
var large *http.MaxBytesError
if errors.As(err, &large) {
return nil, apiError{413, "payload_too_large", "Request body exceeds byte limit"}
}
return nil, invalid()
}
if !validUnicode(b) {
return nil, invalid()
}
d := json.NewDecoder(strings.NewReader(string(b)))
token, err := d.Token()
if err != nil || token != json.Delim('{') {
return nil, invalid()
}
m := map[string]json.RawMessage{}
for d.More() {
t, err := d.Token()
if err != nil {
return nil, invalid()
}
key, ok := t.(string)
if !ok {
return nil, invalid()
}
allowed := false
for _, f := range fields {
if key == f {
allowed = true
}
}
if !allowed || m[key] != nil {
return nil, invalid()
}
var value json.RawMessage
if d.Decode(&value) != nil {
return nil, invalid()
}
m[key] = value
}
if _, err = d.Token(); err != nil {
return nil, invalid()
}
if _, err = d.Token(); err != io.EOF {
return nil, invalid()
}
return m, nil
}
type cursor struct {
Scope string `json:"s"`
Time string `json:"t"`
ID string `json:"i"`
}
func (a *App) encodeCursor(scope, t, id string) string {
b, _ := json.Marshal(cursor{scope, t, id})
h := hmac.New(sha256.New, a.secret)
h.Write(b)
return base64.RawURLEncoding.EncodeToString(b) + "." + base64.RawURLEncoding.EncodeToString(h.Sum(nil))
}
func (a *App) pagination(r *http.Request, scope string) (int, cursor, error) {
values, e := url.ParseQuery(r.URL.RawQuery)
if e != nil {
return 0, cursor{}, invalid()
}
limit := 50
if v, ok := values["limit"]; ok {
if len(v) != 1 || v[0] == "" {
return 0, cursor{}, invalid()
}
for _, c := range v[0] {
if c < '0' || c > '9' {
return 0, cursor{}, invalid()
}
}
limit, e = strconv.Atoi(v[0])
if e != nil || limit < 1 || limit > 100 {
return 0, cursor{}, invalid()
}
}
c := cursor{}
if v, ok := values["cursor"]; ok {
if len(v) != 1 || len(v[0]) > 1024 {
return 0, c, invalid()
}
parts := strings.Split(v[0], ".")
if len(parts) != 2 {
return 0, c, invalid()
}
b, e := base64.RawURLEncoding.DecodeString(parts[0])
if e != nil {
return 0, c, invalid()
}
sig, e := base64.RawURLEncoding.DecodeString(parts[1])
if e != nil {
return 0, c, invalid()
}
h := hmac.New(sha256.New, a.secret)
h.Write(b)
if !hmac.Equal(sig, h.Sum(nil)) || json.Unmarshal(b, &c) != nil || c.Scope != scope {
return 0, c, invalid()
}
}
return limit, c, nil
}
+41
View File
@@ -0,0 +1,41 @@
"""Create local secrets once. Never print values or overwrite existing secrets."""
from pathlib import Path
import secrets
root = Path(__file__).resolve().parents[1]
folder = root / 'secrets'
folder.mkdir(exist_ok=True)
folder.chmod(0o700)
def put(name, value):
path = folder / name
if not path.exists():
with path.open('x', encoding='utf-8', newline='\n') as f:
f.write(value + '\n')
# Parent directory is private; Compose mounts individual files for non-root services.
path.chmod(0o644)
return path.read_text(encoding='utf-8').strip()
pg = put('postgres_password', secrets.token_hex(32))
vk = put('valkey_password', secrets.token_hex(32))
put('database_url', f'postgres://messenger:{pg}@db:5432/messenger?sslmode=disable')
put('cache_url', f'redis://:{vk}@valkey:6379/0')
put('cursor_secret', secrets.token_hex(32))
put('valkey.conf', '\n'.join(['bind 0.0.0.0', 'protected-mode yes', f'requirepass {vk}',
'dir /data', 'appendonly yes', 'appendfsync always', 'maxmemory 128mb',
'maxmemory-policy noeviction', 'save 60 1000']))
broker = put('broker_password', secrets.token_hex(32))
put('broker_url', f'redis://:{broker}@broker:6379/0')
put('broker.conf', '\n'.join(['bind 0.0.0.0', 'protected-mode yes', f'requirepass {broker}',
'dir /data', 'appendonly yes', 'appendfsync always', 'maxmemory 128mb',
'maxmemory-policy noeviction', 'save ""']))
if not (root / '.env').exists():
(root / '.env').write_text((root / '.env.example').read_text(encoding='utf-8'),encoding='utf-8')
put('s3_root_user', 'admin_' + secrets.token_hex(8))
put('s3_root_password', secrets.token_hex(32))
put('s3_access_key_id', 'api_' + secrets.token_hex(8))
put('s3_secret_access_key', secrets.token_hex(32))
put('s3_worker_access_key_id', 'worker_' + secrets.token_hex(8))
put('s3_worker_secret_access_key', secrets.token_hex(32))
for name in ('restart.sh', 'stop-one.sh'):
(root / 'scripts' / name).chmod(0o755)
print('Local .env and secrets ready; existing values preserved.')
+11
View File
@@ -0,0 +1,11 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
mkdir -p evidence
services="db valkey s3 api lb broker dispatcher worker prometheus loki tempo otel-collector grafana"
ids=$(docker compose ps -q $services)
docker inspect --format '{{.Id}} {{index .Config.Labels "com.docker.compose.service"}} {{.State.StartedAt}} {{.State.Status}}' $ids > evidence/restart-before.txt
docker compose restart --no-deps db valkey s3 broker prometheus loki tempo otel-collector grafana
docker compose up -d --no-deps --no-build --wait --wait-timeout 35 db valkey s3 broker
docker compose restart --no-deps api dispatcher worker lb
docker inspect --format '{{.Id}} {{index .Config.Labels "com.docker.compose.service"}} {{.State.StartedAt}} {{.State.Status}}' $ids > evidence/restart-after.txt
+16
View File
@@ -0,0 +1,16 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
: "${TARGET_INSTANCE_ID:?TARGET_INSTANCE_ID is required}"
for id in $(docker compose ps -q api); do
hostname=$(docker inspect --format '{{.Config.Hostname}}' "$id")
if [ "$hostname" = "$TARGET_INSTANCE_ID" ]; then
mkdir -p evidence
docker inspect --format '{{.Id}} {{.State.Status}} {{.State.StartedAt}}' "$id" > evidence/stopped-before.txt
docker stop --time 9 "$id" > /dev/null
docker inspect --format '{{.Id}} {{.State.Status}} {{.State.FinishedAt}} {{.State.ExitCode}}' "$id" > evidence/stopped-after.txt
exit 0
fi
done
echo 'Target is not a running API container' >&2
exit 1
+223
View File
@@ -0,0 +1,223 @@
package main
import (
"context"
"crypto/sha256"
"embed"
"encoding/hex"
"errors"
"fmt"
"sort"
"time"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgconn"
"github.com/jackc/pgx/v5/pgxpool"
)
//go:embed migrations/*.sql
var migrations embed.FS
func migrate(ctx context.Context, db *pgxpool.Pool) error {
conn, err := db.Acquire(ctx)
if err != nil {
return err
}
defer conn.Release()
if _, err = conn.Exec(ctx, "SELECT pg_advisory_lock(202602)"); err != nil {
return err
}
defer conn.Exec(context.Background(), "SELECT pg_advisory_unlock(202602)")
if _, err = conn.Exec(ctx, "CREATE TABLE IF NOT EXISTS schema_migrations(version text PRIMARY KEY, checksum text NOT NULL, applied_at timestamptz NOT NULL DEFAULT clock_timestamp())"); err != nil {
return err
}
entries, err := migrations.ReadDir("migrations")
if err != nil {
return err
}
sort.Slice(entries, func(i, j int) bool { return entries[i].Name() < entries[j].Name() })
for _, entry := range entries {
raw, err := migrations.ReadFile("migrations/" + entry.Name())
if err != nil {
return err
}
sum := sha256.Sum256(raw)
checksum := hex.EncodeToString(sum[:])
var previous string
err = conn.QueryRow(ctx, "SELECT checksum FROM schema_migrations WHERE version=$1", entry.Name()).Scan(&previous)
if err == nil {
if previous != checksum {
return fmt.Errorf("migration checksum mismatch: %s", entry.Name())
}
continue
}
if !errors.Is(err, pgx.ErrNoRows) {
return err
}
tx, err := conn.Begin(ctx)
if err != nil {
return err
}
if _, err = tx.Exec(ctx, string(raw)); err == nil {
_, err = tx.Exec(ctx, "INSERT INTO schema_migrations(version,checksum) VALUES($1,$2)", entry.Name(), checksum)
}
if err != nil {
tx.Rollback(ctx)
return err
}
if err = tx.Commit(ctx); err != nil {
return err
}
}
return nil
}
func dbError(err error) error {
if err == nil {
return nil
}
if errors.Is(err, pgx.ErrNoRows) {
return missing()
}
var pe *pgconn.PgError
if errors.As(err, &pe) {
switch pe.Code {
case "23505":
return apiError{409, "conflict", "Username already exists"}
case "23503", "23514", "22P02", "22021":
return invalid()
}
}
return unavailable()
}
func stamp(t time.Time) string { return t.UTC().Format("2006-01-02T15:04:05.000000Z") }
// pgx BytesScanner preserves every validated UTF-8 code point, including NUL.
type utf8String struct{ dst *string }
func (s utf8String) ScanBytes(value []byte) error {
*s.dst = string(value)
return nil
}
func scanUser(row pgx.Row) (User, error) {
var u User
var t time.Time
err := row.Scan(&u.ID, &u.Username, utf8String{&u.DisplayName}, &t)
u.CreatedAt = stamp(t)
return u, dbError(err)
}
func (a *App) user(ctx context.Context, id string) (User, error) {
return scanUser(a.db.QueryRow(ctx, "SELECT id::text,username,display_name,created_at FROM users WHERE id=$1", id))
}
func (a *App) createChat(ctx context.Context, title string, ids []string) (Chat, error) {
c := Chat{ID: uuid(), Title: title, MemberIDs: ids}
var t time.Time
tx, err := a.db.Begin(ctx)
if err != nil {
return c, unavailable()
}
defer tx.Rollback(ctx)
err = tx.QueryRow(ctx, "INSERT INTO chats(id,title) VALUES($1,$2) RETURNING created_at", c.ID, []byte(title)).Scan(&t)
if err != nil {
return c, dbError(err)
}
// FK failure on any participant rolls back the chat and earlier participants.
for _, id := range ids {
if _, err = tx.Exec(ctx, "INSERT INTO chat_members(chat_id,user_id) VALUES($1,$2)", c.ID, id); err != nil {
return c, dbError(err)
}
}
if err = tx.Commit(ctx); err != nil {
return c, unavailable()
}
c.CreatedAt = stamp(t)
return c, nil
}
func (a *App) chat(ctx context.Context, id, user string) (Chat, error) {
c := Chat{}
var t time.Time
err := a.db.QueryRow(ctx, `SELECT c.id::text,c.title,c.created_at,
ARRAY(SELECT user_id::text FROM chat_members WHERE chat_id=c.id ORDER BY user_id)
FROM chats c WHERE c.id=$1 AND EXISTS(SELECT 1 FROM chat_members WHERE chat_id=c.id AND user_id=$2)`, id, user).Scan(&c.ID, utf8String{&c.Title}, &t, &c.MemberIDs)
c.CreatedAt = stamp(t)
return c, dbError(err)
}
func boundary(c cursor) (time.Time, string, error) {
if c.ID == "" {
return time.Date(1, 1, 1, 0, 0, 0, 0, time.UTC), "00000000-0000-0000-0000-000000000000", nil
}
t, err := time.Parse(time.RFC3339Nano, c.Time)
if err != nil {
return t, "", invalid()
}
id, err := normalizeID(c.ID)
return t, id, err
}
func (a *App) history(ctx context.Context, id string, limit int, c cursor, scope string) (Page[Message], error) {
result := Page[Message]{Items: []Message{}}
t, last, err := boundary(c)
if err != nil {
return result, err
}
rows, err := a.db.Query(ctx, `SELECT id::text,chat_id::text,sender_id::text,text,created_at,ARRAY(SELECT unnest(attachment_ids)::text) FROM messages
WHERE chat_id=$1 AND (created_at,id)>($2,$3) ORDER BY messages.created_at,messages.id LIMIT $4`, id, t, last, limit+1)
if err != nil {
return result, unavailable()
}
defer rows.Close()
for rows.Next() {
var m Message
var created time.Time
if err = rows.Scan(&m.ID, &m.ChatID, &m.SenderID, utf8String{&m.Text}, &created, &m.AttachmentIDs); err != nil {
return result, unavailable()
}
m.CreatedAt = stamp(created)
result.Items = append(result.Items, m)
}
if rows.Err() != nil {
return result, unavailable()
}
if len(result.Items) > limit {
result.Items = result.Items[:limit]
m := result.Items[limit-1]
next := a.encodeCursor(scope, m.CreatedAt, m.ID)
result.NextCursor = &next
}
return result, nil
}
func (a *App) listChats(ctx context.Context, user string, limit int, c cursor, scope string) (Page[Chat], error) {
result := Page[Chat]{Items: []Chat{}}
t, last, err := boundary(c)
if err != nil {
return result, err
}
rows, err := a.db.Query(ctx, `SELECT c.id::text,c.title,c.created_at,
ARRAY(SELECT user_id::text FROM chat_members WHERE chat_id=c.id ORDER BY user_id)
FROM chats c JOIN chat_members own ON own.chat_id=c.id AND own.user_id=$1
WHERE (c.created_at,c.id)>($2,$3) ORDER BY c.created_at,c.id LIMIT $4`, user, t, last, limit+1)
if err != nil {
return result, unavailable()
}
defer rows.Close()
for rows.Next() {
var chat Chat
var created time.Time
if err = rows.Scan(&chat.ID, utf8String{&chat.Title}, &created, &chat.MemberIDs); err != nil {
return result, unavailable()
}
chat.CreatedAt = stamp(created)
result.Items = append(result.Items, chat)
}
if rows.Err() != nil {
return result, unavailable()
}
if len(result.Items) > limit {
result.Items = result.Items[:limit]
v := result.Items[limit-1]
next := a.encodeCursor(scope, v.CreatedAt, v.ID)
result.NextCursor = &next
}
return result, nil
}
+82
View File
@@ -0,0 +1,82 @@
"""Check image edge cases through the public API using the worker's pinned Pillow."""
import base64
import json
from pathlib import Path
import struct
import subprocess
import sys
import zlib
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, check
def pillow(code, data=""):
result = subprocess.run(
["docker", "compose", "exec", "-T", "worker", "python3", "-c", code],
cwd=ROOT, input=data, capture_output=True, text=True, encoding="utf-8", timeout=30, check=True,
)
return json.loads(result.stdout)
def main():
client = Client()
alice, bob, _, chat = client.bootstrap()
path = f"/api/v1/chats/{chat['id']}/attachments?filename=synthetic"
fixtures = pillow("""
import base64, io, json
from PIL import Image
jpeg = Image.new("RGB", (300, 100), "red")
exif = Image.Exif()
exif[274] = 6
exif[271] = "Synthetic EXIF"
png = Image.new("RGBA", (8, 4), (255, 0, 0, 0))
def encode(image, kind, **options):
buffer = io.BytesIO()
image.save(buffer, kind, **options)
return base64.b64encode(buffer.getvalue()).decode()
print(json.dumps([encode(jpeg, "JPEG", exif=exif), encode(png, "PNG")]))
""")
downloads = {}
for fixture, mime, dimensions in zip(fixtures, ("image/jpeg", "image/png"),
([(85, 256), (128, 128), (100, 300)],
[(8, 4), (128, 128), (8, 4)])):
raw = base64.b64decode(fixture)
att = client.call("POST", path, 202, actor=alice, raw=raw,
headers={"Content-Type": mime}, schema="Attachment")["json"]
ready = client.wait_ready(bob, att)
content = f"/api/v1/attachments/{att['id']}/content"
check(client.call("GET", content, actor=bob)["bytes"] == raw, "original changed")
for (width, height), name in zip(dimensions, ("thumbnail", "crop", "optimized")):
download = client.call("GET", content + "?variant=" + name, actor=bob)
downloads[mime + "/" + name] = [base64.b64encode(download["bytes"]).decode(), width, height]
print(f"{mime} attachment={att['id']} dimensions={dimensions}")
probes = pillow("""
import base64, io, json, sys
from PIL import Image
results = {}
for key, (body, width, height) in json.load(sys.stdin).items():
with Image.open(io.BytesIO(base64.b64decode(body))) as image:
image.load()
white = all(channel >= 250 for channel in image.convert("RGB").getpixel((0, 0)))
results[key] = image.size == (width, height) and not image.getexif() and (white or not key.startswith("image/png"))
print(json.dumps(results))
""", json.dumps(downloads))
check(all(probes.values()), "wrong actual dimensions, leaked EXIF or invalid alpha composition")
fixture = (ROOT / "tests/fixtures/sample.png").read_bytes()
ihdr = struct.pack(">II", 10_000, 4_001) + fixture[24:29]
too_many_pixels = fixture[:16] + ihdr + struct.pack(">I", zlib.crc32(b"IHDR" + ihdr)) + fixture[33:]
for body, mime, status in (
(too_many_pixels, "image/png", 413),
(b"0" * (10 * 1024 * 1024 + 1), "image/png", 413),
(fixture, "image/jpeg", 415),
(fixture[:33], "image/png", 415),
):
client.call("POST", path, status, actor=alice, raw=body, headers={"Content-Type": mime})
print("PASS EXIF, fit, crop, WebP, transparency, metadata removal, byte/pixel/format limits")
if __name__ == "__main__":
main()
+355
View File
@@ -0,0 +1,355 @@
"""Black-box checks on the dedicated, local Lab6 Compose project."""
import argparse
import base64
import json
import os
from pathlib import Path
import re
import subprocess
import sys
import time
from urllib.error import HTTPError
from urllib.parse import urlencode, urlsplit
from urllib.request import urlopen
from uuid import uuid4
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client, check
def local_port(name, default):
return int(os.environ.get(name, default))
BASE = os.environ.get('BASE_URL', f'http://127.0.0.1:{local_port("APP_PORT", 8080)}')
PROM = f'http://127.0.0.1:{local_port("PROMETHEUS_PORT", 9090)}'
LOKI = f'http://127.0.0.1:{local_port("LOKI_PORT", 3100)}'
TEMPO = f'http://127.0.0.1:{local_port("TEMPO_PORT", 3200)}'
GRAFANA = f'http://127.0.0.1:{local_port("GRAFANA_PORT", 3000)}'
COMPOSE = ['docker', 'compose', '--project-directory', str(ROOT)]
FIXTURE = (ROOT / 'tests/fixtures/sample.png').read_bytes()
def compose(*args):
return subprocess.run([*COMPOSE, *args], cwd=ROOT, check=True, capture_output=True,
text=True, encoding='utf-8', errors='replace', timeout=60).stdout.strip()
def backend(base, path, **params):
query = '?' + urlencode(params) if params else ''
with urlopen(base + path + query, timeout=5) as response:
return json.load(response)
def tempo_ready():
with urlopen(TEMPO + '/ready', timeout=5) as response:
return response.status == 200
def prometheus(expr, at=None):
params = {'query': expr}
if at is not None:
params['time'] = at
vector = backend(PROM, '/api/v1/query', **params)['data']['result']
return [{**item['metric'], 'value': float(item['value'][1])} for item in vector]
def wait_for(probe, message, timeout=45):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
try:
result = probe()
if result:
return result
except (HTTPError, OSError):
pass
time.sleep(1)
raise AssertionError(message)
def save(output, name, result):
output.mkdir(parents=True, exist_ok=True)
(output / name).write_text(json.dumps(result, ensure_ascii=False, indent=2, allow_nan=False) + '\n',
encoding='utf-8')
def lab_only():
check(urlsplit(BASE).hostname in {'localhost', '127.0.0.1', '::1'}, 'use the dedicated local lab')
project = json.loads(compose('config', '--format', 'json'))['name']
check(project.startswith('mtusi-lab6'), 'Compose project name must begin with mtusi-lab6')
for service in ('worker', 'dispatcher', 'db', 'tempo', 'otel-collector'):
check(bool(compose('ps', '-q', '--status', 'running', service)), f'{service} must be running')
def actor_and_chat(client, label):
actor = client.user(label)
chat = client.call('POST', '/api/v1/chats', 201, actor=actor,
body={'title': 'Синтетическая Lab6', 'member_ids': [actor['id']]}, schema='Chat')['json']
return actor, chat
def upload(client, actor, chat, filename='synthetic.png'):
return client.call('POST', f'/api/v1/chats/{chat["id"]}/attachments?filename={filename}', 202,
actor=actor, raw=FIXTURE, headers={'Content-Type': 'image/png'}, schema='Attachment')
def loki_records(expression, lookback=300):
now = time.time_ns()
data = backend(LOKI, '/loki/api/v1/query_range', query=expression, start=now - lookback * 10**9,
end=now, limit=100)['data']['result']
result = []
for stream in data:
for timestamp, line, *extra in stream['values']:
metadata = extra[0] if extra else {}
result.append({**stream['stream'], **metadata, 'timestamp_ns': timestamp, 'body': line})
return result
def api_log(request_id, lookback=300):
logs = loki_records(f'{{service_name="messenger-api"}} | request_id="{request_id}"', lookback)
return logs[0] if logs else None
def otlp_value(item):
return next(iter(item.values()))
def span_id(value):
if not value or re.fullmatch('[0-9a-f]{16}', value):
return value
return base64.b64decode(value, validate=True).hex()
def trace_by_id(trace_id):
data = backend(TEMPO, '/api/traces/' + trace_id)
spans = []
for batch in data.get('batches', data.get('resourceSpans', [])):
resource = {a['key']: otlp_value(a['value']) for a in batch['resource']['attributes']}
for scope in batch.get('scopeSpans', batch.get('instrumentationLibrarySpans', [])):
for span in scope['spans']:
spans.append({
'name': span['name'], 'service': resource['service.name'],
'service_version': resource['service.version'],
'environment': resource['deployment.environment.name'],
'id': span_id(span['spanId']), 'parent': span_id(span.get('parentSpanId')),
'kind': span['kind'], 'status': span.get('status', {}),
'duration_ms': round((int(span['endTimeUnixNano']) - int(span['startTimeUnixNano'])) / 1e6, 3),
'attributes': {a['key']: otlp_value(a['value']) for a in span.get('attributes', [])},
})
return spans
def assert_privacy(records, secrets):
serialized = json.dumps(records, ensure_ascii=False)
check(not any(secret in serialized for secret in secrets), 'unexpected synthetic sensitive value in telemetry')
forbidden = {'user_id', 'chat_id', 'message_id', 'attachment_id', 'request_id', 'filename',
'url', 'url_full', 'url_path', 'text', 'trace_id', 'span_id'}
sample = prometheus('count by (__name__) ({__name__=~"messenger_.+"})')
check(bool(sample), 'no application metric series')
series = backend(PROM, '/api/v1/series', **{'match[]': '{__name__=~"messenger_.+"}'})['data']
check(not any(forbidden & entry.keys() for entry in series), 'forbidden metric label')
return len(series)
def correlation(output):
client = Client(BASE)
actor, chat = actor_and_chat(client, 'trace')
canary = 'private-' + uuid4().hex
message = client.message(actor, chat, canary)
compose('stop', 'worker')
extra_worker = 'mtusi-lab6-trace-' + uuid4().hex[:8]
try:
received = upload(client, actor, chat, canary + '.png')
attachment = received['json']['id']
compose('run', '-d', '--no-deps', '--name', extra_worker, '-e', 'LAB5_FAULT=temporary_once',
'-e', 'LAB5_FAULT_ATTACHMENT=' + attachment, 'worker')
wait_for(lambda: client.call('GET', '/api/v1/attachments/' + attachment, actor=actor,
schema='Attachment')['json']['status'] == 'ready',
'retry did not produce a ready result')
request_id = received['headers']['X-Request-Id']
log = wait_for(lambda: api_log(request_id), 'no upload log in Loki')
trace_id = log['trace_id']
check(bool(re.fullmatch('[0-9a-f]{32}', trace_id)), 'invalid Loki trace ID')
logs = wait_for(lambda: enough_attempts(trace_id), 'no log for both worker attempts')
spans = wait_for(lambda: enough_spans(trace_id), 'missing worker trace or S3 spans')
server = next(span for span in spans if span['name'].startswith('POST /api/v1/chats/'))
check(log['span_id'] == server['id'], 'API log has the wrong span')
producers = {int(s['attributes']['job.attempt']): s for s in spans if s['name'].endswith(' publish')}
consumers = {int(s['attributes']['job.attempt']): s for s in spans if s['name'].endswith(' process')}
check(set(producers) == set(consumers) == {1, 2}, 'expected one failed attempt and one retry')
for attempt in (1, 2):
check(producers[attempt]['parent'] == server['id'], 'producer parent is not the upload')
check(consumers[attempt]['parent'] == producers[attempt]['id'], 'consumer parent is not the producer')
check(any(s['span_id'] == consumers[attempt]['id'] for s in logs), 'worker log span mismatch')
check({log['outcome'] for log in logs} == {'queued', 'ready'}, 'missing retry/ready outcomes')
check(any(s['name'] == 'postgresql query' and s['parent'] == consumers[2]['id'] for s in spans),
'missing SQL span for the successful worker attempt')
message_log = wait_for(lambda: api_log(message['headers']['X-Request-Id']), 'missing message log')
message_spans = wait_for(lambda: trace_by_id(message_log['trace_id']), 'missing message trace')
series = assert_privacy([log, logs, spans, message_log, message_spans],
[canary, actor['registration']['password'], actor['token']])
dashboard = backend(GRAFANA, '/api/dashboards/uid/messenger-lab6')['dashboard']
sources = {ds['uid'] for ds in backend(GRAFANA, '/api/datasources')}
check({'lab-prom', 'lab-loki', 'lab-tempo'} <= sources, 'missing provisioned datasource')
save(output, 'correlation.json', {
'trace_id': trace_id, 'request_id': request_id, 'logs': [log, *logs], 'spans': spans,
'privacy': {'synthetic_canaries_absent': True, 'metric_series_checked': series},
'grafana': {'dashboard_uid': dashboard['uid'], 'panels': len(dashboard['panels']),
'datasources': sorted(sources)},
})
finally:
subprocess.run(['docker', 'stop', '-t', '5', extra_worker], capture_output=True, timeout=20)
subprocess.run(['docker', 'rm', extra_worker], capture_output=True, timeout=20)
compose('start', 'worker')
def enough_attempts(trace_id):
logs = loki_records(f'{{service_name="messenger-worker"}} | trace_id="{trace_id}" | event="image.attempt"')
return logs if len(logs) >= 2 else None
def enough_spans(trace_id):
spans = trace_by_id(trace_id)
return spans if sum(span['name'] == 's3 put result' for span in spans) == 4 else None
def queue_restart(output):
def stream_entry(attachment):
raw = compose('exec', '-T', 'broker', 'sh', '-c',
'REDISCLI_AUTH="$(cat /run/secrets/broker_password)" '
'exec valkey-cli --json XRANGE attachment-jobs - +')
for identifier, fields in json.loads(raw):
values = dict(zip(fields[::2], fields[1::2]))
if json.loads(values.get('job', '{}')).get('attachment_id') == attachment:
return identifier
return None
def started_at():
containers = compose('ps', '-q', 'api', 'broker').splitlines()
check(len(containers) >= 3, 'expected two API replicas and a broker running')
raw = subprocess.check_output(
['docker', 'inspect', '--format', '{{.Id}} {{.State.StartedAt}}', *containers], text=True, timeout=15)
return dict(line.split() for line in raw.splitlines())
def api_ready():
with urlopen(BASE + '/health/ready', timeout=5) as response:
return response.status == 200
client = Client(BASE)
actor, chat = actor_and_chat(client, 'restart')
compose('stop', 'worker')
try:
accepted = upload(client, actor, chat)['json']
identifier = accepted['id']
path = '/api/v1/attachments/' + identifier
message = client.message(actor, chat, 'synthetic queued attachment', attachments=[identifier])['json']
entry = wait_for(lambda: stream_entry(identifier), 'outbox did not publish the queued job')
before = started_at()
compose('restart', 'api', 'broker')
wait_for(api_ready, 'API did not recover after restart')
after = started_at()
check(before.keys() == after.keys() and all(before[key] != after[key] for key in before),
'restart did not affect every API replica and the broker')
check(not compose('ps', '-q', '--status', 'running', 'worker'), 'worker processed the job during restart')
check(wait_for(lambda: stream_entry(identifier), 'Stream lost the published job') == entry,
'original Stream entry was lost')
check(client.call('GET', path, actor=actor, schema='Attachment')['json']['status'] == 'queued',
'attachment did not stay queued')
history = client.call('GET', '/api/v1/chats/' + chat['id'] + '/messages', actor=actor)['bytes']
check(message['id'] in history.decode('utf-8'), 'message with queued attachment disappeared')
finally:
compose('start', 'worker')
ready = client.wait_ready(actor, accepted)
original = client.call('GET', path + '/content', actor=actor)['bytes']
thumbnail = client.call('GET', path + '/content?variant=thumbnail', actor=actor)['bytes']
check(original == FIXTURE and thumbnail.startswith(b'\xff\xd8'), 'published file or thumbnail is invalid')
save(output, 'queue-restart.json', {
'api_and_broker_started_at_before': before, 'api_and_broker_started_at_after': after,
'stream_entry_before_and_after': entry, 'worker_stopped_during_restart': True,
'attachment_status_before': 'queued', 'attachment_status_after': ready['status'],
'message_preserved': True, 'original_equal': True, 'thumbnail_jpeg': True,
})
def outage(output):
loss = 'sum({__name__=~"otelcol_exporter_send_failed_spans(_total)?",exporter="otlphttp/tempo"})'
queue = 'max(otelcol_exporter_queue_size{exporter="otlphttp/tempo"})'
refused = 'sum({__name__=~"otelcol_exporter_enqueue_failed_spans(_total)?",exporter="otlphttp/tempo"})'
queries = {'send_failed': loss, 'enqueue_failed': refused, 'queue': queue,
'queue_capacity': 'max(otelcol_exporter_queue_capacity{exporter="otlphttp/tempo"})',
'collector_rss_bytes': 'max({__name__=~"otelcol_process_memory_rss(_bytes)?"})'}
def sample():
return {name: next(iter(prometheus(expr)), {}).get('value', 0) for name, expr in queries.items()}
client = Client(BASE)
actor, chat = actor_and_chat(client, 'backend')
baseline = sample()
collector = compose('ps', '-q', 'otel-collector')
inspect = ['docker', 'inspect', '--format',
'{{.State.StartedAt}} {{.RestartCount}} {{.State.OOMKilled}} {{.HostConfig.Memory}}', collector]
identity = subprocess.check_output(inspect, text=True).strip()
timeline = []
delivered = 0
compose('stop', 'tempo')
started = time.monotonic()
try:
snapshot = started
while time.monotonic() - started < 35:
client.message(actor, chat, 'synthetic during Tempo outage')
delivered += 1
if time.monotonic() >= snapshot:
timeline.append({'elapsed_s': round(time.monotonic() - started, 3), **sample()})
snapshot += 2
time.sleep(.1)
attachment = upload(client, actor, chat)['json']['id']
wait_for(lambda: client.call('GET', '/api/v1/attachments/' + attachment, actor=actor,
schema='Attachment')['json']['status'] == 'ready', 'worker blocked on Tempo')
timeline.append({'elapsed_s': round(time.monotonic() - started, 3), **sample()})
check(delivered >= 50, 'not enough successful writes during outage')
check(max(s['send_failed'] for s in timeline) > baseline['send_failed'], 'no accounted span loss')
check(subprocess.check_output(inspect, text=True).strip() == identity, 'collector restarted or OOMed')
finally:
recovery_started = time.monotonic()
compose('start', 'tempo')
wait_for(tempo_ready, 'Tempo did not become ready after restart')
probes = []
def resumed_trace():
resumed = client.message(actor, chat, 'synthetic after Tempo restart')
probes.append(resumed['headers']['X-Request-Id'])
for request_id in probes[-6:]:
log = api_log(request_id)
if log:
try:
if trace_by_id(log['trace_id']):
return log['trace_id']
except HTTPError as error:
if error.code != 404:
raise
return None
recovered = wait_for(resumed_trace, 'traces did not recover')
save(output, 'backend-outage.json', {
'fault': 'tempo stopped', 'baseline': baseline, 'samples': timeline,
'successful_message_writes': delivered, 'upload_ready_while_tempo_stopped': True,
'collector_started_at_restarts_oom_limit_bytes': identity,
'recovery_trace_id': recovered,
'fault_duration_s': round(recovery_started - started, 3),
'new_trace_after_restart_s': round(time.monotonic() - recovery_started, 3),
})
def main():
args = argparse.ArgumentParser(description=__doc__)
args.add_argument('scenario', choices=['correlation', 'outage', 'queue-restart'])
args.add_argument('--output', type=Path, default=ROOT / 'evidence')
options = args.parse_args()
lab_only()
{'correlation': correlation, 'outage': outage, 'queue-restart': queue_restart}[options.scenario](options.output)
print(options.scenario + ' passed')
if __name__ == '__main__':
main()
+254
View File
@@ -0,0 +1,254 @@
"""Drive the local Lab6 workload and record the three phases of the SLO experiment."""
import argparse
from collections import Counter
from concurrent.futures import ThreadPoolExecutor
from contextlib import contextmanager
import json
import math
from pathlib import Path
import subprocess
import threading
import time
from uuid import uuid4
from observability import (
BASE, COMPOSE, ROOT, Client, actor_and_chat, api_log, check, compose, lab_only,
prometheus, save, trace_by_id, upload, wait_for,
)
ALERTS = ('APIFastBurn', 'APISlowBurn', 'ReadyFastBurn', 'ReadySlowBurn', 'QueueLagHigh')
MESSAGE_RATE = 4
UPLOAD_RATE = 1
@contextmanager
def locked_messages(seconds):
# A lock on one business table keeps SELECT 1/readiness and other routes alive.
# pg_sleep bounds the fault even if the client running the experiment dies.
cmd = [*COMPOSE, 'exec', '-T', 'db', 'psql', '-X', '-q', '-v', 'ON_ERROR_STOP=1',
'-U', 'messenger', '-d', 'messenger']
with subprocess.Popen(cmd, cwd=ROOT, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT, text=True, encoding='utf-8') as session:
assert session.stdin is not None and session.stdout is not None
session.stdin.write("SET application_name = 'lab6_slo_lock';\n"
"SET lock_timeout = '3s';\n"
"BEGIN; LOCK TABLE messages IN ACCESS EXCLUSIVE MODE;\n"
"\\echo LAB6_LOCKED\n"
f"SELECT pg_sleep({seconds});\nCOMMIT;\n\\echo LAB6_RELEASED\n")
session.stdin.close()
check(session.stdout.readline().strip() == 'LAB6_LOCKED', 'database lock setup failed')
started = time.time()
try:
yield started
finally:
session.wait(timeout=seconds + 10)
check('LAB6_RELEASED' in session.stdout.read() and session.returncode == 0, 'database lock not released')
class Workload:
def __init__(self, actor, chat):
self.actor = actor
self.chat = chat
self.client = threading.local()
self.shutdown = threading.Event()
self.rows = []
self.dropped = []
self.pool = ThreadPoolExecutor(max_workers=12)
self.thread = threading.Thread(target=self.schedule)
def one(self, kind):
if not hasattr(self.client, 'value'):
self.client.value = Client(BASE, timeout=4)
client = self.client.value
started = time.time()
try:
if kind == 'message':
response = client.call(
'POST', f'/api/v1/chats/{self.chat["id"]}/messages', [201, 503], actor=self.actor,
body={'text': 'Synthetic constant-rate SLO study'}, headers={'Idempotency-Key': uuid4().hex},
schema='Message',
)
else:
response = upload(client, self.actor, self.chat)
self.rows.append({'t': started, 'kind': kind, 'status': response['status'],
'request_id': response['headers']['X-Request-Id'],
'latency_s': round(time.time() - started, 6),
**({'attachment': response['json']['id']} if kind == 'upload' else {})})
except (AssertionError, OSError):
self.rows.append({'t': started, 'kind': kind, 'status': 'transport_or_contract_error',
'latency_s': round(time.time() - started, 6)})
def schedule(self):
tick = time.monotonic()
i = 0
inflight = set()
while not self.shutdown.is_set():
for kind in ('message', 'upload') if i % (MESSAGE_RATE // UPLOAD_RATE) == 0 else ('message',):
inflight = {task for task in inflight if not task.done()}
if len(inflight) >= 12:
self.dropped.append({'t': time.time(), 'kind': kind})
else:
inflight.add(self.pool.submit(self.one, kind))
i += 1
tick += 1 / MESSAGE_RATE
self.shutdown.wait(max(0, tick - time.monotonic()))
def __enter__(self):
self.thread.start()
return self
def __exit__(self, *_):
self.shutdown.set()
self.thread.join()
self.pool.shutdown()
def snapshot():
active = prometheus('ALERTS{alertname=~"' + '|'.join(ALERTS) + '"}')
return {
't': time.time(),
'alerts': {item['alertname']: item['alertstate'] for item in active},
'depth': next(iter(prometheus('messenger_queue_depth')), {}).get('value'),
'oldest_seconds': next(iter(prometheus('messenger_queue_oldest_age_seconds')), {}).get('value'),
'worker_cpu_cores': next(iter(prometheus('sum(rate(messenger_container_cpu_seconds_total{service_name="messenger-worker"}[15s]))')),
{}).get('value'),
'api_cpu_cores': next(iter(prometheus('sum(rate(messenger_container_cpu_seconds_total{service_name="messenger-api"}[15s]))')),
{}).get('value'),
'worker_ram_bytes': next(iter(prometheus('sum(messenger_container_memory_bytes{service_name="messenger-worker"})')),
{}).get('value'),
'api_ram_bytes': next(iter(prometheus('sum(messenger_container_memory_bytes{service_name="messenger-api"})')),
{}).get('value'),
}
def observe(samples, seconds):
until = time.monotonic() + seconds
while time.monotonic() < until:
tick = time.monotonic()
samples.append(snapshot())
time.sleep(max(0, min(2 - (time.monotonic() - tick), until - time.monotonic())))
def during(rows, start, stop):
return [row for row in rows if start <= row['t'] < stop]
def quantiles(values):
ordered = sorted(values)
return {f'p{p}_ms': round(1000 * ordered[math.ceil(len(ordered) * p / 100) - 1], 3)
for p in (50, 95, 99)} if ordered else {}
def summary(rows, start, stop):
result = {}
for kind in ('message', 'upload'):
items = [r for r in during(rows, start, stop) if r['kind'] == kind]
statuses = Counter(str(r['status']) for r in items)
result[kind] = {'statuses': dict(statuses), 'successful_per_s':
round((statuses['201'] + statuses['202']) / (stop - start), 3),
**quantiles([row['latency_s'] for row in items])}
return result
def alert_timing(samples, start, stop, names):
result = {}
for name in names:
firing = [s['t'] for s in samples if s['t'] >= start and s['alerts'].get(name) == 'firing']
if not firing:
result[name] = None
continue
resolved = next((s['t'] for s in samples if s['t'] > max(firing) and name not in s['alerts']), None)
result[name] = {
'firing_after_s': round(min(firing) - start, 3),
'resolved_after_restoration_s': round(resolved - stop, 3) if resolved is not None else None,
}
return result
def diagnose_db(rows):
failing = [row for row in rows if row['status'] == 503]
check(bool(failing), 'no application 503 from the database lock')
for row in failing[:5]:
log = wait_for(lambda: api_log(row['request_id']), 'missing incident log', timeout=8)
trace = wait_for(lambda: trace_by_id(log['trace_id']), 'missing incident trace', timeout=8)
sql = [span for span in trace if span['name'] == 'postgresql query' and span['duration_ms'] >= 900]
if sql:
return {'log': log, 'slow_postgresql_spans': sql, 'trace_id': log['trace_id']}
raise AssertionError('incident trace lacks a slow PostgreSQL child span')
def validate_result(result):
check(not result['firing_during_baseline'] and not result['firing_after_short_spike'],
'unexpected firing during baseline or brief fault')
summaries = result['client_summary']
check(summaries['short_db_spike']['message']['statuses'].get('503', 0) > 0, 'short spike did not generate errors')
events = [*result['database_alerts'].values(), *result['worker_alerts'].values()]
check(all(event and event['resolved_after_restoration_s'] is not None for event in events),
'expected alert was not detected and resolved')
recovery = result['worker_recovery']
check(recovery['uploads_ready'] == recovery['uploads_accepted'] >= 30 and recovery['max_backlog'] >= 30,
'backlog did not grow and drain as expected')
check(result['prometheus_samples'][-1]['depth'] in (0, 1), 'unfinished recovery')
# Count the sampled phases as a conservative lower bound for offered traffic.
observed = sum(sum(item['statuses'].values()) for phase in summaries.values() for item in phase.values())
check(result['dropped_by_generator'] <= observed * .01, 'load generator skipped more than 1% of requests')
def experiment(output):
lab_only()
client = Client(BASE)
actor, chat = actor_and_chat(client, 'slo')
samples, phases = [], {}
with Workload(actor, chat) as load:
baseline = time.time()
observe(samples, 82)
phases['baseline'] = [baseline, time.time()]
with locked_messages(2.5) as start:
observe(samples, 2)
phases['short_db_spike'] = [start, time.time()]
observe(samples, 30)
with locked_messages(65) as start:
observe(samples, 63)
phases['database_lock'] = [start, time.time()]
observe(samples, 36)
compose('stop', 'worker')
start = time.time()
try:
observe(samples, 62)
finally:
compose('start', 'worker')
phases['worker_stopped'] = [start, time.time()]
observe(samples, 50)
at = [item['attachment'] for item in during(load.rows, *phases['worker_stopped']) if 'attachment' in item]
ready = sum(client.call('GET', '/api/v1/attachments/' + identifier, actor=actor,
schema='Attachment')['json']['status'] == 'ready' for identifier in at)
db = alert_timing(samples, *phases['database_lock'], ALERTS[:2])
worker = alert_timing(samples, *phases['worker_stopped'], ALERTS[2:])
short_firing = sorted({name for s in during(samples, phases['short_db_spike'][0], phases['database_lock'][0])
for name, state in s['alerts'].items() if state == 'firing'})
baseline_firing = sorted({name for s in during(samples, *phases['baseline'])
for name, state in s['alerts'].items() if state == 'firing'})
diagnosis = diagnose_db(during(load.rows, *phases['database_lock']))
worker_samples = during(samples, *phases['worker_stopped'])
result = {
'profile': {'messages_per_s': MESSAGE_RATE, 'uploads_per_s': UPLOAD_RATE, 'max_inflight': 12,
'png_bytes': len((ROOT / 'tests/fixtures/sample.png').read_bytes()), 'monitor_interval_s': 2},
'phases_utc_unix': phases, 'client_summary': {phase: summary(load.rows, *times) for phase, times in phases.items()},
'dropped_by_generator': len(load.dropped),
'firing_during_baseline': baseline_firing, 'firing_after_short_spike': short_firing,
'database_alerts': db, 'worker_alerts': worker,
'worker_recovery': {'uploads_accepted': len(at), 'uploads_ready': ready,
'max_backlog': max(s['depth'] or 0 for s in worker_samples),
'max_oldest_s': max(s['oldest_seconds'] or 0 for s in worker_samples)},
'database_diagnosis': diagnosis, 'prometheus_samples': samples,
}
save(output, 'slo-experiment.json', result)
validate_result(result)
print(json.dumps({key: result[key] for key in ('database_alerts', 'worker_alerts', 'worker_recovery')}, indent=2))
if __name__ == '__main__':
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--output', type=Path, default=ROOT / 'evidence')
experiment(parser.parse_args().output)
+273
View File
@@ -0,0 +1,273 @@
package main
import (
"context"
"errors"
"fmt"
"log/slog"
"os"
"strconv"
"strings"
"sync/atomic"
"time"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp"
"go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp"
"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp"
otellog "go.opentelemetry.io/otel/log"
"go.opentelemetry.io/otel/metric"
"go.opentelemetry.io/otel/propagation"
sdklog "go.opentelemetry.io/otel/sdk/log"
sdkmetric "go.opentelemetry.io/otel/sdk/metric"
"go.opentelemetry.io/otel/sdk/resource"
sdktrace "go.opentelemetry.io/otel/sdk/trace"
"go.opentelemetry.io/otel/trace"
)
const telemetryScope = "mtusi/messenger"
var traceContext propagation.TraceContext
type telemetry struct {
logger otellog.Logger
requests metric.Int64Counter
httpTime metric.Float64Histogram
jobs metric.Int64Counter
jobTime metric.Float64Histogram
readyTime metric.Float64Histogram
workerBusy atomic.Int64
}
func startTelemetry(role string, db *pgxpool.Pool, poolTrace *databaseTracer) (*telemetry, func(), error) {
if env("OTEL_SDK_DISABLED", "false") == "true" {
return nil, func() {}, nil
}
ctx := context.Background()
host, err := os.Hostname()
if err != nil {
return nil, nil, errors.New("cannot identify telemetry instance")
}
res := resource.NewSchemaless(
attribute.String("service.name", "messenger-"+role),
attribute.String("service.version", "6.0.0"),
attribute.String("service.instance.id", host),
attribute.String("deployment.environment.name", env("APP_ENV", "lab")),
)
const exportTimeout = 2 * time.Second
tracer, err := otlptracehttp.New(ctx, otlptracehttp.WithTimeout(exportTimeout),
otlptracehttp.WithRetry(otlptracehttp.RetryConfig{Enabled: false}))
if err != nil {
return nil, nil, errors.New("invalid OTLP trace configuration")
}
tp := sdktrace.NewTracerProvider(sdktrace.WithResource(res), sdktrace.WithSampler(sdktrace.AlwaysSample()),
sdktrace.WithBatcher(tracer, sdktrace.WithMaxQueueSize(1024), sdktrace.WithMaxExportBatchSize(128),
sdktrace.WithBatchTimeout(time.Second), sdktrace.WithExportTimeout(exportTimeout)))
var mp *sdkmetric.MeterProvider
var lp *sdklog.LoggerProvider
shutdown := func() {
end, cancel := context.WithTimeout(context.Background(), exportTimeout)
defer cancel()
if lp != nil {
_ = lp.Shutdown(end)
}
if mp != nil {
_ = mp.Shutdown(end)
}
_ = tp.Shutdown(end)
}
metrics, err := otlpmetrichttp.New(ctx, otlpmetrichttp.WithTimeout(exportTimeout),
otlpmetrichttp.WithRetry(otlpmetrichttp.RetryConfig{Enabled: false}))
if err != nil {
shutdown()
return nil, nil, errors.New("invalid OTLP metric configuration")
}
reader := sdkmetric.NewPeriodicReader(metrics, sdkmetric.WithInterval(2*time.Second),
sdkmetric.WithTimeout(exportTimeout))
mp = sdkmetric.NewMeterProvider(sdkmetric.WithResource(res), sdkmetric.WithReader(reader))
logs, err := otlploghttp.New(ctx, otlploghttp.WithTimeout(exportTimeout),
otlploghttp.WithRetry(otlploghttp.RetryConfig{Enabled: false}))
if err != nil {
shutdown()
return nil, nil, errors.New("invalid OTLP log configuration")
}
lp = sdklog.NewLoggerProvider(sdklog.WithResource(res),
sdklog.WithProcessor(sdklog.NewBatchProcessor(logs, sdklog.WithMaxQueueSize(1024),
sdklog.WithExportMaxBatchSize(128), sdklog.WithExportInterval(time.Second),
sdklog.WithExportBufferSize(1), sdklog.WithExportTimeout(exportTimeout))))
t, err := makeTelemetry(mp.Meter(telemetryScope))
if err == nil {
err = observeRuntime(mp.Meter(telemetryScope), role, db, poolTrace)
}
if err == nil && role == "worker" {
err = t.observeWorker(mp.Meter(telemetryScope))
}
if err != nil {
shutdown()
return nil, nil, fmt.Errorf("register telemetry instruments: %w", err)
}
t.logger = lp.Logger(telemetryScope)
otel.SetTracerProvider(tp)
// Exporter errors can contain the configured endpoint. Local diagnostics
// report the fixed event only, and never feed it back into the OTLP queue.
otel.SetErrorHandler(otel.ErrorHandlerFunc(func(error) { slog.Warn("telemetry export failed") }))
return t, shutdown, nil
}
func makeTelemetry(m metric.Meter) (*telemetry, error) {
t := &telemetry{}
var err, errHTTP, errJobs, errJobTime, errReady error
t.requests, err = m.Int64Counter("messenger_http_requests")
t.httpTime, errHTTP = m.Float64Histogram("messenger_http_duration", metric.WithUnit("s"),
metric.WithExplicitBucketBoundaries(.002, .005, .01, .025, .05, .1, .25, .5, 1, 2, 5, 10, 30))
t.jobs, errJobs = m.Int64Counter("messenger_job_attempts")
t.jobTime, errJobTime = m.Float64Histogram("messenger_job_duration", metric.WithUnit("s"),
metric.WithExplicitBucketBoundaries(.01, .025, .05, .1, .25, .5, 1, 2, 3, 5, 10, 20, 30, 60, 120))
t.readyTime, errReady = m.Float64Histogram("messenger_attachment_time_to_ready", metric.WithUnit("s"),
metric.WithExplicitBucketBoundaries(.25, .5, 1, 2, 3, 5, 10, 20, 30, 60, 120, 300))
return t, errors.Join(err, errHTTP, errJobs, errJobTime, errReady)
}
func (t *telemetry) event(ctx context.Context, event string, level otellog.Severity, fields ...otellog.KeyValue) {
if t == nil {
return
}
var record otellog.Record
record.SetTimestamp(time.Now())
record.SetSeverity(level)
record.SetSeverityText(level.String())
record.SetBody(otellog.StringValue(event))
record.AddAttributes(otellog.String("event", event))
record.AddAttributes(fields...)
t.logger.Emit(ctx, record)
}
func (t *telemetry) finishHTTP(ctx context.Context, route, method, requestID string, status int, start time.Time) {
if t == nil {
return
}
class := strconv.Itoa(status/100) + "xx"
dimensions := metric.WithAttributes(attribute.String("method", method), attribute.String("route", route),
attribute.String("status_class", class))
duration := time.Since(start).Seconds()
t.requests.Add(ctx, 1, dimensions)
if class != "5xx" {
t.requests.Add(ctx, 0, metric.WithAttributes(attribute.String("method", method), attribute.String("route", route),
attribute.String("status_class", "5xx")))
}
t.httpTime.Record(ctx, duration, dimensions)
span := trace.SpanFromContext(ctx)
span.SetAttributes(attribute.Int("http.response.status_code", status))
severity := otellog.SeverityInfo
if status >= 500 {
span.SetStatus(codes.Error, "server_error")
severity = otellog.SeverityError
}
t.event(ctx, "http.request", severity, otellog.String("request_id", requestID),
otellog.String("route", route), otellog.String("method", method), otellog.Int("status", status),
otellog.Float64("duration_seconds", duration))
}
func (t *telemetry) finishJob(ctx context.Context, j imageJob, outcome string, started time.Time) {
if t == nil {
return
}
duration := time.Since(started).Seconds()
dimensions := metric.WithAttributes(attribute.String("job_type", j.Type), attribute.String("outcome", outcome))
t.jobs.Add(ctx, 1, dimensions)
t.jobTime.Record(ctx, duration, dimensions)
severity := otellog.SeverityInfo
if outcome != "ready" {
trace.SpanFromContext(ctx).SetStatus(codes.Error, outcome)
severity = otellog.SeverityError
}
t.event(ctx, "image.attempt", severity, otellog.String("job_id", j.JobID),
otellog.Int("attempt", j.Attempt), otellog.String("outcome", outcome),
otellog.Float64("duration_seconds", duration))
}
func safeHTTPRoute(method, path string) (string, string) {
switch method {
case "GET", "POST", "DELETE", "PUT", "PATCH", "HEAD", "OPTIONS":
default:
method = "OTHER"
}
switch path {
case "/api/v1/users", "/api/v1/users/me", "/api/v1/auth/sessions",
"/api/v1/auth/sessions/current", "/api/v1/chats":
return method, path
}
parts := strings.SplitN(path, "/", 7)
if len(parts) >= 5 && parts[0] == "" && parts[1] == "api" && parts[2] == "v1" {
switch parts[3] {
case "chats":
if len(parts) == 5 {
return method, "/api/v1/chats/{chat_id}"
}
if len(parts) == 6 && (parts[5] == "messages" || parts[5] == "attachments") {
return method, "/api/v1/chats/{chat_id}/" + parts[5]
}
case "attachments":
if len(parts) == 5 {
return method, "/api/v1/attachments/{attachment_id}"
}
if len(parts) == 6 && parts[5] == "content" {
return method, "/api/v1/attachments/{attachment_id}/content"
}
}
}
return method, "unmatched"
}
func startSpan(ctx context.Context, name string, opts ...trace.SpanStartOption) (context.Context, trace.Span) {
return otel.Tracer(telemetryScope).Start(ctx, name, opts...)
}
func endSpan(span trace.Span, err error) {
if err != nil {
span.SetStatus(codes.Error, "operation_failed")
}
span.End()
}
func jobSpanAttributes(j imageJob) trace.SpanStartOption {
return trace.WithAttributes(attribute.String("messaging.system", "valkey"),
attribute.String("messaging.destination.name", imageStream),
attribute.String("messaging.message.id", j.JobID), attribute.Int("job.attempt", j.Attempt))
}
func (t *telemetry) startJob(ctx context.Context, original propagation.MapCarrier, j imageJob, producer bool) (context.Context, trace.Span) {
if t != nil {
ctx = traceContext.Extract(ctx, original)
}
if producer {
return startSpan(ctx, "attachment-jobs publish", trace.WithSpanKind(trace.SpanKindProducer), jobSpanAttributes(j))
}
return startSpan(ctx, "attachment-jobs process", trace.WithSpanKind(trace.SpanKindConsumer), jobSpanAttributes(j))
}
// pgx receives full SQL and bind values. Export a fixed span name and status
// only, and skip calls outside a traced request or consumer.
func (*databaseTracer) TraceQueryStart(ctx context.Context, _ *pgx.Conn, _ pgx.TraceQueryStartData) context.Context {
if !trace.SpanFromContext(ctx).IsRecording() {
return ctx
}
child, span := startSpan(ctx, "postgresql query", trace.WithSpanKind(trace.SpanKindClient),
trace.WithAttributes(attribute.String("db.system.name", "postgresql")))
return context.WithValue(child, querySpanKey{}, span)
}
type querySpanKey struct{}
func (*databaseTracer) TraceQueryEnd(ctx context.Context, _ *pgx.Conn, data pgx.TraceQueryEndData) {
if span, ok := ctx.Value(querySpanKey{}).(trace.Span); ok {
if errors.Is(data.Err, pgx.ErrNoRows) {
data.Err = nil
}
endSpan(span, data.Err)
}
}
+110
View File
@@ -0,0 +1,110 @@
package main
import (
"context"
"errors"
"os"
"strconv"
"strings"
"sync/atomic"
"time"
"github.com/jackc/pgx/v5/pgxpool"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/metric"
)
type databaseTracer struct {
acquiring atomic.Int64
}
func (d *databaseTracer) TraceAcquireStart(ctx context.Context, _ *pgxpool.Pool, _ pgxpool.TraceAcquireStartData) context.Context {
d.acquiring.Add(1)
return ctx
}
func (d *databaseTracer) TraceAcquireEnd(context.Context, *pgxpool.Pool, pgxpool.TraceAcquireEndData) {
d.acquiring.Add(-1)
}
func observeRuntime(m metric.Meter, role string, db *pgxpool.Pool, dbTrace *databaseTracer) error {
used, errUsed := m.Int64ObservableGauge("messenger_db_pool_used")
capacity, errMax := m.Int64ObservableGauge("messenger_db_pool_capacity")
waiting, errWaiting := m.Int64ObservableGauge("messenger_db_pool_acquiring")
waits, errWaits := m.Int64ObservableCounter("messenger_db_pool_waits")
cpu, errCPU := m.Float64ObservableCounter("messenger_container_cpu", metric.WithUnit("s"))
memory, errMemory := m.Int64ObservableGauge("messenger_container_memory", metric.WithUnit("By"))
depth, errDepth := m.Int64ObservableGauge("messenger_queue_depth")
age, errAge := m.Float64ObservableGauge("messenger_queue_oldest_age", metric.WithUnit("s"))
slo, errSLO := m.Int64ObservableCounter("messenger_attachment_slo_jobs")
if err := errors.Join(errUsed, errMax, errWaiting, errWaits, errCPU, errMemory, errDepth, errAge, errSLO); err != nil {
return err
}
instruments := []metric.Observable{used, capacity, waiting, waits, cpu, memory}
if role == "dispatcher" {
instruments = append(instruments, depth, age, slo)
}
_, err := m.RegisterCallback(func(ctx context.Context, o metric.Observer) error {
stat := db.Stat()
o.ObserveInt64(used, int64(stat.AcquiredConns()))
o.ObserveInt64(capacity, int64(stat.MaxConns()))
o.ObserveInt64(waiting, dbTrace.acquiring.Load())
o.ObserveInt64(waits, stat.EmptyAcquireCount())
if data, err := os.ReadFile("/sys/fs/cgroup/cpu.stat"); err == nil {
for _, line := range strings.Split(string(data), "\n") {
if micros, found := strings.CutPrefix(line, "usage_usec "); found {
if value, err := strconv.ParseFloat(micros, 64); err == nil {
o.ObserveFloat64(cpu, value/1e6)
}
break
}
}
}
if data, err := os.ReadFile("/sys/fs/cgroup/memory.current"); err == nil {
if value, err := strconv.ParseInt(strings.TrimSpace(string(data)), 10, 64); err == nil {
o.ObserveInt64(memory, value)
}
}
if role != "dispatcher" {
return nil
}
sample, cancel := context.WithTimeout(ctx, 700*time.Millisecond)
defer cancel()
var pending, good, matured int64
var oldest float64
// Measure SQL backlog including jobs waiting for a retry or for XADD.
// Count each first publication exactly once, at its SLO deadline,
// even if the worker is down or its final state is failed.
err := db.QueryRow(sample, `
WITH sample AS (SELECT statement_timestamp() AS now)
SELECT count(*) FILTER (WHERE state='queued'),
coalesce(extract(epoch FROM (sample.now - min(created_at) FILTER (WHERE state='queued'))), 0)::float8,
count(*) FILTER (WHERE version=1 AND created_at <= sample.now - interval '5 seconds'),
count(*) FILTER (WHERE version=1 AND created_at <= sample.now - interval '5 seconds'
AND ready_at <= created_at + interval '5 seconds')
FROM sample LEFT JOIN image_jobs ON true GROUP BY sample.now`).Scan(&pending, &oldest, &matured, &good)
if err == nil {
queue := metric.WithAttributes(attribute.String("queue", imageStream))
o.ObserveInt64(depth, pending, queue)
o.ObserveFloat64(age, oldest, queue)
o.ObserveInt64(slo, good, metric.WithAttributes(attribute.String("outcome", "good")))
o.ObserveInt64(slo, matured-good, metric.WithAttributes(attribute.String("outcome", "missed")))
}
return nil
}, instruments...)
return err
}
func (t *telemetry) observeWorker(m metric.Meter) error {
used, errUsed := m.Int64ObservableGauge("messenger_worker_busy")
capacity, errCapacity := m.Int64ObservableGauge("messenger_worker_capacity")
if err := errors.Join(errUsed, errCapacity); err != nil {
return err
}
_, err := m.RegisterCallback(func(_ context.Context, o metric.Observer) error {
o.ObserveInt64(used, t.workerBusy.Load())
o.ObserveInt64(capacity, 1)
return nil
}, used, capacity)
return err
}