forked from mtusi2026/mtusi2026lab6
Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
4262a16ada |
+7
-2
@@ -7,7 +7,7 @@ SESSION_TTL_SECONDS=3600
|
||||
# Choose credentials locally. File paths contain no secret values.
|
||||
DATABASE_URL_FILE=/run/secrets/database_url
|
||||
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
|
||||
SESSION_STORE=postgres
|
||||
SESSION_STORE=redis
|
||||
CACHE_URL_FILE=/run/secrets/cache_url
|
||||
API_REPLICAS=2
|
||||
IDEMPOTENCY_TTL_SECONDS=86400
|
||||
@@ -19,7 +19,7 @@ S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
|
||||
MAX_UPLOAD_BYTES=10485760
|
||||
MAX_IMAGE_PIXELS=40000000
|
||||
BROKER_URL_FILE=/run/secrets/broker_url
|
||||
WORKER_CONCURRENCY=2
|
||||
WORKER_CONCURRENCY=1
|
||||
JOB_MAX_ATTEMPTS=5
|
||||
JOB_TIMEOUT_SECONDS=60
|
||||
STAGING_RETENTION_SECONDS=86400
|
||||
@@ -28,3 +28,8 @@ OTEL_SERVICE_NAME=messenger-api
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318
|
||||
OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
|
||||
OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab
|
||||
|
||||
DB_TIMEOUT_MS=3000
|
||||
DB_MAX_CONNS=10
|
||||
CLEANUP_GRACE_SECONDS=3600
|
||||
GRAFANA_PORT=13000
|
||||
|
||||
@@ -1,18 +1,47 @@
|
||||
## Лабораторная 6
|
||||
|
||||
Автор / группа: TODO
|
||||
Целевая оценка (3/4/5): TODO
|
||||
Трек 5, если заявлен: TODO
|
||||
Автор / группа: **Меркулова М. А. / БВТ2303**
|
||||
Целевая оценка: **4**. Трек 5 не заявлен.
|
||||
|
||||
Что реализовано и что перенесено: TODO
|
||||
Команды запуска из чистого clone: TODO
|
||||
Результаты проверок и ссылка на REPORT.md: TODO
|
||||
Ограничения / известные ошибки: TODO
|
||||
Источники и использование ИИ: TODO
|
||||
### Реализация
|
||||
|
||||
- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
|
||||
- [ ] Контракт и публичные тесты не ослаблены.
|
||||
- [ ] make check и make test выполнены; приложен реальный вывод.
|
||||
- [ ] Сценарии отказов/безопасности своего уровня воспроизведены.
|
||||
- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
|
||||
- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
|
||||
Перенесена собственная lab5: две Go API-реплики, Nginx, PostgreSQL, Redis sessions, MinIO, Redis Streams/outbox/worker, варианты изображений и обработка отказов.
|
||||
|
||||
Добавлены OTel Go SDK и отдельный Collector; Prometheus, Loki, Tempo и Grafana с persistent volumes и provisioning. HTTP/API, producer/consumer и S3/DB/image spans связаны W3C context через durable job и broker, включая retry. Структурные logs связаны с trace/request IDs; секреты и business payload не экспортируются. Dashboard показывает histogram percentiles, errors/RPS, queue lag/depth, DB pool, worker saturation/heartbeat и CPU/RAM. Есть SLI/SLO, два alert rules, memory limiter/batch/bounded queues и runbook.
|
||||
|
||||
### Запуск
|
||||
|
||||
```sh
|
||||
sh scripts/init-local.sh
|
||||
docker compose --env-file .env up -d --build --wait
|
||||
make check
|
||||
make test
|
||||
python3 student-tests/verify_observability.py
|
||||
python3 student-tests/verify_incidents.py
|
||||
python3 student-tests/verify_resilience.py
|
||||
```
|
||||
|
||||
API по умолчанию 8080, Grafana 13000. При занятом 8080 изменить APP_PORT в .env и передать BASE_URL тестам. Локальная проверка: API 18086, Grafana 13000; Grafana admin password создаётся в secrets и не коммитится.
|
||||
|
||||
### Доказательства
|
||||
|
||||
- [Отчёт](../REPORT.md)
|
||||
- [Команды и результаты](../evidence/lab6-results.md)
|
||||
- [Сигналы, trace и logs](../evidence/observability-grade4.json)
|
||||
- [Инциденты и alert firing/resolved](../evidence/incidents-grade4.json)
|
||||
- [Dashboard JSON](../infra/otel/grafana/dashboards/messenger.json)
|
||||
- [Runbook и SLO](../infra/otel/runbook.md)
|
||||
- [Перезапуски и persistent история](../evidence/resilience.json)
|
||||
|
||||
Worker stop → queue lag firing/resolved; PostgreSQL stop → DB error spans и error-rate firing/resolved. Tempo stop под нагрузкой → бизнес-записи сохранены, queue/RAM ограничены, drops посчитаны. Чистый запуск с новыми secrets и8 пустыми volumes прошёл.
|
||||
|
||||
### Ограничения и источники
|
||||
|
||||
Один Docker-host, нет HA хранилищ; telemetry при отказах может теряться. Sampling ParentBased/AlwaysSample для локального стенда; треки 5 не заявлены. Источники — собственная lab5 и официальные OTel/Grafana/Prometheus/W3C docs; помощь ИИ Codex описана в REPORT.md.
|
||||
|
||||
- [x] Минимум lab5 и бизнес-контракт сохранены.
|
||||
- [x] Обязательные contracts/tests/README/Makefile не изменены.
|
||||
- [x] make check, make test, Go tests/vet и собственные проверки выполнены.
|
||||
- [x] Три сигнала, W3C chain и provisioning проверены на реальных данных.
|
||||
- [x] Alerts и telemetry backend outage воспроизведены.
|
||||
- [x] Секреты исключены из Git.
|
||||
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
FROM golang:1.26.8-alpine3.23@sha256:a8fa79c5bd40d880b52bd3b6d7669ecdcfd00e85facdd427d279efb5ddd79cb1 AS build
|
||||
WORKDIR /src
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
COPY cmd/ ./cmd/
|
||||
COPY internal/ ./internal/
|
||||
RUN CGO_ENABLED=0 go build -trimpath -ldflags="-s -w" -o /messenger ./cmd/messenger
|
||||
|
||||
FROM scratch
|
||||
COPY --from=build /messenger /messenger
|
||||
USER 65532:65532
|
||||
ENV APP_HOST=0.0.0.0 APP_PORT=8080
|
||||
EXPOSE 8080
|
||||
HEALTHCHECK --interval=10s --timeout=3s --start-period=5s --retries=3 CMD ["/messenger", "healthcheck"]
|
||||
ENTRYPOINT ["/messenger"]
|
||||
@@ -1,37 +1,191 @@
|
||||
# Отчёт — лабораторная 6
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
- Автор: **Меркулова М. А., БВТ2303**.
|
||||
- Целевая оценка: **4** — весь минимум и все пункты уровня 4 README. Трек 5 не заявлен.
|
||||
- Ветка: `solution/BVT2303-Merkulova`, создана от `main` skeleton lab6.
|
||||
- Основа: собственная реализация lab5. Бизнес-контракт, README задания, COURSE, Makefile, public tests и contracts сохранены.
|
||||
- Commit/PR заполняется при публикации. Использовалась помощь ИИ Codex при реализации, проверках и документации; источники перечислены ниже.
|
||||
|
||||
## Запуск из чистого clone
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
Требования: Docker Desktop с Linux containers, Compose **2.22+** / BuildKit, Python 3.9+, make, OpenSSL и интернет для образов/Go modules/PyPI/исходников MinIO. На Windows — WSL2 с Docker integration; отдельная Windows-проверка не выполнялась. Локальный Go нужен только для Go unit/vet.
|
||||
|
||||
## Архитектура и решения
|
||||
```sh
|
||||
cd mtusi2026lab6
|
||||
sh scripts/init-local.sh
|
||||
docker compose --env-file .env config --quiet
|
||||
docker compose --env-file .env up -d --build --wait
|
||||
make check
|
||||
make test
|
||||
```
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
`.env` и `secrets/` создаются локально, исключены из Git, не копируются из предыдущей лабы. Миграции выполняет `migrate`, приватный bucket и прикладные права — `s3-init`. Успешные init-контейнеры завершаются с exit 0. Отдельно установленный Redis/БД/MinIO не требуется. Локальные application images собираются из этого репозитория; worker использует build context `application: service:migrate`.
|
||||
|
||||
По умолчанию API опубликован на `http://localhost:8080`, Grafana — `http://localhost:13000`. В проверенном стенде API использует **18086**, поскольку предыдущие лабы занимают 8080/18085. Для этого APP_PORT изменён только в локальном `.env`:
|
||||
|
||||
```sh
|
||||
make test BASE_URL=http://localhost:18086
|
||||
BASE_URL=http://localhost:18086 python3 student-tests/verify_observability.py
|
||||
BASE_URL=http://localhost:18086 python3 student-tests/verify_incidents.py
|
||||
```
|
||||
|
||||
Вход в Grafana: пользователь `admin`, пароль — содержимое локального `secrets/grafana_admin_password`. Пароль не вставляется в отчёт/команды PR. В папке **MTUSI** автоматически появляется dashboard **MTUSI Messenger — lab6**; вручную создавать datasource/panels не нужно. Explore: Prometheus / Loki / Tempo. При изменении GRAFANA_PORT тестам передать `GRAFANA_URL=http://localhost:<port>`.
|
||||
|
||||
`up --wait` проверяет readiness API/хранилищ основного приложения; доступность наблюдаемости дополнительно проверяет `verify_observability.py` через реальные backend API.
|
||||
|
||||
### Проверки
|
||||
|
||||
```sh
|
||||
make check
|
||||
make test BASE_URL=http://localhost:18086
|
||||
CGO_ENABLED=0 go test ./...
|
||||
CGO_ENABLED=0 go vet ./...
|
||||
BASE_URL=http://localhost:18086 python3 student-tests/verify_variants.py
|
||||
BASE_URL=http://localhost:18086 python3 student-tests/verify_pipeline.py
|
||||
BASE_URL=http://localhost:18086 python3 student-tests/verify_observability.py
|
||||
BASE_URL=http://localhost:18086 python3 student-tests/verify_incidents.py
|
||||
BASE_URL=http://localhost:18086 python3 student-tests/verify_resilience.py
|
||||
```
|
||||
|
||||
Тесты изменяют только синтетические данные своего стенда. Сценарии остановки контейнеров запускать последовательно. `verify_observability` создаёт трассу с retry, `verify_incidents` — два обязательных инцидента и отказ Tempo; finally восстанавливает сервисы. `verify_resilience` использует публичные hooks и проверяет StartedAt, данные и сохранённую историю telemetry.
|
||||
|
||||
Остановка без удаления данных: `docker compose --env-file .env down`. Намеренный полный сброс **учебных** данных: `docker compose --env-file .env down --volumes`.
|
||||
|
||||
## Архитектура
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
Client --> Nginx
|
||||
Nginx --> API1
|
||||
Nginx --> API2
|
||||
API1 --> PG[(PostgreSQL)]
|
||||
API2 --> PG
|
||||
API1 --> R[(Redis sessions)]
|
||||
API2 --> R
|
||||
API1 --> S3[(MinIO)]
|
||||
API2 --> S3
|
||||
PG --> Dispatcher
|
||||
Dispatcher --> Broker[(Redis Streams)]
|
||||
Broker --> Worker
|
||||
Worker --> S3
|
||||
Worker --> PG
|
||||
API1 -. OTLP .-> Collector
|
||||
API2 -. OTLP .-> Collector
|
||||
Dispatcher -. OTLP .-> Collector
|
||||
Worker -. OTLP .-> Collector
|
||||
Collector --> Prometheus
|
||||
Collector --> Loki
|
||||
Collector --> Tempo
|
||||
Grafana --> Prometheus
|
||||
Grafana --> Loki
|
||||
Grafana --> Tempo
|
||||
```
|
||||
|
||||
- **Prometheus** хранит временные ряды, histogram buckets, recording/alert rules.
|
||||
- **Loki** хранит структурные логи, принимает native OTLP через `/otlp` (устаревший Loki exporter не используется).
|
||||
- **Tempo** хранит реальные traces, WAL и blocks.
|
||||
- **Collector** принимает/пакетирует/экспортирует сигналы, не заменяет долговременное хранилище.
|
||||
- **Grafana** визуализирует источники; datasource/dashboard provisioning находится в Git.
|
||||
|
||||
Основной API по-прежнему использует PG + Redis sessions + MinIO. Broker/worker и telemetry backend не входят в синхронную readiness API. Принятие job защищено outbox/staging; недоступность telemetry не отменяет business COMMIT.
|
||||
|
||||
### Изменения относительно lab5
|
||||
|
||||
- `internal/telemetry/`: SDK providers, структурные OTLP logs, безопасные route labels, HTTP/jobs histograms, pool/queue/process/container metrics, DB/Redis tracers.
|
||||
- `server.go`: server span, X-Trace-Id, request/log correlation; нормализованный route вместо UUID/raw URL. Бизнес-поля ответа не менялись.
|
||||
- `attachments.go`, migration `005_trace_context.sql`: W3C context и request_id сохраняются вместе с принятием job.
|
||||
- `pipeline.go`: producer/consumer spans, перенос context через JSON broker, сохранение связи при retry, worker outcome/duration/E2E и busy.
|
||||
- `store.go`: реальные S3 client spans; SQL acquire/query/COMMIT записываются QueryTracer, без SQL text/args.
|
||||
- `main.go`: SDK только для API/worker/dispatcher; ограниченный shutdown, register metrics и resource attributes.
|
||||
- `compose.yaml`, `infra/otel/`: Collector, три backend, Grafana, 4 persistent volumes, provisioning и alerts.
|
||||
- `scripts/init-local.sh`: локальный пароль Grafana. `restart.sh`: рестарт также пяти telemetry-сервисов.
|
||||
|
||||
### Сквозной контекст
|
||||
|
||||
HTTP span извлекает W3C traceparent/tracestate. Upload сохраняет carrier в той же транзакции, что attachment/jobs/outbox. Dispatcher создаёт **producer child** исходного HTTP span и помещает его carrier в разрешённые поля job. Worker создаёт **consumer child** producer; S3/DB/image.process — дочерние операции consumer. Retry образует новую producer/consumer ветку того же trace. `request_id` хранится в PG для связи лога worker с исходным HTTP ответом, не становится metric label.
|
||||
|
||||
Время до ready — от created_at job в durable acceptance transaction до фиксации ready; включает queue wait и retry. Original/checksum, thumbnail/crop/WebP, ACL, sessions, idempotency, lease и replay lab5 сохранены.
|
||||
|
||||
## Сигналы и dashboard
|
||||
|
||||
Полная схема: [infra/otel/signals.md](infra/otel/signals.md). Исполняемые PromQL/units: [dashboard JSON](infra/otel/grafana/dashboards/messenger.json).
|
||||
|
||||
Dashboard показывает RPS, 5xx/all business ratio, p50/p95/p99 HTTP, worker outcomes/duration, E2E p95 до ready, queued depth/oldest age, pool used/waiting/capacity, worker busy/capacity и heartbeat, container CPU/RAM, DB availability, SLI, alerts и Collector failures/queue.
|
||||
|
||||
Percentiles агрегируются **из суммарных histogram buckets обеих реплик**:
|
||||
|
||||
```promql
|
||||
histogram_quantile(0.95,
|
||||
sum by (le) (rate(messenger_http_duration_seconds_bucket{service_name="messenger-api",route!~"/health/.*"}[1m])))
|
||||
```
|
||||
|
||||
Все duration metrics экспортируются в секундах; logs duration_ms явно обозначены. Queue depth — jobs, age — seconds, RAM — bytes, CPU rate — CPU seconds/second (cores).
|
||||
|
||||
Metrics не содержат user_id/chat_id/message_id/attachment_id/request_id/filename/raw URL. В логах и traces допустимы синтетические request/job IDs для корреляции; Authorization, пароли, токены, body и изображение не экспортируются. Это проверено canary-значениями и анализом реальных сохранённых данных.
|
||||
|
||||
## SLI, SLO и alerts
|
||||
|
||||
SLI = доля бизнес HTTP-запросов без 5xx за скользящие **5 минут**; SLO **≥99%**. Health исключены, клиентские 4xx не считаются отказом сервера. При искусственном отказе SLO может нарушаться — это ожидаемое воздействие теста.
|
||||
|
||||
- `MessengerHighErrorRate`: 5xx/all >5% за 1m, `for:10s`.
|
||||
- `MessengerQueueLag`: oldest queued >10s и depth>0, `for:10s`.
|
||||
- Scrape 3s, rule evaluation 5s. Доставка наружу не настроена и не требуется: состояния firing/resolved проверяются локально в Prometheus.
|
||||
|
||||
[Alert rules](infra/otel/alerts.yaml) и [runbook](infra/otel/runbook.md) содержат точные запросы, единицы, пороги и шаги диагностики. Alert error ratio — оперативное 1m предупреждение, не burn-rate исследование трека 5.
|
||||
|
||||
## Ограниченная буферизация
|
||||
|
||||
SDK: batch queue 256 spans/log records, batch 64, interval 500 ms, export timeout 2s, без SDK retry; periodic metrics 2s. HTTP handler не ждёт экспорта. Ошибки SDK считаются, payload ошибки не логируется.
|
||||
|
||||
Collector: memory limiter 128 MiB/spike32, hard container limit256 MiB, batch64/max128, по 32 batches в очередях traces/logs, 2 consumers, block_on_overflow=false, retry deadline5s. После исчерпания попыток telemetry теряется, а не блокирует мессенджер. Финальные dropped_items считаются по JSON-логам Collector; failed-send counter включает неуспешные отправки и не приравнивается к числу уникальных трасс.
|
||||
|
||||
## Версии и ресурсы
|
||||
|
||||
| Компонент | Закреплённая версия | CPU / RAM limit |
|
||||
|---|---|---|
|
||||
| Go API/dispatcher | Go 1.26.8, builder digest в Dockerfile | API: 0.5 / 1536 MiB; dispatcher 0.25 /128 MiB |
|
||||
| Worker | Python3.13.7, Pillow12.0.0 + Go binary | 1 /1536 MiB; decoder RLIMIT_AS1GiB |
|
||||
| PostgreSQL | 17.6-alpine3.22 | 1 /512 MiB |
|
||||
| Redis sessions / broker | 8.2.1-alpine3.22 | каждый 0.25 /128 MiB |
|
||||
| MinIO | RELEASE.2025-10-15T17-29-55Z, official source + checksum | 1 /512 MiB |
|
||||
| Nginx | 1.29.1-alpine3.22 | 0.25 /64 MiB |
|
||||
| Collector contrib | 0.136.0 | 0.5 /256 MiB |
|
||||
| Prometheus | v3.7.0 | 0.5 /512 MiB |
|
||||
| Loki | 3.5.5 | 0.5 /384 MiB |
|
||||
| Tempo | 2.8.2 | 0.5 /384 MiB |
|
||||
| Grafana | 12.2.0 | 0.5 /384 MiB |
|
||||
|
||||
SDK traces/metrics1.38.0, logs0.14.0, resource semantic schema1.37.0; собственная схема сигналов v1. Worker concurrency=1. Хранилища metrics/logs/traces имеют retention 24h и отдельные persistent volumes. Grafana state также в volume. Collector очередь in-memory и после его рестарта не восстанавливается; это не бизнес-очередь.
|
||||
|
||||
.env управляет APP_PORT/APP_HOST/APP_ENV, session/idempotency TTL, DB timeout/max connections, worker concurrency/attempts/timeout, staging retention, cleanup grace и GRAFANA_PORT. Внутренний порт API8080, 2 API-реплики, S3 endpoint/region/bucket и OTLP endpoint заданы Compose; ограничения bytes/pixels зафиксированы кодом. Справочные SESSION_STORE/API_REPLICAS/MAX_* из skeleton не переключают реализацию.
|
||||
|
||||
## Доказательства критериев
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
Актуальные команды/наблюдения — [evidence/lab6-results.md](evidence/lab6-results.md).
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
| Критерий | Проверка | Доказательство |
|
||||
|---|---|---|
|
||||
| Минимум lab5, бизнес-контракт | make test + verify_variants + verify_pipeline | сохранность оригинала, variants, outbox/retry/replay |
|
||||
| Три OTLP pipeline и реальные данные | verify_observability.py | observability-grade4.json |
|
||||
| Server/consumer traces + структурные логи | тот же тест | trace-example.json, correlated-logs.json |
|
||||
| W3C, producer/consumer, retry, S3/DB | тот же тест | минимум две consumer attempts, проверенные parent IDs |
|
||||
| Низкая кардинальность и приватность | тот же тест + go test | labels/canary checks |
|
||||
| Dashboard, percentiles, queue/pool/saturation | provisioning + backend queries | dashboard JSON, реальные metrics |
|
||||
| SLI/SLO и два alerts | verify_incidents.py | incidents-grade4.json: firing/resolved |
|
||||
| Два инцидента и диагностика | worker stopped + PostgreSQL stopped | queue/heartbeat, error ratio/DB spans |
|
||||
| Backend telemetry down, bounded memory/queue | Tempo stopped под нагрузкой | drop counts, memory, business writes |
|
||||
| Persistent история и core persistence | verify_resilience.py | resilience.json, StartedAt и восстановленная telemetry |
|
||||
|
||||
## Эксперименты и отказы
|
||||
## Ограничения
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
Один Docker-host и одиночные backend/БД/MinIO/broker не обеспечивают HA при потере host/volume. HTTP используется только на localhost с синтетическими данными. Sampling — ParentBased AlwaysSample для локальной демонстрации; стоимость и альтернативы не исследуются. Очереди SDK/Collector ограничены, но telemetry при отказах может теряться. SLO задан и проверяем, это не обещание production SLA. Полный backup/архивирование вне объёма этой лабы.
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
## Источники
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
- Собственная лабораторная 5.
|
||||
- [OpenTelemetry Go SDK/exporters](https://opentelemetry.io/docs/languages/go/exporters/).
|
||||
- [W3C Trace Context](https://www.w3.org/TR/trace-context/).
|
||||
- [Collector configuration](https://opentelemetry.io/docs/collector/configuration/).
|
||||
- [Loki native OTLP](https://grafana.com/docs/loki/latest/send-data/otel/).
|
||||
- [Prometheus histograms](https://prometheus.io/docs/practices/histograms/).
|
||||
- [Grafana provisioning](https://grafana.com/docs/grafana/latest/administration/provisioning/).
|
||||
|
||||
@@ -0,0 +1,370 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/redis/go-redis/v9"
|
||||
"mtusi-messenger/internal/database"
|
||||
"mtusi-messenger/internal/messenger"
|
||||
"mtusi-messenger/internal/objectstore"
|
||||
"mtusi-messenger/internal/pipeline"
|
||||
"mtusi-messenger/internal/telemetry"
|
||||
)
|
||||
|
||||
func env(name, fallback string) string {
|
||||
if value := os.Getenv(name); value != "" {
|
||||
return value
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
func secret(envName, fileEnv string) (string, error) {
|
||||
if value := os.Getenv(envName); value != "" {
|
||||
return value, nil
|
||||
}
|
||||
path := os.Getenv(fileEnv)
|
||||
if path == "" {
|
||||
return "", fmt.Errorf("set %s or %s", envName, fileEnv)
|
||||
}
|
||||
b, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
value := strings.TrimSpace(string(b))
|
||||
if value == "" {
|
||||
return "", fmt.Errorf("%s is empty", path)
|
||||
}
|
||||
return value, nil
|
||||
}
|
||||
func positiveInt(name string, fallback int) (int, error) {
|
||||
raw := env(name, strconv.Itoa(fallback))
|
||||
n, err := strconv.Atoi(raw)
|
||||
if err != nil || n < 1 {
|
||||
return 0, fmt.Errorf("invalid %s", name)
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
|
||||
func main() {
|
||||
if len(os.Args) > 1 && os.Args[1] == "memory" {
|
||||
values := map[string]int64{}
|
||||
for _, name := range []string{"memory.current", "memory.peak", "memory.max"} {
|
||||
data, err := os.ReadFile("/sys/fs/cgroup/" + name)
|
||||
if err != nil {
|
||||
log.Fatal("cgroup memory metrics unavailable")
|
||||
}
|
||||
n, err := strconv.ParseInt(strings.TrimSpace(string(data)), 10, 64)
|
||||
if err != nil {
|
||||
log.Fatal("invalid cgroup metric")
|
||||
}
|
||||
values[name] = n
|
||||
}
|
||||
_ = json.NewEncoder(os.Stdout).Encode(values)
|
||||
return
|
||||
}
|
||||
port := env("APP_PORT", "8080")
|
||||
if len(os.Args) > 1 && os.Args[1] == "healthcheck" {
|
||||
client := &http.Client{Timeout: 2 * time.Second}
|
||||
resp, err := client.Get("http://127.0.0.1:" + port + "/health/ready")
|
||||
if err != nil || resp.StatusCode != http.StatusOK {
|
||||
os.Exit(1)
|
||||
}
|
||||
_ = resp.Body.Close()
|
||||
return
|
||||
}
|
||||
host := env("APP_HOST", "0.0.0.0")
|
||||
if _, err := strconv.Atoi(port); err != nil {
|
||||
log.Fatal("invalid APP_PORT")
|
||||
}
|
||||
ttlSeconds, err := positiveInt("SESSION_TTL_SECONDS", 3600)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
idempotencyTTLSeconds, err := positiveInt("IDEMPOTENCY_TTL_SECONDS", 86400)
|
||||
if err != nil || idempotencyTTLSeconds < 86400 {
|
||||
log.Fatal("IDEMPOTENCY_TTL_SECONDS must be at least 86400")
|
||||
}
|
||||
dbTimeoutMS, err := positiveInt("DB_TIMEOUT_MS", 3000)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
maxConns, err := positiveInt("DB_MAX_CONNS", 10)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
databaseURL, err := secret("DATABASE_URL", "DATABASE_URL_FILE")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
logger := log.New(os.Stdout, "", log.LstdFlags|log.LUTC)
|
||||
mode := "api"
|
||||
if len(os.Args) > 1 {
|
||||
mode = os.Args[1]
|
||||
}
|
||||
if mode == "api" || mode == "worker" || mode == "dispatcher" {
|
||||
service := "messenger-" + mode
|
||||
if v := os.Getenv("OTEL_SERVICE_NAME"); v != "" {
|
||||
service = v
|
||||
}
|
||||
initctx, done := context.WithTimeout(context.Background(), 3*time.Second)
|
||||
shutdown, initErr := telemetry.Start(initctx, service, env("INSTANCE_ID", mode+"-1"))
|
||||
done()
|
||||
if initErr != nil {
|
||||
log.Fatal("invalid telemetry configuration")
|
||||
}
|
||||
defer func() {
|
||||
c, done := context.WithTimeout(context.Background(), 3*time.Second)
|
||||
defer done()
|
||||
_ = shutdown(c)
|
||||
}()
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
config, err := pgxpool.ParseConfig(databaseURL)
|
||||
if err != nil {
|
||||
log.Fatal("invalid database configuration")
|
||||
}
|
||||
config.ConnConfig.Tracer = telemetry.DBTracer{}
|
||||
config.MaxConns = int32(maxConns)
|
||||
config.ConnConfig.RuntimeParams["application_name"] = env("INSTANCE_ID", "maintenance")
|
||||
config.MinConns = 1
|
||||
config.MaxConnLifetime = 30 * time.Minute
|
||||
var pool *pgxpool.Pool
|
||||
for {
|
||||
pool, err = pgxpool.NewWithConfig(ctx, config)
|
||||
if err == nil {
|
||||
err = pool.Ping(ctx)
|
||||
}
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
if pool != nil {
|
||||
pool.Close()
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
log.Fatal("database unavailable during startup")
|
||||
}
|
||||
time.Sleep(time.Second)
|
||||
}
|
||||
defer pool.Close()
|
||||
if mode == "api" || mode == "worker" || mode == "dispatcher" {
|
||||
capacity, _ := positiveInt("WORKER_CONCURRENCY", 1)
|
||||
if err = telemetry.Monitor(pool, mode, capacity); err != nil {
|
||||
log.Fatal("invalid telemetry instruments")
|
||||
}
|
||||
}
|
||||
|
||||
if len(os.Args) > 1 && os.Args[1] == "migrate" {
|
||||
if err = database.Migrate(ctx, pool); err != nil {
|
||||
log.Fatalf("migration failed: %v", err)
|
||||
}
|
||||
logger.Print("migrations complete")
|
||||
return
|
||||
}
|
||||
access, err := secret("S3_ACCESS_KEY_ID", "S3_ACCESS_KEY_ID_FILE")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
secretKey, err := secret("S3_SECRET_ACCESS_KEY", "S3_SECRET_ACCESS_KEY_FILE")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
objects, err := objectstore.New(env("S3_ENDPOINT", "http://s3:9000"), env("S3_REGION", "us-east-1"), env("S3_BUCKET", "messenger-attachments"), access, secretKey)
|
||||
if err != nil {
|
||||
log.Fatal("invalid S3 configuration")
|
||||
}
|
||||
if len(os.Args) > 1 && os.Args[1] == "s3-bootstrap" {
|
||||
for {
|
||||
err = objects.Client.MakeBucket(ctx, objects.Bucket, minio.MakeBucketOptions{Region: env("S3_REGION", "us-east-1")})
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
if exists, e := objects.Client.BucketExists(ctx, objects.Bucket); e == nil && exists {
|
||||
break
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
log.Fatal("S3 bootstrap unavailable")
|
||||
}
|
||||
time.Sleep(time.Second)
|
||||
}
|
||||
if err = objects.Client.SetBucketPolicy(ctx, objects.Bucket, ""); err != nil {
|
||||
log.Fatal("cannot enforce private bucket")
|
||||
}
|
||||
apiAccess, err := secret("S3_API_ACCESS_KEY_ID", "S3_API_ACCESS_KEY_ID_FILE")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
apiSecret, err := secret("S3_API_SECRET_ACCESS_KEY", "S3_API_SECRET_ACCESS_KEY_FILE")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
if err = objectstore.Bootstrap(ctx, env("S3_ENDPOINT", "http://s3:9000"), objects.Bucket, access, secretKey, apiAccess, apiSecret); err != nil {
|
||||
log.Fatalf("S3 policy bootstrap: %v", err)
|
||||
}
|
||||
logger.Print("private MinIO bucket and limited API user ready")
|
||||
return
|
||||
}
|
||||
|
||||
if len(os.Args) > 1 && (os.Args[1] == "worker" || os.Args[1] == "dispatcher" || os.Args[1] == "replay" || os.Args[1] == "failed-jobs" || os.Args[1] == "fault") {
|
||||
|
||||
var broker *redis.Client
|
||||
if os.Args[1] == "worker" || os.Args[1] == "dispatcher" {
|
||||
brokerURL, e := secret("BROKER_URL", "BROKER_URL_FILE")
|
||||
if e != nil {
|
||||
log.Fatal(e)
|
||||
}
|
||||
options, e := redis.ParseURL(brokerURL)
|
||||
if e != nil {
|
||||
log.Fatal("invalid broker configuration")
|
||||
}
|
||||
options.DialTimeout = 2 * time.Second
|
||||
options.ReadTimeout = 3 * time.Second
|
||||
options.WriteTimeout = 3 * time.Second
|
||||
broker = redis.NewClient(options)
|
||||
defer broker.Close()
|
||||
}
|
||||
timeout, e := positiveInt("JOB_TIMEOUT_SECONDS", 60)
|
||||
if e != nil || timeout > 120 {
|
||||
log.Fatal("JOB_TIMEOUT_SECONDS must be 1..120")
|
||||
}
|
||||
retention, e := positiveInt("STAGING_RETENTION_SECONDS", 86400)
|
||||
if e != nil {
|
||||
log.Fatal(e)
|
||||
}
|
||||
processor := &pipeline.Pipeline{DB: pool, Broker: broker, Objects: objects, Timeout: time.Duration(timeout) * time.Second, Retention: time.Duration(retention) * time.Second, Logger: logger}
|
||||
runctx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM, os.Interrupt)
|
||||
defer stop()
|
||||
switch os.Args[1] {
|
||||
case "dispatcher":
|
||||
processor.Dispatch(runctx)
|
||||
case "worker":
|
||||
concurrency, e := positiveInt("WORKER_CONCURRENCY", 1)
|
||||
if e != nil {
|
||||
log.Fatal(e)
|
||||
}
|
||||
if e = processor.Worker(runctx, concurrency); e != nil {
|
||||
log.Fatal(e)
|
||||
}
|
||||
default:
|
||||
if e = processor.Command(ctx, os.Args[1:]); e != nil {
|
||||
log.Fatal(e)
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
if len(os.Args) > 1 && (os.Args[1] == "cleanup" || os.Args[1] == "cleanup-loop") {
|
||||
grace, err := positiveInt("CLEANUP_GRACE_SECONDS", 3600)
|
||||
if err != nil || grace < 120 {
|
||||
log.Fatal("CLEANUP_GRACE_SECONDS must be at least 120")
|
||||
}
|
||||
loopctx, stopCleanup := signal.NotifyContext(context.Background(), syscall.SIGTERM, os.Interrupt)
|
||||
defer stopCleanup()
|
||||
for {
|
||||
cleanupctx, cancelCleanup := context.WithTimeout(loopctx, 60*time.Second)
|
||||
result, cleanupErr := messenger.Cleanup(cleanupctx, pool, objects, time.Duration(grace)*time.Second)
|
||||
cancelCleanup()
|
||||
if cleanupErr != nil {
|
||||
logger.Print("cleanup failed; retained objects for next sweep")
|
||||
} else {
|
||||
_ = json.NewEncoder(os.Stdout).Encode(result)
|
||||
}
|
||||
if os.Args[1] == "cleanup" {
|
||||
if cleanupErr != nil {
|
||||
os.Exit(1)
|
||||
}
|
||||
return
|
||||
}
|
||||
select {
|
||||
case <-loopctx.Done():
|
||||
return
|
||||
case <-time.After(60 * time.Second):
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(os.Args) > 2 && os.Args[1] == "s3-audit" {
|
||||
if err = objects.Audit(ctx, os.Args[2]); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
fmt.Println("PASS: anonymous S3 GET=403; API credentials cannot create buckets")
|
||||
return
|
||||
}
|
||||
if len(os.Args) > 2 && os.Args[1] == "s3-stat" {
|
||||
info, err := objects.Client.StatObject(ctx, objects.Bucket, os.Args[2], minio.StatObjectOptions{})
|
||||
if err != nil {
|
||||
if minio.ToErrorResponse(err).Code != "NoSuchKey" {
|
||||
log.Fatal("S3 stat failed")
|
||||
}
|
||||
fmt.Println(`{"exists":false}`)
|
||||
return
|
||||
}
|
||||
_ = json.NewEncoder(os.Stdout).Encode(map[string]any{"exists": true, "size_bytes": info.Size})
|
||||
return
|
||||
}
|
||||
instanceID := env("INSTANCE_ID", "")
|
||||
if instanceID == "" {
|
||||
log.Fatal("INSTANCE_ID is required")
|
||||
}
|
||||
cacheURL, err := secret("CACHE_URL", "CACHE_URL_FILE")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
redisOptions, err := redis.ParseURL(cacheURL)
|
||||
if err != nil {
|
||||
log.Fatal("invalid cache configuration")
|
||||
}
|
||||
sessions := redis.NewClient(redisOptions)
|
||||
sessions.AddHook(telemetry.RedisHook{})
|
||||
defer sessions.Close()
|
||||
for {
|
||||
if err = sessions.Ping(ctx).Err(); err == nil {
|
||||
break
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
log.Fatal("session store unavailable during startup")
|
||||
}
|
||||
time.Sleep(time.Second)
|
||||
}
|
||||
for {
|
||||
if err = objects.Ready(ctx); err == nil {
|
||||
break
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
log.Fatal("S3 unavailable during startup")
|
||||
}
|
||||
time.Sleep(time.Second)
|
||||
}
|
||||
dbTimeout := time.Duration(dbTimeoutMS) * time.Millisecond
|
||||
srv := &http.Server{Addr: host + ":" + port, Handler: messenger.NewHandler(pool, sessions, objects, time.Duration(ttlSeconds)*time.Second, time.Duration(idempotencyTTLSeconds)*time.Second, dbTimeout, instanceID, logger), ReadHeaderTimeout: 5 * time.Second, ReadTimeout: 45 * time.Second, WriteTimeout: 60 * time.Second, IdleTimeout: 30 * time.Second, MaxHeaderBytes: 32 * 1024}
|
||||
stop := make(chan os.Signal, 1)
|
||||
signal.Notify(stop, syscall.SIGTERM, os.Interrupt)
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
<-stop
|
||||
logger.Print("shutdown started")
|
||||
shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 8*time.Second)
|
||||
defer shutdownCancel()
|
||||
if shutdownErr := srv.Shutdown(shutdownCtx); shutdownErr != nil {
|
||||
logger.Printf("shutdown error: %v", shutdownErr)
|
||||
}
|
||||
logger.Print("shutdown complete")
|
||||
}()
|
||||
logger.Printf("listening on %s", srv.Addr)
|
||||
if err = srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
|
||||
logger.Fatal(err)
|
||||
}
|
||||
<-done
|
||||
}
|
||||
+465
@@ -0,0 +1,465 @@
|
||||
name: mtusi-lab6
|
||||
|
||||
x-api-common: &api-common
|
||||
image: mtusi-lab6-api:local
|
||||
pull_policy: never
|
||||
environment: &api-environment
|
||||
APP_HOST: ${APP_HOST:-0.0.0.0}
|
||||
APP_PORT: 8080
|
||||
APP_ENV: ${APP_ENV:-lab}
|
||||
OTEL_SERVICE_NAME: messenger-api
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318
|
||||
OTEL_EXPORTER_OTLP_PROTOCOL: http/protobuf
|
||||
OTEL_RESOURCE_ATTRIBUTES: deployment.environment.name=lab
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
|
||||
DB_TIMEOUT_MS: ${DB_TIMEOUT_MS:-3000}
|
||||
DB_MAX_CONNS: ${DB_MAX_CONNS:-10}
|
||||
GOMEMLIMIT: 1200MiB
|
||||
S3_ENDPOINT: http://s3:9000
|
||||
S3_REGION: us-east-1
|
||||
S3_BUCKET: messenger-attachments
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
||||
BROKER_URL_FILE: /run/secrets/broker_url
|
||||
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
||||
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
|
||||
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-1}
|
||||
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
|
||||
CLEANUP_GRACE_SECONDS: ${CLEANUP_GRACE_SECONDS:-3600}
|
||||
DATABASE_URL_FILE: ${DATABASE_URL_FILE:-/run/secrets/database_url}
|
||||
CACHE_URL_FILE: ${CACHE_URL_FILE:-/run/secrets/cache_url}
|
||||
secrets:
|
||||
- database_url
|
||||
- cache_url
|
||||
- s3_access_key_id
|
||||
- s3_secret_access_key
|
||||
tmpfs:
|
||||
- /tmp:size=20m,mode=1777
|
||||
networks:
|
||||
- frontend
|
||||
- backend
|
||||
healthcheck:
|
||||
test: ["CMD", "/messenger", "healthcheck"]
|
||||
interval: 3s
|
||||
timeout: 2s
|
||||
retries: 20
|
||||
start_period: 3s
|
||||
restart: unless-stopped
|
||||
init: true
|
||||
stop_grace_period: 10s
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.50"
|
||||
memory: 1536M
|
||||
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:17.6-alpine3.22
|
||||
environment:
|
||||
POSTGRES_DB: messenger
|
||||
POSTGRES_USER: messenger
|
||||
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
|
||||
secrets:
|
||||
- postgres_password
|
||||
volumes:
|
||||
- postgres_data:/var/lib/postgresql/data
|
||||
networks:
|
||||
- backend
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"]
|
||||
interval: 3s
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "1.00"
|
||||
memory: 512M
|
||||
|
||||
redis:
|
||||
image: redis:8.2.1-alpine3.22
|
||||
command:
|
||||
- /bin/sh
|
||||
- -ec
|
||||
- exec redis-server --appendonly yes --appendfsync always --requirepass "$$(cat /run/secrets/redis_password)"
|
||||
secrets:
|
||||
- redis_password
|
||||
volumes:
|
||||
- redis_data:/data
|
||||
networks:
|
||||
- backend
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "redis-cli --no-auth-warning -a \"$$(cat /run/secrets/redis_password)\" ping | grep PONG"]
|
||||
interval: 3s
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.25"
|
||||
memory: 128M
|
||||
|
||||
migrate:
|
||||
image: mtusi-lab6-api:local
|
||||
pull_policy: never
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
healthcheck:
|
||||
disable: true
|
||||
command: ["migrate"]
|
||||
environment:
|
||||
<<: *api-environment
|
||||
secrets:
|
||||
- database_url
|
||||
networks:
|
||||
- backend
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
restart: "no"
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.25"
|
||||
memory: 128M
|
||||
|
||||
api1:
|
||||
<<: *api-common
|
||||
environment:
|
||||
<<: *api-environment
|
||||
INSTANCE_ID: api-1
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
redis:
|
||||
condition: service_healthy
|
||||
s3-init:
|
||||
condition: service_completed_successfully
|
||||
|
||||
api2:
|
||||
<<: *api-common
|
||||
environment:
|
||||
<<: *api-environment
|
||||
INSTANCE_ID: api-2
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
redis:
|
||||
condition: service_healthy
|
||||
s3-init:
|
||||
condition: service_completed_successfully
|
||||
|
||||
s3:
|
||||
image: mtusi-lab6-minio:RELEASE.2025-10-15T17-29-55Z
|
||||
build:
|
||||
context: .
|
||||
dockerfile: infra/Dockerfile.minio
|
||||
command: ["server", "/data", "--address", ":9000", "--console-address", "127.0.0.1:9001"]
|
||||
environment:
|
||||
MINIO_ROOT_USER_FILE: /run/secrets/s3_admin_access
|
||||
MINIO_ROOT_PASSWORD_FILE: /run/secrets/s3_admin_secret
|
||||
MINIO_BROWSER: "off"
|
||||
MINIO_UPDATE: "off"
|
||||
volumes:
|
||||
- minio_data:/data
|
||||
secrets:
|
||||
- s3_admin_access
|
||||
- s3_admin_secret
|
||||
networks:
|
||||
- backend
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-q", "-T", "1", "-O", "/dev/null", "http://127.0.0.1:9000/minio/health/ready"]
|
||||
interval: 3s
|
||||
timeout: 2s
|
||||
retries: 20
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "1.00"
|
||||
memory: 512M
|
||||
|
||||
s3-init:
|
||||
image: mtusi-lab6-api:local
|
||||
pull_policy: never
|
||||
healthcheck:
|
||||
disable: true
|
||||
command: ["s3-bootstrap"]
|
||||
environment:
|
||||
<<: *api-environment
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_admin_access
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_admin_secret
|
||||
S3_API_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
|
||||
S3_API_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
||||
secrets:
|
||||
- database_url
|
||||
- s3_access_key_id
|
||||
- s3_secret_access_key
|
||||
- s3_admin_access
|
||||
- s3_admin_secret
|
||||
networks:
|
||||
- backend
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
s3:
|
||||
condition: service_healthy
|
||||
restart: "no"
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.25"
|
||||
memory: 128M
|
||||
|
||||
cleanup:
|
||||
image: mtusi-lab6-api:local
|
||||
pull_policy: never
|
||||
healthcheck:
|
||||
disable: true
|
||||
command: ["cleanup-loop"]
|
||||
environment:
|
||||
<<: *api-environment
|
||||
secrets:
|
||||
- database_url
|
||||
- s3_access_key_id
|
||||
- s3_secret_access_key
|
||||
networks:
|
||||
- backend
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
s3-init:
|
||||
condition: service_completed_successfully
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.25"
|
||||
memory: 128M
|
||||
|
||||
broker:
|
||||
image: redis:8.2.1-alpine3.22
|
||||
command: ["/bin/sh", "-ec", 'exec redis-server --appendonly yes --appendfsync always --maxmemory-policy noeviction --requirepass "$$(cat /run/secrets/broker_password)"']
|
||||
secrets: [broker_password]
|
||||
volumes: [broker_data:/data]
|
||||
networks: [backend]
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", 'redis-cli --no-auth-warning -a "$$(cat /run/secrets/broker_password)" ping | grep PONG']
|
||||
interval: 3s
|
||||
timeout: 2s
|
||||
retries: 20
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.25"
|
||||
memory: 128M
|
||||
|
||||
dispatcher:
|
||||
image: mtusi-lab6-api:local
|
||||
pull_policy: never
|
||||
command: ["dispatcher"]
|
||||
healthcheck:
|
||||
disable: true
|
||||
environment:
|
||||
<<: *api-environment
|
||||
OTEL_SERVICE_NAME: messenger-dispatcher
|
||||
INSTANCE_ID: dispatcher-1
|
||||
secrets: [database_url, s3_access_key_id, s3_secret_access_key, broker_url]
|
||||
networks: [backend]
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
s3-init:
|
||||
condition: service_completed_successfully
|
||||
broker:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.25"
|
||||
memory: 128M
|
||||
|
||||
worker:
|
||||
image: mtusi-lab6-worker:local
|
||||
build:
|
||||
context: .
|
||||
dockerfile: infra/Dockerfile.worker
|
||||
additional_contexts:
|
||||
application: service:migrate
|
||||
command: ["worker"]
|
||||
environment:
|
||||
<<: *api-environment
|
||||
GOMEMLIMIT: 128MiB
|
||||
OTEL_SERVICE_NAME: messenger-worker
|
||||
INSTANCE_ID: worker-1
|
||||
secrets: [database_url, s3_access_key_id, s3_secret_access_key, broker_url]
|
||||
networks: [backend]
|
||||
tmpfs: ["/tmp:size=128m,mode=1777"]
|
||||
depends_on:
|
||||
dispatcher:
|
||||
condition: service_started
|
||||
init: true
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 5s
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "1.00"
|
||||
memory: 1536M
|
||||
|
||||
nginx:
|
||||
image: nginx:1.29.1-alpine3.22
|
||||
command: ["/bin/sh", "/health-aware.sh"]
|
||||
volumes:
|
||||
- ./infra/nginx.conf:/etc/nginx/nginx.conf:ro
|
||||
- ./infra/health-aware.sh:/health-aware.sh:ro
|
||||
networks:
|
||||
- frontend
|
||||
depends_on:
|
||||
api1:
|
||||
condition: service_healthy
|
||||
api2:
|
||||
condition: service_healthy
|
||||
ports:
|
||||
- "127.0.0.1:${APP_PORT:-8080}:8080"
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:8080/health/ready"]
|
||||
interval: 3s
|
||||
timeout: 2s
|
||||
retries: 20
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 5s
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.25"
|
||||
memory: 64M
|
||||
|
||||
|
||||
otel-collector:
|
||||
image: otel/opentelemetry-collector-contrib:0.136.0
|
||||
command: ["--config=/etc/otelcol/config.yaml"]
|
||||
volumes:
|
||||
- ./infra/otel/collector.yaml:/etc/otelcol/config.yaml:ro
|
||||
networks: [backend]
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.50"
|
||||
memory: 256M
|
||||
|
||||
prometheus:
|
||||
image: prom/prometheus:v3.7.0
|
||||
command: ["--config.file=/etc/prometheus/prometheus.yaml", "--storage.tsdb.path=/prometheus", "--storage.tsdb.retention.time=24h"]
|
||||
volumes:
|
||||
- ./infra/otel/prometheus.yaml:/etc/prometheus/prometheus.yaml:ro
|
||||
- ./infra/otel/alerts.yaml:/etc/prometheus/alerts.yaml:ro
|
||||
- prometheus_data:/prometheus
|
||||
networks: [backend]
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.50"
|
||||
memory: 512M
|
||||
|
||||
loki:
|
||||
image: grafana/loki:3.5.5
|
||||
command: ["-config.file=/etc/loki/config.yaml"]
|
||||
volumes:
|
||||
- ./infra/otel/loki.yaml:/etc/loki/config.yaml:ro
|
||||
- loki_data:/loki
|
||||
networks: [backend]
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.50"
|
||||
memory: 384M
|
||||
|
||||
tempo:
|
||||
image: grafana/tempo:2.8.2
|
||||
command: ["-config.file=/etc/tempo/config.yaml"]
|
||||
volumes:
|
||||
- ./infra/otel/tempo.yaml:/etc/tempo/config.yaml:ro
|
||||
- tempo_data:/var/tempo
|
||||
networks: [backend]
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.50"
|
||||
memory: 384M
|
||||
|
||||
grafana:
|
||||
image: grafana/grafana:12.2.0
|
||||
environment:
|
||||
GF_SECURITY_ADMIN_USER: admin
|
||||
GF_SECURITY_ADMIN_PASSWORD__FILE: /run/secrets/grafana_admin_password
|
||||
GF_USERS_ALLOW_SIGN_UP: "false"
|
||||
GF_ANALYTICS_REPORTING_ENABLED: "false"
|
||||
GF_ANALYTICS_CHECK_FOR_UPDATES: "false"
|
||||
GF_ANALYTICS_CHECK_FOR_PLUGIN_UPDATES: "false"
|
||||
GF_PLUGINS_PREINSTALL_DISABLED: "true"
|
||||
GF_AUTH_ANONYMOUS_ENABLED: "false"
|
||||
secrets: [grafana_admin_password]
|
||||
volumes:
|
||||
- ./infra/otel/grafana/provisioning:/etc/grafana/provisioning:ro
|
||||
- ./infra/otel/grafana/dashboards:/var/lib/grafana/dashboards:ro
|
||||
- grafana_data:/var/lib/grafana
|
||||
ports:
|
||||
- "127.0.0.1:${GRAFANA_PORT:-13000}:3000"
|
||||
networks: [frontend, backend]
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.50"
|
||||
memory: 384M
|
||||
|
||||
secrets:
|
||||
grafana_admin_password:
|
||||
file: ./secrets/grafana_admin_password
|
||||
broker_password:
|
||||
file: ./secrets/broker_password
|
||||
broker_url:
|
||||
file: ./secrets/broker_url
|
||||
postgres_password:
|
||||
file: ./secrets/postgres_password
|
||||
redis_password:
|
||||
file: ./secrets/redis_password
|
||||
database_url:
|
||||
file: ./secrets/database_url
|
||||
cache_url:
|
||||
file: ./secrets/cache_url
|
||||
s3_access_key_id:
|
||||
file: ./secrets/s3_access_key_id
|
||||
s3_secret_access_key:
|
||||
file: ./secrets/s3_secret_access_key
|
||||
s3_admin_access:
|
||||
file: ./secrets/s3_admin_access
|
||||
s3_admin_secret:
|
||||
file: ./secrets/s3_admin_secret
|
||||
|
||||
volumes:
|
||||
prometheus_data:
|
||||
loki_data:
|
||||
tempo_data:
|
||||
grafana_data:
|
||||
broker_data:
|
||||
postgres_data:
|
||||
redis_data:
|
||||
minio_data:
|
||||
|
||||
networks:
|
||||
frontend:
|
||||
backend:
|
||||
internal: true
|
||||
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"status": "PASS",
|
||||
"date": "2026-11-08",
|
||||
"source": "isolated source copy excluding .git, .env, secrets and virtual environments",
|
||||
"project": "mtusi-lab6-clean-audit",
|
||||
"api_port": 18087,
|
||||
"grafana_port": 13001,
|
||||
"new_secrets": true,
|
||||
"fresh_volumes": [
|
||||
"postgres_data",
|
||||
"redis_data",
|
||||
"minio_data",
|
||||
"broker_data",
|
||||
"prometheus_data",
|
||||
"loki_data",
|
||||
"tempo_data",
|
||||
"grafana_data"
|
||||
],
|
||||
"build_cache_used": true,
|
||||
"public_smoke": "PASS",
|
||||
"temporary_stack_removed": true
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
[
|
||||
{
|
||||
"attempt": 1,
|
||||
"event": "job.published",
|
||||
"job_id": "4985c362-88c5-47f8-8f4e-32e41aebe60a",
|
||||
"service.name": "messenger-dispatcher",
|
||||
"severity": "INFO",
|
||||
"span_id": "65906c44867999d6",
|
||||
"success": true,
|
||||
"timestamp": "2026-11-08T15:48:09.340450259Z",
|
||||
"trace_id": "75692cc8cb7b4ebb920150c60f690298"
|
||||
},
|
||||
{
|
||||
"attempt": 1,
|
||||
"event": "job.started",
|
||||
"job_id": "4985c362-88c5-47f8-8f4e-32e41aebe60a",
|
||||
"request_id": "c7b498ed-4a65-4858-b5ab-91168efe48b2",
|
||||
"service.name": "messenger-worker",
|
||||
"severity": "INFO",
|
||||
"span_id": "12b2410403dff46d",
|
||||
"timestamp": "2026-11-08T15:48:09.401979384Z",
|
||||
"trace_id": "75692cc8cb7b4ebb920150c60f690298"
|
||||
},
|
||||
{
|
||||
"attempt": 2,
|
||||
"event": "job.published",
|
||||
"job_id": "4985c362-88c5-47f8-8f4e-32e41aebe60a",
|
||||
"service.name": "messenger-dispatcher",
|
||||
"severity": "INFO",
|
||||
"span_id": "b3db2d84c0e78904",
|
||||
"success": true,
|
||||
"timestamp": "2026-11-08T15:48:14.360894344Z",
|
||||
"trace_id": "75692cc8cb7b4ebb920150c60f690298"
|
||||
},
|
||||
{
|
||||
"attempt": 2,
|
||||
"event": "job.started",
|
||||
"job_id": "4985c362-88c5-47f8-8f4e-32e41aebe60a",
|
||||
"request_id": "c7b498ed-4a65-4858-b5ab-91168efe48b2",
|
||||
"service.name": "messenger-worker",
|
||||
"severity": "INFO",
|
||||
"span_id": "2c571d705b0aa552",
|
||||
"timestamp": "2026-11-08T15:48:14.386684094Z",
|
||||
"trace_id": "75692cc8cb7b4ebb920150c60f690298"
|
||||
},
|
||||
{
|
||||
"duration_ms": 52,
|
||||
"event": "job.completed",
|
||||
"job_id": "4985c362-88c5-47f8-8f4e-32e41aebe60a",
|
||||
"outcome": "retry",
|
||||
"request_id": "c7b498ed-4a65-4858-b5ab-91168efe48b2",
|
||||
"service.name": "messenger-worker",
|
||||
"severity": "ERROR",
|
||||
"span_id": "12b2410403dff46d",
|
||||
"timestamp": "2026-11-08T15:48:09.45155505Z",
|
||||
"trace_id": "75692cc8cb7b4ebb920150c60f690298"
|
||||
},
|
||||
{
|
||||
"duration_ms": 8,
|
||||
"event": "http.request",
|
||||
"method": "POST",
|
||||
"request_id": "c7b498ed-4a65-4858-b5ab-91168efe48b2",
|
||||
"route": "/api/v1/chats/{chat_id}/attachments",
|
||||
"service.name": "messenger-api",
|
||||
"severity": "INFO",
|
||||
"span_id": "3a0b3d3655700064",
|
||||
"status": 202,
|
||||
"timestamp": "2026-11-08T15:48:09.087228092Z",
|
||||
"trace_id": "75692cc8cb7b4ebb920150c60f690298"
|
||||
},
|
||||
{
|
||||
"duration_ms": 84,
|
||||
"event": "job.completed",
|
||||
"job_id": "4985c362-88c5-47f8-8f4e-32e41aebe60a",
|
||||
"outcome": "success",
|
||||
"request_id": "c7b498ed-4a65-4858-b5ab-91168efe48b2",
|
||||
"service.name": "messenger-worker",
|
||||
"severity": "INFO",
|
||||
"span_id": "2c571d705b0aa552",
|
||||
"timestamp": "2026-11-08T15:48:14.468845719Z",
|
||||
"trace_id": "75692cc8cb7b4ebb920150c60f690298"
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,135 @@
|
||||
{
|
||||
"batches": [
|
||||
{
|
||||
"resource": {
|
||||
"attributes": [
|
||||
{
|
||||
"key": "deployment.environment.name",
|
||||
"value": {
|
||||
"stringValue": "lab"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "service.instance.id",
|
||||
"value": {
|
||||
"stringValue": "api-2"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "service.name",
|
||||
"value": {
|
||||
"stringValue": "messenger-api"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "service.version",
|
||||
"value": {
|
||||
"stringValue": "lab6-v1"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"scopeSpans": [
|
||||
{
|
||||
"scope": {
|
||||
"name": "mtusi-messenger/telemetry-v1"
|
||||
},
|
||||
"spans": [
|
||||
{
|
||||
"traceId": "epY6ExihVHU9tQe8xnXiIw==",
|
||||
"spanId": "MdlQ/J7piv8=",
|
||||
"parentSpanId": "bqKvTaWUxEg=",
|
||||
"flags": 256,
|
||||
"name": "redis.get",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"startTimeUnixNano": "1794152654758902803",
|
||||
"endTimeUnixNano": "1794152654758953261",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "db.system.name",
|
||||
"value": {
|
||||
"stringValue": "redis"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "db.operation.name",
|
||||
"value": {
|
||||
"stringValue": "get"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {}
|
||||
},
|
||||
{
|
||||
"traceId": "epY6ExihVHU9tQe8xnXiIw==",
|
||||
"spanId": "l0cDzDuR/a4=",
|
||||
"parentSpanId": "bqKvTaWUxEg=",
|
||||
"flags": 256,
|
||||
"name": "db.pool.acquire",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"startTimeUnixNano": "1794152654758972844",
|
||||
"endTimeUnixNano": "1794152654760767552",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "db.system.name",
|
||||
"value": {
|
||||
"stringValue": "postgresql"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "error",
|
||||
"value": {
|
||||
"boolValue": true
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {
|
||||
"message": "dependency unavailable",
|
||||
"code": "STATUS_CODE_ERROR"
|
||||
}
|
||||
},
|
||||
{
|
||||
"traceId": "epY6ExihVHU9tQe8xnXiIw==",
|
||||
"spanId": "bqKvTaWUxEg=",
|
||||
"flags": 256,
|
||||
"name": "POST /api/v1/chats/{chat_id}/messages",
|
||||
"kind": "SPAN_KIND_SERVER",
|
||||
"startTimeUnixNano": "1794152654758897136",
|
||||
"endTimeUnixNano": "1794152654760800886",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "http.route",
|
||||
"value": {
|
||||
"stringValue": "/api/v1/chats/{chat_id}/messages"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "http.request.method",
|
||||
"value": {
|
||||
"stringValue": "POST"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "request.id",
|
||||
"value": {
|
||||
"stringValue": "4cf18920-182a-43f1-9bfc-ab056fed9ce2"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "http.response.status_code",
|
||||
"value": {
|
||||
"intValue": "503"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {
|
||||
"message": "HTTP server error",
|
||||
"code": "STATUS_CODE_ERROR"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
{
|
||||
"warmup_load": {
|
||||
"seconds": 5.001,
|
||||
"workers": 2,
|
||||
"successful_requests": 4947,
|
||||
"failed_requests": 0,
|
||||
"successful_rps": 989.109,
|
||||
"latency_ms_successes": {
|
||||
"p50": 1.937,
|
||||
"p95": 3.175,
|
||||
"p99": 4.776
|
||||
}
|
||||
},
|
||||
"worker_stopped": {
|
||||
"firing": true,
|
||||
"detected_seconds": 25.767,
|
||||
"depth": 3.0,
|
||||
"oldest_age_seconds": 24.0,
|
||||
"worker_busy": 0.0,
|
||||
"worker_heartbeat_age_seconds": 26.03500008583069,
|
||||
"resolved": true,
|
||||
"depth_after": 0.0
|
||||
},
|
||||
"postgres_stopped": {
|
||||
"firing": true,
|
||||
"detected_seconds": 19.539,
|
||||
"failed_requests": 2584,
|
||||
"error_ratio": 0.30580225729713156,
|
||||
"db_available": 0.0,
|
||||
"pool_waiting_peak": 0.0,
|
||||
"trace_id": "7a963a1318a154753db507bcc675e223",
|
||||
"resolved": true,
|
||||
"db_available_after": 1.0
|
||||
},
|
||||
"tempo_stopped": {
|
||||
"workload": {
|
||||
"seconds": 20.003,
|
||||
"workers": 2,
|
||||
"successful_requests": 19732,
|
||||
"failed_requests": 0,
|
||||
"successful_rps": 986.464,
|
||||
"latency_ms_successes": {
|
||||
"p50": 1.958,
|
||||
"p95": 3.153,
|
||||
"p99": 4.537
|
||||
}
|
||||
},
|
||||
"collector_before": {
|
||||
"memory": "71.75MiB / 256MiB",
|
||||
"cpu": "1.60%"
|
||||
},
|
||||
"collector_during": {
|
||||
"memory": "68.8MiB / 256MiB",
|
||||
"cpu": "2.10%"
|
||||
},
|
||||
"failed_span_send_records_delta": 800.0,
|
||||
"max_queue_batches": 32.0,
|
||||
"queue_limit_batches": 32,
|
||||
"dropped_spans_from_collector_logs": 864,
|
||||
"business_api_preserved": true,
|
||||
"export_resumed": true
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,101 @@
|
||||
# Результаты лабораторной 6 — уровень 4
|
||||
|
||||
**8 ноября 2026 года**, macOS arm64, Docker Desktop Linux VM: Engine 29.2.1, Compose 5.0.2, 10 CPU, RAM VM 8 217 059 328 bytes. Основной API localhost:18086, Grafana localhost:13000. Все данные синтетические; секреты и tokens в evidence не сохраняются. Лимиты сервисов указаны в compose.yaml и REPORT.md.
|
||||
|
||||
## Базовые проверки
|
||||
|
||||
```sh
|
||||
sh scripts/init-local.sh
|
||||
docker compose --env-file .env config --quiet
|
||||
docker compose --env-file .env up -d --build --wait
|
||||
make check
|
||||
CGO_ENABLED=0 go test ./...
|
||||
CGO_ENABLED=0 go vet ./...
|
||||
docker compose --env-file .env exec -T prometheus promtool check config /etc/prometheus/prometheus.yaml
|
||||
make test BASE_URL=http://localhost:18086
|
||||
```
|
||||
|
||||
Результат: make check PASS; Go tests messenger/pipeline/telemetry PASS; vet exit0; promtool подтвердил конфигурацию и 4 recording/alert rules; публичный smoke **PASS** при включённой telemetry.
|
||||
|
||||
## Чистый запуск
|
||||
|
||||
Изолированная исходная копия без .env/secrets, новые секреты и **8 пустых volumes**, project `mtusi-lab6-clean-audit`, API 18087/Grafana 13001. `up -d --build --wait` и публичный smoke прошли; BuildKit cache использовался. Проверка выполнена на копии текущих исходников, до публикации ветки в удалённом Git. Временный стенд удалён после проверки, основной сохранён.
|
||||
|
||||
Доказательство: [clean-start.json](clean-start.json).
|
||||
|
||||
## Реальные метрики, логи, трассы и provisioning
|
||||
|
||||
```sh
|
||||
BASE_URL=http://localhost:18086 python3 student-tests/verify_observability.py
|
||||
```
|
||||
|
||||
**PASS**:
|
||||
|
||||
- Контролируемый incoming W3C traceparent сохранён; API ответил тем же X-Trace-Id.
|
||||
- Trace upload содержит producer и **2 consumer attempts** (временный S3 outage → retry → ready), реальные S3/DB/image.process spans.
|
||||
- Каждый consumer имеет producer parent span ID; три service resources принадлежат API/dispatcher/worker.
|
||||
- Структурные события `http.request`, `job.published`, `job.started`, `job.completed` найдены в Loki по одному trace ID; worker log сохранил исходный request_id.
|
||||
- Сохранённые trace/logs не содержат тестовых пароля, токена, message body и filename. Labels реальных metrics проверены на отсутствие пользовательских/ресурсных IDs.
|
||||
- Метрики HTTP counter/histogram, worker success/retry/duration, E2E, queue, pool, capacity и память найдены в Prometheus.
|
||||
- Grafana API подтвердил provisioning dashboard из **23 panels**. Все PromQL targets всех панелей отдельно приняты работающим Prometheus.
|
||||
|
||||
Файлы: [observability-grade4.json](observability-grade4.json), [trace-example.json](trace-example.json), [correlated-logs.json](correlated-logs.json).
|
||||
|
||||
## Два диагностируемых инцидента и отказ telemetry backend
|
||||
|
||||
```sh
|
||||
BASE_URL=http://localhost:18086 python3 student-tests/verify_incidents.py
|
||||
```
|
||||
|
||||
### 1. Worker остановлен
|
||||
|
||||
**Наблюдения из сохранённых сигналов:** queue depth=3, oldest age=24s, busy=0, worker heartbeat age=26.035s. Consumer для новых задач отсутствует, API продолжает сообщения. Это отличает отсутствие worker от медленной обработки внутри него.
|
||||
|
||||
`MessengerQueueLag` перешёл в **firing за 25.767s**. После восстановления те же задания стали ready, depth=0, alert **resolved**.
|
||||
|
||||
### 2. PostgreSQL остановлен
|
||||
|
||||
**Наблюдения:** 5xx ratio **30.58%**, DB availability=0, HTTP trace имеет errored DB child spans. В данном быстром отказе sampled pool waiting peak=0: причина определяется по error spans/availability, а не по предположению, что pool обязательно должен заполниться.
|
||||
|
||||
`MessengerHighErrorRate` перешёл в **firing за 19.539s**; в этом интервале тест получил 2584 ответа 503. После запуска PostgreSQL реальные здоровые запросы восстановились, DB availability=1, alert **resolved**. [Пример DB trace](db-incident-trace.json).
|
||||
|
||||
Диагностика обоих инцидентов опирается на сохранённые Prometheus/Loki/Tempo observations; команды Docker используются для внесения и устранения отказа. SLO: ≥99% бизнес-запросов без 5xx за 5m; контролируемый DB outage намеренно нарушает его.
|
||||
|
||||
### 3. Tempo остановлен
|
||||
|
||||
Во время отказа trace backend выполнялась нагрузка: **20.003s**, 2 closed-loop клиента, **19 732 успешных POST**, **0 HTTP failures**, successful RPS 986.464; успешные latency p50=1.958ms, p95=3.153ms, p99=4.537ms. Перед инцидентами был прогрев 5.001s, 4947 успешных запросов. Это производительность HTTP-подтверждений конкретного локального прогона, не максимальная пропускная способность и не исследование цены telemetry.
|
||||
|
||||
- Максимальная наблюдаемая очередь Collector: **32 batches**, равна настроенному пределу.
|
||||
- Память Collector: **71.75MiB до / 68.8MiB во время**, hard limit 256MiB; CPU snapshots 1.60% /2.10%.
|
||||
- Наблюдаемая дельта failed-span-send counter:800 records; JSON-логи финальных drops зафиксировали **864 dropped spans**. Снимки метрик и логов сделаны в разное время; failed attempts не приравниваются к уникальным потерянным traces.
|
||||
- Потеря signals при bounded retry допускается и посчитана для наблюдаемого Collector drops; бизнес-операции сохранились.
|
||||
- После запуска Tempo найдена свежая трасса: экспорт восстановился.
|
||||
|
||||
Все три сценария: **PASS**. Подробности: [incidents-grade4.json](incidents-grade4.json). Шаги диагностики: [runbook](../infra/otel/runbook.md).
|
||||
|
||||
## Сохранение возможностей lab5
|
||||
|
||||
```sh
|
||||
BASE_URL=http://localhost:18086 python3 student-tests/verify_variants.py
|
||||
BASE_URL=http://localhost:18086 python3 student-tests/verify_pipeline.py
|
||||
```
|
||||
|
||||
Оба **PASS**. Проверены JPEG/WebP, orientation/alpha, bytes/pixels/ACL и checksums; outbox gap, crash после S3 до COMMIT/ack, дубли ready, bounded retry, failed registry, replay, staging cleanup. Expired processing восстановлен за **65.020s**, attempts=2. Пик worker за image suite **704 679 936 bytes** при hard limit1 610 612 736 bytes.
|
||||
|
||||
Файлы: [variants-grade4.json](variants-grade4.json), [pipeline-grade4.json](pipeline-grade4.json).
|
||||
|
||||
## Перезапуск и persistent история
|
||||
|
||||
```sh
|
||||
BASE_URL=http://localhost:18086 python3 student-tests/verify_resilience.py
|
||||
```
|
||||
|
||||
**PASS**. Реально перезапущены **15 core/telemetry сервисов**, StartedAt каждого изменился. Прежние token/data/original сохранились. Ранее сохранённые trace и correlated logs доступны после рестарта Collector/Prometheus/Loki/Tempo/Grafana, метрики снова доступны.
|
||||
|
||||
Отдельно остановлены api1 и api2: подтверждены68/69 сообщений, переходных ошибок0/0, первый успех surviving API0.829s /0.727s, исходный файл сохранён. Обе API-реплики восстановлены. Итоговый `make test` — PASS.
|
||||
|
||||
Доказательство: [resilience.json](resilience.json).
|
||||
|
||||
## Границы проверки
|
||||
|
||||
Проверены уровни 3 и 4 README на одном Docker-host, worker concurrency 1. Полная OpenAPI validation (`make validate`) необязательна локально и в этом прогоне не выполнялась; public smoke проверяет ответы своим schema subset. Показатели latency относятся к успешным запросам закрытого генератора; coordinated omission и влияние клиента не устранены. Отказ host/volume и production HA не заявлены.
|
||||
@@ -0,0 +1,29 @@
|
||||
{
|
||||
"status": "PASS",
|
||||
"trace_id": "75692cc8cb7b4ebb920150c60f690298",
|
||||
"request_id": "c7b498ed-4a65-4858-b5ab-91168efe48b2",
|
||||
"consumer_attempt_spans": 2,
|
||||
"events": [
|
||||
"http.request",
|
||||
"job.completed",
|
||||
"job.published",
|
||||
"job.started"
|
||||
],
|
||||
"metrics": {
|
||||
"http_requests": 40217.0,
|
||||
"http_histogram": 40217.0,
|
||||
"jobs_success": 1.0,
|
||||
"jobs_retry": 1.0,
|
||||
"ready_histogram": 1.0,
|
||||
"queue": 0.0,
|
||||
"pool": 10.0,
|
||||
"worker_capacity": 1.0,
|
||||
"memory": 146468864.0
|
||||
},
|
||||
"dashboard": {
|
||||
"title": "MTUSI Messenger — lab6",
|
||||
"panels": 23
|
||||
},
|
||||
"privacy": "PASS",
|
||||
"low_cardinality": "PASS"
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
{
|
||||
"stopped_worker_and_restart": {
|
||||
"status": "PASS",
|
||||
"before": {
|
||||
"api1": "2026-11-08T15:37:02.178556088Z",
|
||||
"api2": "2026-11-08T15:37:02.177388463Z",
|
||||
"broker": "2026-11-08T15:30:58.77326392Z"
|
||||
},
|
||||
"after": {
|
||||
"api1": "2026-11-08T15:48:29.295995045Z",
|
||||
"api2": "2026-11-08T15:48:29.25367792Z",
|
||||
"broker": "2026-11-08T15:48:29.192814837Z"
|
||||
}
|
||||
},
|
||||
"commit_publish_gap": "PASS: actual dispatcher exit 86 and automatic restart",
|
||||
"crash_after_objects": {
|
||||
"status": "PASS",
|
||||
"recovery_seconds": 65.02,
|
||||
"attempts": 2
|
||||
},
|
||||
"commit_before_ack_and_duplicate": "PASS",
|
||||
"retry_failed_registry_replay": {
|
||||
"status": "PASS",
|
||||
"exhausted_attempts": 2,
|
||||
"error_code": "retry_exhausted",
|
||||
"replay_audit_rows": 1,
|
||||
"ready_replay_rejected": true
|
||||
},
|
||||
"permanent_error": "PASS",
|
||||
"message_replay_after_attachment_failure": "PASS",
|
||||
"crash_points": [
|
||||
{
|
||||
"point": "dispatcher_before_publish",
|
||||
"armed": false,
|
||||
"hit_at": "2026-11-08T15:48:32.928538+00:00"
|
||||
},
|
||||
{
|
||||
"point": "worker_after_objects",
|
||||
"armed": false,
|
||||
"hit_at": "2026-11-08T15:48:34.168865+00:00"
|
||||
},
|
||||
{
|
||||
"point": "worker_before_ack",
|
||||
"armed": false,
|
||||
"hit_at": "2026-11-08T15:49:40.389531+00:00"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,156 @@
|
||||
{
|
||||
"persistence": {
|
||||
"status": "PASS",
|
||||
"before": {
|
||||
"postgres": {
|
||||
"id": "06797da995bb6cc6fd69a2c251fd6204992b74dbcf630110c59172dde5fb401f",
|
||||
"started_at": "2026-11-08T15:44:15.618563595Z"
|
||||
},
|
||||
"redis": {
|
||||
"id": "e2f09b5ae65d8667339960a5649429b6e7abf22d5a20ca9de44846e2394aef6f",
|
||||
"started_at": "2026-11-08T15:30:58.770536504Z"
|
||||
},
|
||||
"s3": {
|
||||
"id": "0326a0c248bc89fa1630815b6824af8e03bf57029c88c297bcdf46c2cdc9deb2",
|
||||
"started_at": "2026-11-08T15:49:47.995548388Z"
|
||||
},
|
||||
"broker": {
|
||||
"id": "632478d263786461c5e7a9179bdc88c4e910712d58cf30ae2ba0efcc5317a1f3",
|
||||
"started_at": "2026-11-08T15:48:29.192814837Z"
|
||||
},
|
||||
"api1": {
|
||||
"id": "b2523eba3a14f02311e1a7dabc2b5e3a0459ce74dc22248559ae919bb33c96bd",
|
||||
"started_at": "2026-11-08T15:48:29.295995045Z"
|
||||
},
|
||||
"api2": {
|
||||
"id": "9c3bccf975647eaa0052335faeab360fc199b8aaddc6f1d437009b6a876a1c6f",
|
||||
"started_at": "2026-11-08T15:48:29.25367792Z"
|
||||
},
|
||||
"dispatcher": {
|
||||
"id": "d6579f5458efcd35d21d93efe9d64ee27661251275a73137b0cc690f2fc65133",
|
||||
"started_at": "2026-11-08T15:48:33.042620172Z"
|
||||
},
|
||||
"worker": {
|
||||
"id": "d5975de80120800bd179bad7284ebf92477094d11b14ae423c159b4d42930f5f",
|
||||
"started_at": "2026-11-08T15:49:50.782883Z"
|
||||
},
|
||||
"cleanup": {
|
||||
"id": "aef4f262490e1e616b6c83d8005a1d13286e1b7be5b0389a252315ad66814fdb",
|
||||
"started_at": "2026-11-08T15:37:02.146939838Z"
|
||||
},
|
||||
"nginx": {
|
||||
"id": "652643855d5b97cea36ab41ada3ebef618fd894c5a3e9d603759de32b040f65c",
|
||||
"started_at": "2026-11-08T15:31:08.673401883Z"
|
||||
},
|
||||
"otel-collector": {
|
||||
"id": "3fcb612ac0c6c3a701d25f66e63b2abfb123ad0390c55f5f2dd795a2f3aa8c2d",
|
||||
"started_at": "2026-11-08T15:37:11.526894135Z"
|
||||
},
|
||||
"prometheus": {
|
||||
"id": "a3bad0ed0a03b0d8edc3450830d449ccb6564dfbc3e913674804b82868a9e838",
|
||||
"started_at": "2026-11-08T15:37:11.547627051Z"
|
||||
},
|
||||
"loki": {
|
||||
"id": "6713f6f19668716725bd24a7c5e6c8ab6240d2455b2775b854802702f8d24826",
|
||||
"started_at": "2026-11-08T15:30:58.761601837Z"
|
||||
},
|
||||
"tempo": {
|
||||
"id": "63377b0fc325bad816738019c191e909751609ca9a5f4ed469896fc6112e0ed2",
|
||||
"started_at": "2026-11-08T15:45:47.664199012Z"
|
||||
},
|
||||
"grafana": {
|
||||
"id": "f25430cfedacb0b2a4935bb8a51b63568efffc491fe672b99d3acdd3d02dd6e1",
|
||||
"started_at": "2026-11-08T15:43:16.869937387Z"
|
||||
}
|
||||
},
|
||||
"after": {
|
||||
"postgres": {
|
||||
"id": "06797da995bb6cc6fd69a2c251fd6204992b74dbcf630110c59172dde5fb401f",
|
||||
"started_at": "2026-11-08T15:49:56.430728377Z"
|
||||
},
|
||||
"redis": {
|
||||
"id": "e2f09b5ae65d8667339960a5649429b6e7abf22d5a20ca9de44846e2394aef6f",
|
||||
"started_at": "2026-11-08T15:49:56.462037627Z"
|
||||
},
|
||||
"s3": {
|
||||
"id": "0326a0c248bc89fa1630815b6824af8e03bf57029c88c297bcdf46c2cdc9deb2",
|
||||
"started_at": "2026-11-08T15:49:56.369172627Z"
|
||||
},
|
||||
"broker": {
|
||||
"id": "632478d263786461c5e7a9179bdc88c4e910712d58cf30ae2ba0efcc5317a1f3",
|
||||
"started_at": "2026-11-08T15:49:56.495397377Z"
|
||||
},
|
||||
"api1": {
|
||||
"id": "b2523eba3a14f02311e1a7dabc2b5e3a0459ce74dc22248559ae919bb33c96bd",
|
||||
"started_at": "2026-11-08T15:50:00.454242921Z"
|
||||
},
|
||||
"api2": {
|
||||
"id": "9c3bccf975647eaa0052335faeab360fc199b8aaddc6f1d437009b6a876a1c6f",
|
||||
"started_at": "2026-11-08T15:50:00.410223588Z"
|
||||
},
|
||||
"dispatcher": {
|
||||
"id": "d6579f5458efcd35d21d93efe9d64ee27661251275a73137b0cc690f2fc65133",
|
||||
"started_at": "2026-11-08T15:50:00.216710712Z"
|
||||
},
|
||||
"worker": {
|
||||
"id": "d5975de80120800bd179bad7284ebf92477094d11b14ae423c159b4d42930f5f",
|
||||
"started_at": "2026-11-08T15:50:01.05319863Z"
|
||||
},
|
||||
"cleanup": {
|
||||
"id": "aef4f262490e1e616b6c83d8005a1d13286e1b7be5b0389a252315ad66814fdb",
|
||||
"started_at": "2026-11-08T15:50:00.312024754Z"
|
||||
},
|
||||
"nginx": {
|
||||
"id": "652643855d5b97cea36ab41ada3ebef618fd894c5a3e9d603759de32b040f65c",
|
||||
"started_at": "2026-11-08T15:50:06.843809007Z"
|
||||
},
|
||||
"otel-collector": {
|
||||
"id": "3fcb612ac0c6c3a701d25f66e63b2abfb123ad0390c55f5f2dd795a2f3aa8c2d",
|
||||
"started_at": "2026-11-08T15:50:06.749767799Z"
|
||||
},
|
||||
"prometheus": {
|
||||
"id": "a3bad0ed0a03b0d8edc3450830d449ccb6564dfbc3e913674804b82868a9e838",
|
||||
"started_at": "2026-11-08T15:50:06.911832507Z"
|
||||
},
|
||||
"loki": {
|
||||
"id": "6713f6f19668716725bd24a7c5e6c8ab6240d2455b2775b854802702f8d24826",
|
||||
"started_at": "2026-11-08T15:50:07.253593966Z"
|
||||
},
|
||||
"tempo": {
|
||||
"id": "63377b0fc325bad816738019c191e909751609ca9a5f4ed469896fc6112e0ed2",
|
||||
"started_at": "2026-11-08T15:50:07.025957216Z"
|
||||
},
|
||||
"grafana": {
|
||||
"id": "f25430cfedacb0b2a4935bb8a51b63568efffc491fe672b99d3acdd3d02dd6e1",
|
||||
"started_at": "2026-11-08T15:50:07.067742257Z"
|
||||
}
|
||||
}
|
||||
},
|
||||
"failover": [
|
||||
{
|
||||
"status": "PASS",
|
||||
"stopped_service": "api1",
|
||||
"confirmed_messages": 68,
|
||||
"transient_failures": 0,
|
||||
"first_survivor_seconds": 0.829,
|
||||
"original_preserved": true
|
||||
},
|
||||
{
|
||||
"status": "PASS",
|
||||
"stopped_service": "api2",
|
||||
"confirmed_messages": 69,
|
||||
"transient_failures": 0,
|
||||
"first_survivor_seconds": 0.727,
|
||||
"original_preserved": true
|
||||
}
|
||||
],
|
||||
"public_failover": {
|
||||
"status": "PASS",
|
||||
"output": "python3 tests/resilience.py --mode failover --action-script \"scripts/stop-one.sh\" --base-url \"http://localhost:18086\" \nExecuting explicit failover hook; hook output suppressed to avoid leaking secrets.\n{\"observed_instances_before\": 2, \"successful_writes\": 131, \"failed_attempts\": 0, \"first_recovery_seconds\": 1.059}\nPASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence."
|
||||
},
|
||||
"telemetry_history_preserved": true,
|
||||
"metric_history_before_restart": {
|
||||
"unix_time": 1794152996.911832,
|
||||
"http_counter_sum": 3161.0
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,125 @@
|
||||
{
|
||||
"orientation": {
|
||||
"status": "PASS",
|
||||
"variants": {
|
||||
"thumbnail": {
|
||||
"content_type": "image/jpeg",
|
||||
"height": 256,
|
||||
"name": "thumbnail",
|
||||
"sha256": "335b4953e981a975642374e12196a4701e353972f223c1a7e850ac5998588f1a",
|
||||
"size_bytes": 1270,
|
||||
"width": 160
|
||||
},
|
||||
"crop": {
|
||||
"content_type": "image/jpeg",
|
||||
"height": 128,
|
||||
"name": "crop",
|
||||
"sha256": "b7f4884df301163117ebb3189f6141a9e376cc6d94804e2f56ce38e0dc3dd047",
|
||||
"size_bytes": 886,
|
||||
"width": 128
|
||||
},
|
||||
"optimized": {
|
||||
"content_type": "image/webp",
|
||||
"height": 320,
|
||||
"name": "optimized",
|
||||
"sha256": "577722d04127a774de08f674717fa03e3058124395f9ee31a949e6f508e86b43",
|
||||
"size_bytes": 210,
|
||||
"width": 200
|
||||
}
|
||||
}
|
||||
},
|
||||
"alpha": {
|
||||
"status": "PASS",
|
||||
"variants": {
|
||||
"thumbnail": {
|
||||
"content_type": "image/jpeg",
|
||||
"height": 4,
|
||||
"name": "thumbnail",
|
||||
"sha256": "a89b0af8a3c520ca079ed01f7a6579b39fdf0a593d5ff94b7a5128c194f3d065",
|
||||
"size_bytes": 631,
|
||||
"width": 8
|
||||
},
|
||||
"crop": {
|
||||
"content_type": "image/jpeg",
|
||||
"height": 128,
|
||||
"name": "crop",
|
||||
"sha256": "257edb1205654cf43e411c9fc333ee5544c2dc8753470ab667cdafbfeae4ff7c",
|
||||
"size_bytes": 883,
|
||||
"width": 128
|
||||
},
|
||||
"optimized": {
|
||||
"content_type": "image/webp",
|
||||
"height": 4,
|
||||
"name": "optimized",
|
||||
"sha256": "f4e61263caab4e749c64fc56091797191901d5affdb15ee2d828d905c429e950",
|
||||
"size_bytes": 44,
|
||||
"width": 8
|
||||
}
|
||||
}
|
||||
},
|
||||
"large": {
|
||||
"status": "PASS",
|
||||
"variants": {
|
||||
"thumbnail": {
|
||||
"content_type": "image/jpeg",
|
||||
"height": 128,
|
||||
"name": "thumbnail",
|
||||
"sha256": "126971107bc487792a7b813c01e037c58140ab0a039d4c7fd0cdbd26a9d9d781",
|
||||
"size_bytes": 1142,
|
||||
"width": 256
|
||||
},
|
||||
"crop": {
|
||||
"content_type": "image/jpeg",
|
||||
"height": 128,
|
||||
"name": "crop",
|
||||
"sha256": "4e67b8bf92db7d696303a433822a79479ca6d8ac5ca6bf2c87e5f4514ed3fa4a",
|
||||
"size_bytes": 886,
|
||||
"width": 128
|
||||
},
|
||||
"optimized": {
|
||||
"content_type": "image/webp",
|
||||
"height": 640,
|
||||
"name": "optimized",
|
||||
"sha256": "8e5a8340d7a7b9c37e935e090bb34494e588b7534a6d091aba33aca5ba83a267",
|
||||
"size_bytes": 1540,
|
||||
"width": 1280
|
||||
}
|
||||
}
|
||||
},
|
||||
"boundary": {
|
||||
"status": "PASS",
|
||||
"variants": {
|
||||
"thumbnail": {
|
||||
"content_type": "image/jpeg",
|
||||
"height": 160,
|
||||
"name": "thumbnail",
|
||||
"sha256": "d8d8358f529fd170984ee7926c54e9f6ead7d6a397fd6b803a6c20af197a2ce2",
|
||||
"size_bytes": 1270,
|
||||
"width": 256
|
||||
},
|
||||
"crop": {
|
||||
"content_type": "image/jpeg",
|
||||
"height": 128,
|
||||
"name": "crop",
|
||||
"sha256": "4e67b8bf92db7d696303a433822a79479ca6d8ac5ca6bf2c87e5f4514ed3fa4a",
|
||||
"size_bytes": 886,
|
||||
"width": 128
|
||||
},
|
||||
"optimized": {
|
||||
"content_type": "image/webp",
|
||||
"height": 800,
|
||||
"name": "optimized",
|
||||
"sha256": "a9bc2a93f44b92bf58d3c1a2f53390b21f0330b073411886eeccac3b12eb6769",
|
||||
"size_bytes": 1904,
|
||||
"width": 1280
|
||||
}
|
||||
}
|
||||
},
|
||||
"byte_limits_corrupt_pixel_limits": "PASS",
|
||||
"private_bucket_limited_credentials": "PASS",
|
||||
"worker_memory_bytes": {
|
||||
"memory.current": 23502848,
|
||||
"memory.max": 1610612736,
|
||||
"memory.peak": 704679936
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
module mtusi-messenger
|
||||
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
github.com/google/uuid v1.6.0
|
||||
github.com/jackc/pgx/v5 v5.7.6
|
||||
github.com/minio/madmin-go/v3 v3.0.95
|
||||
github.com/minio/minio-go/v7 v7.0.95
|
||||
github.com/redis/go-redis/v9 v9.14.0
|
||||
go.opentelemetry.io/otel v1.38.0
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.14.0
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.38.0
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0
|
||||
go.opentelemetry.io/otel/log v0.14.0
|
||||
go.opentelemetry.io/otel/metric v1.38.0
|
||||
go.opentelemetry.io/otel/sdk v1.38.0
|
||||
go.opentelemetry.io/otel/sdk/log v0.14.0
|
||||
go.opentelemetry.io/otel/sdk/metric v1.38.0
|
||||
go.opentelemetry.io/otel/trace v1.38.0
|
||||
golang.org/x/crypto v0.42.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/cenkalti/backoff/v5 v5.0.3 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/go-ini/ini v1.67.0 // indirect
|
||||
github.com/go-logr/logr v1.4.3 // indirect
|
||||
github.com/go-logr/stdr v1.2.2 // indirect
|
||||
github.com/go-ole/go-ole v1.3.0 // indirect
|
||||
github.com/goccy/go-json v0.10.5 // indirect
|
||||
github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
|
||||
github.com/golang/protobuf v1.5.4 // indirect
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 // indirect
|
||||
github.com/jackc/pgpassfile v1.0.0 // indirect
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
|
||||
github.com/jackc/puddle/v2 v2.2.2 // indirect
|
||||
github.com/klauspost/compress v1.18.0 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.2.11 // indirect
|
||||
github.com/lufia/plan9stats v0.0.0-20240909124753-873cd0166683 // indirect
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
|
||||
github.com/minio/crc64nvme v1.0.2 // indirect
|
||||
github.com/minio/md5-simd v1.1.2 // indirect
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
|
||||
github.com/philhofer/fwd v1.2.0 // indirect
|
||||
github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
|
||||
github.com/prometheus/client_model v0.6.1 // indirect
|
||||
github.com/prometheus/common v0.59.1 // indirect
|
||||
github.com/prometheus/procfs v0.15.1 // indirect
|
||||
github.com/prometheus/prom2json v1.4.0 // indirect
|
||||
github.com/prometheus/prometheus v0.54.1 // indirect
|
||||
github.com/rs/xid v1.6.0 // indirect
|
||||
github.com/safchain/ethtool v0.4.1 // indirect
|
||||
github.com/secure-io/sio-go v0.3.1 // indirect
|
||||
github.com/shirou/gopsutil/v3 v3.24.5 // indirect
|
||||
github.com/shoenig/go-m1cpu v0.1.6 // indirect
|
||||
github.com/tinylib/msgp v1.3.0 // indirect
|
||||
github.com/tklauser/go-sysconf v0.3.14 // indirect
|
||||
github.com/tklauser/numcpus v0.8.0 // indirect
|
||||
github.com/yusufpapurcu/wmi v1.2.4 // indirect
|
||||
go.opentelemetry.io/auto/sdk v1.1.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 // indirect
|
||||
go.opentelemetry.io/proto/otlp v1.7.1 // indirect
|
||||
golang.org/x/net v0.43.0 // indirect
|
||||
golang.org/x/sync v0.17.0 // indirect
|
||||
golang.org/x/sys v0.36.0 // indirect
|
||||
golang.org/x/text v0.29.0 // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5 // indirect
|
||||
google.golang.org/grpc v1.75.0 // indirect
|
||||
google.golang.org/protobuf v1.36.8 // indirect
|
||||
)
|
||||
@@ -0,0 +1,176 @@
|
||||
github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
|
||||
github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
|
||||
github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
|
||||
github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
|
||||
github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM=
|
||||
github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw=
|
||||
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
|
||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||
github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
|
||||
github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
|
||||
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
|
||||
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
|
||||
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
|
||||
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
|
||||
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
|
||||
github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
|
||||
github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE=
|
||||
github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78=
|
||||
github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4=
|
||||
github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
|
||||
github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg=
|
||||
github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
|
||||
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
|
||||
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
|
||||
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 h1:8Tjv8EJ+pM1xP8mK6egEbD1OgnVTyacbefKhmbLhIhU=
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2/go.mod h1:pkJQ2tZHJ0aFOVEEot6oZmaVEZcRme73eIFmhiVuRWs=
|
||||
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
|
||||
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
|
||||
github.com/jackc/pgx/v5 v5.7.6 h1:rWQc5FwZSPX58r1OQmkuaNicxdmExaEz5A2DO2hUuTk=
|
||||
github.com/jackc/pgx/v5 v5.7.6/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M=
|
||||
github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
|
||||
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
|
||||
github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
|
||||
github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
|
||||
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU=
|
||||
github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
|
||||
github.com/lufia/plan9stats v0.0.0-20240909124753-873cd0166683 h1:7UMa6KCCMjZEMDtTVdcGu0B1GmmC7QJKiCCjyTAWQy0=
|
||||
github.com/lufia/plan9stats v0.0.0-20240909124753-873cd0166683/go.mod h1:ilwx/Dta8jXAgpFYFvSWEMwxmbWXyiUHkd5FwyKhb5k=
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
|
||||
github.com/minio/crc64nvme v1.0.2 h1:6uO1UxGAD+kwqWWp7mBFsi5gAse66C4NXO8cmcVculg=
|
||||
github.com/minio/crc64nvme v1.0.2/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg=
|
||||
github.com/minio/madmin-go/v3 v3.0.95 h1:KRy4NIpTLKrHcuTQbKrwKs3SvGoe+ZAHT2Pbl1zJxDc=
|
||||
github.com/minio/madmin-go/v3 v3.0.95/go.mod h1:pMLdj9OtN0CANNs5tdm6opvOlDFfj0WhbztboZAjRWE=
|
||||
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
|
||||
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
|
||||
github.com/minio/minio-go/v7 v7.0.95 h1:ywOUPg+PebTMTzn9VDsoFJy32ZuARN9zhB+K3IYEvYU=
|
||||
github.com/minio/minio-go/v7 v7.0.95/go.mod h1:wOOX3uxS334vImCNRVyIDdXX9OsXDm89ToynKgqUKlo=
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
|
||||
github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM=
|
||||
github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU=
|
||||
github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
|
||||
github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=
|
||||
github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY=
|
||||
github.com/prometheus/common v0.59.1 h1:LXb1quJHWm1P6wq/U824uxYi4Sg0oGvNeUm1z5dJoX0=
|
||||
github.com/prometheus/common v0.59.1/go.mod h1:GpWM7dewqmVYcd7SmRaiWVe9SSqjf0UrwnYnpEZNuT0=
|
||||
github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=
|
||||
github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
|
||||
github.com/prometheus/prom2json v1.4.0 h1:2AEOsd1ebqql/p9u0IWgCpUAteAAf9Lnf/SVyieqer4=
|
||||
github.com/prometheus/prom2json v1.4.0/go.mod h1:DmcIMPspQD/fMyFCYti5qJJbuEnqDh3DGoooO0sgr4w=
|
||||
github.com/prometheus/prometheus v0.54.1 h1:vKuwQNjnYN2/mDoWfHXDhAsz/68q/dQDb+YbcEqU7MQ=
|
||||
github.com/prometheus/prometheus v0.54.1/go.mod h1:xlLByHhk2g3ycakQGrMaU8K7OySZx98BzeCR99991NY=
|
||||
github.com/redis/go-redis/v9 v9.14.0 h1:u4tNCjXOyzfgeLN+vAZaW1xUooqWDqVEsZN0U01jfAE=
|
||||
github.com/redis/go-redis/v9 v9.14.0/go.mod h1:huWgSWd8mW6+m0VPhJjSSQ+d6Nh1VICQ6Q5lHuCH/Iw=
|
||||
github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
|
||||
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
|
||||
github.com/safchain/ethtool v0.4.1 h1:S6mEleTADqgynileXoiapt/nKnatyR6bmIHoF+h2ADo=
|
||||
github.com/safchain/ethtool v0.4.1/go.mod h1:XLLnZmy4OCRTkksP/UiMjij96YmIsBfmBQcs7H6tA48=
|
||||
github.com/secure-io/sio-go v0.3.1 h1:dNvY9awjabXTYGsTF1PiCySl9Ltofk9GA3VdWlo7rRc=
|
||||
github.com/secure-io/sio-go v0.3.1/go.mod h1:+xbkjDzPjwh4Axd07pRKSNriS9SCiYksWnZqdnfpQxs=
|
||||
github.com/shirou/gopsutil/v3 v3.24.5 h1:i0t8kL+kQTvpAYToeuiVk3TgDeKOFioZO3Ztz/iZ9pI=
|
||||
github.com/shirou/gopsutil/v3 v3.24.5/go.mod h1:bsoOS1aStSs9ErQ1WWfxllSeS1K5D+U30r2NfcubMVk=
|
||||
github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM=
|
||||
github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ=
|
||||
github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU=
|
||||
github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||
github.com/tinylib/msgp v1.3.0 h1:ULuf7GPooDaIlbyvgAxBV/FI7ynli6LZ1/nVUNu+0ww=
|
||||
github.com/tinylib/msgp v1.3.0/go.mod h1:ykjzy2wzgrlvpDCRc4LA8UXy6D8bzMSuAF3WD57Gok0=
|
||||
github.com/tklauser/go-sysconf v0.3.14 h1:g5vzr9iPFFz24v2KZXs/pvpvh8/V9Fw6vQK5ZZb78yU=
|
||||
github.com/tklauser/go-sysconf v0.3.14/go.mod h1:1ym4lWMLUOhuBOPGtRcJm7tEGX4SCYNEEEtghGG/8uY=
|
||||
github.com/tklauser/numcpus v0.8.0 h1:Mx4Wwe/FjZLeQsK/6kt2EOepwwSl7SmJrK5bV/dXYgY=
|
||||
github.com/tklauser/numcpus v0.8.0/go.mod h1:ZJZlAY+dmR4eut8epnzf0u/VwodKmryxR8txiloSqBE=
|
||||
github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=
|
||||
github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
|
||||
go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA=
|
||||
go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A=
|
||||
go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8=
|
||||
go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.14.0 h1:QQqYw3lkrzwVsoEX0w//EhH/TCnpRdEenKBOOEIMjWc=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.14.0/go.mod h1:gSVQcr17jk2ig4jqJ2DX30IdWH251JcNAecvrqTxH1s=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.38.0 h1:Oe2z/BCg5q7k4iXC3cqJxKYg0ieRiOqF0cecFYdPTwk=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.38.0/go.mod h1:ZQM5lAJpOsKnYagGg/zV2krVqTtaVdYdDkhMoX6Oalg=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 h1:GqRJVj7UmLjCVyVJ3ZFLdPRmhDUp2zFmQe3RHIOsw24=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0/go.mod h1:ri3aaHSmCTVYu2AWv44YMauwAQc0aqI9gHKIcSbI1pU=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0 h1:aTL7F04bJHUlztTsNGJ2l+6he8c+y/b//eR0jjjemT4=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0/go.mod h1:kldtb7jDTeol0l3ewcmd8SDvx3EmIE7lyvqbasU3QC4=
|
||||
go.opentelemetry.io/otel/log v0.14.0 h1:2rzJ+pOAZ8qmZ3DDHg73NEKzSZkhkGIua9gXtxNGgrM=
|
||||
go.opentelemetry.io/otel/log v0.14.0/go.mod h1:5jRG92fEAgx0SU/vFPxmJvhIuDU9E1SUnEQrMlJpOno=
|
||||
go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA=
|
||||
go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI=
|
||||
go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E=
|
||||
go.opentelemetry.io/otel/sdk v1.38.0/go.mod h1:ghmNdGlVemJI3+ZB5iDEuk4bWA3GkTpW+DOoZMYBVVg=
|
||||
go.opentelemetry.io/otel/sdk/log v0.14.0 h1:JU/U3O7N6fsAXj0+CXz21Czg532dW2V4gG1HE/e8Zrg=
|
||||
go.opentelemetry.io/otel/sdk/log v0.14.0/go.mod h1:imQvII+0ZylXfKU7/wtOND8Hn4OpT3YUoIgqJVksUkM=
|
||||
go.opentelemetry.io/otel/sdk/log/logtest v0.14.0 h1:Ijbtz+JKXl8T2MngiwqBlPaHqc4YCaP/i13Qrow6gAM=
|
||||
go.opentelemetry.io/otel/sdk/log/logtest v0.14.0/go.mod h1:dCU8aEL6q+L9cYTqcVOk8rM9Tp8WdnHOPLiBgp0SGOA=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6qT5wthqPoM=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA=
|
||||
go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE=
|
||||
go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs=
|
||||
go.opentelemetry.io/proto/otlp v1.7.1 h1:gTOMpGDb0WTBOP8JaO72iL3auEZhVmAQg4ipjOVAtj4=
|
||||
go.opentelemetry.io/proto/otlp v1.7.1/go.mod h1:b2rVh6rfI/s2pHWNlB7ILJcRALpcNDzKhACevjI+ZnE=
|
||||
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
|
||||
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.42.0 h1:chiH31gIWm57EkTXpwnqf8qeuMUi0yekh6mT2AvFlqI=
|
||||
golang.org/x/crypto v0.42.0/go.mod h1:4+rDnOTJhQCx2q7/j6rAN5XDw8kPjeaXEUR2eL94ix8=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE=
|
||||
golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg=
|
||||
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug=
|
||||
golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k=
|
||||
golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk=
|
||||
golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4=
|
||||
gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
|
||||
gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5 h1:BIRfGDEjiHRrk0QKZe3Xv2ieMhtgRGeLcZQ0mIVn4EY=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5/go.mod h1:j3QtIyytwqGr1JUDtYXwtMXWPKsEa5LtzIFN1Wn5WvE=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5 h1:eaY8u2EuxbRv7c3NiGK0/NedzVsCcV6hDuU5qPX5EGE=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5/go.mod h1:M4/wBTSeyLxupu3W3tJtOgB14jILAS/XWPSSa3TAlJc=
|
||||
google.golang.org/grpc v1.75.0 h1:+TW+dqTd2Biwe6KKfhE5JpiYIBWq865PhKGSXiivqt4=
|
||||
google.golang.org/grpc v1.75.0/go.mod h1:JtPAzKiq4v1xcAB2hydNlWI2RnF85XXcV0mhKXr2ecQ=
|
||||
google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc=
|
||||
google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
@@ -0,0 +1,16 @@
|
||||
# Build MinIO from a pinned commit of the official upstream source.
|
||||
FROM golang:1.26.8-alpine3.23@sha256:a8fa79c5bd40d880b52bd3b6d7669ecdcfd00e85facdd427d279efb5ddd79cb1 AS build
|
||||
ADD --checksum=sha256:be6d0bd3696c3a13a35f02d3a0280b64319c67918b4501c5c3d87f96d000085c https://codeload.github.com/minio/minio/tar.gz/refs/tags/RELEASE.2025-10-15T17-29-55Z /source.tar.gz
|
||||
WORKDIR /src
|
||||
RUN tar -xzf /source.tar.gz --strip-components=1 -C /src
|
||||
RUN --mount=type=cache,target=/go/pkg/mod --mount=type=cache,target=/root/.cache/go-build \
|
||||
CGO_ENABLED=0 GOMAXPROCS=2 go build -p 2 -trimpath -buildvcs=false \
|
||||
-ldflags='-s -w -X github.com/minio/minio/cmd.Version=2025-10-15T17:29:55Z -X github.com/minio/minio/cmd.ReleaseTag=RELEASE.2025-10-15T17-29-55Z -X github.com/minio/minio/cmd.CommitID=9e49d5e7a648f00e26f2246f4dc28e6b07f8c84a -X github.com/minio/minio/cmd.CopyrightYear=2025' \
|
||||
-o /minio .
|
||||
|
||||
FROM busybox:1.37.0-musl
|
||||
COPY --from=build /minio /minio
|
||||
COPY --from=build /src/LICENSE /licenses/MinIO-LICENSE
|
||||
COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
|
||||
RUN mkdir -p /data
|
||||
ENTRYPOINT ["/minio"]
|
||||
@@ -0,0 +1,8 @@
|
||||
FROM application AS app_binary
|
||||
FROM python:3.13.7-slim-bookworm
|
||||
RUN pip install --no-cache-dir Pillow==12.0.0
|
||||
COPY --from=app_binary /messenger /messenger
|
||||
COPY worker/process_image.py /opt/process_image.py
|
||||
USER 65532:65532
|
||||
ENTRYPOINT ["/messenger"]
|
||||
CMD ["worker"]
|
||||
@@ -0,0 +1,43 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
upstream=/run/nginx-upstream.conf
|
||||
|
||||
write_servers() {
|
||||
if [ "$1" = 1 ]; then
|
||||
printf '%s\n' 'server api1:8080 resolve max_fails=0;'
|
||||
fi
|
||||
if [ "$2" = 1 ]; then
|
||||
printf '%s\n' 'server api2:8080 resolve max_fails=0;'
|
||||
fi
|
||||
}
|
||||
|
||||
# With no healthy replica, keep both reachable so API can return its normal
|
||||
# 401/503 JSON responses instead of Nginx reporting "no live upstreams".
|
||||
write_servers 1 1 > "$upstream"
|
||||
|
||||
monitor() {
|
||||
sleep 2
|
||||
while :; do
|
||||
ready1=0
|
||||
ready2=0
|
||||
wget -q -T 2 -O /dev/null http://api1:8080/health/ready && ready1=1 || true
|
||||
wget -q -T 2 -O /dev/null http://api2:8080/health/ready && ready2=1 || true
|
||||
if [ "$ready1" = 0 ] && [ "$ready2" = 0 ]; then
|
||||
ready1=1
|
||||
ready2=1
|
||||
fi
|
||||
next=$(mktemp /run/nginx-upstream.XXXXXX)
|
||||
write_servers "$ready1" "$ready2" > "$next"
|
||||
if cmp -s "$next" "$upstream"; then
|
||||
rm -f "$next"
|
||||
else
|
||||
mv "$next" "$upstream"
|
||||
nginx -t >/dev/null 2>&1 && nginx -s reload
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
}
|
||||
|
||||
monitor &
|
||||
exec nginx -g 'daemon off;'
|
||||
@@ -0,0 +1,35 @@
|
||||
events {}
|
||||
|
||||
http {
|
||||
access_log /dev/stdout;
|
||||
error_log /dev/stderr notice;
|
||||
|
||||
resolver 127.0.0.11 valid=2s ipv6=off;
|
||||
|
||||
upstream messenger_api {
|
||||
zone messenger_api 64k;
|
||||
include /run/nginx-upstream.conf;
|
||||
keepalive 16;
|
||||
}
|
||||
|
||||
server {
|
||||
listen 8080;
|
||||
client_max_body_size 0;
|
||||
client_body_timeout 20s;
|
||||
|
||||
location / {
|
||||
proxy_request_buffering off;
|
||||
proxy_buffering off;
|
||||
proxy_pass http://messenger_api;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Connection "";
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_connect_timeout 500ms;
|
||||
proxy_send_timeout 65s;
|
||||
proxy_read_timeout 65s;
|
||||
proxy_next_upstream error timeout http_502 http_503 http_504;
|
||||
proxy_next_upstream_tries 2;
|
||||
}
|
||||
}
|
||||
}
|
||||
+10
-5
@@ -1,7 +1,12 @@
|
||||
# Конфигурация наблюдаемости — реализовать
|
||||
# Наблюдаемость лабораторной 6
|
||||
|
||||
Добавьте `collector.yaml` с OTLP receivers, processors и exporters выбранной версии Collector. Укажите явные bind endpoints, внутренние сети, memory limits/batch и поведение при backpressure по уровню. Сопоставьте все три pipeline с реально развёрнутыми backend.
|
||||
- `collector.yaml` — OTLP receiver и три pipeline, memory limiter, batch, bounded queues/retry.
|
||||
- `prometheus.yaml`, `alerts.yaml` — scrape, SLI и два локальных alert rules.
|
||||
- `loki.yaml` — OTLP logs с structured metadata и persistent filesystem.
|
||||
- `tempo.yaml` — OTLP traces, WAL и persistent blocks.
|
||||
- `grafana/provisioning` — три datasource и автоматический импорт dashboard.
|
||||
- `grafana/dashboards/messenger.json` — импортируемый dashboard с реальными PromQL-запросами.
|
||||
- [Сигналы и единицы](signals.md).
|
||||
- [Runbook и SLO](runbook.md).
|
||||
|
||||
Добавьте конфигурации storage, Grafana provisioning/datasources, provisioning/dashboards и импортируемые dashboard JSON. На 4 — alert rules и runbooks. Пустой dashboard или debug exporter, который только печатает данные в stdout, не заменяют хранилища/доказательства из `contracts/observability.md`.
|
||||
|
||||
Не публикуйте OTLP/metrics/административные ports в общую сеть без необходимости. В lab7 реализуйте TLS/mTLS по trust matrix; экспорт Authorization и message bodies запрещён.
|
||||
Запуск и проверки — в корневом REPORT.md. Публикуется только Grafana на localhost; OTLP/Prometheus/Loki/Tempo ports остаются внутри Compose.
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
groups:
|
||||
- name: messenger-slo
|
||||
rules:
|
||||
- record: messenger:api_error_ratio:rate1m
|
||||
expr: (sum(rate(messenger_http_requests_total{service_name="messenger-api",route!~"/health/.*",status_class="5xx"}[1m])) or vector(0)) / clamp_min(sum(rate(messenger_http_requests_total{service_name="messenger-api",route!~"/health/.*"}[1m])), 0.001)
|
||||
- record: messenger:api_availability:rate5m
|
||||
expr: 1 - (sum(rate(messenger_http_requests_total{service_name="messenger-api",route!~"/health/.*",status_class="5xx"}[5m])) or vector(0)) / clamp_min(sum(rate(messenger_http_requests_total{service_name="messenger-api",route!~"/health/.*"}[5m])), 0.001)
|
||||
- alert: MessengerHighErrorRate
|
||||
expr: messenger:api_error_ratio:rate1m > 0.05
|
||||
for: 10s
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: Business API 5xx ratio exceeds 5% over 1 minute
|
||||
description: Inspect database spans, DB availability and pool panels; see infra/otel/runbook.md.
|
||||
- alert: MessengerQueueLag
|
||||
expr: max(messenger_queue_oldest_age_seconds) > 10 and max(messenger_queue_depth) > 0
|
||||
for: 10s
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: Oldest queued job exceeds 10 seconds
|
||||
description: Compare worker busy/capacity, job outcomes and queue age; see infra/otel/runbook.md.
|
||||
@@ -0,0 +1,74 @@
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
http:
|
||||
endpoint: 0.0.0.0:4318
|
||||
processors:
|
||||
memory_limiter:
|
||||
check_interval: 1s
|
||||
limit_mib: 128
|
||||
spike_limit_mib: 32
|
||||
batch:
|
||||
timeout: 1s
|
||||
send_batch_size: 64
|
||||
send_batch_max_size: 128
|
||||
exporters:
|
||||
prometheus:
|
||||
endpoint: 0.0.0.0:8889
|
||||
resource_to_telemetry_conversion:
|
||||
enabled: true
|
||||
otlphttp/tempo:
|
||||
endpoint: http://tempo:4318
|
||||
timeout: 2s
|
||||
sending_queue:
|
||||
enabled: true
|
||||
num_consumers: 2
|
||||
queue_size: 32
|
||||
block_on_overflow: false
|
||||
retry_on_failure:
|
||||
enabled: true
|
||||
initial_interval: 1s
|
||||
max_interval: 2s
|
||||
max_elapsed_time: 5s
|
||||
otlphttp/loki:
|
||||
endpoint: http://loki:3100/otlp
|
||||
timeout: 2s
|
||||
sending_queue:
|
||||
enabled: true
|
||||
num_consumers: 2
|
||||
queue_size: 32
|
||||
block_on_overflow: false
|
||||
retry_on_failure:
|
||||
enabled: true
|
||||
initial_interval: 1s
|
||||
max_interval: 2s
|
||||
max_elapsed_time: 5s
|
||||
extensions:
|
||||
health_check:
|
||||
endpoint: 0.0.0.0:13133
|
||||
service:
|
||||
extensions: [health_check]
|
||||
telemetry:
|
||||
logs:
|
||||
encoding: json
|
||||
metrics:
|
||||
level: detailed
|
||||
readers:
|
||||
- pull:
|
||||
exporter:
|
||||
prometheus:
|
||||
host: 0.0.0.0
|
||||
port: 8888
|
||||
pipelines:
|
||||
metrics:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, batch]
|
||||
exporters: [prometheus]
|
||||
logs:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, batch]
|
||||
exporters: [otlphttp/loki]
|
||||
traces:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, batch]
|
||||
exporters: [otlphttp/tempo]
|
||||
@@ -0,0 +1,669 @@
|
||||
{
|
||||
"uid": "mtusi-messenger",
|
||||
"title": "MTUSI Messenger — lab6",
|
||||
"schemaVersion": 39,
|
||||
"version": 1,
|
||||
"refresh": "5s",
|
||||
"tags": [
|
||||
"mtusi",
|
||||
"lab6"
|
||||
],
|
||||
"time": {
|
||||
"from": "now-15m",
|
||||
"to": "now"
|
||||
},
|
||||
"panels": [
|
||||
{
|
||||
"id": 1,
|
||||
"title": "API RPS (business routes)",
|
||||
"type": "timeseries",
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 0,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum by (service_instance_id) (rate(messenger_http_requests_total{service_name=\"messenger-api\",route!~\"/health/.*\"}[1m]))",
|
||||
"legendFormat": "{{service_instance_id}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "reqps"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"title": "Business API error ratio: 5xx / all",
|
||||
"type": "timeseries",
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 0,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "messenger:api_error_ratio:rate1m",
|
||||
"legendFormat": "5xx ratio"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percentunit"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"title": "API p50 from histogram",
|
||||
"type": "timeseries",
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 8,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "histogram_quantile(0.5, sum by (le) (rate(messenger_http_duration_seconds_bucket{service_name=\"messenger-api\",route!~\"/health/.*\"}[1m])))",
|
||||
"legendFormat": "p50"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"title": "API p95 from histogram",
|
||||
"type": "timeseries",
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 8,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_http_duration_seconds_bucket{service_name=\"messenger-api\",route!~\"/health/.*\"}[1m])))",
|
||||
"legendFormat": "p95"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"title": "API p99 from histogram",
|
||||
"type": "timeseries",
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 16,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "histogram_quantile(0.99, sum by (le) (rate(messenger_http_duration_seconds_bucket{service_name=\"messenger-api\",route!~\"/health/.*\"}[1m])))",
|
||||
"legendFormat": "p99"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"title": "Worker job outcomes",
|
||||
"type": "timeseries",
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 16,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum by (outcome) (rate(messenger_jobs_processed_total[1m]))",
|
||||
"legendFormat": "{{outcome}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "ops"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 7,
|
||||
"title": "Worker p95 processing duration",
|
||||
"type": "timeseries",
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 24,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_job_duration_seconds_bucket[5m])))",
|
||||
"legendFormat": "p95"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 8,
|
||||
"title": "End-to-end p95 time to ready",
|
||||
"type": "timeseries",
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 24,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_time_to_ready_seconds_bucket[5m])))",
|
||||
"legendFormat": "p95"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 9,
|
||||
"title": "Queue depth",
|
||||
"type": "timeseries",
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 32,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "max(messenger_queue_depth)",
|
||||
"legendFormat": "queued"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 10,
|
||||
"title": "Oldest queued job age",
|
||||
"type": "timeseries",
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 32,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "max(messenger_queue_oldest_age_seconds)",
|
||||
"legendFormat": "oldest"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 11,
|
||||
"title": "DB pool used / waiting",
|
||||
"type": "timeseries",
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 40,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "messenger_db_pool_used",
|
||||
"legendFormat": "used {{service_instance_id}}"
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"expr": "messenger_db_pool_waiting",
|
||||
"legendFormat": "waiting {{service_instance_id}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 12,
|
||||
"title": "DB pool wait rate",
|
||||
"type": "timeseries",
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 40,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum by (service_instance_id) (rate(messenger_db_pool_acquire_wait_total[1m]))",
|
||||
"legendFormat": "{{service_instance_id}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "ops"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 13,
|
||||
"title": "Worker saturation: busy / capacity",
|
||||
"type": "timeseries",
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 48,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "messenger_worker_busy / messenger_worker_capacity",
|
||||
"legendFormat": "{{service_instance_id}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percentunit"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 14,
|
||||
"title": "Worker capacity",
|
||||
"type": "timeseries",
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 48,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "messenger_worker_capacity",
|
||||
"legendFormat": "{{service_instance_id}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 15,
|
||||
"title": "Container CPU time rate (includes decoder)",
|
||||
"type": "timeseries",
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 56,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "rate(messenger_container_cpu_seconds_total[1m])",
|
||||
"legendFormat": "{{service_instance_id}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "cores"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 16,
|
||||
"title": "Container memory",
|
||||
"type": "timeseries",
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 56,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "messenger_container_memory_bytes",
|
||||
"legendFormat": "{{service_instance_id}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "bytes"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 17,
|
||||
"title": "Database available",
|
||||
"type": "timeseries",
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 64,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "messenger_db_available",
|
||||
"legendFormat": "{{service_instance_id}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 18,
|
||||
"title": "SLI availability, rolling 5m (SLO 99%)",
|
||||
"type": "timeseries",
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 64,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "messenger:api_availability:rate5m",
|
||||
"legendFormat": "availability"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percentunit"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 19,
|
||||
"title": "Local alert states",
|
||||
"type": "timeseries",
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 72,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "ALERTS{alertname=~\"Messenger.*\"}",
|
||||
"legendFormat": "{{alertname}} {{alertstate}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 20,
|
||||
"title": "Collector queued export batches",
|
||||
"type": "timeseries",
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 72,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "otelcol_exporter_queue_size",
|
||||
"legendFormat": "{{exporter}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 21,
|
||||
"title": "Collector failed trace exports",
|
||||
"type": "timeseries",
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 80,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "otelcol_exporter_send_failed_spans_total",
|
||||
"legendFormat": "{{exporter}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 22,
|
||||
"title": "SDK exporter errors",
|
||||
"type": "timeseries",
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 80,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "messenger_telemetry_export_errors",
|
||||
"legendFormat": "{{service_instance_id}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 23,
|
||||
"title": "Worker heartbeat age (stale if stopped)",
|
||||
"type": "timeseries",
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 88,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "time() - messenger_process_last_observed_seconds{service_name=\"messenger-worker\"}",
|
||||
"legendFormat": "worker heartbeat age"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
}
|
||||
],
|
||||
"annotations": {
|
||||
"list": []
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
apiVersion: 1
|
||||
providers:
|
||||
- name: Messenger
|
||||
orgId: 1
|
||||
folder: MTUSI
|
||||
type: file
|
||||
disableDeletion: false
|
||||
editable: false
|
||||
updateIntervalSeconds: 10
|
||||
options:
|
||||
path: /var/lib/grafana/dashboards
|
||||
@@ -0,0 +1,36 @@
|
||||
apiVersion: 1
|
||||
datasources:
|
||||
- name: Prometheus
|
||||
uid: prometheus
|
||||
type: prometheus
|
||||
access: proxy
|
||||
url: http://prometheus:9090
|
||||
isDefault: true
|
||||
editable: false
|
||||
- name: Loki
|
||||
uid: loki
|
||||
type: loki
|
||||
access: proxy
|
||||
url: http://loki:3100
|
||||
editable: false
|
||||
jsonData:
|
||||
derivedFields:
|
||||
- name: TraceID
|
||||
matcherRegex: '"trace_id":"([a-f0-9]{32})"'
|
||||
datasourceUid: tempo
|
||||
url: '$${__value.raw}'
|
||||
- name: Tempo
|
||||
uid: tempo
|
||||
type: tempo
|
||||
access: proxy
|
||||
url: http://tempo:3200
|
||||
editable: false
|
||||
jsonData:
|
||||
tracesToLogsV2:
|
||||
datasourceUid: loki
|
||||
spanStartTimeShift: '-1m'
|
||||
spanEndTimeShift: '1m'
|
||||
filterByTraceID: true
|
||||
filterBySpanID: false
|
||||
serviceMap:
|
||||
datasourceUid: prometheus
|
||||
@@ -0,0 +1,30 @@
|
||||
auth_enabled: false
|
||||
server:
|
||||
http_listen_port: 3100
|
||||
common:
|
||||
instance_addr: 127.0.0.1
|
||||
path_prefix: /loki
|
||||
replication_factor: 1
|
||||
ring:
|
||||
kvstore:
|
||||
store: inmemory
|
||||
storage:
|
||||
filesystem:
|
||||
chunks_directory: /loki/chunks
|
||||
rules_directory: /loki/rules
|
||||
schema_config:
|
||||
configs:
|
||||
- from: 2024-01-01
|
||||
store: tsdb
|
||||
object_store: filesystem
|
||||
schema: v13
|
||||
index:
|
||||
prefix: index_
|
||||
period: 24h
|
||||
limits_config:
|
||||
allow_structured_metadata: true
|
||||
retention_period: 24h
|
||||
compactor:
|
||||
working_directory: /loki/compactor
|
||||
retention_enabled: true
|
||||
delete_request_store: filesystem
|
||||
@@ -0,0 +1,12 @@
|
||||
global:
|
||||
scrape_interval: 3s
|
||||
evaluation_interval: 5s
|
||||
rule_files:
|
||||
- /etc/prometheus/alerts.yaml
|
||||
scrape_configs:
|
||||
- job_name: messenger
|
||||
static_configs:
|
||||
- targets: [otel-collector:8889]
|
||||
- job_name: collector
|
||||
static_configs:
|
||||
- targets: [otel-collector:8888]
|
||||
@@ -0,0 +1,42 @@
|
||||
# Локальная диагностика
|
||||
|
||||
Открыть Grafana → MTUSI → **MTUSI Messenger — lab6**. В Explore доступны Prometheus, Loki и Tempo. Пароль admin хранится локально в `secrets/grafana_admin_password`, в Git его нет.
|
||||
|
||||
## MessengerQueueLag
|
||||
|
||||
Условие: oldest queued job > 10 s при depth > 0 в течение 10 s. Интервал оценки — 5 s, scrape — 3 s.
|
||||
|
||||
1. Посмотреть Queue depth и Oldest queued job age: растут ли они вместе.
|
||||
2. Посмотреть Worker heartbeat age и busy/capacity. Старый heartbeat при busy=0 означает отсутствие свежих наблюдений worker, а не доказательство его здоровья.
|
||||
3. Найти `job.published` в Loki и trace upload в Tempo. Если producer есть, а нового consumer нет, искать недоступный worker/доставку. Если consumer есть с error, посмотреть его S3/DB child spans.
|
||||
4. После устранения причины наблюдать jobs outcome=success, снижение depth/age, свежий heartbeat и resolved (alert исчезает из активных).
|
||||
|
||||
Для демонстрации: `docker compose --env-file .env stop worker`, загрузить изображение; восстановить `docker compose --env-file .env start worker`. Проверки автоматизированы в `student-tests/verify_incidents.py`.
|
||||
|
||||
## MessengerHighErrorRate
|
||||
|
||||
Условие: 5xx / все бизнес-запросы > 5% за 1 минуту в течение 10 s. Health исключены. Ошибки клиента 400/401/404 не входят в числитель.
|
||||
|
||||
1. Посмотреть Business API error ratio и API percentiles.
|
||||
2. По `X-Trace-Id` ответа открыть trace в Tempo. В Loki фильтровать `trace_id` или `request_id`.
|
||||
3. Для отказа PostgreSQL видны error spans `db.pool.acquire`/`db.select`, `messenger_db_available=0` и ожидание пула. Длительность HTTP без DB error сама по себе не доказывает отказ БД.
|
||||
4. Восстановить PostgreSQL. Проверить успешные запросы, DB availability=1 и падение error ratio в движущемся окне. Resolved может наступить после обновления окна и нескольких scrape/evaluation.
|
||||
|
||||
## Tempo недоступен
|
||||
|
||||
1. API продолжает записи; проверить бизнес-операцию, а не только `/health/live`.
|
||||
2. В Prometheus посмотреть `otelcol_exporter_send_failed_spans_total{exporter="otlphttp/tempo"}` и `otelcol_exporter_queue_size`.
|
||||
3. Collector queue ограничена 32 batch requests, retry — 5 s; SDK queue — 256 spans/log records, exporter timeout — 2 s. Ожидание backend не переносится в HTTP handler.
|
||||
4. При переполнении/исчерпании retry данные telemetry могут быть потеряны. В JSON-логах Collector финальная потеря содержит `dropped_items`. Счётчик failed sends не следует считать числом уникальных потерянных трасс.
|
||||
5. После восстановления проверить свежий trace. Старые потерянные spans автоматически не восстанавливаются из бизнес-БД.
|
||||
|
||||
## SLI/SLO
|
||||
|
||||
SLI: доля бизнес HTTP-запросов без 5xx за скользящие **5 минут**. SLO: **≥99%**.
|
||||
|
||||
```promql
|
||||
1 - (sum(rate(messenger_http_requests_total{service_name="messenger-api",route!~"/health/.*",status_class="5xx"}[5m])) or vector(0))
|
||||
/ clamp_min(sum(rate(messenger_http_requests_total{service_name="messenger-api",route!~"/health/.*"}[5m])), 0.001)
|
||||
```
|
||||
|
||||
SLO оценивается на окне с реальным трафиком; при отсутствии запросов правило не должно порождать ошибку деления. Alert использует более короткое 1m окно и порог 5% для локальной диагностики. Это не burn-rate исследование оценки 5.
|
||||
@@ -0,0 +1,53 @@
|
||||
# Схема сигналов v1
|
||||
|
||||
SDK: OpenTelemetry Go traces/metrics **1.38.0**, logs **0.14.0**. Resource schema URL — OTel semantic conventions **1.37.0**; прикладные имена и labels ниже зафиксированы собственной схемой `mtusi-messenger/telemetry-v1`.
|
||||
|
||||
Resources: `service.name` = messenger-api / messenger-worker / messenger-dispatcher; `service.version=lab6-v1`, `service.instance.id` = api-1/api-2/worker-1/dispatcher-1, `deployment.environment.name=lab`. Collector превращает resource attrs в Prometheus labels, например service_name.
|
||||
|
||||
| Prometheus metric | Тип, единица | Labels |
|
||||
|---|---|---|
|
||||
| messenger_http_requests_total | counter, requests | method, route template, status_class |
|
||||
| messenger_http_duration_seconds_bucket/count/sum | histogram, seconds | те же |
|
||||
| messenger_jobs_processed_total | counter, attempts | job_type, outcome |
|
||||
| messenger_job_duration_seconds_bucket/count/sum | histogram, seconds | job_type, outcome |
|
||||
| messenger_time_to_ready_seconds_bucket/count/sum | histogram, seconds | job_type |
|
||||
| messenger_queue_depth | gauge, jobs | queue; только dispatcher |
|
||||
| messenger_queue_oldest_age_seconds | gauge, seconds | queue; только dispatcher |
|
||||
| messenger_db_pool_used/idle/waiting/capacity | gauge, connections/acquires | resource instance |
|
||||
| messenger_db_pool_acquire_wait_total | counter, empty pool acquires | resource instance |
|
||||
| messenger_db_pool_acquire_wait_duration_seconds_total | counter, cumulative seconds | resource instance |
|
||||
| messenger_worker_busy/capacity | gauge, attempts/slots | worker instance |
|
||||
| messenger_process_cpu_seconds_total | counter, Go process CPU seconds | instance |
|
||||
| messenger_container_cpu_seconds_total | counter, cgroup CPU seconds including decoder | instance |
|
||||
| messenger_process_memory_bytes | gauge, Go runtime reserved memory (Sys), не RSS | instance |
|
||||
| messenger_container_memory_bytes/limit_bytes | gauge, cgroup bytes | instance |
|
||||
| messenger_process_last_observed_seconds | gauge, Unix timestamp | instance |
|
||||
| messenger_db_available | gauge, 0/1 | dispatcher |
|
||||
| messenger_telemetry_export_errors | gauge, cumulative SDK export errors | instance |
|
||||
|
||||
`waiting` — незавершённые Acquire; EmptyAcquire counter показывает обращения к пустому пулу. Busy/capacity — текущая занятость обработчиков, а не CPU%. E2E время измерено от created_at job в транзакции durable acceptance до COMMIT ready, включает queue wait и retry. Replay использует исходное время принятия.
|
||||
|
||||
Percentiles вычисляются объединением histogram buckets реплик:
|
||||
|
||||
```promql
|
||||
histogram_quantile(0.95,
|
||||
sum by (le) (rate(messenger_http_duration_seconds_bucket{service_name="messenger-api",route!~"/health/.*"}[1m])))
|
||||
```
|
||||
|
||||
p50 и p99 используют 0.50 и 0.99. Единицы всех duration panels — секунды. Суммируются buckets, не percentiles отдельных реплик. Полные исполняемые запросы находятся в `grafana/dashboards/messenger.json`.
|
||||
|
||||
## Логи и трассы
|
||||
|
||||
API log event `http.request`: timestamp, severity, service.name, method, route template, status, duration_ms, request_id, trace_id, span_id. Worker — `job.started`, `job.completed`; dispatcher — `job.published`.
|
||||
|
||||
Технические request_id/job_id допустимы только в логах/трассах для корреляции синтетических работ. В labels метрик они отсутствуют. SQL text/args, URL/query, filename, headers Authorization, пароли, токены, текст сообщения и содержимое файлов не записываются.
|
||||
|
||||
HTTP server span принимает W3C traceparent/tracestate. Upload транзакционно сохраняет их в job. Producer span публикует свой контекст в разрешённых полях JSON брокера; consumer span извлекает его как parent. Повторы образуют новые producer/consumer ветки того же trace. S3, Redis sessions, DB acquire/query/commit и image.process имеют реальные дочерние spans.
|
||||
|
||||
Log body — структурный JSON, дополнительно отправляемый OTLP Log SDK с trace/span context. В Grafana Loki TraceID derived field открывает Tempo; поиск по trace_id показывает связанные события. Заголовок X-Trace-Id добавлен для диагностики; X-Request-Id и бизнес-контракт сохранены.
|
||||
|
||||
## Буферизация
|
||||
|
||||
SDK: batch queue 256, batch 64, export interval 500 ms, exporter timeout 2 s, retry SDK выключен; metrics interval 2 s. Collector: memory limiter 128 MiB со spike 32 MiB, hard limit контейнера 256 MiB; batch size 64/max128; по 32 batches и 2 consumers для traces/logs, неблокирующее переполнение, retry deadline 5 s. Метрики предоставляются ограниченным набором series на pull endpoint Collector; Prometheus хранит историю в volume. Retention storage — 24 h.
|
||||
|
||||
При отказе telemetry допускаются потери signals. Ошибки SDK учитываются отдельно; Collector показывает failed exports/queue, а финальные drops записывает с dropped_items. Business transaction от успешности экспорта не зависит.
|
||||
@@ -0,0 +1,22 @@
|
||||
server:
|
||||
http_listen_port: 3200
|
||||
distributor:
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
http:
|
||||
endpoint: 0.0.0.0:4318
|
||||
grpc:
|
||||
endpoint: 0.0.0.0:4317
|
||||
ingester:
|
||||
max_block_duration: 1m
|
||||
compactor:
|
||||
compaction:
|
||||
block_retention: 24h
|
||||
storage:
|
||||
trace:
|
||||
backend: local
|
||||
wal:
|
||||
path: /var/tempo/wal
|
||||
local:
|
||||
path: /var/tempo/blocks
|
||||
@@ -0,0 +1,64 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
"embed"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
//go:embed migrations/*.sql
|
||||
var migrations embed.FS
|
||||
|
||||
func Migrate(ctx context.Context, pool *pgxpool.Pool) error {
|
||||
tx, err := pool.BeginTx(ctx, pgx.TxOptions{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
if _, err = tx.Exec(ctx, `SELECT pg_advisory_xact_lock(202602)`); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err = tx.Exec(ctx, `CREATE TABLE IF NOT EXISTS schema_migrations (version integer PRIMARY KEY, applied_at timestamptz NOT NULL DEFAULT clock_timestamp())`); err != nil {
|
||||
return err
|
||||
}
|
||||
entries, err := fs.ReadDir(migrations, "migrations")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
sort.Slice(entries, func(i, j int) bool { return entries[i].Name() < entries[j].Name() })
|
||||
for _, entry := range entries {
|
||||
if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".sql") {
|
||||
continue
|
||||
}
|
||||
part := strings.SplitN(entry.Name(), "_", 2)[0]
|
||||
version, parseErr := strconv.Atoi(part)
|
||||
if parseErr != nil {
|
||||
return fmt.Errorf("invalid migration name %s", entry.Name())
|
||||
}
|
||||
var applied bool
|
||||
if err = tx.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM schema_migrations WHERE version=$1)`, version).Scan(&applied); err != nil {
|
||||
return err
|
||||
}
|
||||
if applied {
|
||||
continue
|
||||
}
|
||||
sqlBytes, readErr := migrations.ReadFile("migrations/" + entry.Name())
|
||||
if readErr != nil {
|
||||
return readErr
|
||||
}
|
||||
if _, err = tx.Exec(ctx, string(sqlBytes)); err != nil {
|
||||
return fmt.Errorf("migration %s: %w", entry.Name(), err)
|
||||
}
|
||||
if _, err = tx.Exec(ctx, `INSERT INTO schema_migrations(version) VALUES($1)`, version); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return tx.Commit(ctx)
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
CREATE TABLE users (
|
||||
id uuid PRIMARY KEY,
|
||||
username text NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'),
|
||||
display_name text NOT NULL CHECK (char_length(btrim(display_name)) > 0 AND char_length(display_name) <= 100),
|
||||
password_hash text NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
|
||||
CREATE TABLE chats (
|
||||
id uuid PRIMARY KEY,
|
||||
title text NOT NULL CHECK (char_length(btrim(title)) > 0 AND char_length(title) <= 100),
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
|
||||
CREATE TABLE chat_members (
|
||||
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
user_id uuid NOT NULL REFERENCES users(id) ON DELETE RESTRICT,
|
||||
PRIMARY KEY (chat_id, user_id)
|
||||
);
|
||||
|
||||
CREATE INDEX chat_members_user_chat_idx ON chat_members (user_id, chat_id);
|
||||
|
||||
CREATE TABLE messages (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
sender_id uuid NOT NULL REFERENCES users(id) ON DELETE RESTRICT,
|
||||
text text NOT NULL CHECK (char_length(btrim(text)) > 0 AND char_length(text) <= 4000),
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
|
||||
CREATE INDEX messages_chat_history_idx ON messages (chat_id, created_at, id);
|
||||
@@ -0,0 +1,13 @@
|
||||
CREATE TABLE message_idempotency (
|
||||
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
method text NOT NULL,
|
||||
path text NOT NULL,
|
||||
idempotency_key text NOT NULL,
|
||||
request_hash bytea NOT NULL,
|
||||
message_id uuid REFERENCES messages(id) ON DELETE CASCADE,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
PRIMARY KEY (user_id, method, path, idempotency_key)
|
||||
);
|
||||
|
||||
CREATE INDEX message_idempotency_created_at_idx
|
||||
ON message_idempotency (created_at);
|
||||
@@ -0,0 +1,30 @@
|
||||
CREATE TABLE attachments (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
owner_id uuid NOT NULL REFERENCES users(id),
|
||||
filename text NOT NULL,
|
||||
object_key text NOT NULL UNIQUE,
|
||||
content_type text NOT NULL CHECK (content_type IN ('image/png','image/jpeg')),
|
||||
size_bytes bigint NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760),
|
||||
sha256 text NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'),
|
||||
status text NOT NULL DEFAULT 'ready' CHECK (status='ready'),
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
updated_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
CREATE INDEX attachments_chat_idx ON attachments(chat_id,created_at,id);
|
||||
|
||||
-- Durable intent is committed BEFORE any S3 PUT. A crashed upload remains
|
||||
-- visible to the cleanup process even if its ready metadata never committed.
|
||||
CREATE TABLE attachment_uploads (
|
||||
id uuid PRIMARY KEY,
|
||||
object_key text NOT NULL UNIQUE,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
CREATE INDEX attachment_uploads_created_idx ON attachment_uploads(created_at);
|
||||
|
||||
ALTER TABLE messages DROP CONSTRAINT messages_text_check;
|
||||
ALTER TABLE messages ADD CONSTRAINT messages_text_check CHECK (char_length(text)<=4000);
|
||||
ALTER TABLE messages ADD COLUMN attachment_ids uuid[] NOT NULL DEFAULT '{}';
|
||||
ALTER TABLE messages ADD CONSTRAINT messages_body_check
|
||||
CHECK (char_length(btrim(text))>0 OR cardinality(attachment_ids)>0);
|
||||
ALTER TABLE messages ADD CONSTRAINT messages_attachments_limit CHECK (cardinality(attachment_ids)<=10);
|
||||
@@ -0,0 +1,37 @@
|
||||
ALTER TABLE attachments DROP CONSTRAINT attachments_status_check;
|
||||
ALTER TABLE attachments ALTER COLUMN status SET DEFAULT 'queued';
|
||||
ALTER TABLE attachments ADD CONSTRAINT attachments_status_check CHECK (status IN ('queued','processing','ready','failed'));
|
||||
ALTER TABLE attachments ADD COLUMN error_code text;
|
||||
ALTER TABLE attachments ADD COLUMN variants jsonb NOT NULL DEFAULT '[]';
|
||||
|
||||
CREATE TABLE attachment_jobs (
|
||||
id uuid PRIMARY KEY,
|
||||
attachment_id uuid NOT NULL UNIQUE REFERENCES attachments(id) ON DELETE CASCADE,
|
||||
pipeline_version integer NOT NULL DEFAULT 1 CHECK (pipeline_version=1),
|
||||
staging_key text NOT NULL UNIQUE,
|
||||
state text NOT NULL DEFAULT 'queued' CHECK (state IN ('queued','processing','ready','failed')),
|
||||
attempts integer NOT NULL DEFAULT 0,
|
||||
max_attempts integer NOT NULL DEFAULT 5 CHECK (max_attempts BETWEEN 1 AND 20),
|
||||
next_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
lease_until timestamptz,
|
||||
lease_token uuid,
|
||||
error_code text,
|
||||
staging_deleted_at timestamptz,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
updated_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
CREATE INDEX attachment_jobs_due_idx ON attachment_jobs(state,next_at);
|
||||
-- Same transaction as attachment + job; repeated publication is intentional.
|
||||
CREATE TABLE attachment_outbox (
|
||||
job_id uuid PRIMARY KEY REFERENCES attachment_jobs(id) ON DELETE CASCADE,
|
||||
published_at timestamptz,
|
||||
publications integer NOT NULL DEFAULT 0
|
||||
);
|
||||
CREATE TABLE attachment_replays (
|
||||
id bigserial PRIMARY KEY,
|
||||
job_id uuid NOT NULL REFERENCES attachment_jobs(id),
|
||||
previous_error text NOT NULL,
|
||||
replayed_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
-- Internal CLI-only, one-shot crash injection. No HTTP endpoint exposes it.
|
||||
CREATE TABLE pipeline_faults (point text PRIMARY KEY, armed boolean NOT NULL DEFAULT true, hit_at timestamptz);
|
||||
@@ -0,0 +1,4 @@
|
||||
-- Persist causal context in the same transaction as durable acceptance/outbox.
|
||||
ALTER TABLE attachment_jobs ADD COLUMN traceparent text NOT NULL DEFAULT '';
|
||||
ALTER TABLE attachment_jobs ADD COLUMN tracestate text NOT NULL DEFAULT '';
|
||||
ALTER TABLE attachment_jobs ADD COLUMN request_id text NOT NULL DEFAULT '';
|
||||
@@ -0,0 +1,263 @@
|
||||
package messenger
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"image"
|
||||
_ "image/jpeg"
|
||||
_ "image/png"
|
||||
"io"
|
||||
"mime"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"mtusi-messenger/internal/telemetry"
|
||||
)
|
||||
|
||||
const MaxUploadBytes int64 = 10 * 1024 * 1024
|
||||
const MaxImagePixels int64 = 40000000
|
||||
|
||||
type Attachment struct {
|
||||
ID string `json:"id"`
|
||||
ChatID string `json:"chat_id"`
|
||||
OwnerID string `json:"owner_id"`
|
||||
Filename string `json:"filename"`
|
||||
ContentType string `json:"content_type"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
SHA256 string `json:"sha256"`
|
||||
Status string `json:"status"`
|
||||
Variants []any `json:"variants"`
|
||||
ErrorCode *string `json:"error_code"`
|
||||
CreatedAt string `json:"created_at"`
|
||||
UpdatedAt string `json:"updated_at"`
|
||||
}
|
||||
|
||||
func tooLarge() *apiError {
|
||||
return &apiError{status: 413, code: "payload_too_large", message: "Превышен лимит изображения"}
|
||||
}
|
||||
func invalidImage() *apiError {
|
||||
return &apiError{status: 415, code: "unsupported_media_type", message: "Требуется корректный PNG или JPEG"}
|
||||
}
|
||||
|
||||
func rollbackUpload(tx pgx.Tx) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
||||
defer cancel()
|
||||
_ = tx.Rollback(ctx)
|
||||
}
|
||||
|
||||
func validFilename(s string) bool {
|
||||
if !utf8.ValidString(s) || strings.TrimSpace(s) == "" || utf8.RuneCountInString(s) > 255 {
|
||||
return false
|
||||
}
|
||||
for _, c := range s {
|
||||
if c == '/' || c == '\\' || unicode.IsControl(c) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (a *API) isMember(r *http.Request, uid, chatID string) *apiError {
|
||||
ctx, cancel := a.dbContext(r.Context())
|
||||
defer cancel()
|
||||
var member bool
|
||||
if err := a.db.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM chat_members WHERE chat_id=$1 AND user_id=$2)`, chatID, uid).Scan(&member); err != nil {
|
||||
return unavailable()
|
||||
}
|
||||
if !member {
|
||||
return missing()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *API) uploadAttachment(r *http.Request, uid, chatID string) (response, *apiError) {
|
||||
if e := a.isMember(r, uid, chatID); e != nil {
|
||||
return response{}, e
|
||||
}
|
||||
query, err := url.ParseQuery(r.URL.RawQuery)
|
||||
if err != nil || len(query) != 1 || len(query["filename"]) != 1 || !validFilename(query.Get("filename")) {
|
||||
return response{}, bad("Неверное имя файла")
|
||||
}
|
||||
kind, _, err := mime.ParseMediaType(r.Header.Get("Content-Type"))
|
||||
if err != nil || (kind != "image/png" && kind != "image/jpeg") {
|
||||
return response{}, invalidImage()
|
||||
}
|
||||
if r.ContentLength > MaxUploadBytes {
|
||||
return response{}, tooLarge()
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 45*time.Second)
|
||||
defer cancel()
|
||||
// One decoder/temporary original per replica bounds RAM and tmpfs usage.
|
||||
select {
|
||||
case a.uploadSlots <- struct{}{}:
|
||||
defer func() { <-a.uploadSlots }()
|
||||
case <-ctx.Done():
|
||||
return response{}, unavailable()
|
||||
}
|
||||
file, err := os.CreateTemp("", "upload-*")
|
||||
if err != nil {
|
||||
return response{}, unavailable()
|
||||
}
|
||||
defer os.Remove(file.Name())
|
||||
defer file.Close()
|
||||
hash := sha256.New()
|
||||
size, err := io.CopyBuffer(io.MultiWriter(file, hash), io.LimitReader(r.Body, MaxUploadBytes+1), make([]byte, 32*1024))
|
||||
if size > MaxUploadBytes {
|
||||
return response{}, tooLarge()
|
||||
}
|
||||
if err != nil {
|
||||
return response{}, bad("Загрузка прервана")
|
||||
}
|
||||
if size == 0 {
|
||||
return response{}, invalidImage()
|
||||
}
|
||||
file.Seek(0, io.SeekStart)
|
||||
config, format, err := image.DecodeConfig(file)
|
||||
if err != nil || (format != "png" && format != "jpeg") || "image/"+format != kind {
|
||||
return response{}, invalidImage()
|
||||
}
|
||||
if config.Width < 1 || config.Height < 1 || int64(config.Width) > MaxImagePixels/int64(config.Height) {
|
||||
return response{}, tooLarge()
|
||||
}
|
||||
file.Seek(0, io.SeekStart)
|
||||
// Decode after the dimension guard to reject corrupt payloads, not just
|
||||
// apparently valid headers. The decoded bitmap is never persisted.
|
||||
if e := validateImage(file); e != nil {
|
||||
return response{}, e
|
||||
}
|
||||
file.Seek(0, io.SeekStart)
|
||||
id := newID()
|
||||
key := "staging/" + id
|
||||
dbctx, dbcancel := a.dbContext(ctx)
|
||||
_, err = a.db.Exec(dbctx, `INSERT INTO attachment_uploads(id,object_key) VALUES($1,$2)`, id, key)
|
||||
dbcancel()
|
||||
if err != nil {
|
||||
return response{}, unavailable()
|
||||
}
|
||||
tx, err := a.db.Begin(ctx)
|
||||
if err != nil {
|
||||
return response{}, unavailable()
|
||||
}
|
||||
defer rollbackUpload(tx)
|
||||
// Cleanup and upload use the same lock. Cleanup cannot delete an object
|
||||
// while its upload/metadata transaction is in progress.
|
||||
if _, err = tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1,4))`, key); err != nil {
|
||||
return response{}, unavailable()
|
||||
}
|
||||
var pending bool
|
||||
if err = tx.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM attachment_uploads WHERE id=$1)`, id).Scan(&pending); err != nil || !pending {
|
||||
return response{}, unavailable()
|
||||
}
|
||||
putctx, putcancel := context.WithTimeout(ctx, 20*time.Second)
|
||||
err = a.objects.Put(putctx, key, file, size, kind)
|
||||
putcancel()
|
||||
if err != nil {
|
||||
return response{}, unavailable()
|
||||
}
|
||||
att := Attachment{ID: id, ChatID: chatID, OwnerID: uid, Filename: query.Get("filename"), ContentType: kind, SizeBytes: size, SHA256: hex.EncodeToString(hash.Sum(nil)), Status: "queued", Variants: []any{}}
|
||||
var created, updated time.Time
|
||||
if err = tx.QueryRow(ctx, `INSERT INTO attachments(id,chat_id,owner_id,filename,object_key,content_type,size_bytes,sha256,status) VALUES($1,$2,$3,$4,$5,$6,$7,$8,'queued') RETURNING created_at,updated_at`, id, chatID, uid, att.Filename, key, kind, size, att.SHA256).Scan(&created, &updated); err != nil {
|
||||
return response{}, unavailable()
|
||||
}
|
||||
if _, err = tx.Exec(ctx, `DELETE FROM attachment_uploads WHERE id=$1`, id); err != nil {
|
||||
return response{}, unavailable()
|
||||
}
|
||||
|
||||
maxAttempts := 5
|
||||
if raw := os.Getenv("JOB_MAX_ATTEMPTS"); raw != "" {
|
||||
n, parseErr := strconv.Atoi(raw)
|
||||
if parseErr != nil || n < 1 || n > 20 {
|
||||
return response{}, unavailable()
|
||||
}
|
||||
maxAttempts = n
|
||||
}
|
||||
parent, state := telemetry.Inject(ctx)
|
||||
if _, err = tx.Exec(ctx, `INSERT INTO attachment_jobs(id,attachment_id,staging_key,max_attempts,traceparent,tracestate,request_id) VALUES($1,$2,$3,$4,$5,$6,$7)`, id, id, key, maxAttempts, parent, state, telemetry.RequestID(ctx)); err != nil {
|
||||
return response{}, unavailable()
|
||||
}
|
||||
if _, err = tx.Exec(ctx, `INSERT INTO attachment_outbox(job_id) VALUES($1)`, id); err != nil {
|
||||
return response{}, unavailable()
|
||||
}
|
||||
if err = tx.Commit(ctx); err != nil {
|
||||
return response{}, unavailable()
|
||||
}
|
||||
att.CreatedAt = formatTime(created)
|
||||
att.UpdatedAt = formatTime(updated)
|
||||
return response{status: 202, body: att}, nil
|
||||
}
|
||||
|
||||
func validateImage(r io.Reader) *apiError {
|
||||
_, _, err := image.Decode(r)
|
||||
if err != nil {
|
||||
return invalidImage()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *API) getAttachment(r *http.Request, uid, id string, content bool) (response, *apiError) {
|
||||
query, err := url.ParseQuery(r.URL.RawQuery)
|
||||
if err != nil {
|
||||
return response{}, bad("Неверные параметры вложения")
|
||||
}
|
||||
if (!content && len(query) > 0) || (content && (len(query) > 1 || (len(query) > 0 && len(query["variant"]) != 1))) {
|
||||
return response{}, bad("Неверные параметры вложения")
|
||||
}
|
||||
ctx, cancel := a.dbContext(r.Context())
|
||||
defer cancel()
|
||||
att := Attachment{Variants: []any{}}
|
||||
var key string
|
||||
var variantsJSON []byte
|
||||
var created, updated time.Time
|
||||
err = a.db.QueryRow(ctx, `SELECT a.id::text,a.chat_id::text,a.owner_id::text,a.filename,a.object_key,a.content_type,a.size_bytes,a.sha256,a.status,a.created_at,a.updated_at,a.variants,a.error_code FROM attachments a JOIN chat_members cm ON cm.chat_id=a.chat_id AND cm.user_id=$2 WHERE a.id=$1`, id, uid).Scan(&att.ID, &att.ChatID, &att.OwnerID, &att.Filename, &key, &att.ContentType, &att.SizeBytes, &att.SHA256, &att.Status, &created, &updated, &variantsJSON, &att.ErrorCode)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return response{}, missing()
|
||||
}
|
||||
if err != nil {
|
||||
return response{}, unavailable()
|
||||
}
|
||||
_ = json.Unmarshal(variantsJSON, &att.Variants)
|
||||
att.CreatedAt = formatTime(created)
|
||||
att.UpdatedAt = formatTime(updated)
|
||||
if !content {
|
||||
return response{status: 200, body: att}, nil
|
||||
}
|
||||
|
||||
if att.Status != "ready" {
|
||||
return response{}, &apiError{status: 409, code: "conflict", message: "Вложение не готово"}
|
||||
}
|
||||
if variant, ok := query["variant"]; ok && variant[0] != "original" {
|
||||
found := false
|
||||
for _, item := range att.Variants {
|
||||
v, ok := item.(map[string]any)
|
||||
if ok && v["name"] == variant[0] {
|
||||
key = "variants/" + id + "/v1/" + variant[0]
|
||||
att.ContentType = v["content_type"].(string)
|
||||
att.SizeBytes = int64(v["size_bytes"].(float64))
|
||||
att.Filename = variant[0] + map[bool]string{true: ".webp", false: ".jpg"}[variant[0] == "optimized"]
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
return response{}, missing()
|
||||
}
|
||||
}
|
||||
|
||||
downloadCtx, downloadCancel := context.WithTimeout(r.Context(), 45*time.Second)
|
||||
object, err := a.objects.Get(downloadCtx, key)
|
||||
if err != nil {
|
||||
downloadCancel()
|
||||
return response{}, unavailable()
|
||||
}
|
||||
return response{status: 200, content: object, contentType: att.ContentType, length: att.SizeBytes, filename: att.Filename, cancel: downloadCancel}, nil
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
package messenger
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
"github.com/minio/minio-go/v7"
|
||||
"mtusi-messenger/internal/objectstore"
|
||||
)
|
||||
|
||||
type CleanupResult struct {
|
||||
RemovedObjects int `json:"removed_objects"`
|
||||
RemovedIntents int64 `json:"removed_intents"`
|
||||
KeptReady int `json:"kept_ready"`
|
||||
SkippedRecent int `json:"skipped_recent"`
|
||||
SkippedBusy int `json:"skipped_busy"`
|
||||
}
|
||||
|
||||
// Cleanup removes abandoned uploads. Accepted pipeline jobs protect staging.
|
||||
func Cleanup(ctx context.Context, db *pgxpool.Pool, store *objectstore.Store, grace time.Duration) (CleanupResult, error) {
|
||||
result := CleanupResult{}
|
||||
cutoff := time.Now().Add(-grace)
|
||||
clean := func(key string) error {
|
||||
if !strings.HasPrefix(key, "originals/") && !strings.HasPrefix(key, "staging/") {
|
||||
return nil
|
||||
}
|
||||
tx, err := db.Begin(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer rollbackUpload(tx)
|
||||
var locked bool
|
||||
if err = tx.QueryRow(ctx, `SELECT pg_try_advisory_xact_lock(hashtextextended($1,4))`, key).Scan(&locked); err != nil {
|
||||
return err
|
||||
}
|
||||
if !locked {
|
||||
result.SkippedBusy++
|
||||
return nil
|
||||
}
|
||||
var ready, recent bool
|
||||
if err = tx.QueryRow(ctx, `SELECT (EXISTS(SELECT 1 FROM attachments WHERE object_key=$1) OR EXISTS(SELECT 1 FROM attachment_jobs WHERE staging_key=$1)),EXISTS(SELECT 1 FROM attachment_uploads WHERE object_key=$1 AND created_at>$2)`, key, cutoff).Scan(&ready, &recent); err != nil {
|
||||
return err
|
||||
}
|
||||
if ready {
|
||||
result.KeptReady++
|
||||
return nil
|
||||
}
|
||||
if recent {
|
||||
result.SkippedRecent++
|
||||
return nil
|
||||
}
|
||||
info, err := store.Client.StatObject(ctx, store.Bucket, key, minio.StatObjectOptions{})
|
||||
exists := err == nil
|
||||
if err != nil && minio.ToErrorResponse(err).Code != "NoSuchKey" {
|
||||
return err
|
||||
}
|
||||
if exists && info.LastModified.After(cutoff) {
|
||||
result.SkippedRecent++
|
||||
return nil
|
||||
}
|
||||
if exists {
|
||||
if err = store.Delete(ctx, key); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
command, err := tx.Exec(ctx, `DELETE FROM attachment_uploads WHERE object_key=$1`, key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err = tx.Commit(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
result.RemovedIntents += command.RowsAffected()
|
||||
if exists {
|
||||
result.RemovedObjects++
|
||||
}
|
||||
return nil
|
||||
}
|
||||
rows, err := db.Query(ctx, `SELECT object_key FROM attachment_uploads WHERE created_at<$1 ORDER BY created_at LIMIT 1000`, cutoff)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
keys := []string{}
|
||||
for rows.Next() {
|
||||
var key string
|
||||
if err = rows.Scan(&key); err != nil {
|
||||
rows.Close()
|
||||
return result, err
|
||||
}
|
||||
keys = append(keys, key)
|
||||
}
|
||||
err = rows.Err()
|
||||
rows.Close()
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
for _, key := range keys {
|
||||
if err = clean(key); err != nil {
|
||||
return result, err
|
||||
}
|
||||
}
|
||||
listctx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
for object := range store.Client.ListObjects(listctx, store.Bucket, minio.ListObjectsOptions{Recursive: true}) {
|
||||
if object.Err != nil {
|
||||
return result, fmt.Errorf("S3 cleanup listing: %w", object.Err)
|
||||
}
|
||||
if object.LastModified.Before(cutoff) {
|
||||
if err = clean(object.Key); err != nil {
|
||||
return result, err
|
||||
}
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
package messenger
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"crypto/subtle"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"golang.org/x/crypto/scrypt"
|
||||
)
|
||||
|
||||
const scryptN, scryptR, scryptP = 32768, 8, 1
|
||||
|
||||
func hashPassword(password string) (string, error) {
|
||||
salt := make([]byte, 16)
|
||||
if _, err := rand.Read(salt); err != nil {
|
||||
return "", err
|
||||
}
|
||||
hash, err := scrypt.Key([]byte(password), salt, scryptN, scryptR, scryptP, 32)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
b64 := base64.RawStdEncoding
|
||||
return fmt.Sprintf("scrypt$%d$%d$%d$%s$%s", scryptN, scryptR, scryptP, b64.EncodeToString(salt), b64.EncodeToString(hash)), nil
|
||||
}
|
||||
|
||||
func verifyPassword(encoded, password string) bool {
|
||||
parts := strings.Split(encoded, "$")
|
||||
if len(parts) != 6 || parts[0] != "scrypt" {
|
||||
return false
|
||||
}
|
||||
n, e1 := strconv.Atoi(parts[1])
|
||||
r, e2 := strconv.Atoi(parts[2])
|
||||
p, e3 := strconv.Atoi(parts[3])
|
||||
salt, e4 := base64.RawStdEncoding.DecodeString(parts[4])
|
||||
expected, e5 := base64.RawStdEncoding.DecodeString(parts[5])
|
||||
if e1 != nil || e2 != nil || e3 != nil || e4 != nil || e5 != nil || len(expected) != 32 {
|
||||
return false
|
||||
}
|
||||
actual, err := scrypt.Key([]byte(password), salt, n, r, p, len(expected))
|
||||
return err == nil && subtle.ConstantTimeCompare(actual, expected) == 1
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
package messenger
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestPasswordHash(t *testing.T) {
|
||||
const password = "long-test-password"
|
||||
first, err := hashPassword(password)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
second, err := hashPassword(password)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if first == password || strings.Contains(first, password) {
|
||||
t.Fatal("hash contains the original password")
|
||||
}
|
||||
if first == second {
|
||||
t.Fatal("independent hashes must use different salts")
|
||||
}
|
||||
if !verifyPassword(first, password) {
|
||||
t.Fatal("correct password was rejected")
|
||||
}
|
||||
if verifyPassword(first, "wrong-test-password") {
|
||||
t.Fatal("wrong password was accepted")
|
||||
}
|
||||
if verifyPassword("damaged-hash", password) {
|
||||
t.Fatal("malformed hash was accepted")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,776 @@
|
||||
package messenger
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"mime"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgconn"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
"github.com/redis/go-redis/v9"
|
||||
"go.opentelemetry.io/otel"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/codes"
|
||||
"go.opentelemetry.io/otel/propagation"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
"mtusi-messenger/internal/objectstore"
|
||||
"mtusi-messenger/internal/telemetry"
|
||||
)
|
||||
|
||||
type User struct {
|
||||
ID string `json:"id"`
|
||||
Username string `json:"username"`
|
||||
DisplayName string `json:"display_name"`
|
||||
CreatedAt string `json:"created_at"`
|
||||
}
|
||||
type Chat struct {
|
||||
ID string `json:"id"`
|
||||
Title string `json:"title"`
|
||||
MemberIDs []string `json:"member_ids"`
|
||||
CreatedAt string `json:"created_at"`
|
||||
}
|
||||
type Message struct {
|
||||
ID string `json:"id"`
|
||||
ChatID string `json:"chat_id"`
|
||||
SenderID string `json:"sender_id"`
|
||||
Text string `json:"text"`
|
||||
CreatedAt string `json:"created_at"`
|
||||
AttachmentIDs []string `json:"attachment_ids"`
|
||||
}
|
||||
type Session struct {
|
||||
Token string `json:"token"`
|
||||
TokenType string `json:"token_type"`
|
||||
ExpiresAt string `json:"expires_at"`
|
||||
User User `json:"user"`
|
||||
}
|
||||
|
||||
type API struct {
|
||||
objects *objectstore.Store
|
||||
uploadSlots chan struct{}
|
||||
db *pgxpool.Pool
|
||||
sessions *redis.Client
|
||||
ttl time.Duration
|
||||
idempotencyTTL time.Duration
|
||||
dbTimeout time.Duration
|
||||
instanceID string
|
||||
logger *log.Logger
|
||||
}
|
||||
|
||||
func NewHandler(db *pgxpool.Pool, sessions *redis.Client, objects *objectstore.Store, ttl, idempotencyTTL, dbTimeout time.Duration, instanceID string, logger *log.Logger) http.Handler {
|
||||
return &API{db: db, sessions: sessions, objects: objects, uploadSlots: make(chan struct{}, 1), ttl: ttl, idempotencyTTL: idempotencyTTL, dbTimeout: dbTimeout, instanceID: instanceID, logger: logger}
|
||||
}
|
||||
|
||||
var uuidRE = regexp.MustCompile(`^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$`)
|
||||
var usernameRE = regexp.MustCompile(`^[a-z0-9_]{3,32}$`)
|
||||
var idempotencyKeyRE = regexp.MustCompile(`^[A-Za-z0-9._:-]{1,128}$`)
|
||||
|
||||
type apiError struct {
|
||||
status int
|
||||
code, message, challenge string
|
||||
}
|
||||
type response struct {
|
||||
status int
|
||||
body any
|
||||
noBody bool
|
||||
content io.ReadCloser
|
||||
contentType string
|
||||
length int64
|
||||
filename string
|
||||
cancel context.CancelFunc
|
||||
}
|
||||
|
||||
func bad(msg string) *apiError { return &apiError{status: 400, code: "invalid_request", message: msg} }
|
||||
func missing() *apiError {
|
||||
return &apiError{status: 404, code: "not_found", message: "Объект не найден"}
|
||||
}
|
||||
func unavailable() *apiError {
|
||||
return &apiError{status: 503, code: "unavailable", message: "Сервис временно недоступен"}
|
||||
}
|
||||
func unauthorized(scheme string) *apiError {
|
||||
challenge := `Bearer realm="messenger"`
|
||||
if scheme == "Basic" {
|
||||
challenge = `Basic realm="messenger", charset="UTF-8"`
|
||||
}
|
||||
return &apiError{status: 401, code: "unauthorized", message: "Неверные учётные данные", challenge: challenge}
|
||||
}
|
||||
|
||||
func (a *API) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
id := newID()
|
||||
route, method := telemetry.Route(r.URL.Path), telemetry.Method(r.Method)
|
||||
parent := otel.GetTextMapPropagator().Extract(r.Context(), propagation.HeaderCarrier(r.Header))
|
||||
ctx, span := telemetry.Span(parent, method+" "+route, trace.SpanKindServer, attribute.String("http.route", route), attribute.String("http.request.method", method), attribute.String("request.id", id))
|
||||
ctx = telemetry.WithRequestID(ctx, id)
|
||||
r = r.WithContext(ctx)
|
||||
w.Header().Set("X-Trace-Id", span.SpanContext().TraceID().String())
|
||||
w.Header().Set("X-Request-Id", id)
|
||||
w.Header().Set("X-Instance-Id", a.instanceID)
|
||||
w.Header().Set("Content-Type", "application/json; charset=utf-8")
|
||||
start, status := time.Now(), 500
|
||||
defer func() {
|
||||
if x := recover(); x != nil {
|
||||
status = 500
|
||||
telemetry.Event(ctx, "http.panic", nil)
|
||||
writeError(w, id, &apiError{status: 500, code: "internal_error", message: "Внутренняя ошибка"})
|
||||
}
|
||||
span.SetAttributes(attribute.Int("http.response.status_code", status))
|
||||
if status >= 500 {
|
||||
span.SetStatus(codes.Error, "HTTP server error")
|
||||
}
|
||||
telemetry.HTTP(ctx, method, route, status, time.Since(start))
|
||||
telemetry.Event(ctx, "http.request", map[string]any{"method": method, "route": route, "status": status, "duration_ms": time.Since(start).Milliseconds()})
|
||||
span.End()
|
||||
}()
|
||||
res, apiErr := a.route(r)
|
||||
if apiErr != nil {
|
||||
status = apiErr.status
|
||||
writeError(w, id, apiErr)
|
||||
return
|
||||
}
|
||||
status = res.status
|
||||
if res.content != nil {
|
||||
if res.cancel != nil {
|
||||
defer res.cancel()
|
||||
}
|
||||
defer res.content.Close()
|
||||
w.Header().Set("Content-Type", res.contentType)
|
||||
w.Header().Set("Content-Length", strconv.FormatInt(res.length, 10))
|
||||
w.Header().Set("Content-Disposition", mime.FormatMediaType("inline", map[string]string{"filename": res.filename}))
|
||||
w.Header().Set("Cache-Control", "private, no-store")
|
||||
w.Header().Set("X-Content-Type-Options", "nosniff")
|
||||
w.WriteHeader(status)
|
||||
if _, err := io.Copy(w, res.content); err != nil {
|
||||
telemetry.Event(ctx, "http.download_interrupted", nil)
|
||||
}
|
||||
return
|
||||
}
|
||||
w.WriteHeader(status)
|
||||
if !res.noBody {
|
||||
_ = json.NewEncoder(w).Encode(res.body)
|
||||
}
|
||||
}
|
||||
|
||||
func writeError(w http.ResponseWriter, requestID string, e *apiError) {
|
||||
if e.challenge != "" {
|
||||
w.Header().Set("WWW-Authenticate", e.challenge)
|
||||
}
|
||||
w.WriteHeader(e.status)
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"error": map[string]string{"code": e.code, "message": e.message, "request_id": requestID}})
|
||||
}
|
||||
|
||||
func decode(r *http.Request, dst any) *apiError {
|
||||
if strings.ToLower(strings.TrimSpace(strings.Split(r.Header.Get("Content-Type"), ";")[0])) != "application/json" {
|
||||
return &apiError{status: 415, code: "unsupported_media_type", message: "Требуется application/json"}
|
||||
}
|
||||
if r.ContentLength > 1<<20 {
|
||||
return &apiError{status: 413, code: "payload_too_large", message: "Слишком большой запрос"}
|
||||
}
|
||||
data, err := io.ReadAll(io.LimitReader(r.Body, (1<<20)+1))
|
||||
if err != nil {
|
||||
return bad("Не удалось прочитать JSON")
|
||||
}
|
||||
if len(data) > 1<<20 {
|
||||
return &apiError{status: 413, code: "payload_too_large", message: "Слишком большой запрос"}
|
||||
}
|
||||
if !utf8.Valid(data) {
|
||||
return bad("JSON должен быть в UTF-8")
|
||||
}
|
||||
var fields map[string]json.RawMessage
|
||||
if json.Unmarshal(data, &fields) == nil {
|
||||
for _, value := range fields {
|
||||
if bytes.Equal(bytes.TrimSpace(value), []byte("null")) {
|
||||
return bad("Поля запроса не должны быть null")
|
||||
}
|
||||
}
|
||||
}
|
||||
d := json.NewDecoder(bytes.NewReader(data))
|
||||
d.DisallowUnknownFields()
|
||||
if err = d.Decode(dst); err != nil {
|
||||
return bad("Неверный JSON или поля")
|
||||
}
|
||||
var rest any
|
||||
if !errors.Is(d.Decode(&rest), io.EOF) {
|
||||
return bad("Ожидался один JSON-объект")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validText(s string, max int) bool {
|
||||
return utf8.ValidString(s) && strings.TrimSpace(s) != "" && utf8.RuneCountInString(s) <= max
|
||||
}
|
||||
func validPassword(s string) bool {
|
||||
n := utf8.RuneCountInString(s)
|
||||
return utf8.ValidString(s) && n >= 12 && n <= 128
|
||||
}
|
||||
func formatTime(t time.Time) string { return t.UTC().Format(time.RFC3339Nano) }
|
||||
func newID() string {
|
||||
var b [16]byte
|
||||
if _, err := rand.Read(b[:]); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
b[6] = (b[6] & 0x0f) | 0x40
|
||||
b[8] = (b[8] & 0x3f) | 0x80
|
||||
return fmt.Sprintf("%x-%x-%x-%x-%x", b[:4], b[4:6], b[6:8], b[8:10], b[10:])
|
||||
}
|
||||
func newToken() string {
|
||||
b := make([]byte, 32)
|
||||
if _, err := rand.Read(b); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return base64.RawURLEncoding.EncodeToString(b)
|
||||
}
|
||||
func tokenKey(token string) string {
|
||||
sum := sha256.Sum256([]byte(token))
|
||||
return "session:" + hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
func cursorFor(resource string, created time.Time, id string) string {
|
||||
b, _ := json.Marshal([]string{resource, formatTime(created), id})
|
||||
return base64.RawURLEncoding.EncodeToString(b)
|
||||
}
|
||||
func parsePage(r *http.Request, resource string) (int, *time.Time, string, *apiError) {
|
||||
q, err := url.ParseQuery(r.URL.RawQuery)
|
||||
if err != nil {
|
||||
return 0, nil, "", bad("Неверные параметры страницы")
|
||||
}
|
||||
for k, v := range q {
|
||||
if (k != "limit" && k != "cursor") || len(v) != 1 {
|
||||
return 0, nil, "", bad("Неверные параметры страницы")
|
||||
}
|
||||
}
|
||||
limit := 50
|
||||
if raw := q.Get("limit"); raw != "" {
|
||||
n, e := strconv.Atoi(raw)
|
||||
if e != nil || n < 1 || n > 100 {
|
||||
return 0, nil, "", bad("limit должен быть от 1 до 100")
|
||||
}
|
||||
limit = n
|
||||
} else if _, ok := q["limit"]; ok {
|
||||
return 0, nil, "", bad("Пустой limit")
|
||||
}
|
||||
if _, ok := q["cursor"]; !ok {
|
||||
return limit, nil, "", nil
|
||||
}
|
||||
raw := q.Get("cursor")
|
||||
if raw == "" || len(raw) > 1024 {
|
||||
return 0, nil, "", bad("Неверный cursor")
|
||||
}
|
||||
b, e := base64.RawURLEncoding.DecodeString(raw)
|
||||
if e != nil {
|
||||
return 0, nil, "", bad("Неверный cursor")
|
||||
}
|
||||
var p []string
|
||||
if json.Unmarshal(b, &p) != nil || len(p) != 3 || p[0] != resource || !uuidRE.MatchString(p[2]) {
|
||||
return 0, nil, "", bad("Неверный cursor")
|
||||
}
|
||||
t, e := time.Parse(time.RFC3339Nano, p[1])
|
||||
if e != nil {
|
||||
return 0, nil, "", bad("Неверный cursor")
|
||||
}
|
||||
return limit, &t, p[2], nil
|
||||
}
|
||||
func (a *API) dbContext(parent context.Context) (context.Context, context.CancelFunc) {
|
||||
return context.WithTimeout(parent, a.dbTimeout)
|
||||
}
|
||||
|
||||
func (a *API) authenticate(r *http.Request) (string, string, *apiError) {
|
||||
h := r.Header.Get("Authorization")
|
||||
if !strings.HasPrefix(h, "Bearer ") || strings.TrimSpace(strings.TrimPrefix(h, "Bearer ")) == "" {
|
||||
return "", "", unauthorized("Bearer")
|
||||
}
|
||||
token := strings.TrimSpace(strings.TrimPrefix(h, "Bearer "))
|
||||
ctx, cancel := context.WithTimeout(r.Context(), a.dbTimeout)
|
||||
defer cancel()
|
||||
uid, err := a.sessions.Get(ctx, tokenKey(token)).Result()
|
||||
if errors.Is(err, redis.Nil) {
|
||||
return "", "", unauthorized("Bearer")
|
||||
}
|
||||
if err != nil {
|
||||
a.logger.Printf("session store unavailable: %v", err)
|
||||
return "", "", unavailable()
|
||||
}
|
||||
if !uuidRE.MatchString(uid) {
|
||||
return "", "", unauthorized("Bearer")
|
||||
}
|
||||
return uid, token, nil
|
||||
}
|
||||
|
||||
func scanUser(row pgx.Row) (User, string, error) {
|
||||
var u User
|
||||
var created time.Time
|
||||
var password string
|
||||
err := row.Scan(&u.ID, &u.Username, &u.DisplayName, &created, &password)
|
||||
u.CreatedAt = formatTime(created)
|
||||
return u, password, err
|
||||
}
|
||||
|
||||
func (a *API) route(r *http.Request) (response, *apiError) {
|
||||
p := r.URL.Path
|
||||
if r.Method == "GET" && p == "/health/live" {
|
||||
return response{status: 200, body: map[string]string{"status": "ok"}}, nil
|
||||
}
|
||||
if r.Method == "GET" && p == "/health/ready" {
|
||||
ctx, cancel := context.WithTimeout(r.Context(), time.Second)
|
||||
defer cancel()
|
||||
dbctx, dbspan := telemetry.Span(ctx, "db.readiness", trace.SpanKindClient, attribute.String("db.system.name", "postgresql"))
|
||||
dbErr := a.db.Ping(dbctx)
|
||||
telemetry.Result(dbspan, dbErr)
|
||||
if dbErr != nil || a.sessions.Ping(ctx).Err() != nil || a.objects.Ready(ctx) != nil {
|
||||
return response{}, unavailable()
|
||||
}
|
||||
return response{status: 200, body: map[string]string{"status": "ok"}}, nil
|
||||
}
|
||||
if r.Method == "POST" && p == "/api/v1/users" {
|
||||
return a.createUser(r)
|
||||
}
|
||||
if r.Method == "POST" && p == "/api/v1/auth/sessions" {
|
||||
return a.createSession(r)
|
||||
}
|
||||
if !strings.HasPrefix(p, "/api/v1/") {
|
||||
return response{}, missing()
|
||||
}
|
||||
uid, token, authErr := a.authenticate(r)
|
||||
if authErr != nil {
|
||||
return response{}, authErr
|
||||
}
|
||||
if r.Method == "DELETE" && p == "/api/v1/auth/sessions/current" {
|
||||
ctx, cancel := context.WithTimeout(r.Context(), a.dbTimeout)
|
||||
defer cancel()
|
||||
if err := a.sessions.Del(ctx, tokenKey(token)).Err(); err != nil {
|
||||
return response{}, unavailable()
|
||||
}
|
||||
return response{status: 204, noBody: true}, nil
|
||||
}
|
||||
if r.Method == "GET" && p == "/api/v1/users/me" {
|
||||
return a.currentUser(r, uid)
|
||||
}
|
||||
if strings.HasPrefix(p, "/api/v1/attachments/") {
|
||||
parts := strings.Split(strings.TrimPrefix(p, "/api/v1/attachments/"), "/")
|
||||
if !uuidRE.MatchString(parts[0]) {
|
||||
return response{}, bad("Неверный attachment_id")
|
||||
}
|
||||
if r.Method == "GET" && (len(parts) == 1 || (len(parts) == 2 && parts[1] == "content")) {
|
||||
return a.getAttachment(r, uid, parts[0], len(parts) == 2)
|
||||
}
|
||||
return response{}, missing()
|
||||
}
|
||||
if p == "/api/v1/chats" {
|
||||
if r.Method == "POST" {
|
||||
return a.createChat(r, uid)
|
||||
}
|
||||
if r.Method == "GET" {
|
||||
return a.listChats(r, uid)
|
||||
}
|
||||
}
|
||||
if !strings.HasPrefix(p, "/api/v1/chats/") {
|
||||
return response{}, missing()
|
||||
}
|
||||
parts := strings.Split(strings.TrimPrefix(p, "/api/v1/chats/"), "/")
|
||||
if len(parts) < 1 || !uuidRE.MatchString(parts[0]) {
|
||||
return response{}, bad("Неверный chat_id")
|
||||
}
|
||||
chatID := parts[0]
|
||||
if len(parts) == 2 && parts[1] == "attachments" && r.Method == "POST" {
|
||||
return a.uploadAttachment(r, uid, chatID)
|
||||
}
|
||||
if len(parts) == 1 && r.Method == "GET" {
|
||||
return a.getChat(r, uid, chatID)
|
||||
}
|
||||
if len(parts) == 2 && parts[1] == "messages" {
|
||||
if r.Method == "POST" {
|
||||
return a.createMessage(r, uid, chatID)
|
||||
}
|
||||
if r.Method == "GET" {
|
||||
return a.listMessages(r, uid, chatID)
|
||||
}
|
||||
}
|
||||
return response{}, missing()
|
||||
}
|
||||
|
||||
func (a *API) createUser(r *http.Request) (response, *apiError) {
|
||||
var in struct {
|
||||
Username string `json:"username"`
|
||||
DisplayName string `json:"display_name"`
|
||||
Password string `json:"password"`
|
||||
}
|
||||
if e := decode(r, &in); e != nil {
|
||||
return response{}, e
|
||||
}
|
||||
if !usernameRE.MatchString(in.Username) || !validText(in.DisplayName, 100) || !validPassword(in.Password) {
|
||||
return response{}, bad("Неверные данные пользователя")
|
||||
}
|
||||
passwordHash, err := hashPassword(in.Password)
|
||||
if err != nil {
|
||||
return response{}, &apiError{status: 500, code: "internal_error", message: "Внутренняя ошибка"}
|
||||
}
|
||||
ctx, cancel := a.dbContext(r.Context())
|
||||
defer cancel()
|
||||
var u User
|
||||
var created time.Time
|
||||
err = a.db.QueryRow(ctx, `INSERT INTO users(id,username,display_name,password_hash) VALUES($1,$2,$3,$4) RETURNING id::text,username,display_name,created_at`, newID(), in.Username, in.DisplayName, passwordHash).Scan(&u.ID, &u.Username, &u.DisplayName, &created)
|
||||
if err != nil {
|
||||
var pgErr *pgconn.PgError
|
||||
if errors.As(err, &pgErr) && pgErr.Code == "23505" {
|
||||
return response{}, &apiError{status: 409, code: "conflict", message: "Имя пользователя занято"}
|
||||
}
|
||||
return response{}, unavailable()
|
||||
}
|
||||
u.CreatedAt = formatTime(created)
|
||||
return response{status: 201, body: u}, nil
|
||||
}
|
||||
|
||||
func (a *API) createSession(r *http.Request) (response, *apiError) {
|
||||
if r.ContentLength > 0 {
|
||||
return response{}, bad("Тело запроса не допускается")
|
||||
}
|
||||
h := r.Header.Get("Authorization")
|
||||
if !strings.HasPrefix(h, "Basic ") {
|
||||
return response{}, unauthorized("Basic")
|
||||
}
|
||||
raw, err := base64.StdEncoding.DecodeString(strings.TrimSpace(strings.TrimPrefix(h, "Basic ")))
|
||||
if err != nil {
|
||||
return response{}, unauthorized("Basic")
|
||||
}
|
||||
parts := strings.SplitN(string(raw), ":", 2)
|
||||
if len(parts) != 2 || !usernameRE.MatchString(parts[0]) {
|
||||
return response{}, unauthorized("Basic")
|
||||
}
|
||||
ctx, cancel := a.dbContext(r.Context())
|
||||
defer cancel()
|
||||
u, passwordHash, err := scanUser(a.db.QueryRow(ctx, `SELECT id::text,username,display_name,created_at,password_hash FROM users WHERE username=$1`, parts[0]))
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return response{}, unauthorized("Basic")
|
||||
}
|
||||
if err != nil {
|
||||
return response{}, unavailable()
|
||||
}
|
||||
if !verifyPassword(passwordHash, parts[1]) {
|
||||
return response{}, unauthorized("Basic")
|
||||
}
|
||||
token := newToken()
|
||||
expires := time.Now().UTC().Add(a.ttl)
|
||||
if err = a.sessions.Set(ctx, tokenKey(token), u.ID, a.ttl).Err(); err != nil {
|
||||
return response{}, unavailable()
|
||||
}
|
||||
return response{status: 201, body: Session{Token: token, TokenType: "Bearer", ExpiresAt: formatTime(expires), User: u}}, nil
|
||||
}
|
||||
|
||||
func (a *API) currentUser(r *http.Request, uid string) (response, *apiError) {
|
||||
ctx, cancel := a.dbContext(r.Context())
|
||||
defer cancel()
|
||||
u, _, err := scanUser(a.db.QueryRow(ctx, `SELECT id::text,username,display_name,created_at,password_hash FROM users WHERE id=$1`, uid))
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return response{}, unauthorized("Bearer")
|
||||
}
|
||||
if err != nil {
|
||||
return response{}, unavailable()
|
||||
}
|
||||
return response{status: 200, body: u}, nil
|
||||
}
|
||||
|
||||
func (a *API) createChat(r *http.Request, uid string) (response, *apiError) {
|
||||
var in struct {
|
||||
Title string `json:"title"`
|
||||
MemberIDs []string `json:"member_ids"`
|
||||
}
|
||||
if e := decode(r, &in); e != nil {
|
||||
return response{}, e
|
||||
}
|
||||
if !validText(in.Title, 100) || len(in.MemberIDs) < 1 || len(in.MemberIDs) > 100 {
|
||||
return response{}, bad("Неверные данные чата")
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, id := range in.MemberIDs {
|
||||
if !uuidRE.MatchString(id) || seen[id] {
|
||||
return response{}, bad("Неверные участники чата")
|
||||
}
|
||||
seen[id] = true
|
||||
}
|
||||
if !seen[uid] {
|
||||
return response{}, bad("Создатель должен быть участником")
|
||||
}
|
||||
ctx, cancel := a.dbContext(r.Context())
|
||||
defer cancel()
|
||||
tx, err := a.db.Begin(ctx)
|
||||
if err != nil {
|
||||
return response{}, unavailable()
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
c := Chat{ID: newID(), Title: in.Title, MemberIDs: append([]string{}, in.MemberIDs...)}
|
||||
var created time.Time
|
||||
if err = tx.QueryRow(ctx, `INSERT INTO chats(id,title) VALUES($1,$2) RETURNING created_at`, c.ID, c.Title).Scan(&created); err != nil {
|
||||
return response{}, unavailable()
|
||||
}
|
||||
for _, id := range c.MemberIDs {
|
||||
if _, err = tx.Exec(ctx, `INSERT INTO chat_members(chat_id,user_id) VALUES($1,$2)`, c.ID, id); err != nil {
|
||||
var pgErr *pgconn.PgError
|
||||
if errors.As(err, &pgErr) && pgErr.Code == "23503" {
|
||||
return response{}, bad("Неверные участники чата")
|
||||
}
|
||||
return response{}, unavailable()
|
||||
}
|
||||
}
|
||||
if err = tx.Commit(ctx); err != nil {
|
||||
return response{}, unavailable()
|
||||
}
|
||||
c.CreatedAt = formatTime(created)
|
||||
return response{status: 201, body: c}, nil
|
||||
}
|
||||
|
||||
func scanChat(row pgx.Row) (Chat, time.Time, error) {
|
||||
var c Chat
|
||||
var t time.Time
|
||||
err := row.Scan(&c.ID, &c.Title, &c.MemberIDs, &t)
|
||||
c.CreatedAt = formatTime(t)
|
||||
return c, t, err
|
||||
}
|
||||
func chatSelect() string {
|
||||
return `SELECT c.id::text,c.title,ARRAY(SELECT cm2.user_id::text FROM chat_members cm2 WHERE cm2.chat_id=c.id ORDER BY cm2.user_id),c.created_at FROM chats c`
|
||||
}
|
||||
|
||||
func (a *API) getChat(r *http.Request, uid, chatID string) (response, *apiError) {
|
||||
ctx, cancel := a.dbContext(r.Context())
|
||||
defer cancel()
|
||||
c, _, err := scanChat(a.db.QueryRow(ctx, chatSelect()+` WHERE c.id=$1 AND EXISTS(SELECT 1 FROM chat_members cm WHERE cm.chat_id=c.id AND cm.user_id=$2)`, chatID, uid))
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return response{}, missing()
|
||||
}
|
||||
if err != nil {
|
||||
return response{}, unavailable()
|
||||
}
|
||||
return response{status: 200, body: c}, nil
|
||||
}
|
||||
|
||||
func (a *API) listChats(r *http.Request, uid string) (response, *apiError) {
|
||||
limit, afterTime, afterID, e := parsePage(r, "chats:"+uid)
|
||||
if e != nil {
|
||||
return response{}, e
|
||||
}
|
||||
ctx, cancel := a.dbContext(r.Context())
|
||||
defer cancel()
|
||||
if afterTime != nil {
|
||||
var ok bool
|
||||
err := a.db.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM chats c JOIN chat_members cm ON cm.chat_id=c.id WHERE cm.user_id=$1 AND c.id=$2 AND c.created_at=$3)`, uid, afterID, *afterTime).Scan(&ok)
|
||||
if err != nil {
|
||||
return response{}, unavailable()
|
||||
}
|
||||
if !ok {
|
||||
return response{}, bad("Неверный cursor")
|
||||
}
|
||||
}
|
||||
var rows pgx.Rows
|
||||
var err error
|
||||
base := chatSelect() + ` JOIN chat_members mine ON mine.chat_id=c.id WHERE mine.user_id=$1`
|
||||
if afterTime == nil {
|
||||
rows, err = a.db.Query(ctx, base+` ORDER BY c.created_at,c.id LIMIT $2`, uid, limit+1)
|
||||
} else {
|
||||
rows, err = a.db.Query(ctx, base+` AND (c.created_at,c.id)>($2,$3) ORDER BY c.created_at,c.id LIMIT $4`, uid, *afterTime, afterID, limit+1)
|
||||
}
|
||||
if err != nil {
|
||||
return response{}, unavailable()
|
||||
}
|
||||
defer rows.Close()
|
||||
items := make([]Chat, 0, limit+1)
|
||||
times := make([]time.Time, 0, limit+1)
|
||||
for rows.Next() {
|
||||
c, t, scanErr := scanChat(rows)
|
||||
if scanErr != nil {
|
||||
return response{}, unavailable()
|
||||
}
|
||||
items = append(items, c)
|
||||
times = append(times, t)
|
||||
}
|
||||
if rows.Err() != nil {
|
||||
return response{}, unavailable()
|
||||
}
|
||||
var next any
|
||||
if len(items) > limit {
|
||||
last := items[limit-1]
|
||||
next = cursorFor("chats:"+uid, times[limit-1], last.ID)
|
||||
items = items[:limit]
|
||||
}
|
||||
return response{status: 200, body: map[string]any{"items": items, "next_cursor": next}}, nil
|
||||
}
|
||||
|
||||
func (a *API) createMessage(r *http.Request, uid, chatID string) (response, *apiError) {
|
||||
key := r.Header.Get("Idempotency-Key")
|
||||
if !idempotencyKeyRE.MatchString(key) {
|
||||
return response{}, bad("Требуется корректный Idempotency-Key")
|
||||
}
|
||||
var in struct {
|
||||
Text *string `json:"text"`
|
||||
AttachmentIDs []string `json:"attachment_ids"`
|
||||
}
|
||||
if e := decode(r, &in); e != nil {
|
||||
return response{}, e
|
||||
}
|
||||
if in.Text == nil {
|
||||
return response{}, bad("Требуется поле text")
|
||||
}
|
||||
text := *in.Text
|
||||
if !utf8.ValidString(text) || utf8.RuneCountInString(text) > 4000 || len(in.AttachmentIDs) > 10 || (strings.TrimSpace(text) == "" && len(in.AttachmentIDs) == 0) {
|
||||
return response{}, bad("Текст сообщения должен содержать от 1 до 4000 символов")
|
||||
}
|
||||
if in.AttachmentIDs == nil {
|
||||
in.AttachmentIDs = []string{}
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for i, id := range in.AttachmentIDs {
|
||||
id = strings.ToLower(id)
|
||||
if !uuidRE.MatchString(id) || seen[id] {
|
||||
return response{}, bad("Неверные attachment_ids")
|
||||
}
|
||||
seen[id] = true
|
||||
in.AttachmentIDs[i] = id
|
||||
}
|
||||
ctx, cancel := a.dbContext(r.Context())
|
||||
defer cancel()
|
||||
tx, err := a.db.Begin(ctx)
|
||||
if err != nil {
|
||||
return response{}, unavailable()
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
var member bool
|
||||
if err = tx.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM chat_members WHERE chat_id=$1 AND user_id=$2)`, chatID, uid).Scan(&member); err != nil {
|
||||
return response{}, unavailable()
|
||||
}
|
||||
if !member {
|
||||
return response{}, missing()
|
||||
}
|
||||
requestHash := sha256.Sum256([]byte(text))
|
||||
if len(in.AttachmentIDs) > 0 {
|
||||
canonical, _ := json.Marshal(in)
|
||||
requestHash = sha256.Sum256(canonical)
|
||||
}
|
||||
if _, err = tx.Exec(ctx, `DELETE FROM message_idempotency WHERE user_id=$1 AND method=$2 AND path=$3 AND idempotency_key=$4 AND created_at < clock_timestamp() - ($5 * interval '1 second')`, uid, r.Method, r.URL.Path, key, int64(a.idempotencyTTL/time.Second)); err != nil {
|
||||
return response{}, unavailable()
|
||||
}
|
||||
result, err := tx.Exec(ctx, `INSERT INTO message_idempotency(user_id,method,path,idempotency_key,request_hash) VALUES($1,$2,$3,$4,$5) ON CONFLICT DO NOTHING`, uid, r.Method, r.URL.Path, key, requestHash[:])
|
||||
if err != nil {
|
||||
return response{}, unavailable()
|
||||
}
|
||||
inserted := result.RowsAffected() == 1
|
||||
if !inserted {
|
||||
var storedHash []byte
|
||||
var messageID string
|
||||
if err = tx.QueryRow(ctx, `SELECT request_hash,message_id::text FROM message_idempotency WHERE user_id=$1 AND method=$2 AND path=$3 AND idempotency_key=$4 FOR UPDATE`, uid, r.Method, r.URL.Path, key).Scan(&storedHash, &messageID); err != nil {
|
||||
return response{}, unavailable()
|
||||
}
|
||||
if !bytes.Equal(storedHash, requestHash[:]) {
|
||||
return response{}, &apiError{status: 409, code: "conflict", message: "Idempotency-Key уже использован с другим запросом"}
|
||||
}
|
||||
var m Message
|
||||
var created time.Time
|
||||
if err = tx.QueryRow(ctx, `SELECT id::text,chat_id::text,sender_id::text,text,created_at,attachment_ids::text[] FROM messages WHERE id=$1`, messageID).Scan(&m.ID, &m.ChatID, &m.SenderID, &m.Text, &created, &m.AttachmentIDs); err != nil {
|
||||
return response{}, unavailable()
|
||||
}
|
||||
m.CreatedAt = formatTime(created)
|
||||
if err = tx.Commit(ctx); err != nil {
|
||||
return response{}, unavailable()
|
||||
}
|
||||
return response{status: 201, body: m}, nil
|
||||
}
|
||||
for _, id := range in.AttachmentIDs {
|
||||
var state string
|
||||
err = tx.QueryRow(ctx, `SELECT status FROM attachments WHERE id=$1 AND chat_id=$2`, id, chatID).Scan(&state)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return response{}, missing()
|
||||
}
|
||||
if err != nil {
|
||||
return response{}, unavailable()
|
||||
}
|
||||
if state == "failed" {
|
||||
return response{}, &apiError{status: 409, code: "conflict", message: "Вложение не готово"}
|
||||
}
|
||||
}
|
||||
m := Message{ID: newID(), ChatID: chatID, SenderID: uid, Text: text, AttachmentIDs: in.AttachmentIDs}
|
||||
var created time.Time
|
||||
if err = tx.QueryRow(ctx, `INSERT INTO messages(id,chat_id,sender_id,text,attachment_ids) VALUES($1,$2,$3,$4,$5) RETURNING created_at`, m.ID, m.ChatID, m.SenderID, m.Text, m.AttachmentIDs).Scan(&created); err != nil {
|
||||
return response{}, unavailable()
|
||||
}
|
||||
if _, err = tx.Exec(ctx, `UPDATE message_idempotency SET message_id=$1 WHERE user_id=$2 AND method=$3 AND path=$4 AND idempotency_key=$5`, m.ID, uid, r.Method, r.URL.Path, key); err != nil {
|
||||
return response{}, unavailable()
|
||||
}
|
||||
if err = tx.Commit(ctx); err != nil {
|
||||
return response{}, unavailable()
|
||||
}
|
||||
m.CreatedAt = formatTime(created)
|
||||
return response{status: 201, body: m}, nil
|
||||
}
|
||||
|
||||
func (a *API) listMessages(r *http.Request, uid, chatID string) (response, *apiError) {
|
||||
limit, afterTime, afterID, e := parsePage(r, "messages:"+chatID)
|
||||
if e != nil {
|
||||
return response{}, e
|
||||
}
|
||||
ctx, cancel := a.dbContext(r.Context())
|
||||
defer cancel()
|
||||
var member bool
|
||||
if err := a.db.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM chat_members WHERE chat_id=$1 AND user_id=$2)`, chatID, uid).Scan(&member); err != nil {
|
||||
return response{}, unavailable()
|
||||
}
|
||||
if !member {
|
||||
return response{}, missing()
|
||||
}
|
||||
if afterTime != nil {
|
||||
var ok bool
|
||||
err := a.db.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM messages WHERE chat_id=$1 AND id=$2 AND created_at=$3)`, chatID, afterID, *afterTime).Scan(&ok)
|
||||
if err != nil {
|
||||
return response{}, unavailable()
|
||||
}
|
||||
if !ok {
|
||||
return response{}, bad("Неверный cursor")
|
||||
}
|
||||
}
|
||||
var rows pgx.Rows
|
||||
var err error
|
||||
base := `SELECT id::text,chat_id::text,sender_id::text,text,created_at,attachment_ids::text[] FROM messages WHERE chat_id=$1`
|
||||
if afterTime == nil {
|
||||
rows, err = a.db.Query(ctx, base+` ORDER BY created_at,id LIMIT $2`, chatID, limit+1)
|
||||
} else {
|
||||
rows, err = a.db.Query(ctx, base+` AND (created_at,id)>($2,$3) ORDER BY created_at,id LIMIT $4`, chatID, *afterTime, afterID, limit+1)
|
||||
}
|
||||
if err != nil {
|
||||
return response{}, unavailable()
|
||||
}
|
||||
defer rows.Close()
|
||||
items := make([]Message, 0, limit+1)
|
||||
times := make([]time.Time, 0, limit+1)
|
||||
for rows.Next() {
|
||||
var m Message
|
||||
var t time.Time
|
||||
if rows.Scan(&m.ID, &m.ChatID, &m.SenderID, &m.Text, &t, &m.AttachmentIDs) != nil {
|
||||
return response{}, unavailable()
|
||||
}
|
||||
m.CreatedAt = formatTime(t)
|
||||
items = append(items, m)
|
||||
times = append(times, t)
|
||||
}
|
||||
if rows.Err() != nil {
|
||||
return response{}, unavailable()
|
||||
}
|
||||
var next any
|
||||
if len(items) > limit {
|
||||
last := items[limit-1]
|
||||
next = cursorFor("messages:"+chatID, times[limit-1], last.ID)
|
||||
items = items[:limit]
|
||||
}
|
||||
return response{status: 200, body: map[string]any{"items": items, "next_cursor": next}}, nil
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
package objectstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/url"
|
||||
|
||||
madmin "github.com/minio/madmin-go/v3"
|
||||
)
|
||||
|
||||
// Bootstrap receives root credentials only in the one-shot s3-init process.
|
||||
// The API and cleaner receive a separate user limited to this bucket/prefix.
|
||||
func Bootstrap(ctx context.Context, endpoint, bucket, rootAccess, rootSecret, apiAccess, apiSecret string) error {
|
||||
u, err := url.Parse(endpoint)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
admin, err := madmin.New(u.Host, rootAccess, rootSecret, u.Scheme == "https")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err = admin.AddUser(ctx, apiAccess, apiSecret); err != nil {
|
||||
return err
|
||||
}
|
||||
policy := map[string]any{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": []any{
|
||||
map[string]any{"Effect": "Allow", "Action": []string{"s3:GetBucketLocation", "s3:ListBucket"}, "Resource": []string{"arn:aws:s3:::" + bucket}},
|
||||
map[string]any{"Effect": "Allow", "Action": []string{"s3:GetObject", "s3:PutObject", "s3:DeleteObject"}, "Resource": []string{"arn:aws:s3:::" + bucket + "/originals/*", "arn:aws:s3:::" + bucket + "/staging/*", "arn:aws:s3:::" + bucket + "/variants/*"}},
|
||||
},
|
||||
}
|
||||
encoded, err := json.Marshal(policy)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err = admin.AddCannedPolicy(ctx, "messenger-api", encoded); err != nil {
|
||||
return err
|
||||
}
|
||||
return admin.SetPolicy(ctx, "messenger-api", apiAccess, false)
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
package objectstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
"mtusi-messenger/internal/telemetry"
|
||||
)
|
||||
|
||||
type Store struct {
|
||||
Client *minio.Client
|
||||
Bucket string
|
||||
endpoint string
|
||||
}
|
||||
|
||||
func New(endpoint, region, bucket, access, secret string) (*Store, error) {
|
||||
u, err := url.Parse(endpoint)
|
||||
if err != nil || u.Host == "" || (u.Scheme != "http" && u.Scheme != "https") || u.Path != "" {
|
||||
return nil, fmt.Errorf("invalid S3 endpoint")
|
||||
}
|
||||
transport := &http.Transport{
|
||||
DialContext: (&net.Dialer{Timeout: 2 * time.Second, KeepAlive: 30 * time.Second}).DialContext,
|
||||
MaxIdleConns: 16, MaxIdleConnsPerHost: 8, MaxConnsPerHost: 8,
|
||||
ResponseHeaderTimeout: 10 * time.Second, IdleConnTimeout: 30 * time.Second,
|
||||
TLSHandshakeTimeout: 3 * time.Second,
|
||||
}
|
||||
client, err := minio.New(u.Host, &minio.Options{
|
||||
Creds: credentials.NewStaticV4(access, secret, ""), Secure: u.Scheme == "https",
|
||||
Region: region, Transport: transport, MaxRetries: 1,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &Store{Client: client, Bucket: bucket, endpoint: endpoint}, nil
|
||||
}
|
||||
|
||||
func (s *Store) Ready(ctx context.Context) (err error) {
|
||||
ctx, span := telemetry.Span(ctx, "s3.bucket.exists", trace.SpanKindClient, attribute.String("db.system.name", "s3"))
|
||||
defer func() { telemetry.Result(span, err) }()
|
||||
ok, err := s.Client.BucketExists(ctx, s.Bucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !ok {
|
||||
return fmt.Errorf("S3 bucket missing")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) Put(ctx context.Context, key string, r io.Reader, size int64, kind string) (err error) {
|
||||
ctx, span := telemetry.Span(ctx, "s3.put", trace.SpanKindClient, attribute.String("db.system.name", "s3"))
|
||||
defer func() { telemetry.Result(span, err) }()
|
||||
_, err = s.Client.PutObject(ctx, s.Bucket, key, r, size, minio.PutObjectOptions{
|
||||
ContentType: kind, DisableMultipart: true,
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *Store) Get(ctx context.Context, key string) (object *minio.Object, err error) {
|
||||
ctx, span := telemetry.Span(ctx, "s3.get", trace.SpanKindClient, attribute.String("db.system.name", "s3"))
|
||||
defer func() { telemetry.Result(span, err) }()
|
||||
object, err = s.Client.GetObject(ctx, s.Bucket, key, minio.GetObjectOptions{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err = object.Stat(); err != nil {
|
||||
object.Close()
|
||||
return nil, err
|
||||
}
|
||||
return object, nil
|
||||
}
|
||||
|
||||
func (s *Store) Delete(ctx context.Context, key string) (err error) {
|
||||
ctx, span := telemetry.Span(ctx, "s3.delete", trace.SpanKindClient, attribute.String("db.system.name", "s3"))
|
||||
defer func() { telemetry.Result(span, err) }()
|
||||
return s.Client.RemoveObject(ctx, s.Bucket, key, minio.RemoveObjectOptions{})
|
||||
}
|
||||
|
||||
// Audit performs an actual unauthenticated GET and an out-of-scope operation
|
||||
// with API credentials. Neither check prints credentials or object URLs.
|
||||
func (s *Store) Audit(ctx context.Context, key string) error {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", s.endpoint+"/"+s.Bucket+"/"+key, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
response, err := (&http.Client{Timeout: 5 * time.Second}).Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
response.Body.Close()
|
||||
if response.StatusCode != 403 {
|
||||
return fmt.Errorf("anonymous GET returned %d", response.StatusCode)
|
||||
}
|
||||
err = s.Client.MakeBucket(ctx, "forbidden-audit-bucket", minio.MakeBucketOptions{})
|
||||
if minio.ToErrorResponse(err).Code != "AccessDenied" {
|
||||
return fmt.Errorf("API credentials are not restricted: %v", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,549 @@
|
||||
// Package pipeline implements durable intent, at-least-once delivery and fenced effects.
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"math/rand/v2"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
"github.com/redis/go-redis/v9"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/codes"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
"mtusi-messenger/internal/objectstore"
|
||||
"mtusi-messenger/internal/telemetry"
|
||||
)
|
||||
|
||||
const Stream = "attachment.jobs.v1"
|
||||
const Group = "image-workers"
|
||||
|
||||
type Pipeline struct {
|
||||
DB *pgxpool.Pool
|
||||
Broker *redis.Client
|
||||
Objects *objectstore.Store
|
||||
Timeout time.Duration
|
||||
Retention time.Duration
|
||||
Logger *log.Logger
|
||||
}
|
||||
type Job struct {
|
||||
SchemaVersion int `json:"schema_version"`
|
||||
ID string `json:"job_id"`
|
||||
Type string `json:"type"`
|
||||
AttachmentID string `json:"attachment_id"`
|
||||
Version int `json:"pipeline_version"`
|
||||
Attempt int `json:"attempt"`
|
||||
Created string `json:"created_at"`
|
||||
Traceparent string `json:"traceparent,omitempty"`
|
||||
Tracestate string `json:"tracestate,omitempty"`
|
||||
}
|
||||
type Variant struct {
|
||||
Name string `json:"name"`
|
||||
ContentType string `json:"content_type"`
|
||||
Size int64 `json:"size_bytes"`
|
||||
Width int `json:"width"`
|
||||
Height int `json:"height"`
|
||||
SHA string `json:"sha256"`
|
||||
}
|
||||
|
||||
func (p *Pipeline) Checkpoint(ctx context.Context, point string) {
|
||||
var armed bool
|
||||
err := p.DB.QueryRow(ctx, `UPDATE pipeline_faults SET armed=false,hit_at=clock_timestamp() WHERE point=$1 AND armed RETURNING true`, point).Scan(&armed)
|
||||
if err == nil && armed {
|
||||
p.Logger.Printf("injected crash: %s", point)
|
||||
os.Exit(86)
|
||||
}
|
||||
}
|
||||
func pause(ctx context.Context, d time.Duration) {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case <-time.After(d):
|
||||
}
|
||||
}
|
||||
|
||||
// Dispatch re-publishes unfinished intents if a broker confirmation/connection is lost.
|
||||
// Publication is deliberately outside the PostgreSQL transaction; duplicates are safe.
|
||||
func (p *Pipeline) Dispatch(ctx context.Context) {
|
||||
for ctx.Err() == nil {
|
||||
if err := p.dispatchOnce(ctx); err != nil {
|
||||
p.Logger.Print("dispatcher unavailable; intent retained")
|
||||
}
|
||||
sweepctx, cancelSweep := context.WithTimeout(ctx, 5*time.Second)
|
||||
if err := p.Sweep(sweepctx); err != nil {
|
||||
p.Logger.Print("staging sweep deferred")
|
||||
}
|
||||
cancelSweep()
|
||||
pause(ctx, time.Second)
|
||||
}
|
||||
}
|
||||
func (p *Pipeline) dispatchOnce(ctx context.Context) error {
|
||||
rows, err := p.DB.Query(ctx, `SELECT j.id::text,j.attachment_id::text,j.pipeline_version,j.attempts+1,j.created_at,j.traceparent,j.tracestate
|
||||
FROM attachment_jobs j JOIN attachment_outbox o ON o.job_id=j.id
|
||||
WHERE ((j.state='queued' AND j.next_at<=clock_timestamp()) OR (j.state='processing' AND j.lease_until<clock_timestamp()))
|
||||
AND (o.published_at IS NULL OR o.published_at<clock_timestamp()-interval '5 seconds') ORDER BY j.created_at LIMIT 50`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
jobs := []Job{}
|
||||
for rows.Next() {
|
||||
j := Job{SchemaVersion: 1, Type: "attachment.process.v1"}
|
||||
var created time.Time
|
||||
if err = rows.Scan(&j.ID, &j.AttachmentID, &j.Version, &j.Attempt, &created, &j.Traceparent, &j.Tracestate); err != nil {
|
||||
rows.Close()
|
||||
return err
|
||||
}
|
||||
j.Created = created.UTC().Format(time.RFC3339Nano)
|
||||
jobs = append(jobs, j)
|
||||
}
|
||||
err = rows.Err()
|
||||
rows.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, j := range jobs {
|
||||
p.Checkpoint(ctx, "dispatcher_before_publish")
|
||||
|
||||
parent := telemetry.Extract(ctx, j.Traceparent, j.Tracestate)
|
||||
pubctx, pubspan := telemetry.Span(parent, "attachment.publish", trace.SpanKindProducer, attribute.String("messaging.system", "redis"), attribute.String("messaging.destination.name", Stream), attribute.Int("job.attempt", j.Attempt))
|
||||
j.Traceparent, j.Tracestate = telemetry.Inject(pubctx)
|
||||
b, _ := json.Marshal(j)
|
||||
err = p.Broker.XAdd(pubctx, &redis.XAddArgs{Stream: Stream, Values: map[string]any{"job": string(b)}}).Err()
|
||||
telemetry.Event(pubctx, "job.published", map[string]any{"job_id": j.ID, "attempt": j.Attempt, "success": err == nil})
|
||||
telemetry.Result(pubspan, err)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
p.Checkpoint(ctx, "dispatcher_after_publish")
|
||||
if _, err = p.DB.Exec(ctx, `UPDATE attachment_outbox SET published_at=clock_timestamp(),publications=publications+1 WHERE job_id=$1`, j.ID); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func (p *Pipeline) ack(ctx context.Context, id string) {
|
||||
// Stream belongs to one group. Delete only after acknowledged; PG remains recovery authority.
|
||||
if p.Broker.XAck(ctx, Stream, Group, id).Err() == nil {
|
||||
_ = p.Broker.XDel(ctx, Stream, id).Err()
|
||||
}
|
||||
}
|
||||
func (p *Pipeline) Worker(ctx context.Context, concurrency int) error {
|
||||
if concurrency < 1 || concurrency > 4 {
|
||||
return fmt.Errorf("WORKER_CONCURRENCY must be 1..4")
|
||||
}
|
||||
// One message at a time per consumer: prefetch=1.
|
||||
done := make(chan struct{}, concurrency)
|
||||
for i := 0; i < concurrency; i++ {
|
||||
go func() { defer func() { done <- struct{}{} }(); p.consume(ctx, uuid.NewString()) }()
|
||||
}
|
||||
for i := 0; i < concurrency; i++ {
|
||||
<-done
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func (p *Pipeline) consume(ctx context.Context, consumer string) {
|
||||
reclaimIdle := p.Timeout + 10*time.Second
|
||||
for ctx.Err() == nil {
|
||||
_ = p.Broker.XGroupCreateMkStream(ctx, Stream, Group, "0").Err()
|
||||
pending, _, err := p.Broker.XAutoClaim(ctx, &redis.XAutoClaimArgs{Stream: Stream, Group: Group, Consumer: consumer, MinIdle: reclaimIdle, Start: "0-0", Count: 1}).Result()
|
||||
if err == nil && len(pending) > 0 {
|
||||
p.handle(ctx, pending[0])
|
||||
continue
|
||||
}
|
||||
messages, err := p.Broker.XReadGroup(ctx, &redis.XReadGroupArgs{Group: Group, Consumer: consumer, Streams: []string{Stream, ">"}, Count: 1, Block: time.Second}).Result()
|
||||
if err != nil {
|
||||
pause(ctx, 200*time.Millisecond)
|
||||
continue
|
||||
}
|
||||
for _, stream := range messages {
|
||||
for _, m := range stream.Messages {
|
||||
p.handle(ctx, m)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
func (p *Pipeline) handle(ctx context.Context, m redis.XMessage) {
|
||||
raw, ok := m.Values["job"].(string)
|
||||
var j Job
|
||||
if !ok || json.Unmarshal([]byte(raw), &j) != nil || j.SchemaVersion != 1 || j.Version != 1 || j.Type != "attachment.process.v1" || uuid.Validate(j.ID) != nil || uuid.Validate(j.AttachmentID) != nil {
|
||||
// Malformed envelopes go to a diagnostic dead-letter stream, never interpreted as storage paths.
|
||||
if p.Broker.XAdd(ctx, &redis.XAddArgs{Stream: "attachment.invalid.dlq", Values: map[string]any{"stream_id": m.ID, "error_code": "invalid_envelope"}}).Err() == nil {
|
||||
p.ack(ctx, m.ID)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
ctx = telemetry.Extract(ctx, j.Traceparent, j.Tracestate)
|
||||
ctx, consumer := telemetry.Span(ctx, "attachment.consume", trace.SpanKindConsumer, attribute.String("messaging.system", "redis"), attribute.String("messaging.destination.name", Stream), attribute.String("job.id", j.ID))
|
||||
began := time.Now()
|
||||
attempted := false
|
||||
outcome := "abandoned"
|
||||
defer func() {
|
||||
consumer.SetAttributes(attribute.String("job.outcome", outcome))
|
||||
if outcome == "retry" || outcome == "failed" || outcome == "abandoned" {
|
||||
consumer.SetStatus(codes.Error, outcome)
|
||||
}
|
||||
if attempted {
|
||||
telemetry.Busy.Add(-1)
|
||||
telemetry.Job(ctx, outcome, time.Since(began))
|
||||
}
|
||||
telemetry.Event(ctx, "job.completed", map[string]any{"job_id": j.ID, "outcome": outcome, "duration_ms": time.Since(began).Milliseconds()})
|
||||
consumer.End()
|
||||
}()
|
||||
jobctx, cancel := context.WithTimeout(ctx, p.Timeout)
|
||||
defer cancel()
|
||||
conn, err := p.DB.Acquire(jobctx)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer conn.Release()
|
||||
var locked bool
|
||||
if err = conn.QueryRow(jobctx, `SELECT pg_try_advisory_lock(hashtextextended($1,5))`, j.ID).Scan(&locked); err != nil || !locked {
|
||||
return
|
||||
}
|
||||
defer func() {
|
||||
unlockctx, done := context.WithTimeout(context.Background(), 3*time.Second)
|
||||
defer done()
|
||||
_, e := conn.Exec(unlockctx, `SELECT pg_advisory_unlock(hashtextextended($1,5))`, j.ID)
|
||||
if e != nil {
|
||||
_ = conn.Conn().Close(unlockctx)
|
||||
}
|
||||
}()
|
||||
tx, err := conn.Begin(jobctx)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer tx.Rollback(context.Background())
|
||||
var state, staging, attachment, requestID string
|
||||
var accepted time.Time
|
||||
var attempt, maxAttempts int
|
||||
var due bool
|
||||
err = tx.QueryRow(jobctx, `SELECT state,staging_key,attachment_id::text,attempts,max_attempts,
|
||||
(state='queued' AND next_at<=clock_timestamp()) OR (state='processing' AND lease_until<clock_timestamp())
|
||||
,created_at,request_id FROM attachment_jobs WHERE id=$1 FOR UPDATE`, j.ID).Scan(&state, &staging, &attachment, &attempt, &maxAttempts, &due, &accepted, &requestID)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
p.ack(ctx, m.ID)
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if attachment != j.AttachmentID {
|
||||
return
|
||||
}
|
||||
ctx = telemetry.WithRequestID(ctx, requestID)
|
||||
jobctx = telemetry.WithRequestID(jobctx, requestID)
|
||||
if state == "ready" || state == "failed" {
|
||||
outcome = "duplicate"
|
||||
_ = tx.Rollback(jobctx)
|
||||
p.ack(ctx, m.ID)
|
||||
return
|
||||
}
|
||||
if !due {
|
||||
outcome = "deferred"
|
||||
_ = tx.Rollback(jobctx)
|
||||
p.ack(ctx, m.ID)
|
||||
return
|
||||
}
|
||||
if attempt >= maxAttempts {
|
||||
if p.failTx(jobctx, tx, j.ID, attachment, "retry_exhausted") == nil && tx.Commit(jobctx) == nil {
|
||||
p.ack(ctx, m.ID)
|
||||
}
|
||||
return
|
||||
}
|
||||
token := uuid.NewString()
|
||||
consumer.SetAttributes(attribute.Int("job.attempt", attempt+1))
|
||||
attempt++
|
||||
_, err = tx.Exec(jobctx, `UPDATE attachment_jobs SET state='processing',attempts=$2,lease_token=$3,lease_until=clock_timestamp()+($4*interval '1 second'),updated_at=clock_timestamp() WHERE id=$1`, j.ID, attempt, token, int(p.Timeout/time.Second)+5)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if _, err = tx.Exec(jobctx, `UPDATE attachments SET status='processing',updated_at=clock_timestamp() WHERE id=$1`, attachment); err != nil {
|
||||
return
|
||||
}
|
||||
if tx.Commit(jobctx) != nil {
|
||||
return
|
||||
}
|
||||
attempted = true
|
||||
telemetry.Busy.Add(1)
|
||||
telemetry.Event(jobctx, "job.started", map[string]any{"job_id": j.ID, "attempt": attempt})
|
||||
p.Checkpoint(jobctx, "worker_after_claim")
|
||||
variants, code := p.process(jobctx, attachment, staging)
|
||||
p.Checkpoint(jobctx, "worker_after_objects")
|
||||
// A fresh transaction completes after job timeout/cancel; token fences stale attempts.
|
||||
finishctx, finishcancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
|
||||
defer finishcancel()
|
||||
finish, err := conn.Begin(finishctx)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer finish.Rollback(context.Background())
|
||||
var current string
|
||||
if err = finish.QueryRow(finishctx, `SELECT lease_token::text FROM attachment_jobs WHERE id=$1 AND state='processing' FOR UPDATE`, j.ID).Scan(¤t); err != nil || current != token {
|
||||
return
|
||||
}
|
||||
if code == "" {
|
||||
outcome = "success"
|
||||
encoded, _ := json.Marshal(variants)
|
||||
_, err = finish.Exec(finishctx, `UPDATE attachments SET status='ready',object_key=$2,variants=$3,error_code=NULL,updated_at=clock_timestamp() WHERE id=$1`, attachment, "originals/"+attachment, encoded)
|
||||
if err == nil {
|
||||
_, err = finish.Exec(finishctx, `UPDATE attachment_jobs SET state='ready',lease_until=NULL,lease_token=NULL,error_code=NULL,updated_at=clock_timestamp() WHERE id=$1`, j.ID)
|
||||
}
|
||||
} else if code == "invalid_image" || attempt >= maxAttempts {
|
||||
outcome = "failed"
|
||||
if code != "invalid_image" {
|
||||
code = "retry_exhausted"
|
||||
}
|
||||
err = p.failTx(finishctx, finish, j.ID, attachment, code)
|
||||
} else {
|
||||
outcome = "retry"
|
||||
delay := RetryDelay(attempt)
|
||||
_, err = finish.Exec(finishctx, `UPDATE attachment_jobs SET state='queued',error_code=$2,next_at=clock_timestamp()+($3*interval '1 millisecond'),lease_until=NULL,lease_token=NULL,updated_at=clock_timestamp() WHERE id=$1`, j.ID, code, delay.Milliseconds())
|
||||
if err == nil {
|
||||
_, err = finish.Exec(finishctx, `UPDATE attachments SET status='queued',updated_at=clock_timestamp() WHERE id=$1`, attachment)
|
||||
}
|
||||
}
|
||||
if err != nil || finish.Commit(finishctx) != nil {
|
||||
outcome = "abandoned"
|
||||
return
|
||||
}
|
||||
if outcome == "success" {
|
||||
telemetry.Ready(ctx, time.Since(accepted))
|
||||
}
|
||||
p.Checkpoint(finishctx, "worker_before_ack")
|
||||
p.ack(finishctx, m.ID)
|
||||
}
|
||||
func RetryDelay(attempt int) time.Duration {
|
||||
if attempt > 6 {
|
||||
attempt = 6
|
||||
}
|
||||
base := time.Second * time.Duration(1<<uint(attempt-1))
|
||||
return base + time.Duration(rand.Int64N(int64(base/2)+1))
|
||||
}
|
||||
func (p *Pipeline) failTx(ctx context.Context, tx pgx.Tx, id, attachment, code string) error {
|
||||
_, err := tx.Exec(ctx, `UPDATE attachment_jobs SET state='failed',error_code=$2,lease_until=NULL,lease_token=NULL,updated_at=clock_timestamp() WHERE id=$1`, id, code)
|
||||
if err == nil {
|
||||
_, err = tx.Exec(ctx, `UPDATE attachments SET status='failed',error_code=$2,updated_at=clock_timestamp() WHERE id=$1`, attachment, code)
|
||||
}
|
||||
return err
|
||||
}
|
||||
func (p *Pipeline) process(ctx context.Context, id, key string) ([]Variant, string) {
|
||||
dir, err := os.MkdirTemp("", "job-*")
|
||||
if err != nil {
|
||||
return nil, "storage_unavailable"
|
||||
}
|
||||
defer os.RemoveAll(dir)
|
||||
object, err := p.Objects.Get(ctx, key)
|
||||
if err != nil {
|
||||
return nil, "storage_unavailable"
|
||||
}
|
||||
file, err := os.Create(filepath.Join(dir, "source"))
|
||||
if err != nil {
|
||||
object.Close()
|
||||
return nil, "storage_unavailable"
|
||||
}
|
||||
hash := sha256.New()
|
||||
n, copyErr := io.Copy(io.MultiWriter(file, hash), io.LimitReader(object, 10485761))
|
||||
object.Close()
|
||||
file.Close()
|
||||
if copyErr != nil {
|
||||
return nil, "storage_unavailable"
|
||||
}
|
||||
if n > 10485760 {
|
||||
return nil, "invalid_image"
|
||||
}
|
||||
var expected string
|
||||
if p.DB.QueryRow(ctx, `SELECT sha256 FROM attachments WHERE id=$1`, id).Scan(&expected) != nil {
|
||||
return nil, "storage_unavailable"
|
||||
}
|
||||
if hex.EncodeToString(hash.Sum(nil)) != expected {
|
||||
return nil, "invalid_image"
|
||||
}
|
||||
cmd := exec.CommandContext(ctx, "python3", "/opt/process_image.py", filepath.Join(dir, "source"), dir)
|
||||
_, imagespan := telemetry.Span(ctx, "image.process", trace.SpanKindInternal)
|
||||
output, err := cmd.Output()
|
||||
telemetry.Result(imagespan, err)
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return nil, "processing_timeout"
|
||||
}
|
||||
var exit *exec.ExitError
|
||||
if errors.As(err, &exit) && exit.ExitCode() == 2 {
|
||||
return nil, "invalid_image"
|
||||
}
|
||||
return nil, "processing_failed"
|
||||
}
|
||||
variants := []Variant{}
|
||||
if json.Unmarshal(output, &variants) != nil || len(variants) != 3 {
|
||||
return nil, "processing_failed"
|
||||
}
|
||||
var kind string
|
||||
if p.DB.QueryRow(ctx, `SELECT content_type FROM attachments WHERE id=$1`, id).Scan(&kind) != nil {
|
||||
return nil, "storage_unavailable"
|
||||
}
|
||||
source, err := os.Open(filepath.Join(dir, "source"))
|
||||
if err != nil {
|
||||
return nil, "storage_unavailable"
|
||||
}
|
||||
err = p.Objects.Put(ctx, "originals/"+id, source, n, kind)
|
||||
source.Close()
|
||||
if err != nil {
|
||||
return nil, "storage_unavailable"
|
||||
}
|
||||
for _, v := range variants {
|
||||
f, err := os.Open(filepath.Join(dir, v.Name))
|
||||
if err != nil {
|
||||
return nil, "processing_failed"
|
||||
}
|
||||
err = p.Objects.Put(ctx, "variants/"+id+"/v1/"+v.Name, f, v.Size, v.ContentType)
|
||||
f.Close()
|
||||
if err != nil {
|
||||
return nil, "storage_unavailable"
|
||||
}
|
||||
}
|
||||
return variants, ""
|
||||
}
|
||||
|
||||
// Failed registry is PostgreSQL; explicit replay is audited and refuses successful jobs.
|
||||
func (p *Pipeline) Replay(ctx context.Context, id string) error {
|
||||
if uuid.Validate(id) != nil {
|
||||
return fmt.Errorf("invalid job id")
|
||||
}
|
||||
conn, err := p.DB.Acquire(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer conn.Release()
|
||||
tx, err := conn.Begin(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
var attachment, state, code string
|
||||
var retained bool
|
||||
err = tx.QueryRow(ctx, `SELECT attachment_id::text,state,COALESCE(error_code,''),staging_deleted_at IS NULL FROM attachment_jobs WHERE id=$1 FOR UPDATE`, id).Scan(&attachment, &state, &code, &retained)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if state != "failed" || !retained {
|
||||
return fmt.Errorf("only failed jobs with retained staging can be replayed")
|
||||
}
|
||||
var key string
|
||||
if err = tx.QueryRow(ctx, `SELECT staging_key FROM attachment_jobs WHERE id=$1`, id).Scan(&key); err != nil {
|
||||
return err
|
||||
}
|
||||
object, err := p.Objects.Get(ctx, key)
|
||||
if err != nil {
|
||||
return fmt.Errorf("staging unavailable")
|
||||
}
|
||||
object.Close()
|
||||
if _, err = tx.Exec(ctx, `INSERT INTO attachment_replays(job_id,previous_error) VALUES($1,$2)`, id, code); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err = tx.Exec(ctx, `UPDATE attachment_jobs SET state='queued',attempts=0,error_code=NULL,next_at=clock_timestamp(),lease_until=NULL,lease_token=NULL,updated_at=clock_timestamp() WHERE id=$1`, id); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err = tx.Exec(ctx, `UPDATE attachments SET status='queued',error_code=NULL,updated_at=clock_timestamp() WHERE id=$1`, attachment); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err = tx.Exec(ctx, `UPDATE attachment_outbox SET published_at=NULL WHERE job_id=$1`, id); err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.Commit(ctx)
|
||||
}
|
||||
|
||||
func (p *Pipeline) Sweep(ctx context.Context) error {
|
||||
rows, err := p.DB.Query(ctx, `SELECT id::text,staging_key FROM attachment_jobs WHERE staging_deleted_at IS NULL AND (state='ready' OR (state='failed' AND updated_at<clock_timestamp()-($1*interval '1 second'))) LIMIT 20`, int64(p.Retention/time.Second))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
type item struct{ id, key string }
|
||||
items := []item{}
|
||||
for rows.Next() {
|
||||
var i item
|
||||
if err = rows.Scan(&i.id, &i.key); err != nil {
|
||||
rows.Close()
|
||||
return err
|
||||
}
|
||||
items = append(items, i)
|
||||
}
|
||||
err = rows.Err()
|
||||
rows.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, i := range items {
|
||||
tx, err := p.DB.Begin(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Replay and retention serialize on this row. Never delete a newly queued job.
|
||||
var allowed bool
|
||||
err = tx.QueryRow(ctx, `SELECT state='ready' OR (state='failed' AND updated_at<clock_timestamp()-($2*interval '1 second')) FROM attachment_jobs WHERE id=$1 FOR UPDATE`, i.id, int64(p.Retention/time.Second)).Scan(&allowed)
|
||||
if err != nil || !allowed {
|
||||
_ = tx.Rollback(ctx)
|
||||
continue
|
||||
}
|
||||
err = p.Objects.Delete(ctx, i.key)
|
||||
if err == nil {
|
||||
_, err = tx.Exec(ctx, `UPDATE attachment_jobs SET staging_deleted_at=clock_timestamp() WHERE id=$1`, i.id)
|
||||
}
|
||||
if err != nil {
|
||||
_ = tx.Rollback(ctx)
|
||||
return err
|
||||
}
|
||||
if err = tx.Commit(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Operator commands are available only through docker exec, not through HTTP.
|
||||
func (p *Pipeline) Command(ctx context.Context, args []string) error {
|
||||
if len(args) == 2 && args[0] == "replay" {
|
||||
return p.Replay(ctx, args[1])
|
||||
}
|
||||
if len(args) == 1 && args[0] == "failed-jobs" {
|
||||
rows, err := p.DB.Query(ctx, `SELECT id::text,error_code,attempts FROM attachment_jobs WHERE state='failed' ORDER BY created_at`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var id, code string
|
||||
var attempts int
|
||||
if err = rows.Scan(&id, &code, &attempts); err != nil {
|
||||
return err
|
||||
}
|
||||
_ = json.NewEncoder(os.Stdout).Encode(map[string]any{"job_id": id, "error_code": code, "attempts": attempts})
|
||||
}
|
||||
return rows.Err()
|
||||
}
|
||||
if len(args) == 3 && args[0] == "fault" && (args[1] == "arm" || args[1] == "clear") {
|
||||
allowed := map[string]bool{"dispatcher_before_publish": true, "dispatcher_after_publish": true, "worker_after_claim": true, "worker_after_objects": true, "worker_before_ack": true}
|
||||
if !allowed[args[2]] {
|
||||
return fmt.Errorf("unknown fault point")
|
||||
}
|
||||
if args[1] == "clear" {
|
||||
_, err := p.DB.Exec(ctx, `DELETE FROM pipeline_faults WHERE point=$1`, args[2])
|
||||
return err
|
||||
}
|
||||
_, err := p.DB.Exec(ctx, `INSERT INTO pipeline_faults(point) VALUES($1) ON CONFLICT(point) DO UPDATE SET armed=true,hit_at=NULL`, args[2])
|
||||
return err
|
||||
}
|
||||
return fmt.Errorf("unknown pipeline command: %s", strconv.Itoa(len(args)))
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestRetryDelayBounds(t *testing.T) {
|
||||
for attempt := 1; attempt <= 20; attempt++ {
|
||||
bounded := attempt
|
||||
if bounded > 6 {
|
||||
bounded = 6
|
||||
}
|
||||
base := time.Second * time.Duration(1<<uint(bounded-1))
|
||||
for i := 0; i < 100; i++ {
|
||||
got := RetryDelay(attempt)
|
||||
if got < base || got > base+base/2 {
|
||||
t.Fatalf("attempt %d: delay %s outside [%s,%s]", attempt, got, base, base+base/2)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
package telemetry
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
"github.com/redis/go-redis/v9"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
"net"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Queries are named by operation only. SQL text/args never enter telemetry.
|
||||
type DBTracer struct{}
|
||||
|
||||
func (DBTracer) TraceQueryStart(ctx context.Context, _ *pgx.Conn, d pgx.TraceQueryStartData) context.Context {
|
||||
if !trace.SpanContextFromContext(ctx).IsValid() {
|
||||
return ctx
|
||||
}
|
||||
op := "OTHER"
|
||||
f := strings.Fields(d.SQL)
|
||||
if len(f) > 0 {
|
||||
switch strings.ToUpper(f[0]) {
|
||||
case "SELECT", "INSERT", "UPDATE", "DELETE", "BEGIN", "COMMIT", "ROLLBACK", "CREATE", "ALTER":
|
||||
op = strings.ToUpper(f[0])
|
||||
}
|
||||
}
|
||||
c, _ := Span(ctx, "db."+strings.ToLower(op), trace.SpanKindClient, attribute.String("db.system.name", "postgresql"), attribute.String("db.operation.name", op))
|
||||
return c
|
||||
}
|
||||
func (DBTracer) TraceQueryEnd(ctx context.Context, _ *pgx.Conn, d pgx.TraceQueryEndData) {
|
||||
if trace.SpanContextFromContext(ctx).IsValid() {
|
||||
Result(trace.SpanFromContext(ctx), d.Err)
|
||||
}
|
||||
}
|
||||
func (DBTracer) TraceAcquireStart(ctx context.Context, _ *pgxpool.Pool, _ pgxpool.TraceAcquireStartData) context.Context {
|
||||
Waiting.Add(1)
|
||||
if trace.SpanContextFromContext(ctx).IsValid() {
|
||||
c, _ := Span(ctx, "db.pool.acquire", trace.SpanKindClient, attribute.String("db.system.name", "postgresql"))
|
||||
return c
|
||||
}
|
||||
return ctx
|
||||
}
|
||||
func (DBTracer) TraceAcquireEnd(ctx context.Context, _ *pgxpool.Pool, data pgxpool.TraceAcquireEndData) {
|
||||
Waiting.Add(-1)
|
||||
if trace.SpanContextFromContext(ctx).IsValid() {
|
||||
Result(trace.SpanFromContext(ctx), data.Err)
|
||||
}
|
||||
}
|
||||
|
||||
type RedisHook struct{}
|
||||
|
||||
func (RedisHook) DialHook(next redis.DialHook) redis.DialHook {
|
||||
return func(ctx context.Context, network, addr string) (net.Conn, error) { return next(ctx, network, addr) }
|
||||
}
|
||||
func (RedisHook) ProcessHook(next redis.ProcessHook) redis.ProcessHook {
|
||||
return func(ctx context.Context, cmd redis.Cmder) error {
|
||||
if !trace.SpanContextFromContext(ctx).IsValid() {
|
||||
return next(ctx, cmd)
|
||||
}
|
||||
c, s := Span(ctx, "redis."+cmd.Name(), trace.SpanKindClient, attribute.String("db.system.name", "redis"), attribute.String("db.operation.name", cmd.Name()))
|
||||
err := next(c, cmd)
|
||||
if err == redis.Nil {
|
||||
s.End()
|
||||
} else {
|
||||
Result(s, err)
|
||||
}
|
||||
return err
|
||||
}
|
||||
}
|
||||
func (RedisHook) ProcessPipelineHook(next redis.ProcessPipelineHook) redis.ProcessPipelineHook {
|
||||
return next
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
package telemetry
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
"go.opentelemetry.io/otel"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/metric"
|
||||
"os"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
func Monitor(pool *pgxpool.Pool, role string, capacity int) error {
|
||||
m := otel.Meter(Scope)
|
||||
names := []string{"messenger_db_pool_used", "messenger_db_pool_idle", "messenger_db_pool_waiting", "messenger_db_pool_capacity", "messenger_worker_busy", "messenger_worker_capacity", "messenger_queue_depth", "messenger_queue_oldest_age_seconds", "messenger_db_available", "messenger_process_memory_bytes", "messenger_container_memory_bytes", "messenger_container_memory_limit_bytes", "messenger_telemetry_export_errors", "messenger_process_last_observed_seconds"}
|
||||
gauges := map[string]metric.Int64ObservableGauge{}
|
||||
for _, n := range names {
|
||||
g, err := m.Int64ObservableGauge(n)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
gauges[n] = g
|
||||
}
|
||||
containerCPU, _ := m.Float64ObservableCounter("messenger_container_cpu", metric.WithUnit("s"))
|
||||
cpu, _ := m.Float64ObservableCounter("messenger_process_cpu", metric.WithUnit("s"))
|
||||
acquired, _ := m.Int64ObservableCounter("messenger_db_pool_acquire_wait")
|
||||
duration, _ := m.Float64ObservableCounter("messenger_db_pool_acquire_wait_duration", metric.WithUnit("s"))
|
||||
instruments := []metric.Observable{cpu, containerCPU, acquired, duration}
|
||||
for _, g := range gauges {
|
||||
instruments = append(instruments, g)
|
||||
}
|
||||
_, err := m.RegisterCallback(func(ctx context.Context, o metric.Observer) error {
|
||||
stat := pool.Stat()
|
||||
observeTime := time.Now().Unix()
|
||||
observe := func(n string, value int64) { o.ObserveInt64(gauges[n], value) }
|
||||
observe("messenger_db_pool_used", int64(stat.AcquiredConns()))
|
||||
observe("messenger_db_pool_idle", int64(stat.IdleConns()))
|
||||
observe("messenger_db_pool_waiting", Waiting.Load())
|
||||
observe("messenger_db_pool_capacity", int64(stat.MaxConns()))
|
||||
o.ObserveInt64(acquired, stat.EmptyAcquireCount())
|
||||
o.ObserveFloat64(duration, stat.EmptyAcquireWaitTime().Seconds())
|
||||
observe("messenger_telemetry_export_errors", ExportErrors.Load())
|
||||
observe("messenger_process_last_observed_seconds", observeTime)
|
||||
if role == "worker" {
|
||||
observe("messenger_worker_busy", Busy.Load())
|
||||
observe("messenger_worker_capacity", int64(capacity))
|
||||
}
|
||||
var mem runtime.MemStats
|
||||
runtime.ReadMemStats(&mem)
|
||||
observe("messenger_process_memory_bytes", int64(mem.Sys))
|
||||
for file, name := range map[string]string{"memory.current": "messenger_container_memory_bytes", "memory.max": "messenger_container_memory_limit_bytes"} {
|
||||
if b, e := os.ReadFile("/sys/fs/cgroup/" + file); e == nil {
|
||||
if n, e := strconv.ParseInt(strings.TrimSpace(string(b)), 10, 64); e == nil {
|
||||
observe(name, n)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if raw, e := os.ReadFile("/sys/fs/cgroup/cpu.stat"); e == nil {
|
||||
for _, line := range strings.Split(string(raw), "\n") {
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) == 2 && fields[0] == "usage_usec" {
|
||||
if n, e := strconv.ParseFloat(fields[1], 64); e == nil {
|
||||
o.ObserveFloat64(containerCPU, n/1e6)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
var usage syscall.Rusage
|
||||
if syscall.Getrusage(syscall.RUSAGE_SELF, &usage) == nil {
|
||||
o.ObserveFloat64(cpu, float64(usage.Utime.Sec+usage.Stime.Sec)+float64(usage.Utime.Usec+usage.Stime.Usec)/1e6)
|
||||
}
|
||||
if role == "dispatcher" {
|
||||
c, cancel := context.WithTimeout(ctx, time.Second)
|
||||
defer cancel()
|
||||
var depth, age int64
|
||||
e := pool.QueryRow(c, `SELECT count(*),COALESCE(EXTRACT(EPOCH FROM clock_timestamp()-min(created_at))::bigint,0) FROM attachment_jobs WHERE state='queued'`).Scan(&depth, &age)
|
||||
if e == nil {
|
||||
o.ObserveInt64(gauges["messenger_queue_depth"], depth, metric.WithAttributes(attribute.String("queue", "attachment.jobs.v1")))
|
||||
o.ObserveInt64(gauges["messenger_queue_oldest_age_seconds"], age, metric.WithAttributes(attribute.String("queue", "attachment.jobs.v1")))
|
||||
observe("messenger_db_available", 1)
|
||||
} else {
|
||||
observe("messenger_db_available", 0)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}, instruments...)
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,205 @@
|
||||
// Package telemetry exports bounded, asynchronous OTLP signals without business payloads.
|
||||
package telemetry
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"go.opentelemetry.io/otel"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/codes"
|
||||
"go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp"
|
||||
"go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp"
|
||||
"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp"
|
||||
otellog "go.opentelemetry.io/otel/log"
|
||||
logglobal "go.opentelemetry.io/otel/log/global"
|
||||
"go.opentelemetry.io/otel/metric"
|
||||
"go.opentelemetry.io/otel/propagation"
|
||||
sdklog "go.opentelemetry.io/otel/sdk/log"
|
||||
sdkmetric "go.opentelemetry.io/otel/sdk/metric"
|
||||
"go.opentelemetry.io/otel/sdk/resource"
|
||||
sdktrace "go.opentelemetry.io/otel/sdk/trace"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
)
|
||||
|
||||
const Scope = "mtusi-messenger/telemetry-v1"
|
||||
|
||||
var Service = "messenger-api"
|
||||
var Busy atomic.Int64
|
||||
var Waiting atomic.Int64
|
||||
var ExportErrors atomic.Int64
|
||||
var signals *Signals
|
||||
|
||||
type Signals struct {
|
||||
Requests metric.Int64Counter
|
||||
HTTPDuration metric.Float64Histogram
|
||||
Jobs metric.Int64Counter
|
||||
JobDuration metric.Float64Histogram
|
||||
ReadyDuration metric.Float64Histogram
|
||||
}
|
||||
|
||||
func Start(ctx context.Context, service, instance string) (func(context.Context) error, error) {
|
||||
Service = service
|
||||
otel.SetTextMapPropagator(propagation.TraceContext{})
|
||||
attrs := []attribute.KeyValue{attribute.String("service.name", service), attribute.String("service.version", "lab6-v1"), attribute.String("service.instance.id", instance), attribute.String("deployment.environment.name", "lab")}
|
||||
res := resource.NewWithAttributes("https://opentelemetry.io/schemas/1.37.0", attrs...)
|
||||
traceExp, err := otlptracehttp.New(ctx, otlptracehttp.WithTimeout(2*time.Second), otlptracehttp.WithRetry(otlptracehttp.RetryConfig{Enabled: false}))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
metricExp, err := otlpmetrichttp.New(ctx, otlpmetrichttp.WithTimeout(2*time.Second), otlpmetrichttp.WithRetry(otlpmetrichttp.RetryConfig{Enabled: false}))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
logExp, err := otlploghttp.New(ctx, otlploghttp.WithTimeout(2*time.Second), otlploghttp.WithRetry(otlploghttp.RetryConfig{Enabled: false}))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tp := sdktrace.NewTracerProvider(sdktrace.WithResource(res), sdktrace.WithSampler(sdktrace.ParentBased(sdktrace.AlwaysSample())), sdktrace.WithBatcher(traceExp, sdktrace.WithMaxQueueSize(256), sdktrace.WithMaxExportBatchSize(64), sdktrace.WithBatchTimeout(500*time.Millisecond), sdktrace.WithExportTimeout(2*time.Second)))
|
||||
mp := sdkmetric.NewMeterProvider(sdkmetric.WithResource(res), sdkmetric.WithReader(sdkmetric.NewPeriodicReader(metricExp, sdkmetric.WithInterval(2*time.Second), sdkmetric.WithTimeout(2*time.Second))))
|
||||
lp := sdklog.NewLoggerProvider(sdklog.WithResource(res), sdklog.WithProcessor(sdklog.NewBatchProcessor(logExp, sdklog.WithMaxQueueSize(256), sdklog.WithExportMaxBatchSize(64), sdklog.WithExportInterval(500*time.Millisecond), sdklog.WithExportTimeout(2*time.Second))))
|
||||
otel.SetTracerProvider(tp)
|
||||
otel.SetMeterProvider(mp)
|
||||
logglobal.SetLoggerProvider(lp)
|
||||
// No exporter/network details, payloads or credentials are logged on errors.
|
||||
otel.SetErrorHandler(otel.ErrorHandlerFunc(func(error) { ExportErrors.Add(1) }))
|
||||
meter := otel.Meter(Scope)
|
||||
s := &Signals{}
|
||||
s.Requests, err = meter.Int64Counter("messenger_http_requests")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
buckets := []float64{.005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10, 30, 60, 120}
|
||||
s.HTTPDuration, _ = meter.Float64Histogram("messenger_http_duration", metric.WithUnit("s"), metric.WithExplicitBucketBoundaries(buckets...))
|
||||
s.Jobs, _ = meter.Int64Counter("messenger_jobs_processed")
|
||||
s.JobDuration, _ = meter.Float64Histogram("messenger_job_duration", metric.WithUnit("s"), metric.WithExplicitBucketBoundaries(buckets...))
|
||||
s.ReadyDuration, _ = meter.Float64Histogram("messenger_time_to_ready", metric.WithUnit("s"), metric.WithExplicitBucketBoundaries(.1, .5, 1, 5, 10, 30, 60, 120, 300, 900, 3600, 86400))
|
||||
signals = s
|
||||
return func(c context.Context) error {
|
||||
// Shared deadline bounds shutdown across all signals.
|
||||
a := tp.Shutdown(c)
|
||||
b := mp.Shutdown(c)
|
||||
d := lp.Shutdown(c)
|
||||
if a != nil || b != nil || d != nil {
|
||||
return fmt.Errorf("telemetry shutdown incomplete")
|
||||
}
|
||||
return nil
|
||||
}, nil
|
||||
}
|
||||
func Span(ctx context.Context, name string, kind trace.SpanKind, attrs ...attribute.KeyValue) (context.Context, trace.Span) {
|
||||
return otel.Tracer(Scope).Start(ctx, name, trace.WithSpanKind(kind), trace.WithAttributes(attrs...))
|
||||
}
|
||||
func Result(span trace.Span, err error) {
|
||||
if err != nil {
|
||||
span.SetStatus(codes.Error, "dependency unavailable")
|
||||
span.SetAttributes(attribute.Bool("error", true))
|
||||
}
|
||||
span.End()
|
||||
}
|
||||
func Inject(ctx context.Context) (string, string) {
|
||||
c := propagation.MapCarrier{}
|
||||
otel.GetTextMapPropagator().Inject(ctx, c)
|
||||
return c.Get("traceparent"), c.Get("tracestate")
|
||||
}
|
||||
func Extract(ctx context.Context, parent, state string) context.Context {
|
||||
return otel.GetTextMapPropagator().Extract(ctx, propagation.MapCarrier{"traceparent": parent, "tracestate": state})
|
||||
}
|
||||
|
||||
type requestKey struct{}
|
||||
|
||||
func WithRequestID(ctx context.Context, id string) context.Context {
|
||||
return context.WithValue(ctx, requestKey{}, id)
|
||||
}
|
||||
func RequestID(ctx context.Context) string { v, _ := ctx.Value(requestKey{}).(string); return v }
|
||||
func Event(ctx context.Context, event string, fields map[string]any) {
|
||||
sc := trace.SpanContextFromContext(ctx)
|
||||
body := map[string]any{"timestamp": time.Now().UTC().Format(time.RFC3339Nano), "severity": "INFO", "service.name": Service, "event": event}
|
||||
if sc.IsValid() {
|
||||
body["trace_id"] = sc.TraceID().String()
|
||||
body["span_id"] = sc.SpanID().String()
|
||||
}
|
||||
if id, ok := ctx.Value(requestKey{}).(string); ok {
|
||||
body["request_id"] = id
|
||||
}
|
||||
for k, v := range fields {
|
||||
body[k] = v
|
||||
}
|
||||
severity := otellog.SeverityInfo
|
||||
if status, ok := fields["status"].(int); ok && status >= 500 {
|
||||
severity = otellog.SeverityError
|
||||
}
|
||||
if outcome, ok := fields["outcome"].(string); ok && (outcome == "failed" || outcome == "retry" || outcome == "abandoned") {
|
||||
severity = otellog.SeverityError
|
||||
}
|
||||
if severity == otellog.SeverityError {
|
||||
body["severity"] = "ERROR"
|
||||
}
|
||||
encoded, _ := json.Marshal(body)
|
||||
_, _ = os.Stdout.Write(append(encoded, '\n'))
|
||||
var record otellog.Record
|
||||
record.SetTimestamp(time.Now())
|
||||
record.SetSeverity(severity)
|
||||
record.SetSeverityText(body["severity"].(string))
|
||||
record.SetBody(otellog.StringValue(string(encoded)))
|
||||
record.AddAttributes(otellog.String("event", event))
|
||||
logglobal.GetLoggerProvider().Logger(Scope).Emit(ctx, record)
|
||||
}
|
||||
func HTTP(ctx context.Context, method, route string, status int, elapsed time.Duration) {
|
||||
if signals == nil {
|
||||
return
|
||||
}
|
||||
a := metric.WithAttributes(attribute.String("method", method), attribute.String("route", route), attribute.String("status_class", fmt.Sprintf("%dxx", status/100)))
|
||||
signals.Requests.Add(ctx, 1, a)
|
||||
signals.HTTPDuration.Record(ctx, elapsed.Seconds(), a)
|
||||
}
|
||||
func Job(ctx context.Context, outcome string, d time.Duration) {
|
||||
if signals == nil {
|
||||
return
|
||||
}
|
||||
a := metric.WithAttributes(attribute.String("job_type", "attachment.process.v1"), attribute.String("outcome", outcome))
|
||||
signals.Jobs.Add(ctx, 1, a)
|
||||
signals.JobDuration.Record(ctx, d.Seconds(), a)
|
||||
}
|
||||
func Ready(ctx context.Context, d time.Duration) {
|
||||
if signals != nil {
|
||||
signals.ReadyDuration.Record(ctx, d.Seconds(), metric.WithAttributes(attribute.String("job_type", "attachment.process.v1")))
|
||||
}
|
||||
}
|
||||
func Method(s string) string {
|
||||
switch s {
|
||||
case "GET", "POST", "DELETE", "PUT", "PATCH", "HEAD", "OPTIONS":
|
||||
return s
|
||||
}
|
||||
return "OTHER"
|
||||
}
|
||||
func Route(path string) string {
|
||||
switch path {
|
||||
case "/health/live", "/health/ready", "/api/v1/users", "/api/v1/users/me", "/api/v1/auth/sessions", "/api/v1/auth/sessions/current", "/api/v1/chats":
|
||||
return path
|
||||
}
|
||||
parts := strings.Split(strings.Trim(path, "/"), "/")
|
||||
if len(parts) >= 4 && parts[0] == "api" && parts[1] == "v1" {
|
||||
if parts[2] == "chats" {
|
||||
if len(parts) == 4 {
|
||||
return "/api/v1/chats/{chat_id}"
|
||||
}
|
||||
if len(parts) == 5 && (parts[4] == "messages" || parts[4] == "attachments") {
|
||||
return "/api/v1/chats/{chat_id}/" + parts[4]
|
||||
}
|
||||
}
|
||||
if parts[2] == "attachments" {
|
||||
if len(parts) == 4 {
|
||||
return "/api/v1/attachments/{attachment_id}"
|
||||
}
|
||||
if len(parts) == 5 && parts[4] == "content" {
|
||||
return "/api/v1/attachments/{attachment_id}/content"
|
||||
}
|
||||
}
|
||||
}
|
||||
return "/unmatched"
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
package telemetry
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestRouteLabelsNeverContainResourceIDs(t *testing.T) {
|
||||
ids := []string{"11111111-1111-4111-8111-111111111111", "22222222-2222-4222-8222-222222222222"}
|
||||
for _, id := range ids {
|
||||
cases := map[string]string{
|
||||
"/api/v1/chats/" + id: "/api/v1/chats/{chat_id}",
|
||||
"/api/v1/chats/" + id + "/messages": "/api/v1/chats/{chat_id}/messages",
|
||||
"/api/v1/chats/" + id + "/attachments": "/api/v1/chats/{chat_id}/attachments",
|
||||
"/api/v1/attachments/" + id + "/content": "/api/v1/attachments/{attachment_id}/content",
|
||||
"/arbitrary/" + id: "/unmatched",
|
||||
}
|
||||
for input, want := range cases {
|
||||
got := Route(input)
|
||||
if got != want || strings.Contains(got, id) {
|
||||
t.Fatalf("unsafe route label %q for %q", got, input)
|
||||
}
|
||||
}
|
||||
}
|
||||
if Method("CUSTOM-USER-METHOD") != "OTHER" {
|
||||
t.Fatal("unbounded method label")
|
||||
}
|
||||
}
|
||||
Executable
+28
@@ -0,0 +1,28 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
mkdir -p secrets
|
||||
if [ ! -f secrets/postgres_password ]; then
|
||||
umask 077
|
||||
openssl rand -hex 24 > secrets/postgres_password
|
||||
fi
|
||||
if [ ! -f secrets/redis_password ]; then
|
||||
umask 077
|
||||
openssl rand -hex 24 > secrets/redis_password
|
||||
fi
|
||||
postgres_password=$(cat secrets/postgres_password)
|
||||
redis_password=$(cat secrets/redis_password)
|
||||
umask 077
|
||||
printf 'postgres://messenger:%s@postgres:5432/messenger?sslmode=disable\n' "$postgres_password" > secrets/database_url
|
||||
printf 'redis://:%s@redis:6379/0\n' "$redis_password" > secrets/cache_url
|
||||
if [ ! -f .env ]; then cp .env.example .env; fi
|
||||
for secret_name in s3_admin_access s3_admin_secret s3_access_key_id s3_secret_access_key; do
|
||||
if [ ! -f "secrets/$secret_name" ]; then openssl rand -hex 24 > "secrets/$secret_name"; fi
|
||||
done
|
||||
printf '%s\n' 'S3 credentials ready; bucket and policy are bootstrapped by s3-init.'
|
||||
|
||||
if [ ! -f secrets/broker_password ]; then openssl rand -hex 24 > secrets/broker_password; fi
|
||||
broker_password=$(cat secrets/broker_password)
|
||||
printf 'redis://:%s@broker:6379/0\n' "$broker_password" > secrets/broker_url
|
||||
|
||||
if [ ! -f secrets/grafana_admin_password ]; then openssl rand -hex 24 > secrets/grafana_admin_password; fi
|
||||
Executable
+28
@@ -0,0 +1,28 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
wait_healthy() {
|
||||
service=$1
|
||||
container_id=$(docker compose --env-file .env ps -q "$service")
|
||||
attempts=0
|
||||
until [ "$(docker inspect --format '{{.State.Health.Status}}' "$container_id" 2>/dev/null || true)" = healthy ]; do
|
||||
attempts=$((attempts + 1))
|
||||
if [ "$attempts" -ge 60 ]; then
|
||||
printf '%s\n' "$service did not become healthy" >&2
|
||||
exit 1
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
}
|
||||
|
||||
docker compose --env-file .env restart postgres redis s3 broker >/dev/null
|
||||
wait_healthy postgres
|
||||
wait_healthy redis
|
||||
wait_healthy s3
|
||||
wait_healthy broker
|
||||
docker compose --env-file .env restart api1 api2 cleanup dispatcher worker >/dev/null
|
||||
wait_healthy api1
|
||||
wait_healthy api2
|
||||
docker compose --env-file .env restart nginx otel-collector prometheus loki tempo grafana >/dev/null
|
||||
wait_healthy nginx
|
||||
Executable
+17
@@ -0,0 +1,17 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
docker compose --env-file .env start api1 api2 >/dev/null
|
||||
for service in api1 api2; do
|
||||
container_id=$(docker compose --env-file .env ps -q "$service")
|
||||
attempts=0
|
||||
until [ "$(docker inspect --format '{{.State.Health.Status}}' "$container_id" 2>/dev/null || true)" = healthy ]; do
|
||||
attempts=$((attempts + 1))
|
||||
if [ "$attempts" -ge 60 ]; then
|
||||
printf '%s\n' "$service did not become healthy" >&2
|
||||
exit 1
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
done
|
||||
Executable
+12
@@ -0,0 +1,12 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
|
||||
|
||||
case "$TARGET_INSTANCE_ID" in
|
||||
api-1) service=api1 ;;
|
||||
api-2) service=api2 ;;
|
||||
*) printf '%s\n' "Unknown API instance alias" >&2; exit 2 ;;
|
||||
esac
|
||||
|
||||
docker compose --env-file .env stop --timeout 3 "$service" >/dev/null
|
||||
@@ -0,0 +1,65 @@
|
||||
"""Read stored signals through internal backend APIs; no public admin ports."""
|
||||
import json
|
||||
import time
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
import base64
|
||||
from pipeline_helpers import *
|
||||
|
||||
|
||||
def backend(url, text=False):
|
||||
# Query via Grafana proxy: observation must work even while worker is stopped.
|
||||
target=urllib.parse.urlsplit(url)
|
||||
sources={'prometheus:9090':'prometheus','loki:3100':'loki','tempo:3200':'tempo'}
|
||||
check(target.netloc in sources,'unknown telemetry backend')
|
||||
password=(ROOT/'secrets/grafana_admin_password').read_text().strip()
|
||||
base=os.environ.get('GRAFANA_URL','http://localhost:13000').rstrip('/')
|
||||
address=base+'/api/datasources/proxy/uid/'+sources[target.netloc]+target.path
|
||||
if target.query:address+='?'+target.query
|
||||
auth=base64.b64encode(('admin:'+password).encode()).decode()
|
||||
request=urllib.request.Request(address,headers={'Accept':'application/json','Authorization':'Basic '+auth})
|
||||
try:
|
||||
with urllib.request.urlopen(request,timeout=8) as response:output=response.read().decode()
|
||||
return output if text else json.loads(output)
|
||||
except (urllib.error.URLError,OSError,ValueError):
|
||||
raise Failure('telemetry backend not ready: '+sources[target.netloc]) from None
|
||||
|
||||
|
||||
def prom(expr):
|
||||
result=backend('http://prometheus:9090/api/v1/query?'+urllib.parse.urlencode({'query':expr}))
|
||||
check(result['status']=='success','Prometheus query failed')
|
||||
return result['data']['result']
|
||||
|
||||
|
||||
def number(expr, default=0):
|
||||
values=prom(expr)
|
||||
return float(values[0]['value'][1]) if values else default
|
||||
|
||||
|
||||
def trace_data(trace_id):
|
||||
return backend('http://tempo:3200/api/traces/'+trace_id)
|
||||
|
||||
|
||||
def spans(value):
|
||||
found=[]
|
||||
if isinstance(value,dict):
|
||||
if 'spanId' in value and 'name' in value:found.append(value)
|
||||
for child in value.values():found.extend(spans(child))
|
||||
elif isinstance(value,list):
|
||||
for child in value:found.extend(spans(child))
|
||||
return found
|
||||
|
||||
|
||||
def logs(trace_id=None, service=None):
|
||||
selector='{service_name="'+service+'"}' if service else '{service_name=~"messenger-api|messenger-worker|messenger-dispatcher"}'
|
||||
query=selector+' | json'
|
||||
if trace_id:query+=' | trace_id="'+trace_id+'"'
|
||||
data=backend('http://loki:3100/loki/api/v1/query_range?'+urllib.parse.urlencode({'query':query,'start':str(time.time_ns()-900_000_000_000),'limit':'1000'}))
|
||||
return [json.loads(row[1]) for stream in data['data']['result'] for row in stream['values']]
|
||||
|
||||
|
||||
def alert(name):
|
||||
result=backend('http://prometheus:9090/api/v1/alerts')
|
||||
matches=[a for a in result['data']['alerts'] if a['labels'].get('alertname')==name]
|
||||
return matches[0]['state'] if matches else 'resolved'
|
||||
@@ -0,0 +1,82 @@
|
||||
"""Local synthetic tests; credentials stay in memory and never enter evidence."""
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / 'tests'))
|
||||
from client import Client, Failure, check, run
|
||||
|
||||
|
||||
def compose(*args, data=None):
|
||||
result = subprocess.run(['docker', 'compose', '--env-file', '.env', *args], cwd=ROOT,
|
||||
input=data, capture_output=True, text=True)
|
||||
check(result.returncode == 0, 'compose command failed: ' + ' '.join(args[:3]))
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
def psql(sql):
|
||||
return compose('exec', '-T', 'postgres', 'psql', '-XAtq', '-v', 'ON_ERROR_STOP=1',
|
||||
'-U', 'messenger', '-d', 'messenger', data=sql)
|
||||
|
||||
|
||||
def wait_for(test, message, timeout=100):
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
value = test()
|
||||
if value:
|
||||
return value
|
||||
except Failure:
|
||||
pass
|
||||
time.sleep(.3)
|
||||
raise Failure(message)
|
||||
|
||||
|
||||
def ready(client):
|
||||
wait_for(lambda: client.call('GET', '/health/ready', schema='Health'), 'API not ready')
|
||||
|
||||
|
||||
def upload(client, actor, chat, raw=None):
|
||||
if raw is None:
|
||||
raw = (ROOT / 'tests/fixtures/sample.png').read_bytes()
|
||||
result = client.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=test.png',
|
||||
202, actor=actor, raw=raw, headers={'Content-Type': 'image/png'}, schema='Attachment')['json']
|
||||
check(result['status'] == 'queued', '202 must contain queued snapshot')
|
||||
return result
|
||||
|
||||
|
||||
def state(client, actor, attachment):
|
||||
return client.call('GET', '/api/v1/attachments/' + attachment['id'], actor=actor, schema='Attachment')['json']
|
||||
|
||||
|
||||
def wait_state(client, actor, attachment, wanted='ready', timeout=100):
|
||||
def poll():
|
||||
result = state(client, actor, attachment)
|
||||
if wanted == 'ready':
|
||||
check(result['status'] != 'failed', 'job unexpectedly failed')
|
||||
return result if result['status'] == wanted else None
|
||||
return wait_for(poll, 'attachment did not become ' + wanted, timeout)
|
||||
|
||||
|
||||
def fault(point):
|
||||
compose('exec', '-T', 'api1', '/messenger', 'fault', 'arm', point)
|
||||
|
||||
|
||||
def hit(point):
|
||||
wait_for(lambda: psql("SELECT hit_at IS NOT NULL FROM pipeline_faults WHERE point='" + point + "';") == 't',
|
||||
'fault not reached: ' + point)
|
||||
|
||||
|
||||
def dump_evidence(name, value):
|
||||
(ROOT / 'evidence').mkdir(exist_ok=True)
|
||||
(ROOT / 'evidence' / name).write_text(json.dumps(value, ensure_ascii=False, indent=2) + '\n')
|
||||
|
||||
|
||||
def start_only(service):
|
||||
# Compose start can also start dependencies; fault tests need exactly this container.
|
||||
container = compose('ps', '--all', '-q', service)
|
||||
result = subprocess.run(['docker', 'start', container], capture_output=True, text=True)
|
||||
check(result.returncode == 0, 'cannot start container ' + service)
|
||||
@@ -0,0 +1,129 @@
|
||||
"""Two diagnosed incidents + bounded telemetry outage, using stored observations."""
|
||||
import concurrent.futures
|
||||
import datetime
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import time
|
||||
import uuid
|
||||
from obs_helpers import *
|
||||
|
||||
|
||||
def send_load(client,actor,chat,seconds,workers=2):
|
||||
deadline=time.monotonic()+seconds
|
||||
path='/api/v1/chats/'+chat['id']+'/messages'
|
||||
def task(_):
|
||||
c=Client(client.base,timeout=8);ok=failed=0;durations=[]
|
||||
while time.monotonic()<deadline:
|
||||
begin=time.monotonic()
|
||||
try:
|
||||
c.call('POST',path,201,actor=actor,body={'text':'synthetic load'},headers={'Idempotency-Key':uuid.uuid4().hex})
|
||||
ok+=1;durations.append(time.monotonic()-begin)
|
||||
except Failure:failed+=1
|
||||
return ok,failed,durations
|
||||
start=time.monotonic()
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as pool:rows=list(pool.map(task,range(workers)))
|
||||
elapsed=time.monotonic()-start
|
||||
durations=sorted(d for r in rows for d in r[2]);count=sum(r[0] for r in rows)
|
||||
def percentile(fraction):
|
||||
import math
|
||||
return round(durations[max(0,math.ceil(len(durations)*fraction)-1)]*1000,3) if durations else None
|
||||
return dict(seconds=round(elapsed,3),workers=workers,successful_requests=count,failed_requests=sum(r[1] for r in rows),
|
||||
successful_rps=round(count/elapsed,3),latency_ms_successes={k:percentile(f) for k,f in [('p50',.5),('p95',.95),('p99',.99)]})
|
||||
|
||||
|
||||
def collector_stats():
|
||||
container=compose('ps','-q','otel-collector')
|
||||
r=subprocess.run(['docker','stats','--no-stream','--format','{{json .}}',container],capture_output=True,text=True,check=True)
|
||||
value=json.loads(r.stdout)
|
||||
return {'memory':value['MemUsage'],'cpu':value['CPUPerc']}
|
||||
|
||||
|
||||
def main():
|
||||
client=Client(timeout=8);actor,_,_,chat=client.bootstrap()
|
||||
output={'warmup_load':send_load(client,actor,chat,5)}
|
||||
try:
|
||||
# 1. Queue lag: no worker; HTTP remains usable; stored heartbeat reveals cause.
|
||||
compose('stop','worker');began=time.monotonic()
|
||||
pending=[upload(client,actor,chat) for _ in range(3)]
|
||||
wait_for(lambda:alert('MessengerQueueLag')=='firing','queue lag alert did not fire',60)
|
||||
queue=dict(firing=True,detected_seconds=round(time.monotonic()-began,3),
|
||||
depth=number('max(messenger_queue_depth)'),oldest_age_seconds=number('max(messenger_queue_oldest_age_seconds)'),
|
||||
worker_busy=number('max(messenger_worker_busy)'),
|
||||
worker_heartbeat_age_seconds=number('max(time()-messenger_process_last_observed_seconds{service_name="messenger-worker"})'))
|
||||
check(queue['depth']>=3 and queue['oldest_age_seconds']>10,'queue metrics do not explain lag')
|
||||
check(queue['worker_heartbeat_age_seconds']>10,'worker absence not observable')
|
||||
client.message(actor,chat,'messages continue with worker stopped')
|
||||
start_only('worker')
|
||||
for item in pending:wait_state(client,actor,item)
|
||||
wait_for(lambda:alert('MessengerQueueLag')=='resolved','queue alert did not resolve',40)
|
||||
queue.update(resolved=True,depth_after=number('max(messenger_queue_depth)'))
|
||||
output['worker_stopped']=queue
|
||||
print('PASS: queue lag firing -> resolved, heartbeat stale, backlog drained',flush=True)
|
||||
|
||||
# 2. PostgreSQL unavailable; identify dependency spans, not generic slow handler.
|
||||
compose('stop','postgres');begin=time.monotonic();error_trace=None;errors=0
|
||||
while time.monotonic()-begin<50:
|
||||
response=client.call('POST','/api/v1/chats/'+chat['id']+'/messages',503,actor=actor,
|
||||
body={'text':'synthetic outage'},headers={'Idempotency-Key':uuid.uuid4().hex})
|
||||
error_trace=response['headers']['X-Trace-Id'];errors+=1
|
||||
if alert('MessengerHighErrorRate')=='firing' and time.monotonic()-begin>=15:break
|
||||
check(alert('MessengerHighErrorRate')=='firing','5xx alert did not fire')
|
||||
database=dict(firing=True,detected_seconds=round(time.monotonic()-begin,3),failed_requests=errors,
|
||||
error_ratio=number('messenger:api_error_ratio:rate1m'),
|
||||
db_available=number('min_over_time(messenger_db_available[1m])'),
|
||||
pool_waiting_peak=number('max(max_over_time(messenger_db_pool_waiting[1m]))'),trace_id=error_trace)
|
||||
data=wait_for(lambda:trace_data(error_trace),'DB failure trace not stored',25)
|
||||
children=[s for s in spans(data) if s['name'].startswith('db.')]
|
||||
check(children,'no DB child span explains HTTP failure')
|
||||
check(any(s.get('status',{}).get('code') in (2,'STATUS_CODE_ERROR') for s in children),'DB spans lack error status')
|
||||
check(database['db_available']==0,'DB availability signal did not show outage')
|
||||
dump_evidence('db-incident-trace.json',data)
|
||||
compose('start','postgres');ready(client)
|
||||
# Real recovered traffic, no time-series manipulation.
|
||||
deadline=time.monotonic()+90
|
||||
while alert('MessengerHighErrorRate')!='resolved' and time.monotonic()<deadline:
|
||||
for _ in range(10):client.message(actor,chat,'healthy recovery')
|
||||
time.sleep(.5)
|
||||
check(alert('MessengerHighErrorRate')=='resolved','5xx alert did not resolve')
|
||||
database.update(resolved=True,db_available_after=number('messenger_db_available'))
|
||||
output['postgres_stopped']=database
|
||||
print('PASS: DB error spans + 5xx alert firing -> resolved',flush=True)
|
||||
|
||||
# 3. A telemetry backend down: business writes still work; buffers are finite.
|
||||
before_failed=number('sum(otelcol_exporter_send_failed_spans_total{exporter="otlphttp/tempo"})')
|
||||
before_stats=collector_stats()
|
||||
since=datetime.datetime.now(datetime.timezone.utc).isoformat()
|
||||
compose('stop','tempo')
|
||||
result=send_load(client,actor,chat,20)
|
||||
check(result['successful_requests']>0 and result['failed_requests']==0,'telemetry backend broke business API')
|
||||
# wait beyond bounded retry deadline so final drops can be observed.
|
||||
time.sleep(8)
|
||||
failed=number('sum(otelcol_exporter_send_failed_spans_total{exporter="otlphttp/tempo"})')-before_failed
|
||||
queued=number('max(max_over_time(otelcol_exporter_queue_size{exporter="otlphttp/tempo"}[1m]))')
|
||||
check(failed>0,'Collector failures not observable')
|
||||
check(queued<=32,'Collector exceeded configured buffer')
|
||||
after_stats=collector_stats()
|
||||
raw_logs=compose('logs','--no-log-prefix','--since',since,'otel-collector')
|
||||
dropped=0
|
||||
for line in raw_logs.splitlines():
|
||||
try:record=json.loads(line)
|
||||
except ValueError:continue
|
||||
if record.get('otelcol.signal')=='traces':dropped+=int(record.get('dropped_items',0))
|
||||
check(dropped>0,'actual dropped trace records not counted')
|
||||
output['tempo_stopped']=dict(workload=result,collector_before=before_stats,collector_during=after_stats,
|
||||
failed_span_send_records_delta=failed,max_queue_batches=queued,queue_limit_batches=32,
|
||||
dropped_spans_from_collector_logs=dropped,business_api_preserved=True)
|
||||
compose('start','tempo')
|
||||
wait_for(lambda:backend('http://tempo:3200/ready',text=True),'Tempo did not recover',40)
|
||||
probe=client.call('GET','/api/v1/users/me',actor=actor)
|
||||
wait_for(lambda:trace_data(probe['headers']['X-Trace-Id']),'traces did not resume',40)
|
||||
output['tempo_stopped']['export_resumed']=True
|
||||
print('PASS: telemetry backend outage preserves writes, queues bounded, dropped spans counted',flush=True)
|
||||
dump_evidence('incidents-grade4.json',output)
|
||||
finally:
|
||||
compose('start','postgres','worker','tempo');ready(client)
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
run(main)
|
||||
@@ -0,0 +1,94 @@
|
||||
"""Grade 4: real stored signals, W3C chain with retry, dashboard and privacy."""
|
||||
import base64
|
||||
import hashlib
|
||||
import os
|
||||
import time
|
||||
import uuid
|
||||
from obs_helpers import *
|
||||
|
||||
|
||||
def normalized_id(value):
|
||||
if not value:return ''
|
||||
if all(c in '0123456789abcdef' for c in value.lower()) and len(value) in (16,32):return value.lower()
|
||||
return base64.b64decode(value).hex()
|
||||
|
||||
|
||||
def main():
|
||||
client=Client(timeout=30)
|
||||
actor,_,_,chat=client.bootstrap()
|
||||
marker='PRIVATE-BODY-'+uuid.uuid4().hex
|
||||
client.message(actor,chat,marker)
|
||||
trace_id=uuid.uuid4().hex
|
||||
raw=(ROOT/'tests/fixtures/sample.png').read_bytes()
|
||||
try:
|
||||
compose('stop','worker')
|
||||
response=client.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=private-canary.png',202,
|
||||
actor=actor,raw=raw,headers={'Content-Type':'image/png','traceparent':'00-'+trace_id+'-0123456789abcdef-01','tracestate':'lab=synthetic'},schema='Attachment')
|
||||
item=response['json'];request_id=response['headers']['X-Request-Id']
|
||||
check(response['headers']['X-Trace-Id']==trace_id,'incoming W3C traceparent not honored')
|
||||
compose('stop','s3');start_only('worker')
|
||||
wait_for(lambda: psql("SELECT attempts>=1 AND state='queued' FROM attachment_jobs WHERE id='"+item['id']+"';")=='t','retry not observed',35)
|
||||
compose('start','s3');ready(client)
|
||||
wait_state(client,actor,item)
|
||||
check(client.call('GET','/api/v1/attachments/'+item['id']+'/content',actor=actor)['bytes']==raw,'original changed')
|
||||
def linked():
|
||||
data=trace_data(trace_id);found=spans(data)
|
||||
names={s['name'] for s in found}
|
||||
consumers=[s for s in found if s['name']=='attachment.consume']
|
||||
if len(consumers)>=2 and {'attachment.publish','image.process','s3.put','db.update','db.commit'}.issubset(names):return data
|
||||
return None
|
||||
data=wait_for(linked,'complete upload/publish/retry/consume/result trace missing',60)
|
||||
found=spans(data)
|
||||
publishers={normalized_id(s['spanId']) for s in found if s['name']=='attachment.publish'}
|
||||
consumers=[s for s in found if s['name']=='attachment.consume']
|
||||
check(all(normalized_id(s.get('parentSpanId','')) in publishers for s in consumers),'consumer parent is not producer')
|
||||
correlated=wait_for(lambda: [entry for entry in logs(trace_id) if entry.get('event')=='job.completed' and entry.get('outcome')=='success'], 'worker log missing',40)
|
||||
all_logs=logs(trace_id)
|
||||
events={entry['event'] for entry in all_logs}
|
||||
check({'http.request','job.published','job.started','job.completed'}.issubset(events),'API/publisher/worker logs not correlated')
|
||||
check(any(e.get('request_id')==request_id and e['event']=='http.request' for e in all_logs),'request id correlation missing')
|
||||
check(correlated[0]['request_id']==request_id,'worker lost originating request id')
|
||||
encoded=json.dumps(data)+json.dumps(all_logs)+json.dumps(logs(service='messenger-api'))
|
||||
for secret in (marker,actor['token'],actor['registration']['password'],'private-canary.png'):
|
||||
check(secret not in encoded,'sensitive payload/header/filename entered telemetry')
|
||||
# Actual stored metric names, not assumptions from SDK naming.
|
||||
expressions={
|
||||
'http_requests':'sum(messenger_http_requests_total)',
|
||||
'http_histogram':'sum(messenger_http_duration_seconds_count)',
|
||||
'jobs_success':'sum(messenger_jobs_processed_total{outcome="success"})',
|
||||
'jobs_retry':'sum(messenger_jobs_processed_total{outcome="retry"})',
|
||||
'ready_histogram':'sum(messenger_time_to_ready_seconds_count)',
|
||||
'queue':'max(messenger_queue_depth)',
|
||||
'pool':'max(messenger_db_pool_capacity)',
|
||||
'worker_capacity':'max(messenger_worker_capacity)',
|
||||
'memory':'max(messenger_container_memory_bytes)',
|
||||
}
|
||||
values={}
|
||||
for name,expression in expressions.items():
|
||||
wait_for(lambda: prom(expression),'missing metric '+name,20)
|
||||
values[name]=number(expression)
|
||||
check(values['jobs_success']>0 and values['jobs_retry']>0,'worker outcomes absent')
|
||||
metric_rows=prom('{__name__=~"messenger_.*"}')
|
||||
forbidden={'user_id','chat_id','message_id','attachment_id','request_id','filename','url','path','job_id'}
|
||||
check(not any(forbidden.intersection(row['metric']) for row in metric_rows),'high-cardinality metric labels')
|
||||
password=(ROOT/'secrets/grafana_admin_password').read_text().strip()
|
||||
code='''
|
||||
import base64,json,sys,urllib.request
|
||||
p=sys.stdin.read().strip();req=urllib.request.Request('http://grafana:3000/api/dashboards/uid/mtusi-messenger',headers={'Authorization':'Basic '+base64.b64encode(('admin:'+p).encode()).decode()})
|
||||
with urllib.request.urlopen(req,timeout=10) as r:d=json.load(r)
|
||||
print(json.dumps({'title':d['dashboard']['title'],'panels':len(d['dashboard']['panels'])}))
|
||||
'''
|
||||
dashboard=json.loads(compose('exec','-T','worker','python3','-c',code,data=password))
|
||||
check(dashboard['panels']>=18,'provisioned dashboard missing')
|
||||
dump_evidence('trace-example.json',data)
|
||||
dump_evidence('correlated-logs.json',all_logs)
|
||||
output=dict(status='PASS',trace_id=trace_id,request_id=request_id,consumer_attempt_spans=len(consumers),
|
||||
events=sorted(events),metrics=values,dashboard=dashboard,privacy='PASS',low_cardinality='PASS')
|
||||
dump_evidence('observability-grade4.json',output)
|
||||
print('PASS:',json.dumps(output))
|
||||
finally:
|
||||
compose('start','s3','worker');ready(client)
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
run(main)
|
||||
@@ -0,0 +1,138 @@
|
||||
"""Grade 4: durable acceptance, outbox, crashes, lease, retries, explicit replay."""
|
||||
import hashlib
|
||||
import json
|
||||
import time
|
||||
import uuid
|
||||
from pipeline_helpers import *
|
||||
|
||||
|
||||
def main():
|
||||
client = Client(timeout=30)
|
||||
ready(client)
|
||||
actor, _, _, chat = client.bootstrap()
|
||||
raw = (ROOT / 'tests/fixtures/sample.png').read_bytes()
|
||||
results = {}
|
||||
try:
|
||||
# Worker absent: no resize in API, accepted job survives real restarts.
|
||||
compose('stop', 'worker')
|
||||
attachment = upload(client, actor, chat)
|
||||
client.call('GET', '/api/v1/attachments/' + attachment['id'] + '/content', 409, actor=actor)
|
||||
client.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', 201, actor=actor,
|
||||
body={'text': '', 'attachment_ids': [attachment['id']]},
|
||||
headers={'Idempotency-Key': uuid.uuid4().hex}, schema='Message')
|
||||
services = ('api1', 'api2', 'broker')
|
||||
def starts():
|
||||
values = {}
|
||||
for service in services:
|
||||
container = compose('ps', '-q', service)
|
||||
data = subprocess.run(['docker', 'inspect', container], capture_output=True, text=True, check=True)
|
||||
values[service] = json.loads(data.stdout)[0]['State']['StartedAt']
|
||||
return values
|
||||
before = starts()
|
||||
compose('restart', *services)
|
||||
ready(client)
|
||||
after = starts()
|
||||
check(all(before[s] != after[s] for s in services), 'actual restart not observed')
|
||||
check(state(client, actor, attachment)['status'] == 'queued', 'worker stopped but state changed')
|
||||
start_only('worker')
|
||||
result = wait_state(client, actor, attachment)
|
||||
check({v['name'] for v in result['variants']} == {'thumbnail', 'crop', 'optimized'}, 'missing grade4 variants')
|
||||
check(client.call('GET', '/api/v1/attachments/' + attachment['id'] + '/content', actor=actor)['bytes'] == raw, 'original changed')
|
||||
wait_for(lambda: psql("SELECT staging_deleted_at IS NOT NULL FROM attachment_jobs WHERE id='" + attachment['id'] + "';") == 't', 'staging not cleaned')
|
||||
results['stopped_worker_and_restart'] = dict(status='PASS', before=before, after=after)
|
||||
|
||||
# Committed outbox remains when dispatcher dies before XADD.
|
||||
compose('stop', 'dispatcher')
|
||||
fault('dispatcher_before_publish')
|
||||
item = upload(client, actor, chat)
|
||||
check(psql("SELECT count(*) FROM attachment_outbox WHERE job_id='" + item['id'] + "';") == '1', 'outbox absent')
|
||||
compose('start', 'dispatcher')
|
||||
hit('dispatcher_before_publish')
|
||||
wait_state(client, actor, item)
|
||||
results['commit_publish_gap'] = 'PASS: actual dispatcher exit 86 and automatic restart'
|
||||
|
||||
# Crash after S3 objects, before final PG COMMIT; reclaim expired processing.
|
||||
fault('worker_after_objects')
|
||||
item = upload(client, actor, chat)
|
||||
hit('worker_after_objects')
|
||||
check(psql("SELECT state FROM attachment_jobs WHERE id='" + item['id'] + "';") == 'processing', 'crash did not leave processing')
|
||||
started = time.monotonic()
|
||||
result = wait_state(client, actor, item, timeout=110)
|
||||
check(psql("SELECT attempts FROM attachment_jobs WHERE id='" + item['id'] + "';") == '2', 'expired lease not recovered')
|
||||
check(len(result['variants']) == 3, 'duplicate variant metadata')
|
||||
results['crash_after_objects'] = dict(status='PASS', recovery_seconds=round(time.monotonic()-started, 3), attempts=2)
|
||||
|
||||
# COMMIT ready then death before ack: ready must never regress/reprocess.
|
||||
fault('worker_before_ack')
|
||||
item = upload(client, actor, chat)
|
||||
hit('worker_before_ack')
|
||||
result = wait_state(client, actor, item)
|
||||
attempts = psql("SELECT attempts FROM attachment_jobs WHERE id='" + item['id'] + "';")
|
||||
duplicate = dict(schema_version=1, job_id=item['id'], type='attachment.process.v1', attachment_id=item['id'],
|
||||
pipeline_version=1, attempt=1, created_at=item['created_at'])
|
||||
# Use broker password only inside the container, never in evidence.
|
||||
duplicate_id = compose('exec', '-T', 'broker', 'sh', '-ec',
|
||||
'redis-cli --no-auth-warning -a "$(cat /run/secrets/broker_password)" XADD attachment.jobs.v1 "*" job "$1"',
|
||||
'sh', json.dumps(duplicate))
|
||||
wait_for(lambda: compose('exec', '-T', 'broker', 'sh', '-ec',
|
||||
'redis-cli --raw --no-auth-warning -a "$(cat /run/secrets/broker_password)" XRANGE attachment.jobs.v1 "$1" "$1"',
|
||||
'sh', duplicate_id) == '', 'ready duplicate was not acknowledged')
|
||||
check(state(client, actor, item) == result, 'ready duplicate changed result')
|
||||
check(psql("SELECT attempts FROM attachment_jobs WHERE id='" + item['id'] + "';") == attempts, 'ready duplicate processed again')
|
||||
results['commit_before_ack_and_duplicate'] = 'PASS'
|
||||
|
||||
# Temporary S3 failure: attempt budget/backoff, failed registry, audited replay.
|
||||
compose('stop', 'worker')
|
||||
item = upload(client, actor, chat)
|
||||
psql("UPDATE attachment_jobs SET max_attempts=2 WHERE id='" + item['id'] + "';")
|
||||
compose('stop', 's3')
|
||||
start_only('worker')
|
||||
check(compose('ps', '--status', 'running', '-q', 's3') == '', 'fault test unexpectedly restarted S3')
|
||||
wait_for(lambda: psql("SELECT state FROM attachment_jobs WHERE id='" + item['id'] + "';") == 'failed', 'temporary errors did not exhaust retries', 100)
|
||||
compose('start', 's3')
|
||||
ready(client)
|
||||
failed = state(client, actor, item)
|
||||
check(failed['error_code'] == 'retry_exhausted', 'unsafe or missing final error')
|
||||
client.call('GET', '/api/v1/attachments/' + item['id'] + '/content', 409, actor=actor)
|
||||
client.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', 409, actor=actor,
|
||||
body={'text':'failed', 'attachment_ids':[item['id']]}, headers={'Idempotency-Key':uuid.uuid4().hex})
|
||||
registry = compose('exec', '-T', 'dispatcher', '/messenger', 'failed-jobs')
|
||||
check(item['id'] in registry, 'failed registry missing job')
|
||||
compose('exec', '-T', 'dispatcher', '/messenger', 'replay', item['id'])
|
||||
wait_state(client, actor, item)
|
||||
check(psql("SELECT count(*) FROM attachment_replays WHERE job_id='" + item['id'] + "';") == '1', 'replay audit missing')
|
||||
proc = subprocess.run(['docker','compose','--env-file','.env','exec','-T','dispatcher','/messenger','replay',item['id']], cwd=ROOT, capture_output=True)
|
||||
check(proc.returncode != 0, 'ready replay must be refused')
|
||||
results['retry_failed_registry_replay'] = dict(status='PASS', exhausted_attempts=2, error_code='retry_exhausted', replay_audit_rows=1, ready_replay_rejected=True)
|
||||
|
||||
# Permanent invalid staging: no endless retries, safe machine error.
|
||||
compose('stop', 'worker')
|
||||
item = upload(client, actor, chat)
|
||||
original_key = uuid.uuid4().hex
|
||||
original_body = {'text':'accepted while queued', 'attachment_ids':[item['id']]}
|
||||
original_message = client.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', 201,
|
||||
actor=actor, body=original_body, headers={'Idempotency-Key':original_key}, schema='Message')['json']
|
||||
# Integrity metadata mismatch represents invalid durable source.
|
||||
psql("UPDATE attachments SET sha256=repeat('0',64) WHERE id='" + item['id'] + "';")
|
||||
start_only('worker')
|
||||
failed = wait_state(client, actor, item, 'failed')
|
||||
check(failed['error_code'] == 'invalid_image', 'permanent invalid source must fail safely')
|
||||
check(psql("SELECT attempts FROM attachment_jobs WHERE id='" + item['id'] + "';") == '1', 'permanent error retried')
|
||||
repeated = client.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', 201,
|
||||
actor=actor, body=original_body, headers={'Idempotency-Key':original_key}, schema='Message')['json']
|
||||
check(repeated == original_message, 'committed message replay changed after attachment failed')
|
||||
client.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', 409,
|
||||
actor=actor, body=original_body, headers={'Idempotency-Key':uuid.uuid4().hex})
|
||||
results['permanent_error'] = 'PASS'
|
||||
results['message_replay_after_attachment_failure'] = 'PASS'
|
||||
results['crash_points'] = json.loads(psql("SELECT json_agg(json_build_object('point',point,'armed',armed,'hit_at',hit_at) ORDER BY point) FROM pipeline_faults WHERE hit_at IS NOT NULL;"))
|
||||
dump_evidence('pipeline-grade4.json', results)
|
||||
print('PASS:', json.dumps(results))
|
||||
finally:
|
||||
compose('start', 's3', 'broker', 'dispatcher', 'worker', 'api1', 'api2')
|
||||
psql('DELETE FROM pipeline_faults;')
|
||||
ready(client)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run(main)
|
||||
@@ -0,0 +1,92 @@
|
||||
"""Public persistence/failover hooks plus actual container restart evidence."""
|
||||
import json
|
||||
import datetime
|
||||
import subprocess
|
||||
import uuid
|
||||
from pipeline_helpers import *
|
||||
|
||||
|
||||
def snapshots():
|
||||
services=('postgres','redis','s3','broker','api1','api2','dispatcher','worker','cleanup','nginx','otel-collector','prometheus','loki','tempo','grafana')
|
||||
result={}
|
||||
for service in services:
|
||||
container=compose('ps','-q',service)
|
||||
info=subprocess.run(['docker','inspect',container],capture_output=True,text=True,check=True)
|
||||
data=json.loads(info.stdout)[0]
|
||||
result[service]=dict(id=data['Id'],started_at=data['State']['StartedAt'])
|
||||
return result
|
||||
|
||||
|
||||
def main():
|
||||
client=Client();ready(client)
|
||||
before=snapshots()
|
||||
base=client.base
|
||||
command=['make','test-persistence','ACTION_SCRIPT=scripts/restart.sh','BASE_URL='+base]
|
||||
result=subprocess.run(command,cwd=ROOT,capture_output=True,text=True)
|
||||
check(result.returncode==0,'public persistence failed: '+result.stdout[-500:])
|
||||
after=snapshots()
|
||||
check(all(before[s]['started_at']!=after[s]['started_at'] for s in before),'hook did not restart all required services')
|
||||
|
||||
output={'persistence':dict(status='PASS',before=before,after=after),'failover':[]}
|
||||
public=subprocess.run(['make','test-failover','ACTION_SCRIPT=scripts/stop-one.sh','BASE_URL='+base],cwd=ROOT,capture_output=True,text=True)
|
||||
check(public.returncode==0,'public failover failed: '+public.stdout[-500:])
|
||||
output['public_failover']=dict(status='PASS',output=public.stdout.strip())
|
||||
try:
|
||||
compose('start','api1','api2');ready(client)
|
||||
actor,_,_,chat=client.bootstrap()
|
||||
item=upload(client,actor,chat);wait_state(client,actor,item)
|
||||
raw=(ROOT/'tests/fixtures/sample.png').read_bytes()
|
||||
path='/api/v1/chats/'+chat['id']+'/messages'
|
||||
for target in ('api-1','api-2'):
|
||||
compose('start','api1','api2');ready(client)
|
||||
seen=set()
|
||||
wait_for(lambda: (seen.add(client.call('GET','/health/ready')['headers']['X-Instance-Id']) or len(seen)==2),'two replicas not visible')
|
||||
process=subprocess.Popen([str(ROOT/'scripts/stop-one.sh')],cwd=ROOT,
|
||||
env=dict(os.environ,TARGET_INSTANCE_ID=target),stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
|
||||
began=time.monotonic();deadline=began+8;first=None;confirmed=set();failures=0
|
||||
key=uuid.uuid4().hex
|
||||
while time.monotonic()<deadline:
|
||||
try:
|
||||
response=client.call('POST',path,201,actor=actor,body={'text':key},headers={'Idempotency-Key':key},schema='Message')
|
||||
confirmed.add(response['json']['id'])
|
||||
if process.poll() is not None and response['headers']['X-Instance-Id']!=target and first is None:
|
||||
first=time.monotonic()-began
|
||||
key=uuid.uuid4().hex
|
||||
except Failure:
|
||||
failures+=1
|
||||
time.sleep(.1)
|
||||
check(process.wait(timeout=10)==0,'stop hook failed')
|
||||
check(first is not None and first<=10,'surviving API did not recover within 10 seconds')
|
||||
service={'api-1':'api1','api-2':'api2'}[target]
|
||||
check(not compose('ps','--status','running','-q',service),'target was not actually stopped')
|
||||
page=client.call('GET',path+'?limit=100',actor=actor,schema='MessagePage')['json']
|
||||
visible={m['id'] for m in page['items']}
|
||||
# At most 80 requests per case: walk pages to include both cases.
|
||||
while page['next_cursor']:
|
||||
from urllib.parse import quote
|
||||
page=client.call('GET',path+'?limit=100&cursor='+quote(page['next_cursor']),actor=actor,schema='MessagePage')['json']
|
||||
visible.update(m['id'] for m in page['items'])
|
||||
check(confirmed.issubset(visible),'confirmed messages lost')
|
||||
check(client.call('GET','/api/v1/attachments/'+item['id']+'/content',actor=actor)['bytes']==raw,'original lost on API failover')
|
||||
output['failover'].append(dict(status='PASS',stopped_service=service,confirmed_messages=len(confirmed),
|
||||
transient_failures=failures,first_survivor_seconds=round(first,3),original_preserved=True))
|
||||
finally:
|
||||
compose('start','api1','api2');ready(client)
|
||||
saved=ROOT/'evidence/observability-grade4.json'
|
||||
if saved.exists():
|
||||
from obs_helpers import trace_data, logs, number
|
||||
trace_id=json.loads(saved.read_text())['trace_id']
|
||||
wait_for(lambda:trace_data(trace_id),'stored trace lost after telemetry restart',60)
|
||||
wait_for(lambda:logs(trace_id),'stored logs lost after telemetry restart',60)
|
||||
check(number('sum(messenger_http_requests_total)')>0,'metrics unavailable after restart')
|
||||
stamp=datetime.datetime.fromisoformat(after['prometheus']['started_at'].replace('Z','+00:00')).timestamp()-10
|
||||
historical=number('sum(messenger_http_requests_total @ '+str(stamp)+')')
|
||||
check(historical>0,'historical metrics lost after Prometheus restart')
|
||||
output['metric_history_before_restart']={'unix_time':stamp,'http_counter_sum':historical}
|
||||
output['telemetry_history_preserved']=True
|
||||
dump_evidence('resilience.json',output)
|
||||
print('PASS: all fifteen core/telemetry services restarted, same token/data/original preserved; both API failures covered')
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
run(main)
|
||||
@@ -0,0 +1,100 @@
|
||||
"""Grade 4 image contract: actual decoding, EXIF, alpha, limits and private storage."""
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import struct
|
||||
import uuid
|
||||
import zlib
|
||||
from pipeline_helpers import *
|
||||
|
||||
|
||||
def make_fixture(kind):
|
||||
code = '''
|
||||
import base64,io,sys
|
||||
from PIL import Image
|
||||
kind=sys.argv[1]
|
||||
if kind=='orientation':
|
||||
im=Image.new('RGB',(320,200),'red');exif=Image.Exif();exif[274]=6;exif[315]='synthetic test';kw=dict(format='JPEG',exif=exif)
|
||||
elif kind=='alpha':
|
||||
im=Image.new('RGBA',(8,4),(255,0,0,0));kw=dict(format='PNG')
|
||||
elif kind=='boundary':
|
||||
im=Image.new('RGB',(8000,5000),'blue');kw=dict(format='PNG')
|
||||
else:
|
||||
im=Image.new('RGB',(1601,801),'blue');kw=dict(format='PNG')
|
||||
b=io.BytesIO();im.save(b,**kw);print(base64.b64encode(b.getvalue()).decode())
|
||||
'''
|
||||
return base64.b64decode(compose('exec', '-T', 'worker', 'python3', '-c', code, kind))
|
||||
|
||||
|
||||
def inspect_image(raw):
|
||||
code = '''
|
||||
import base64,io,json,sys
|
||||
from PIL import Image
|
||||
im=Image.open(io.BytesIO(base64.b64decode(sys.stdin.read())));im.load()
|
||||
print(json.dumps(dict(format=im.format,width=im.width,height=im.height,exif=len(im.getexif()),pixel=list(im.convert('RGB').getpixel((0,0))))))
|
||||
'''
|
||||
return json.loads(compose('exec', '-T', 'worker', 'python3', '-c', code, data=base64.b64encode(raw).decode()))
|
||||
|
||||
|
||||
def chunk(kind, payload):
|
||||
return struct.pack('>I', len(payload)) + kind + payload + struct.pack('>I', zlib.crc32(kind+payload) & 0xffffffff)
|
||||
|
||||
|
||||
def main():
|
||||
client=Client(timeout=60)
|
||||
actor, _, outsider, chat=client.bootstrap()
|
||||
results={}
|
||||
for kind in ('orientation','alpha','large','boundary'):
|
||||
raw=make_fixture(kind)
|
||||
content_type='image/jpeg' if kind=='orientation' else 'image/png'
|
||||
item=client.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=synthetic',202,actor=actor,raw=raw,
|
||||
headers={'Content-Type':content_type},schema='Attachment')['json']
|
||||
result=wait_state(client,actor,item)
|
||||
variants={v['name']:v for v in result['variants']}
|
||||
expected={'orientation':(160,256),'alpha':(8,4),'large':(256,128),'boundary':(256,160)}[kind]
|
||||
check((variants['thumbnail']['width'],variants['thumbnail']['height'])==expected,'thumbnail dimensions/orientation')
|
||||
check((variants['crop']['width'],variants['crop']['height'])==(128,128),'crop dimensions')
|
||||
optimized={'orientation':(200,320),'alpha':(8,4),'large':(1280,640),'boundary':(1280,800)}[kind]
|
||||
check((variants['optimized']['width'],variants['optimized']['height'])==optimized,'optimized fit dimensions')
|
||||
for name,meta in variants.items():
|
||||
blob=client.call('GET','/api/v1/attachments/'+item['id']+'/content?variant='+name,actor=actor)['bytes']
|
||||
check(hashlib.sha256(blob).hexdigest()==meta['sha256'] and len(blob)==meta['size_bytes'],'variant checksum/size')
|
||||
decoded=inspect_image(blob)
|
||||
check(decoded['format']==('WEBP' if name=='optimized' else 'JPEG'),'actual variant encoding')
|
||||
check(decoded['exif']==0,'EXIF leaked into variant')
|
||||
if kind=='alpha':
|
||||
check(all(v>=250 for v in decoded['pixel']),'alpha not composited on white')
|
||||
check(client.call('GET','/api/v1/attachments/'+item['id']+'/content',actor=actor)['bytes']==raw,'original changed')
|
||||
client.call('GET','/api/v1/attachments/'+item['id'],404,actor=outsider)
|
||||
client.call('GET','/api/v1/attachments/'+item['id']+'/content?variant=crop',404,actor=outsider)
|
||||
client.call('GET','/api/v1/attachments/'+item['id']+'/content?variant=missing',404,actor=actor)
|
||||
results[kind]={'status':'PASS','variants':variants}
|
||||
original=(ROOT/'tests/fixtures/sample.png').read_bytes()
|
||||
for size in (6294458,10485760):
|
||||
extra=size-len(original)-12
|
||||
raw=original[:-12]+chunk(b'paDd',bytes(extra))+original[-12:]
|
||||
check(len(raw)==size,'fixture exact size')
|
||||
item=upload(client,actor,chat,raw)
|
||||
wait_state(client,actor,item)
|
||||
check(client.call('GET','/api/v1/attachments/'+item['id']+'/content',actor=actor)['bytes']==raw,'large original changed')
|
||||
path='/api/v1/chats/'+chat['id']+'/attachments?filename=bad.png'
|
||||
client.call('POST',path,413,actor=actor,raw=bytes(10485761),headers={'Content-Type':'image/png'})
|
||||
for raw in (original[:40],original[:-12],b'not an image'):
|
||||
client.call('POST',path,415,actor=actor,raw=raw,headers={'Content-Type':'image/png'})
|
||||
huge=b'\x89PNG\r\n\x1a\n'+chunk(b'IHDR',struct.pack('>IIBBBBB',20000,10000,8,2,0,0,0))+chunk(b'IEND',b'')
|
||||
client.call('POST',path,413,actor=actor,raw=huge,headers={'Content-Type':'image/png'})
|
||||
results['byte_limits_corrupt_pixel_limits']='PASS'
|
||||
item=upload(client,actor,chat)
|
||||
wait_state(client,actor,item)
|
||||
audit=compose('exec','-T','api1','/messenger','s3-audit','originals/'+item['id'])
|
||||
check('PASS' in audit,'S3 credential audit failed')
|
||||
results['private_bucket_limited_credentials']='PASS'
|
||||
metrics=json.loads(compose('exec','-T','worker','/messenger','memory'))
|
||||
check(metrics['memory.peak'] < metrics['memory.max'],'worker reached memory limit')
|
||||
results['worker_memory_bytes']=metrics
|
||||
dump_evidence('variants-grade4.json',results)
|
||||
print('PASS: actual JPEG/WebP, EXIF orientation, white alpha, bounds, ACL, checksums, private S3')
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
run(main)
|
||||
@@ -0,0 +1,60 @@
|
||||
"""Bounded subprocess: trusted local paths, no URLs, no credentials."""
|
||||
import hashlib
|
||||
import json
|
||||
import resource
|
||||
import sys
|
||||
import warnings
|
||||
from pathlib import Path
|
||||
from PIL import Image, ImageOps, UnidentifiedImageError
|
||||
|
||||
# Bound decoder subprocess independently of container and Go memory.
|
||||
resource.setrlimit(resource.RLIMIT_AS, (1024 * 1024 * 1024, 1024 * 1024 * 1024))
|
||||
Image.MAX_IMAGE_PIXELS = 40_000_000
|
||||
warnings.simplefilter('error', Image.DecompressionBombWarning)
|
||||
|
||||
|
||||
def contain(image, bound):
|
||||
w, h = image.size
|
||||
if max(w, h) <= bound:
|
||||
return image.copy()
|
||||
if w >= h:
|
||||
size = (bound, max(1, h * bound // w))
|
||||
else:
|
||||
size = (max(1, w * bound // h), bound)
|
||||
return image.resize(size, Image.Resampling.LANCZOS)
|
||||
|
||||
|
||||
def main():
|
||||
source, target = map(Path, sys.argv[1:3])
|
||||
if source.stat().st_size > 10_485_760:
|
||||
raise ValueError('size')
|
||||
with Image.open(source) as original:
|
||||
if original.format not in ('PNG', 'JPEG') or original.width * original.height > 40_000_000:
|
||||
raise ValueError('format or pixels')
|
||||
original.load()
|
||||
oriented = ImageOps.exif_transpose(original)
|
||||
rgba = oriented.convert('RGBA')
|
||||
image = Image.new('RGB', rgba.size, 'white')
|
||||
image.paste(rgba, mask=rgba.getchannel('A'))
|
||||
# New pixel images + empty info ensure EXIF/ICC/text are never propagated.
|
||||
variants = []
|
||||
for name, result, kind, quality in (
|
||||
('thumbnail', contain(image, 256), 'JPEG', 85),
|
||||
('crop', ImageOps.fit(image, (128, 128), method=Image.Resampling.LANCZOS, centering=(.5, .5)), 'JPEG', 85),
|
||||
('optimized', contain(image, 1280), 'WEBP', 80),
|
||||
):
|
||||
result.info.clear()
|
||||
path = target / name
|
||||
result.save(path, format=kind, quality=quality, **({'method': 4} if kind == 'WEBP' else {}))
|
||||
data = path.read_bytes()
|
||||
variants.append(dict(name=name, content_type='image/webp' if kind == 'WEBP' else 'image/jpeg',
|
||||
size_bytes=len(data), width=result.width, height=result.height,
|
||||
sha256=hashlib.sha256(data).hexdigest()))
|
||||
print(json.dumps(variants))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
main()
|
||||
except (UnidentifiedImageError, OSError, ValueError, Image.DecompressionBombError, Image.DecompressionBombWarning):
|
||||
sys.exit(2) # permanent invalid_image; details never reach public API
|
||||
Reference in New Issue
Block a user