10 Commits
83 changed files with 4379 additions and 392 deletions
+9 -3
View File
@@ -16,11 +16,13 @@ jobs:
runs-on: ubuntu-latest
timeout-minutes: 45
env:
COMPOSE_PROJECT_NAME: mtusi2026lab5-${{ github.run_id }}
COMPOSE_PROJECT_NAME: mtusi2026lab6-${{ github.run_id }}
API_BIND_ADDRESS: 127.0.0.1:0
SWAGGER_BIND_ADDRESS: 127.0.0.1:0
PROMETHEUS_BIND_ADDRESS: 127.0.0.1:0
GRAFANA_BIND_ADDRESS: 127.0.0.1:0
LOKI_BIND_ADDRESS: 127.0.0.1:0
TEMPO_BIND_ADDRESS: 127.0.0.1:0
steps:
- name: Checkout exact commit
run: |
@@ -46,7 +48,8 @@ jobs:
- name: Check formatting and shell scripts
run: |
test -z "$(gofmt -l cmd internal migrations)"
sh -n infra/minio/bootstrap.sh scripts/restart.sh scripts/rolling-update.sh scripts/stop-one.sh scripts/test-lab5.sh scripts/test-lab5-acceptance.sh
sh -n infra/minio/bootstrap.sh scripts/restart.sh scripts/rolling-update.sh scripts/stop-one.sh scripts/test-lab5.sh scripts/test-lab5-acceptance.sh scripts/test-lab6.sh scripts/incident-worker.sh scripts/incident-database.sh scripts/incident-telemetry.sh
python3 -m py_compile scripts/lab6-traffic.py scripts/lab6-observe.py scripts/lab6-alert.py scripts/lab6-correlation.py scripts/validate-lab6-evidence.py
- name: Lint
uses: golangci/golangci-lint-action@v7
@@ -62,11 +65,12 @@ jobs:
- name: Prepare local stack
run: |
cp lab5.env.example .env
cp lab6.env.example .env
docker compose config --quiet
docker compose up -d --build --scale api=2 --wait --wait-timeout 150
docker network connect "${COMPOSE_PROJECT_NAME}_application" "$HOSTNAME"
docker network connect "${COMPOSE_PROJECT_NAME}_data" "$HOSTNAME"
docker network connect "${COMPOSE_PROJECT_NAME}_monitoring" "$HOSTNAME"
- name: Integration checks
run: |
@@ -82,6 +86,7 @@ jobs:
docker compose up -d --scale api=2 --wait --wait-timeout 90
make test-lab5-acceptance
make test-lab5
ENV_FILE=.env PROMETHEUS_URL=http://prometheus:9090 TEMPO_URL=http://tempo:3200 LOKI_URL=http://loki:3100 make test-lab6-smoke BASE_URL=http://nginx:8080
- name: Container logs
if: failure()
@@ -92,4 +97,5 @@ jobs:
run: |
docker network disconnect "${COMPOSE_PROJECT_NAME}_application" "$HOSTNAME" 2>/dev/null || true
docker network disconnect "${COMPOSE_PROJECT_NAME}_data" "$HOSTNAME" 2>/dev/null || true
docker network disconnect "${COMPOSE_PROJECT_NAME}_monitoring" "$HOSTNAME" 2>/dev/null || true
if [ -f .env ]; then docker compose --profile tests down -v --remove-orphans; fi
+19 -13
View File
@@ -22,19 +22,21 @@ jobs:
s3_image: ${{ steps.publish.outputs.s3_image }}
s3_bootstrap_image: ${{ steps.publish.outputs.s3_bootstrap_image }}
env:
COMPOSE_PROJECT_NAME: mtusi2026lab5-deploy-${{ github.run_id }}
COMPOSE_PROJECT_NAME: mtusi2026lab6-deploy-${{ github.run_id }}
COMPOSE_FILE: compose.yaml:deploy/compose.images.yaml
API_BIND_ADDRESS: 127.0.0.1:0
SWAGGER_BIND_ADDRESS: 127.0.0.1:0
PROMETHEUS_BIND_ADDRESS: 127.0.0.1:0
GRAFANA_BIND_ADDRESS: 127.0.0.1:0
API_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab5-api:${{ github.sha }}
NGINX_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab5-nginx:${{ github.sha }}
SWAGGER_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab5-swagger:${{ github.sha }}
PROMETHEUS_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab5-prometheus:${{ github.sha }}
GRAFANA_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab5-grafana:${{ github.sha }}
S3_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab5-minio:${{ github.sha }}
S3_BOOTSTRAP_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab5-minio-client:${{ github.sha }}
LOKI_BIND_ADDRESS: 127.0.0.1:0
TEMPO_BIND_ADDRESS: 127.0.0.1:0
API_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab6-api:${{ github.sha }}
NGINX_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab6-nginx:${{ github.sha }}
SWAGGER_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab6-swagger:${{ github.sha }}
PROMETHEUS_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab6-prometheus:${{ github.sha }}
GRAFANA_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab6-grafana:${{ github.sha }}
S3_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab6-minio:${{ github.sha }}
S3_BOOTSTRAP_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab6-minio-client:${{ github.sha }}
steps:
- name: Checkout exact commit
run: |
@@ -60,7 +62,8 @@ jobs:
- name: Check formatting and shell scripts
run: |
test -z "$(gofmt -l cmd internal migrations)"
sh -n infra/minio/bootstrap.sh scripts/restart.sh scripts/rolling-update.sh scripts/stop-one.sh scripts/test-lab5.sh scripts/test-lab5-acceptance.sh
sh -n infra/minio/bootstrap.sh scripts/restart.sh scripts/rolling-update.sh scripts/stop-one.sh scripts/test-lab5.sh scripts/test-lab5-acceptance.sh scripts/test-lab6.sh scripts/incident-worker.sh scripts/incident-database.sh scripts/incident-telemetry.sh
python3 -m py_compile scripts/lab6-traffic.py scripts/lab6-observe.py scripts/lab6-alert.py scripts/lab6-correlation.py scripts/validate-lab6-evidence.py
- name: Lint
uses: golangci/golangci-lint-action@v7
@@ -76,11 +79,12 @@ jobs:
- name: Prepare local stack
run: |
cp lab5.env.example .env
cp lab6.env.example .env
docker compose config --quiet
docker compose up -d --build --scale api=2 --wait --wait-timeout 150
docker network connect "${COMPOSE_PROJECT_NAME}_application" "$HOSTNAME"
docker network connect "${COMPOSE_PROJECT_NAME}_data" "$HOSTNAME"
docker network connect "${COMPOSE_PROJECT_NAME}_monitoring" "$HOSTNAME"
- name: Integration checks
run: |
@@ -95,6 +99,7 @@ jobs:
make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=http://nginx:8080
make test-lab5-acceptance
make test-lab5
ENV_FILE=.env PROMETHEUS_URL=http://prometheus:9090 TEMPO_URL=http://tempo:3200 LOKI_URL=http://loki:3100 make test-lab6-smoke BASE_URL=http://nginx:8080
- name: Publish verified images
id: publish
@@ -132,6 +137,7 @@ jobs:
run: |
docker network disconnect "${COMPOSE_PROJECT_NAME}_application" "$HOSTNAME" 2>/dev/null || true
docker network disconnect "${COMPOSE_PROJECT_NAME}_data" "$HOSTNAME" 2>/dev/null || true
docker network disconnect "${COMPOSE_PROJECT_NAME}_monitoring" "$HOSTNAME" 2>/dev/null || true
if [ -f .env ]; then docker compose --profile tests down -v --remove-orphans; fi
deploy:
@@ -171,17 +177,17 @@ jobs:
run: printf '%s' "$REGISTRY_PASSWORD" | ssh "$DEPLOY_USER@$DEPLOY_HOST" "docker login gitea.privcloud.tech --username '$REGISTRY_USERNAME' --password-stdin"
- name: Upload image manifest
run: scp deploy-images.env "$DEPLOY_USER@$DEPLOY_HOST:/tmp/mtusi2026lab5-${{ github.run_id }}.env"
run: scp deploy-images.env "$DEPLOY_USER@$DEPLOY_HOST:/tmp/mtusi2026lab6-${{ github.run_id }}.env"
- name: Deploy and verify
run: |
ssh "$DEPLOY_USER@$DEPLOY_HOST" "
set -eu
cd ~/mtusi2026lab5
cd ~/mtusi2026lab6
git fetch origin main
git cat-file -e '$DEPLOY_SHA^{commit}'
git reset --hard '$DEPLOY_SHA'
manifest='/tmp/mtusi2026lab5-${{ github.run_id }}.env'
manifest='/tmp/mtusi2026lab6-${{ github.run_id }}.env'
if DEPLOY_GENERATION='${{ github.run_id }}' scripts/rolling-update.sh \"\$manifest\"; then
rm -f \"\$manifest\"
else
+73
View File
@@ -0,0 +1,73 @@
name: Lab 6 incidents
on:
workflow_dispatch:
concurrency:
group: lab6-incidents-${{ github.repository }}-${{ github.ref }}
cancel-in-progress: false
jobs:
incidents:
runs-on: ubuntu-latest
timeout-minutes: 35
env:
COMPOSE_PROJECT_NAME: mtusi2026lab6-incidents-${{ github.run_id }}
API_BIND_ADDRESS: 127.0.0.1:0
SWAGGER_BIND_ADDRESS: 127.0.0.1:0
PROMETHEUS_BIND_ADDRESS: 127.0.0.1:0
GRAFANA_BIND_ADDRESS: 127.0.0.1:0
LOKI_BIND_ADDRESS: 127.0.0.1:0
TEMPO_BIND_ADDRESS: 127.0.0.1:0
steps:
- name: Checkout exact commit
run: |
git init .
git remote add origin https://gitea.privcloud.tech/${{ github.repository }}.git
git fetch --depth=1 origin ${{ github.sha }}
git checkout --detach FETCH_HEAD
- uses: actions/setup-go@v5
with:
go-version-file: go.mod
cache: false
- name: Prepare stack
run: |
cp lab6.env.example .env
docker compose config --quiet
docker compose up -d --build --scale api=2 --wait --wait-timeout 180
docker network connect "${COMPOSE_PROJECT_NAME}_application" "$HOSTNAME"
docker network connect "${COMPOSE_PROJECT_NAME}_data" "$HOSTNAME"
docker network connect "${COMPOSE_PROJECT_NAME}_monitoring" "$HOSTNAME"
- name: Reproduce SLO incidents
run: |
ENV_FILE=.env \
BASE_URL=http://nginx:8080 \
PROMETHEUS_URL=http://prometheus:9090 \
TEMPO_URL=http://tempo:3200 \
LOKI_URL=http://loki:3100 \
EVIDENCE_DIR=evidence/runtime/ci \
make test-lab6
- name: Upload sanitized evidence
if: always()
uses: actions/upload-artifact@v4
with:
name: lab6-incidents-${{ github.sha }}
path: evidence/runtime/ci
if-no-files-found: warn
retention-days: 14
- name: Container logs
if: failure()
run: docker compose logs --no-color --tail=200
- name: Stop local stack
if: always()
run: |
docker network disconnect "${COMPOSE_PROJECT_NAME}_application" "$HOSTNAME" 2>/dev/null || true
docker network disconnect "${COMPOSE_PROJECT_NAME}_data" "$HOSTNAME" 2>/dev/null || true
docker network disconnect "${COMPOSE_PROJECT_NAME}_monitoring" "$HOSTNAME" 2>/dev/null || true
docker compose down -v --remove-orphans
+2
View File
@@ -30,3 +30,5 @@ coverage/
.vscode/
# Keep small, sanitized evidence summaries in evidence/ under version control.
# Inspect git diff --cached: ignore rules are not a secret scanner.
evidence/runtime/
+2 -1
View File
@@ -20,8 +20,9 @@ ENV SWAGGER_JSON=/spec/openapi.json
FROM prom/prometheus:v3.13.3 AS prometheus
COPY monitoring/prometheus.yml /etc/prometheus/prometheus.yml
COPY monitoring/rules /etc/prometheus/rules
FROM grafana/grafana:13.1.6 AS grafana
FROM grafana/grafana:13.2.4 AS grafana
COPY monitoring/grafana/provisioning /etc/grafana/provisioning
COPY monitoring/grafana/dashboards /etc/grafana/dashboards
+10 -1
View File
@@ -4,7 +4,7 @@ CA_FILE ?=
ACTION_SCRIPT ?=
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
.PHONY: help check validate test load test-persistence test-failover test-lab5 test-lab5-acceptance
.PHONY: help check validate test load test-persistence test-failover test-lab5 test-lab5-acceptance lab6-load test-lab6-smoke test-lab6
help:
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
@@ -34,3 +34,12 @@ test-lab5:
test-lab5-acceptance:
./scripts/test-lab5-acceptance.sh
lab6-load:
$(PYTHON) scripts/lab6-traffic.py --base-url "$(BASE_URL)" --mode mixed --duration 60 --concurrency 4 --wait-ready
test-lab6-smoke:
BASE_URL="$(BASE_URL)" ./scripts/test-lab6.sh smoke
test-lab6:
BASE_URL="$(BASE_URL)" ./scripts/test-lab6.sh full
+243 -20
View File
@@ -1,37 +1,260 @@
# Отчёт — лабораторная 6
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
- Автор, группа: Пластинин Денис, БВТ2303
- Целевая оценка: 5
- Выбранный трек: 2 — SLO и обнаружение инцидентов
## Запуск из чистого clone
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
Требуются Docker Desktop с Compose v2, Go из `go.mod`, Python 3.9+ и `curl`. Все образы закреплены в Compose или Dockerfile. Основной observability stack: OpenTelemetry Collector `0.160.0`, Prometheus `3.13.3`, Loki `3.7.9`, Tempo `2.10.10`, Grafana `13.2.4`.
## Архитектура и решения
```bash
cp lab6.env.example .env
docker compose --env-file .env config --quiet
docker compose --env-file .env up -d --build --scale api=2 --wait --wait-timeout 180
make test
make test-lab5-acceptance
make test-lab5
make test-lab6-smoke
```
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
Интерфейсы локального стенда:
- API: `http://127.0.0.1:8080`;
- Swagger: `http://127.0.0.1:8081`;
- Grafana: `http://127.0.0.1:3000`;
- Prometheus: `http://127.0.0.1:9090`;
- Loki API: `http://127.0.0.1:3100`;
- Tempo API: `http://127.0.0.1:3200`.
Безопасная остановка сохраняет volumes:
```bash
docker compose --env-file .env down
```
Намеренное удаление всех учебных данных выполняется отдельно:
```bash
docker compose --env-file .env down -v
```
## Архитектура
Возможности Lab 5 сохранены: две API-реплики за Nginx, PostgreSQL как источник бизнес-состояния, Valkey для сессий, MinIO/S3 для изображений, transactional outbox, Redis Streams и отдельный worker. Lab 6 добавляет единый telemetry pipeline.
```text
Client → Nginx → API replicas → PostgreSQL / Valkey / MinIO
│
PostgreSQL outbox → Dispatcher → Redis Streams → Worker → PostgreSQL / MinIO
│ │
└──────── OTLP ──────────┤
API ────────────────────────────── OTLP ──────────┤
▼
OpenTelemetry Collector
├─ metrics → Prometheus
├─ logs → Loki
└─ traces → Tempo
│
▼
Grafana
```
Приложения экспортируют OTLP только в Collector. Они не знают адреса Loki и Tempo. Collector выполняет `memory_limiter` до `batch`, ограничивает очереди exporter и прекращает retry через 30 секунд. Evidence проверяет queue size/capacity, failures и drops отдельно для `otlp_grpc/tempo` и `otlp_http/loki`, а память — по пиковому RSS за двухминутное окно. Недоступность telemetry backend поэтому не блокирует бизнес-запросы и не вызывает неограниченный рост памяти.
Prometheus, Loki, Tempo и Grafana используют отдельные volumes. Retention локальных logs/traces — 7 суток; Prometheus хранит 7 суток. Collector является транспортом и процессором, а не долговременным хранилищем.
### Resource attributes и среды
Каждый процесс объявляет:
```text
service.name = messenger-api | messenger-dispatcher | messenger-worker
service.version = значение SERVICE_VERSION
deployment.environment.name = lab
service.instance.id = стабильный ID процесса
```
В учебном Compose одна среда `lab`. Атрибут среды всё равно передаётся: один и тот же dashboard и TraceQL остаются пригодны для `test`/`production`, если стенд будет развёрнут там позднее. Среда и service name имеют низкую кардинальность.
### Middleware и сквозной контекст
HTTP middleware создаёт server span, нормализует route template, считает request counter/histogram и добавляет в JSON-лог `request_id`, `trace_id` и `span_id`. Реальный UUID из URL не становится label.
При upload текущий W3C context сохраняется в outbox envelope как `traceparent`/`tracestate`. Dispatcher создаёт producer span. Worker извлекает context и создаёт consumer span. Поэтому один trace показывает цепочку:
```text
HTTP upload → staging S3 → PostgreSQL/outbox → Redis publish
→ Redis consume → image transform → S3 variants → PostgreSQL ready
```
Идентификаторы пользователя, чата, сообщения, вложения, filename, raw URL и request ID запрещены в metric labels. Ограниченные технические ID допустимы в logs/spans синтетического стенда для расследования. `Authorization`, session token, пароль, тело сообщения, байты изображения, S3 credentials и signed URL не экспортируются.
## Хранилища и назначение сигналов
| Сигнал | Хранилище | На какой вопрос отвечает |
| --- | --- | --- |
| Метрика | Prometheus TSDB | Есть ли массовая деградация, когда началась и каков масштаб |
| Лог | Loki | Какое событие и ошибка произошли в конкретном процессе |
| Trace | Tempo | На каком участке сквозного запроса потрачено время или возникла ошибка |
| Dashboard / Explore | Grafana | Единый поиск, корреляция и переходы между тремя сигналами |
Grafana provisioning создаёт Prometheus, Loki и Tempo без ручной настройки. Tempo datasource включает поиск TraceQL, node graph, service map, переходы trace → logs и trace → metrics. Loki derived field открывает trace по `trace_id`; Prometheus exemplars ведут из histogram в Tempo.
## Карта сигналов
| Контрактный сигнал | Реализация и единица |
| --- | --- |
| HTTP requests | `messenger_http_requests_total{method,route,status_class}`, requests |
| HTTP duration | `messenger_http_request_duration_seconds_bucket`, seconds |
| Delivery outcomes | `messenger_attachment_worker_jobs_total{job_type,result}`, deliveries |
| Terminal job transitions | `messenger_attachment_terminal_transitions_total{status}`, first durable outcome per job |
| Job duration | `messenger_attachment_worker_job_duration_seconds_bucket`, seconds |
| Time to ready | `messenger_attachment_time_to_ready_seconds_bucket`, seconds |
| Queue states | `messenger_attachment_jobs_count{status}`, jobs |
| Oldest job | `messenger_attachment_jobs_oldest_age_seconds{status}`, seconds |
| Worker saturation | `messenger_attachment_worker_jobs_in_flight` / `messenger_attachment_worker_capacity`, jobs |
| Outbox | `messenger_outbox_pending`, events |
| Logs | OTLP log records: timestamp, severity, service, event, trace/span IDs |
| Traces | HTTP server, producer, consumer, PostgreSQL, Redis and S3 spans |
p50/p95/p99 вычисляются `histogram_quantile` из суммы bucket всех реплик. Перцентили реплик не усредняются. Queue depth и oldest age показываются отдельно: количество элементов не измеряет длительность ожидания.
## SLO
### Доступность API
SLI — доля бизнес-запросов без `5xx`. Health и `/metrics` исключены. Ожидаемые клиентские `4xx` не считаются отказом сервиса.
```text
bad = business responses 5xx
total = all business responses
SLI = 1 - bad / total
SLO = 99.5% за 30 дней
error budget = 0.5%
```
Цель 99,5% выбрана для учебного single-host стенда без multi-zone HA: она требует реагировать на деградацию, но не заявляет недостижимую для локального Compose доступность.
### Время готовности вложений
SLI — доля уникальных durable attachment jobs, чей первый terminal outcome равен `ready` и наступил не позднее 30 секунд после acceptance. Время acceptance назначает PostgreSQL при создании job, поэтому начало и конец интервала принадлежат одному clock domain. PostgreSQL записывает первый outcome в append-only ledger в той же транзакции, что и переход job. Миграция переносит уже завершённые jobs с `accepted_at = created_at` и `terminal_at = updated_at`. Повторная delivery и ручной replay не меняют уже учтённый результат: job, впервые завершившаяся `failed`, остаётся bad event даже после успешного replay.
```text
bad = failed + ready later than 30s
total = ready + failed
SLI = 1 - bad / total
SLO = 99% за 30 дней
error budget = 1%
```
`messenger_attachment_terminal_transitions_total` и cumulative buckets `messenger_attachment_time_to_ready_seconds_bucket` экспортируются из ledger. Bucket `le="+Inf"` является durable количеством первых outcomes `ready` и используется как знаменатель ready jobs.
30 секунд отделяют нормальную асинхронную обработку от видимой пользователю задержки и существенно превышают обычное время небольшого синтетического изображения.
### Multi-window burn-rate alerts
Для обоих SLO используются две пары окон:
| Alert | Burn rate | Короткое окно | Длинное окно | Назначение |
| --- | ---: | ---: | ---: | --- |
| Fast | 14,4× | 5 минут | 1 час | быстро обнаружить сильный инцидент |
| Slow | 6× | 30 минут | 6 часов | обнаружить устойчивую деградацию |
Условие должно выполняться в обоих окнах пары. Короткое окно быстро реагирует, длинное отсекает одиночный шум. Отдельные operational alerts контролируют отсутствие worker, рост oldest queued age и Redis Stream lag.
## Dashboard и поиск
Dashboard **Messenger Lab 6 overview** содержит:
- API RPS, error ratio, p50/p95/p99;
- состояния API/worker/dispatcher и DB pool;
- accepted/ready/failed jobs, processing duration и end-to-end ready time;
- queue depth, oldest age, Redis lag/PEL, outbox pending;
- worker busy/capacity;
- SLO и firing alerts;
- Collector export failures и переходы к Explore.
Поиск trace выполняется в Grafana Explore → Tempo по `service.name`, span name, route, status и duration. Из trace доступен переход в Loki по тому же `trace_id`. Пошаговые процедуры:
- `monitoring/runbooks/api-availability.md`;
- `monitoring/runbooks/attachment-pipeline.md`.
## Доказательства критериев
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| Критерий | Команда/сценарий | Проверяемый результат | Доказательство |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
| Offline contract | `make check` | структура и обязательные файлы не ослаблены | stdout CI |
| OpenAPI | `make validate` | OpenAPI 3.1 проходит полную проверку | stdout CI |
| Публичный smoke | `make test` | бизнес-API предыдущих лабораторных работает | stdout CI |
| Lab 5 regression | `make test-lab5-acceptance && make test-lab5` | durable jobs, retry, replay и idempotency сохранены | stdout CI |
| Три telemetry signal | `make test-lab6-smoke` | нагрузка создаёт metrics/logs/traces, attachments доходят до terminal state | `evidence/runtime/.../baseline-*.json` |
| Worker incident | `scripts/incident-worker.sh short` | alert firing, после запуска worker backlog дренирован и alert resolved | `worker-short-*/` |
| DB incident | `scripts/incident-database.sh` | видны 5xx и DB error spans, burn-rate firing, API восстанавливается, оперативный alert resolved | `database-*/` |
| Длительная деградация | `scripts/incident-worker.sh long` | растут oldest age/lag, срабатывает backlog alert, все jobs затем ready | `worker-long-*/` |
| Telemetry outage | `scripts/incident-telemetry.sh` | API readiness остаётся 200, Collector queue/retry ограничены, backend восстанавливается | `telemetry-*/` |
| Полный трек 2 | `make test-lab6` | выполнены short/long incidents, firing/resolved и schema validation | `evidence/runtime/lab6-*/` |
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
Runtime evidence создаётся локально и игнорируется Git до проверки содержимого. В нём нет session tokens, cookies, passwords, payload envelope и пользовательских текстов. Для публикации выбираются только небольшие агрегированные JSON.
## Эксперименты и отказы
## Методика экспериментов
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
`scripts/lab6-traffic.py` создаёт изолированных синтетических пользователей и чат, затем генерирует сообщения и PNG-вложения. Нагрузка closed-loop, concurrency и duration фиксируются в JSON; HTTP status и p50/p95/p99 рассчитываются клиентом только как вспомогательные данные. Серверные SLO используют Prometheus histograms.
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
Полный сценарий:
## Ограничения и следующие шаги
```bash
make test-lab6
```
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
Он выполняет:
1. baseline mixed load;
2. короткую остановку worker;
3. краткую недоступность PostgreSQL под message traffic;
4. длительную остановку worker;
5. остановку Loki и Tempo под mixed traffic;
6. проверку evidence и переходов alert `firing → resolved`.
Каждый incident script устанавливает trap и восстанавливает затронутые сервисы. `docker compose down -v` не используется. Baseline/degraded/recovered snapshots содержат одни и те же PromQL, поэтому сравнение не зависит от ручного выбора панели.
Гипотезы трека:
- короткий сильный отказ расходует бюджет быстрее 14,4× и обнаруживается fast alert;
- длительная деградация выявляется ростом oldest queued age и multi-window SLO;
- после восстановления worker backlog уменьшается, все принятые jobs достигают terminal state, alerts переходят в resolved;
- отказ Loki/Tempo не влияет на бизнес-ответы, а ограниченные очереди Collector отражают потери telemetry без неограниченного роста RAM.
Фактические времена обнаружения и восстановления берутся из `started_at`, `alert-firing.json`, `recovery_started_at` и `alert-resolved.json`. Отсутствие перехода считается падением сценария, а не успешным экспериментом. Ложное срабатывание проверяется baseline snapshot: до fault operational и burn-rate alerts не должны находиться в `firing`.
## Результаты прогона
Полный прогон `2026-11-04` сохранён локально в `evidence/runtime/lab6-20261104T174049Z` и прошёл schema validation без ошибок. Обезличенная сводка включена в `evidence/lab6-results.json`. Время обнаружения является верхней границей: сценарий опрашивает alert после выдержки заданной длительности отказа.
| Инцидент | Наблюдение | Обнаружение | Восстановление |
| --- | --- | ---: | ---: |
| Worker, короткий | `AttachmentWorkerUnavailable` firing, очередь затем обработана | не более 76 с | alert resolved через 13 с, полное наблюдение 33 с |
| PostgreSQL | operational 5xx и fast burn-rate firing; сохранены 3 DB error traces, один связан с API-логом | не более 78 с | operational alert resolved через 134 с, API стабилен через 154 с |
| Worker, длительный | oldest queued age рос; backlog, fast и slow burn alerts firing | не более 181 с | alert resolved через 20 с, backlog обработан за 91 с |
| Loki и Tempo | очереди выросли до 15/512 и 16/512, затем дренировались до нуля; peak RSS 226.7 MB | 30 с отказа | `/health/ready` вернул 200, backends healthy через 31 с |
Перед первым fault список firing alerts был пуст. Burn-rate alerts после инцидента закономерно остаются активными дольше оперативных alerts: их длинные окна сохраняют уже потраченный error budget. Это не считается ложным срабатыванием следующей фазы.
## Ограничения
- Стенд single-host и не проверяет потерю всего Docker host или volume.
- Loki и Tempo работают в single-binary режиме без репликации.
- 30-дневный SLO определяется production-подобными окнами, но учебный эксперимент создаёт искусственно высокий error ratio и не заменяет месячное наблюдение.
- Closed-loop generator подвержен coordinated omission; capacity conclusions по его RPS не делаются.
- Остановка telemetry backend проверяет ограниченное поведение exporter, но не гарантирует отсутствие потерь: после заполнения bounded queue часть telemetry намеренно отбрасывается.
- Внешняя доставка alerts не настраивается; проверяются локальные состояния Prometheus `firing` и `resolved`.
## Источники
ИИ-ассистент использовался при подготовке кода, конфигурации и текста; результат проверен тестами, CI-сценариями и воспроизводимым fault-прогоном.
- OpenTelemetry Collector configuration и Go SDK;
- W3C Trace Context;
- Prometheus histograms и multi-window burn-rate alerts;
- Grafana provisioning, Loki OTLP ingestion и Tempo TraceQL;
- документация Redis Streams, PostgreSQL, MinIO и Docker Compose.
+1 -1
View File
@@ -17,7 +17,7 @@ import (
"github.com/google/uuid"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
)
type apiState struct {
+9 -9
View File
@@ -21,15 +21,15 @@ import (
"github.com/google/uuid"
"github.com/redis/go-redis/v9"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/broker/redisstream"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/config"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/database"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment/transform"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment/worker"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/objectstore"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/outbox"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/postgres"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/broker/redisstream"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/config"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/database"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment/transform"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment/worker"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/objectstore"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/outbox"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/postgres"
)
const (
+2 -2
View File
@@ -5,8 +5,8 @@ import (
"log/slog"
"os"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/bootstrap"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/logger"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/bootstrap"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/logger"
)
func main() {
+94 -9
View File
@@ -4,6 +4,11 @@ x-messenger-config-volume: &messenger-config-volume
target: /etc/messenger/messenger.yaml
read_only: true
x-telemetry-environment: &telemetry-environment
OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT}
OTEL_EXPORTER_OTLP_PROTOCOL: ${OTEL_EXPORTER_OTLP_PROTOCOL}
OTEL_RESOURCE_ATTRIBUTES: deployment.environment.name=${DEPLOYMENT_ENVIRONMENT},service.version=${SERVICE_VERSION}
services:
postgres:
image: postgres:17.6-alpine3.22
@@ -118,7 +123,7 @@ services:
networks: [data]
migrate:
image: ${API_IMAGE:-mtusi-messenger:lab5}
image: ${API_IMAGE:-mtusi-messenger:lab6}
command: ["/app/messenger", "migrate"]
environment:
MESSENGER_CONFIG_PATH: /etc/messenger/messenger.yaml
@@ -137,8 +142,10 @@ services:
api:
build: .
image: ${API_IMAGE:-mtusi-messenger:lab5}
image: ${API_IMAGE:-mtusi-messenger:lab6}
environment:
<<: *telemetry-environment
OTEL_SERVICE_NAME: messenger-api
MESSENGER_CONFIG_PATH: /etc/messenger/messenger.yaml
APP_HOST: ${APP_HOST}
APP_PORT: ${APP_PORT}
@@ -176,9 +183,11 @@ services:
networks: [application, data]
dispatcher:
image: ${API_IMAGE:-mtusi-messenger:lab5}
image: ${API_IMAGE:-mtusi-messenger:lab6}
command: ["/app/messenger", "dispatch"]
environment:
<<: *telemetry-environment
OTEL_SERVICE_NAME: messenger-dispatcher
MESSENGER_CONFIG_PATH: /etc/messenger/messenger.yaml
DATABASE_URL: ${DATABASE_URL}
BROKER_URL: ${BROKER_URL}
@@ -202,9 +211,11 @@ services:
networks: [data, monitoring]
worker:
image: ${API_IMAGE:-mtusi-messenger:lab5}
image: ${API_IMAGE:-mtusi-messenger:lab6}
command: ["/app/messenger", "worker"]
environment:
<<: *telemetry-environment
OTEL_SERVICE_NAME: messenger-worker
MESSENGER_CONFIG_PATH: /etc/messenger/messenger.yaml
DATABASE_URL: ${DATABASE_URL}
S3_ENDPOINT: ${S3_ENDPOINT}
@@ -239,7 +250,7 @@ services:
build:
context: .
target: loadbalancer
image: ${NGINX_IMAGE:-mtusi-nginx:lab5}
image: ${NGINX_IMAGE:-mtusi-nginx:lab6}
ports:
- "${API_BIND_ADDRESS}:8080"
depends_on:
@@ -259,7 +270,7 @@ services:
build:
context: .
target: swagger
image: ${SWAGGER_IMAGE:-mtusi-swagger:lab5}
image: ${SWAGGER_IMAGE:-mtusi-swagger:lab6}
ports:
- "${SWAGGER_BIND_ADDRESS}:8080"
depends_on:
@@ -289,7 +300,7 @@ services:
lab5-experiment:
profiles: ["experiments"]
image: ${API_IMAGE:-mtusi-messenger:lab5}
image: ${API_IMAGE:-mtusi-messenger:lab6}
command: ["/app/lab5experiment"]
environment:
MESSENGER_CONFIG_PATH: /etc/messenger/messenger.yaml
@@ -375,11 +386,12 @@ services:
build:
context: .
target: prometheus
image: ${PROMETHEUS_IMAGE:-mtusi-prometheus:lab5}
image: ${PROMETHEUS_IMAGE:-mtusi-prometheus:lab6}
command:
- --config.file=/etc/prometheus/prometheus.yml
- --storage.tsdb.path=/prometheus
- --storage.tsdb.retention.time=7d
- --web.enable-remote-write-receiver
volumes:
- prometheus_data:/prometheus
ports:
@@ -398,6 +410,9 @@ services:
- dispatcher
- worker
- s3
- otel-collector
- loki
- tempo
cpus: ${PROMETHEUS_CPUS}
mem_limit: ${PROMETHEUS_MEMORY}
restart: unless-stopped
@@ -407,28 +422,98 @@ services:
build:
context: .
target: grafana
image: ${GRAFANA_IMAGE:-mtusi-grafana:lab5}
image: ${GRAFANA_IMAGE:-mtusi-grafana:lab6}
environment:
GF_SECURITY_ADMIN_USER: admin
GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD}
GF_USERS_ALLOW_SIGN_UP: "false"
GF_EXPLORE_ENABLED: "true"
volumes:
- grafana_data:/var/lib/grafana
ports:
- "${GRAFANA_BIND_ADDRESS}:3000"
depends_on:
- prometheus
- loki
- tempo
healthcheck:
test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:3000/api/health || exit 1"]
interval: 5s
timeout: 3s
retries: 20
cpus: ${GRAFANA_CPUS}
mem_limit: ${GRAFANA_MEMORY}
restart: unless-stopped
networks: [edge, monitoring]
loki:
image: grafana/loki:3.7.9
command: ["-config.file=/etc/loki/loki.yaml"]
volumes:
- ./infra/loki/loki.yaml:/etc/loki/loki.yaml:ro
- loki_data:/loki
ports:
- "${LOKI_BIND_ADDRESS}:3100"
healthcheck:
test: ["CMD", "/usr/bin/loki", "-version"]
interval: 10s
timeout: 3s
retries: 10
cpus: ${LOKI_CPUS}
mem_limit: ${LOKI_MEMORY}
restart: unless-stopped
networks: [edge, monitoring]
tempo:
image: grafana/tempo:2.10.10
command: ["-config.file=/etc/tempo/tempo.yaml"]
volumes:
- ./infra/tempo/tempo.yaml:/etc/tempo/tempo.yaml:ro
- tempo_data:/var/tempo
ports:
- "${TEMPO_BIND_ADDRESS}:3200"
healthcheck:
test: ["CMD", "/tempo", "-version"]
interval: 10s
timeout: 3s
retries: 10
cpus: ${TEMPO_CPUS}
mem_limit: ${TEMPO_MEMORY}
restart: unless-stopped
networks: [edge, monitoring]
otel-collector:
image: otel/opentelemetry-collector-contrib:0.160.0
command: ["--config=/etc/otelcol-contrib/collector.yaml"]
volumes:
- ./infra/otel/collector.yaml:/etc/otelcol-contrib/collector.yaml:ro
expose:
- "4317"
- "4318"
- "8888"
- "8889"
- "13133"
depends_on:
- loki
- tempo
healthcheck:
test: ["CMD", "/otelcol-contrib", "components"]
interval: 10s
timeout: 3s
retries: 10
cpus: ${OTEL_COLLECTOR_CPUS}
mem_limit: ${OTEL_COLLECTOR_MEMORY}
restart: unless-stopped
networks: [application, data, monitoring]
volumes:
postgres_data:
valkey_data:
redis_broker_data:
prometheus_data:
grafana_data:
loki_data:
tempo_data:
s3_data:
networks:
+80
View File
@@ -0,0 +1,80 @@
{
"schema_version": 1,
"captured_at": "2026-11-04T17:53:52Z",
"baseline": {
"api_request_rate_per_second": 1.014325,
"firing_alerts": 0,
"traces_by_service": {
"messenger-api": 100,
"messenger-worker": 72,
"messenger-dispatcher": 64
},
"logs_by_service": {
"messenger-api": 207,
"messenger-worker": 41,
"messenger-dispatcher": 1
},
"api_worker_shared_traces": 22,
"traces_with_correlated_logs": 101
},
"incidents": [
{
"name": "worker_unavailable_short",
"injected_duration_seconds": 76,
"alert_detection_upper_bound_seconds": 76,
"alert_resolution_seconds": 13,
"recovery_observation_seconds": 33
},
{
"name": "postgres_unavailable",
"injected_duration_seconds": 78,
"alert_detection_upper_bound_seconds": 78,
"operational_alert_resolution_seconds": 134,
"recovery_observation_seconds": 154,
"fast_burn_alert_observed": true,
"database_error_traces": 3,
"database_error_traces_with_logs": 1
},
{
"name": "worker_unavailable_long",
"injected_duration_seconds": 181,
"alert_detection_upper_bound_seconds": 181,
"alert_resolution_seconds": 20,
"recovery_observation_seconds": 91,
"fast_burn_alert_observed": true,
"slow_burn_alert_observed": true
},
{
"name": "telemetry_backends_unavailable",
"injected_duration_seconds": 30,
"api_ready_http_status": 200,
"recovery_observation_seconds": 31,
"exporters": {
"otlp_grpc/tempo": {
"capacity": 512,
"degraded_queue_size": 15,
"recovered_queue_size": 0,
"export_failures": 16,
"dropped_records": 0
},
"otlp_http/loki": {
"capacity": 512,
"degraded_queue_size": 16,
"recovered_queue_size": 0,
"export_failures": 8,
"dropped_records": 0
}
},
"collector_peak_rss_bytes": {
"baseline": 225923072,
"degraded": 226549760,
"recovered": 226746368
}
}
],
"validation": {
"valid": true,
"errors": [],
"incident_summaries": 4
}
}
+37
View File
@@ -0,0 +1,37 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://gitea.privcloud.tech/mtusi2026/mtusi2026lab6/evidence/lab6-schema.json",
"title": "Lab 6 incident summary",
"type": "object",
"required": [
"schema_version",
"incident",
"started_at",
"recovery_started_at",
"recovered_at",
"injected_duration_seconds",
"recovery_observation_seconds",
"artifacts"
],
"properties": {
"schema_version": {"const": 1},
"incident": {
"enum": ["worker_unavailable", "postgres_unavailable", "telemetry_backends_unavailable"]
},
"profile": {"enum": ["short", "long"]},
"started_at": {"type": "string", "format": "date-time"},
"recovery_started_at": {"type": "string", "format": "date-time"},
"recovered_at": {"type": "string", "format": "date-time"},
"planned_injection_seconds": {"type": "integer", "minimum": 1},
"injected_duration_seconds": {"type": "integer", "minimum": 1},
"recovery_observation_seconds": {"type": "integer", "minimum": 0},
"api_ready_http_status_during_outage": {"type": "integer", "minimum": 100, "maximum": 599},
"artifacts": {
"type": "array",
"items": {"type": "string", "pattern": "^[A-Za-z0-9._-]+$"},
"minItems": 4,
"uniqueItems": true
}
},
"additionalProperties": false
}
+27 -2
View File
@@ -1,4 +1,4 @@
module gitea.privcloud.tech/dizi/mtusi2026lab5
module gitea.privcloud.tech/dizi/mtusi2026lab6
go 1.26.0
@@ -12,19 +12,36 @@ require (
github.com/spf13/viper v1.21.0
github.com/stretchr/testify v1.11.1
github.com/valkey-io/valkey-go v1.0.78
go.opentelemetry.io/contrib/bridges/otelslog v0.13.0
go.opentelemetry.io/contrib/instrumentation/runtime v0.60.0
go.opentelemetry.io/otel v1.38.0
go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.14.0
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.38.0
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.38.0
go.opentelemetry.io/otel/exporters/prometheus v0.60.0
go.opentelemetry.io/otel/metric v1.38.0
go.opentelemetry.io/otel/sdk v1.38.0
go.opentelemetry.io/otel/sdk/log v0.14.0
go.opentelemetry.io/otel/sdk/metric v1.38.0
go.opentelemetry.io/otel/trace v1.38.0
golang.org/x/crypto v0.57.0
golang.org/x/sync v0.23.0
)
require (
github.com/beorn7/perks v1.0.1 // indirect
github.com/cenkalti/backoff/v5 v5.0.3 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/fsnotify/fsnotify v1.9.0 // indirect
github.com/go-ini/ini v1.67.0 // indirect
github.com/go-logr/logr v1.4.3 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
github.com/jackc/puddle/v2 v2.2.2 // indirect
@@ -39,7 +56,8 @@ require (
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/client_model v0.6.2 // indirect
github.com/prometheus/common v0.66.1 // indirect
github.com/prometheus/procfs v0.16.1 // indirect
github.com/prometheus/otlptranslator v0.0.2 // indirect
github.com/prometheus/procfs v0.17.0 // indirect
github.com/rs/xid v1.6.0 // indirect
github.com/sagikazarmark/locafero v0.11.0 // indirect
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect
@@ -48,11 +66,18 @@ require (
github.com/spf13/pflag v1.0.10 // indirect
github.com/subosito/gotenv v1.6.0 // indirect
github.com/tinylib/msgp v1.6.1 // indirect
go.opentelemetry.io/auto/sdk v1.1.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 // indirect
go.opentelemetry.io/otel/log v0.14.0 // indirect
go.opentelemetry.io/proto/otlp v1.7.1 // indirect
go.yaml.in/yaml/v2 v2.4.2 // indirect
go.yaml.in/yaml/v3 v3.0.5 // indirect
golang.org/x/net v0.58.0 // indirect
golang.org/x/sys v0.48.0 // indirect
golang.org/x/text v0.42.0 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5 // indirect
google.golang.org/grpc v1.75.0 // indirect
google.golang.org/protobuf v1.36.8 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
+61 -4
View File
@@ -4,6 +4,8 @@ github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM=
github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -21,12 +23,23 @@ github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S
github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs=
github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc h1:GN2Lv3MGO7AS6PrRoT6yV5+wkrOpcszoIsO4+4ds248=
github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc/go.mod h1:+JKpmjMGhpgPL+rXZ5nsZieVzvarn86asRlBg4uNGnk=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 h1:8Tjv8EJ+pM1xP8mK6egEbD1OgnVTyacbefKhmbLhIhU=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2/go.mod h1:pkJQ2tZHJ0aFOVEEot6oZmaVEZcRme73eIFmhiVuRWs=
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
@@ -70,12 +83,14 @@ github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNw
github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs=
github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA=
github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg=
github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is=
github.com/prometheus/otlptranslator v0.0.2 h1:+1CdeLVrRQ6Psmhnobldo0kTp96Rj80DRXRd5OSnMEQ=
github.com/prometheus/otlptranslator v0.0.2/go.mod h1:P8AwMgdD7XEr6QRUJ2QWLpiAZTgTE2UYgjlu3svompI=
github.com/prometheus/procfs v0.17.0 h1:FuLQ+05u4ZI+SS/w9+BWEM2TXiHKsUQ9TADiRH7DuK0=
github.com/prometheus/procfs v0.17.0/go.mod h1:oPQLaDAMRbA+u8H5Pbfq+dl3VDAvHxMUOVhe0wYB2zw=
github.com/redis/go-redis/v9 v9.17.2 h1:P2EGsA4qVIM3Pp+aPocCJ7DguDHhqrXNhVcEp4ViluI=
github.com/redis/go-redis/v9 v9.17.2/go.mod h1:u410H11HMLoB+TP67dz8rL9s6QW2j76l0//kSOd3370=
github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII=
github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o=
github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc=
@@ -101,6 +116,40 @@ github.com/tinylib/msgp v1.6.1 h1:ESRv8eL3u+DNHUoSAAQRE50Hm162zqAnBoGv9PzScPY=
github.com/tinylib/msgp v1.6.1/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA=
github.com/valkey-io/valkey-go v1.0.78 h1:1Ce8hRaaNXpKfqmqZQRhnbVh1eW8MGR873pdXVvE4y8=
github.com/valkey-io/valkey-go v1.0.78/go.mod h1:gvC/r2m3eW4Hbj0YnjogTzNtFdPSM/D+NCqen+5OABM=
go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA=
go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A=
go.opentelemetry.io/contrib/bridges/otelslog v0.13.0 h1:bwnLpizECbPr1RrQ27waeY2SPIPeccCx/xLuoYADZ9s=
go.opentelemetry.io/contrib/bridges/otelslog v0.13.0/go.mod h1:3nWlOiiqA9UtUnrcNk82mYasNxD8ehOspL0gOfEo6Y4=
go.opentelemetry.io/contrib/instrumentation/runtime v0.60.0 h1:0NgN/3SYkqYJ9NBlDfl/2lzVlwos/YQLvi8sUrzJRBE=
go.opentelemetry.io/contrib/instrumentation/runtime v0.60.0/go.mod h1:oxpUfhTkhgQaYIjtBt3T3w135dLoxq//qo3WPlPIKkE=
go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8=
go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM=
go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.14.0 h1:OMqPldHt79PqWKOMYIAQs3CxAi7RLgPxwfFSwr4ZxtM=
go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.14.0/go.mod h1:1biG4qiqTxKiUCtoWDPpL3fB3KxVwCiGw81j3nKMuHE=
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.38.0 h1:vl9obrcoWVKp/lwl8tRE33853I8Xru9HFbw/skNeLs8=
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.38.0/go.mod h1:GAXRxmLJcVM3u22IjTg74zWBrRCKq8BnOqUVLodpcpw=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 h1:GqRJVj7UmLjCVyVJ3ZFLdPRmhDUp2zFmQe3RHIOsw24=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0/go.mod h1:ri3aaHSmCTVYu2AWv44YMauwAQc0aqI9gHKIcSbI1pU=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.38.0 h1:lwI4Dc5leUqENgGuQImwLo4WnuXFPetmPpkLi2IrX54=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.38.0/go.mod h1:Kz/oCE7z5wuyhPxsXDuaPteSWqjSBD5YaSdbxZYGbGk=
go.opentelemetry.io/otel/exporters/prometheus v0.60.0 h1:cGtQxGvZbnrWdC2GyjZi0PDKVSLWP/Jocix3QWfXtbo=
go.opentelemetry.io/otel/exporters/prometheus v0.60.0/go.mod h1:hkd1EekxNo69PTV4OWFGZcKQiIqg0RfuWExcPKFvepk=
go.opentelemetry.io/otel/log v0.14.0 h1:2rzJ+pOAZ8qmZ3DDHg73NEKzSZkhkGIua9gXtxNGgrM=
go.opentelemetry.io/otel/log v0.14.0/go.mod h1:5jRG92fEAgx0SU/vFPxmJvhIuDU9E1SUnEQrMlJpOno=
go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA=
go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI=
go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E=
go.opentelemetry.io/otel/sdk v1.38.0/go.mod h1:ghmNdGlVemJI3+ZB5iDEuk4bWA3GkTpW+DOoZMYBVVg=
go.opentelemetry.io/otel/sdk/log v0.14.0 h1:JU/U3O7N6fsAXj0+CXz21Czg532dW2V4gG1HE/e8Zrg=
go.opentelemetry.io/otel/sdk/log v0.14.0/go.mod h1:imQvII+0ZylXfKU7/wtOND8Hn4OpT3YUoIgqJVksUkM=
go.opentelemetry.io/otel/sdk/log/logtest v0.14.0 h1:Ijbtz+JKXl8T2MngiwqBlPaHqc4YCaP/i13Qrow6gAM=
go.opentelemetry.io/otel/sdk/log/logtest v0.14.0/go.mod h1:dCU8aEL6q+L9cYTqcVOk8rM9Tp8WdnHOPLiBgp0SGOA=
go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6qT5wthqPoM=
go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA=
go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE=
go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs=
go.opentelemetry.io/proto/otlp v1.7.1 h1:gTOMpGDb0WTBOP8JaO72iL3auEZhVmAQg4ipjOVAtj4=
go.opentelemetry.io/proto/otlp v1.7.1/go.mod h1:b2rVh6rfI/s2pHWNlB7ILJcRALpcNDzKhACevjI+ZnE=
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI=
@@ -117,6 +166,14 @@ golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI=
golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E=
gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E=
google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5 h1:BIRfGDEjiHRrk0QKZe3Xv2ieMhtgRGeLcZQ0mIVn4EY=
google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5/go.mod h1:j3QtIyytwqGr1JUDtYXwtMXWPKsEa5LtzIFN1Wn5WvE=
google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5 h1:eaY8u2EuxbRv7c3NiGK0/NedzVsCcV6hDuU5qPX5EGE=
google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5/go.mod h1:M4/wBTSeyLxupu3W3tJtOgB14jILAS/XWPSSa3TAlJc=
google.golang.org/grpc v1.75.0 h1:+TW+dqTd2Biwe6KKfhE5JpiYIBWq865PhKGSXiivqt4=
google.golang.org/grpc v1.75.0/go.mod h1:JtPAzKiq4v1xcAB2hydNlWI2RnF85XXcV0mhKXr2ecQ=
google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc=
google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+48
View File
@@ -0,0 +1,48 @@
auth_enabled: false
server:
http_listen_address: 0.0.0.0
http_listen_port: 3100
grpc_listen_address: 0.0.0.0
grpc_listen_port: 9096
common:
path_prefix: /loki
replication_factor: 1
ring:
kvstore:
store: inmemory
storage:
filesystem:
chunks_directory: /loki/chunks
rules_directory: /loki/rules
schema_config:
configs:
- from: 2024-01-01
store: tsdb
object_store: filesystem
schema: v13
index:
prefix: index_
period: 24h
limits_config:
allow_structured_metadata: true
retention_period: 168h
volume_enabled: true
otlp_config:
resource_attributes:
attributes_config:
- action: index_label
attributes:
- service.name
- deployment.environment.name
compactor:
working_directory: /loki/compactor
retention_enabled: true
delete_request_store: filesystem
analytics:
reporting_enabled: false
+80
View File
@@ -0,0 +1,80 @@
extensions:
health_check:
endpoint: 0.0.0.0:13133
receivers:
otlp:
protocols:
grpc:
endpoint: 0.0.0.0:4317
http:
endpoint: 0.0.0.0:4318
processors:
memory_limiter:
check_interval: 1s
limit_mib: 192
spike_limit_mib: 48
batch:
send_batch_size: 1024
send_batch_max_size: 2048
timeout: 2s
exporters:
prometheus:
endpoint: 0.0.0.0:8889
resource_constant_labels:
included:
- service.name
- service.version
- deployment.environment.name
otlp_grpc/tempo:
endpoint: tempo:4317
tls:
insecure: true
sending_queue:
enabled: true
num_consumers: 2
queue_size: 512
retry_on_failure:
enabled: true
initial_interval: 1s
max_interval: 5s
max_elapsed_time: 30s
otlp_http/loki:
endpoint: http://loki:3100/otlp
sending_queue:
enabled: true
num_consumers: 2
queue_size: 512
retry_on_failure:
enabled: true
initial_interval: 1s
max_interval: 5s
max_elapsed_time: 30s
service:
extensions: [health_check]
telemetry:
logs:
level: info
metrics:
readers:
- pull:
exporter:
prometheus:
host: 0.0.0.0
port: 8888
pipelines:
metrics:
receivers: [otlp]
processors: [memory_limiter, batch]
exporters: [prometheus]
logs:
receivers: [otlp]
processors: [memory_limiter, batch]
exporters: [otlp_http/loki]
traces:
receivers: [otlp]
processors: [memory_limiter, batch]
exporters: [otlp_grpc/tempo]
+48
View File
@@ -0,0 +1,48 @@
server:
http_listen_address: 0.0.0.0
http_listen_port: 3200
grpc_listen_address: 0.0.0.0
grpc_listen_port: 9095
distributor:
receivers:
otlp:
protocols:
grpc:
endpoint: 0.0.0.0:4317
http:
endpoint: 0.0.0.0:4318
ingester:
max_block_duration: 5m
compactor:
compaction:
block_retention: 168h
storage:
trace:
backend: local
wal:
path: /var/tempo/wal
local:
path: /var/tempo/traces
metrics_generator:
registry:
external_labels:
source: tempo
deployment_environment: lab
storage:
path: /var/tempo/generator/wal
remote_write:
- url: http://prometheus:9090/api/v1/write
send_exemplars: true
overrides:
defaults:
metrics_generator:
processors: [service-graphs, span-metrics]
usage_report:
reporting_enabled: false
+2 -2
View File
@@ -3,8 +3,8 @@ package bootstrap
import (
"context"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/config"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/database"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/config"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/database"
)
func RunMigrations() error {
+36 -15
View File
@@ -14,15 +14,16 @@ import (
"github.com/google/uuid"
"github.com/redis/go-redis/v9"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/broker/redisstream"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/config"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/database"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment/pipeline"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment/transform"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment/worker"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/objectstore"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/outbox"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/postgres"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/broker/redisstream"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/config"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/database"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment/pipeline"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment/transform"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment/worker"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/objectstore"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/outbox"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/postgres"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/telemetry"
)
func RunDispatcher(logger *slog.Logger) error {
@@ -30,11 +31,23 @@ func RunDispatcher(logger *slog.Logger) error {
if err != nil {
return err
}
instanceID, err := os.Hostname()
if err != nil {
return err
}
observability, logger, err := startTelemetry("messenger-dispatcher", instanceID)
if err != nil {
return err
}
defer shutdownTelemetry(observability)
pool, err := database.Open(context.Background(), dispatcherDatabaseConfig(settings))
if err != nil {
return err
}
defer pool.Close()
if err := telemetry.ObserveDatabasePool(observability.MeterProvider().Meter("messenger-dispatcher/database"), pool); err != nil {
return err
}
client, err := newRedisClient(settings.BrokerURL)
if err != nil {
return err
@@ -51,7 +64,7 @@ func RunDispatcher(logger *slog.Logger) error {
if err := broker.Ensure(context.Background()); err != nil {
return err
}
metrics := pipeline.NewMetrics()
metrics := pipeline.NewMetrics(observability.MeterProvider().Meter("messenger-dispatcher"), observability.MetricsHandler(), 0)
dispatcher, err := outbox.New(postgres.New(pool), broker, metrics, outbox.Config{
BatchSize: settings.BatchSize, LeaseDuration: settings.LeaseDuration,
PollInterval: settings.PollInterval, RecoveryInterval: settings.RecoveryInterval,
@@ -70,11 +83,23 @@ func RunWorker(logger *slog.Logger) error {
if err != nil {
return err
}
hostname, err := os.Hostname()
if err != nil {
return err
}
observability, logger, err := startTelemetry("messenger-worker", hostname)
if err != nil {
return err
}
defer shutdownTelemetry(observability)
pool, err := database.Open(context.Background(), workerDatabaseConfig(settings))
if err != nil {
return err
}
defer pool.Close()
if err := telemetry.ObserveDatabasePool(observability.MeterProvider().Meter("messenger-worker/database"), pool); err != nil {
return err
}
objects, err := objectstore.New(objectstore.Config{
Endpoint: settings.S3Endpoint, Region: settings.S3Region, Bucket: settings.S3Bucket,
AccessKey: settings.S3AccessKey, SecretKey: settings.S3SecretKey,
@@ -88,10 +113,6 @@ func RunWorker(logger *slog.Logger) error {
return err
}
defer func() { _ = client.Close() }()
hostname, err := os.Hostname()
if err != nil {
return err
}
consumer := hostname + "-" + uuid.NewString()
broker, err := redisstream.New(client, redisstream.Config{
Stream: settings.Stream, Group: settings.ConsumerGroup, Consumer: consumer,
@@ -108,7 +129,7 @@ func RunWorker(logger *slog.Logger) error {
if err != nil {
return err
}
metrics := pipeline.NewMetrics()
metrics := pipeline.NewMetrics(observability.MeterProvider().Meter("messenger-worker"), observability.MetricsHandler(), settings.Concurrency)
processorWorker, err := worker.New(
logger, postgres.New(pool), broker, objects, processor, metrics,
worker.Config{
+5 -5
View File
@@ -10,11 +10,11 @@ import (
"syscall"
"time"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/config"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/database"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/objectstore"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/postgres"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/reconcile"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/config"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/database"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/objectstore"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/postgres"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/reconcile"
)
type reconciliationOutput struct {
+3 -3
View File
@@ -6,9 +6,9 @@ import (
"fmt"
"io"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/config"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/database"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/postgres"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/config"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/database"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/postgres"
)
func RunAttachmentReplay(arguments []string, output io.Writer) error {
+18 -8
View File
@@ -9,14 +9,15 @@ import (
"os/signal"
"syscall"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/config"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/database"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/httpapi"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/objectstore"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/password"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/postgres"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/session"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/config"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/database"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/httpapi"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/objectstore"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/password"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/postgres"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/session"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/telemetry"
)
func RunServer(logger *slog.Logger) error {
@@ -28,11 +29,19 @@ func RunServer(logger *slog.Logger) error {
if err != nil {
return err
}
observability, logger, err := startTelemetry("messenger-api", instanceID)
if err != nil {
return err
}
defer shutdownTelemetry(observability)
pool, err := database.Open(context.Background(), databaseConfig(settings))
if err != nil {
return err
}
defer pool.Close()
if err := telemetry.ObserveDatabasePool(observability.MeterProvider().Meter("messenger-api/database"), pool); err != nil {
return err
}
sessions, err := session.New(settings.CacheURL)
if err != nil {
return err
@@ -62,6 +71,7 @@ func RunServer(logger *slog.Logger) error {
app := httpapi.NewServer(logger, httpapi.Dependencies{
Users: repository, Chats: repository, Messages: repository, Attachments: attachments,
Sessions: sessions, Passwords: passwords, Database: repository, Objects: objects,
Meter: observability.MeterProvider().Meter("messenger-api"), Metrics: observability.MetricsHandler(),
}, httpapi.Config{
SessionTTL: settings.SessionTTL, IdempotencyTTL: settings.IdempotencyTTL,
RequestTTL: settings.RequestTimeout, InstanceID: instanceID, SwaggerPort: settings.SwaggerPort,
+23
View File
@@ -0,0 +1,23 @@
package bootstrap
import (
"context"
"log/slog"
"time"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/telemetry"
)
func startTelemetry(serviceName, instanceID string) (*telemetry.Runtime, *slog.Logger, error) {
runtime, err := telemetry.New(context.Background(), telemetry.ConfigFromEnvironment(serviceName, instanceID))
if err != nil {
return nil, nil, err
}
return runtime, runtime.Logger(), nil
}
func shutdownTelemetry(runtime *telemetry.Runtime) {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
_ = runtime.Shutdown(ctx)
}
+19 -1
View File
@@ -11,7 +11,12 @@ import (
"github.com/redis/go-redis/v9"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/outbox"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/trace"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/outbox"
)
const (
@@ -103,6 +108,10 @@ func isBusyGroup(err error) bool {
}
func (b *Broker) PublishConfirmed(ctx context.Context, payload []byte) (string, error) {
ctx, span := otel.Tracer("redisstream").Start(ctx, "redis XADD", trace.WithSpanKind(trace.SpanKindClient),
trace.WithAttributes(attribute.String("db.system", "redis"), attribute.String("db.operation.name", "XADD"),
attribute.String("messaging.destination.name", b.config.Stream)))
defer span.End()
envelope, err := DecodeEnvelope(payload)
if err != nil {
return "", err
@@ -117,12 +126,19 @@ func (b *Broker) PublishConfirmed(ctx context.Context, payload []byte) (string,
Values: map[string]any{payloadField: string(payload)},
}).Result()
if err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, "XADD failed")
return "", fmt.Errorf("publish redis stream job: %w", err)
}
span.SetAttributes(attribute.String("messaging.message.id", id))
return id, nil
}
func (b *Broker) Read(ctx context.Context) ([]Delivery, error) {
ctx, span := otel.Tracer("redisstream").Start(ctx, "redis XREADGROUP", trace.WithSpanKind(trace.SpanKindClient),
trace.WithAttributes(attribute.String("db.system", "redis"), attribute.String("db.operation.name", "XREADGROUP"),
attribute.String("messaging.destination.name", b.config.Stream)))
defer span.End()
streams, err := b.client.XReadGroup(ctx, &redis.XReadGroupArgs{
Group: b.config.Group,
Consumer: b.config.Consumer,
@@ -134,6 +150,8 @@ func (b *Broker) Read(ctx context.Context) ([]Delivery, error) {
return nil, nil
}
if err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, "XREADGROUP failed")
return nil, fmt.Errorf("read redis stream jobs: %w", err)
}
return b.deliveries(streams)
+2 -1
View File
@@ -8,7 +8,7 @@ import (
"github.com/jackc/pgx/v5/pgxpool"
"gitea.privcloud.tech/dizi/mtusi2026lab5/migrations"
"gitea.privcloud.tech/dizi/mtusi2026lab6/migrations"
)
var (
@@ -40,6 +40,7 @@ func Open(ctx context.Context, config Config) (*pgxpool.Pool, error) {
if err != nil {
return nil, fmt.Errorf("parse database configuration: %w", err)
}
poolConfig.ConnConfig.Tracer = queryTracer{}
poolConfig.MinConns = config.MinConnections
poolConfig.MaxConns = config.MaxConnections
poolConfig.MaxConnLifetime = config.MaxConnectionAge
+28
View File
@@ -0,0 +1,28 @@
package database
import (
"context"
"github.com/jackc/pgx/v5"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/trace"
)
type queryTracer struct{}
func (queryTracer) TraceQueryStart(ctx context.Context, _ *pgx.Conn, _ pgx.TraceQueryStartData) context.Context {
ctx, _ = otel.Tracer("postgres").Start(ctx, "postgres query", trace.WithSpanKind(trace.SpanKindClient),
trace.WithAttributes(attribute.String("db.system", "postgresql")))
return ctx
}
func (queryTracer) TraceQueryEnd(ctx context.Context, _ *pgx.Conn, data pgx.TraceQueryEndData) {
span := trace.SpanFromContext(ctx)
if data.Err != nil {
span.RecordError(data.Err)
span.SetStatus(codes.Error, "query failed")
}
span.End()
}
@@ -1,104 +1,97 @@
package pipeline
import (
"context"
"net/http"
"time"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/collectors"
"github.com/prometheus/client_golang/prometheus/promhttp"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/metric"
)
var workerDurationBuckets = []float64{0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 15, 20, 30, 60}
type Metrics struct {
workerInFlight prometheus.Gauge
workerJobs *prometheus.CounterVec
workerDuration *prometheus.HistogramVec
workerRetries prometheus.Counter
workerReclaimed prometheus.Counter
outboxPublished prometheus.Counter
outboxPublishFailure prometheus.Counter
outboxRecovered prometheus.Counter
meter metric.Meter
workerInFlight metric.Int64UpDownCounter
workerJobs metric.Int64Counter
workerDuration metric.Float64Histogram
workerRetries metric.Int64Counter
workerReclaimed metric.Int64Counter
outboxPublished metric.Int64Counter
outboxPublishFailure metric.Int64Counter
outboxRecovered metric.Int64Counter
handler http.Handler
}
func NewMetrics() *Metrics {
registry := prometheus.NewRegistry()
metrics := &Metrics{
workerInFlight: prometheus.NewGauge(prometheus.GaugeOpts{
Namespace: "messenger", Subsystem: "attachment_worker", Name: "jobs_in_flight",
Help: "Attachment jobs currently processed by this worker.",
}),
workerJobs: prometheus.NewCounterVec(prometheus.CounterOpts{
Namespace: "messenger", Subsystem: "attachment_worker", Name: "jobs_total",
Help: "Completed attachment job deliveries by result.",
}, []string{"result"}),
workerDuration: prometheus.NewHistogramVec(prometheus.HistogramOpts{
Namespace: "messenger", Subsystem: "attachment_worker", Name: "job_duration_seconds",
Help: "Attachment job delivery processing duration.", Buckets: workerDurationBuckets,
}, []string{"result"}),
workerRetries: prometheus.NewCounter(prometheus.CounterOpts{
Namespace: "messenger", Subsystem: "attachment_worker", Name: "retries_total",
Help: "Attachment jobs scheduled for another attempt.",
}),
workerReclaimed: prometheus.NewCounter(prometheus.CounterOpts{
Namespace: "messenger", Subsystem: "attachment_worker", Name: "reclaimed_deliveries_total",
Help: "Redis Stream deliveries reclaimed after their idle timeout.",
}),
outboxPublished: prometheus.NewCounter(prometheus.CounterOpts{
Namespace: "messenger", Subsystem: "outbox", Name: "published_total",
Help: "Outbox events confirmed by Redis Streams.",
}),
outboxPublishFailure: prometheus.NewCounter(prometheus.CounterOpts{
Namespace: "messenger", Subsystem: "outbox", Name: "publish_failures_total",
Help: "Outbox events whose Redis Streams publish failed.",
}),
outboxRecovered: prometheus.NewCounter(prometheus.CounterOpts{
Namespace: "messenger", Subsystem: "outbox", Name: "recovered_leases_total",
Help: "Expired outbox leases returned for publication.",
}),
func NewMetrics(meter metric.Meter, handler http.Handler, workerCapacity int) *Metrics {
if meter == nil {
meter = otel.Meter("messenger-pipeline")
}
registry.MustRegister(
metrics.workerInFlight, metrics.workerJobs, metrics.workerDuration,
metrics.workerRetries, metrics.workerReclaimed, metrics.outboxPublished,
metrics.outboxPublishFailure, metrics.outboxRecovered,
collectors.NewGoCollector(), collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}),
if handler == nil {
handler = http.NotFoundHandler()
}
workerInFlight, _ := meter.Int64UpDownCounter("messenger_attachment_worker_jobs_in_flight", metric.WithDescription("Attachment jobs currently processed by this worker."))
workerJobs, _ := meter.Int64Counter("messenger_attachment_worker_jobs", metric.WithDescription("Completed attachment job deliveries by result."))
workerDuration, _ := meter.Float64Histogram("messenger_attachment_worker_job_duration",
metric.WithDescription("Attachment job delivery processing duration."), metric.WithUnit("s"),
metric.WithExplicitBucketBoundaries(workerDurationBuckets...),
)
metrics.handler = promhttp.HandlerFor(registry, promhttp.HandlerOpts{})
return metrics
workerRetries, _ := meter.Int64Counter("messenger_attachment_worker_retries", metric.WithDescription("Attachment jobs scheduled for another attempt."))
workerReclaimed, _ := meter.Int64Counter("messenger_attachment_worker_reclaimed_deliveries", metric.WithDescription("Redis Stream deliveries reclaimed after their idle timeout."))
outboxPublished, _ := meter.Int64Counter("messenger_outbox_published", metric.WithDescription("Outbox events confirmed by Redis Streams."))
outboxPublishFailure, _ := meter.Int64Counter("messenger_outbox_publish_failures", metric.WithDescription("Outbox events whose Redis Streams publish failed."))
outboxRecovered, _ := meter.Int64Counter("messenger_outbox_recovered_leases", metric.WithDescription("Expired outbox leases returned for publication."))
if workerCapacity > 0 {
_, _ = meter.Int64ObservableGauge("messenger_attachment_worker_capacity",
metric.WithDescription("Maximum concurrent attachment jobs for this worker."),
metric.WithInt64Callback(func(_ context.Context, observer metric.Int64Observer) error {
observer.Observe(int64(workerCapacity))
return nil
}),
)
}
return &Metrics{
meter: meter, workerInFlight: workerInFlight, workerJobs: workerJobs,
workerDuration: workerDuration,
workerRetries: workerRetries, workerReclaimed: workerReclaimed,
outboxPublished: outboxPublished, outboxPublishFailure: outboxPublishFailure,
outboxRecovered: outboxRecovered, handler: handler,
}
}
func (m *Metrics) Handler() http.Handler {
return m.handler
}
func (m *Metrics) ObserveStarted() {
m.workerInFlight.Inc()
func (m *Metrics) ObserveStarted(ctx context.Context) {
m.workerInFlight.Add(ctx, 1)
}
func (m *Metrics) ObserveFinished(result string, duration time.Duration) {
m.workerInFlight.Dec()
m.workerJobs.WithLabelValues(result).Inc()
m.workerDuration.WithLabelValues(result).Observe(duration.Seconds())
func (m *Metrics) ObserveFinished(ctx context.Context, result string, duration time.Duration) {
attributes := metric.WithAttributes(attribute.String("job_type", "attachment.process"), attribute.String("result", result))
m.workerInFlight.Add(ctx, -1)
m.workerJobs.Add(ctx, 1, attributes)
m.workerDuration.Record(ctx, duration.Seconds(), attributes)
}
func (m *Metrics) ObserveRetry() {
m.workerRetries.Inc()
func (m *Metrics) ObserveRetry(ctx context.Context) {
m.workerRetries.Add(ctx, 1)
}
func (m *Metrics) ObserveReclaimed(count int) {
m.workerReclaimed.Add(float64(count))
func (m *Metrics) ObserveReclaimed(ctx context.Context, count int) {
m.workerReclaimed.Add(ctx, int64(count))
}
func (m *Metrics) ObservePublished() {
m.outboxPublished.Inc()
func (m *Metrics) ObservePublished(ctx context.Context) {
m.outboxPublished.Add(ctx, 1)
}
func (m *Metrics) ObservePublishFailure() {
m.outboxPublishFailure.Inc()
func (m *Metrics) ObservePublishFailure(ctx context.Context) {
m.outboxPublishFailure.Add(ctx, 1)
}
func (m *Metrics) ObserveRecovered(count int) {
m.outboxRecovered.Add(float64(count))
func (m *Metrics) ObserveRecovered(ctx context.Context, count int) {
m.outboxRecovered.Add(ctx, int64(count))
}
+10 -10
View File
@@ -19,7 +19,8 @@ import (
"github.com/google/uuid"
"golang.org/x/sync/semaphore"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/telemetry"
)
type Repository interface {
@@ -53,8 +54,7 @@ type Upload struct {
ContentType string
ContentLength int64
Body io.Reader
Traceparent string
Tracestate string
TraceContext telemetry.MessageCarrier
}
type Content struct {
@@ -112,27 +112,27 @@ func (s *Service) Upload(ctx context.Context, input Upload) (model.Attachment, e
return model.Attachment{}, mapDependencyError(err)
}
if err := s.objects.Put(ctx, stagingKey, contentType, data); err != nil {
s.logger.Error("attachment staging write failed", "attachment_id", pending.ID, "error", err)
s.deletePending(pending.ID, stagingKey)
s.logger.ErrorContext(ctx, "attachment staging write failed", "attachment_id", pending.ID, "error", err)
s.deletePending(ctx, pending.ID, stagingKey)
return model.Attachment{}, model.ErrUnavailable
}
createdAt := time.Now().UTC()
queued, err := s.repository.QueueAttachmentWithJobAndOutbox(ctx, pending.ID, stagingKey, model.AttachmentJob{
ID: uuid.NewString(), AttachmentID: pending.ID, PipelineVersion: s.config.PipelineVersion,
Attempt: 1, Status: model.AttachmentJobQueued, AvailableAt: createdAt, CreatedAt: createdAt,
}, input.Traceparent, input.Tracestate)
}, input.TraceContext.Traceparent, input.TraceContext.Tracestate)
if err != nil {
s.logger.Error("attachment queue transaction failed", "attachment_id", pending.ID, "error", err)
s.logger.ErrorContext(ctx, "attachment queue transaction failed", "attachment_id", pending.ID, "error", err)
return model.Attachment{}, mapDependencyError(err)
}
return queued, nil
}
func (s *Service) deletePending(attachmentID, stagingKey string) {
cleanupCtx, cancel := context.WithTimeout(context.Background(), s.config.CleanupTimeout)
func (s *Service) deletePending(ctx context.Context, attachmentID, stagingKey string) {
cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), s.config.CleanupTimeout)
defer cancel()
if err := s.repository.DeletePendingAttachment(cleanupCtx, attachmentID, stagingKey); err != nil {
s.logger.Error("pending attachment cleanup failed", "attachment_id", attachmentID, "error", err)
s.logger.ErrorContext(cleanupCtx, "pending attachment cleanup failed", "attachment_id", attachmentID, "error", err)
}
}
@@ -15,7 +15,7 @@ import (
"github.com/stretchr/testify/require"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
)
func TestUploadLifecycle(t *testing.T) {
+57 -25
View File
@@ -18,10 +18,16 @@ import (
"golang.org/x/sync/semaphore"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/broker/redisstream"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment/transform"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/objectstore"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/trace"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/broker/redisstream"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment/transform"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/objectstore"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/telemetry"
)
type Repository interface {
@@ -29,7 +35,7 @@ type Repository interface {
ClaimAttachmentJob(context.Context, string, string, int, int, string, time.Duration) (model.AttachmentJobLease, error)
RenewAttachmentJobLease(context.Context, model.AttachmentJobLease, time.Duration) (bool, error)
CompleteAttachmentJob(context.Context, model.AttachmentJobLease, []model.Variant) (model.JobMutationResult, error)
RetryAttachmentJob(context.Context, model.AttachmentJobLease, string, time.Time) (model.JobMutationResult, error)
RetryAttachmentJob(context.Context, model.AttachmentJobLease, string, time.Time, string, string) (model.JobMutationResult, error)
FailAttachmentJob(context.Context, model.AttachmentJobLease, string) (model.JobMutationResult, error)
RecoverExpiredAttachmentJobs(context.Context, int, int, time.Time) (model.RecoveryResult, error)
}
@@ -50,10 +56,10 @@ type ImageProcessor interface {
}
type Metrics interface {
ObserveStarted()
ObserveFinished(string, time.Duration)
ObserveRetry()
ObserveReclaimed(int)
ObserveStarted(context.Context)
ObserveFinished(context.Context, string, time.Duration)
ObserveRetry(context.Context)
ObserveReclaimed(context.Context, int)
}
type Config struct {
@@ -166,7 +172,7 @@ func (w *Worker) Run(ctx context.Context) error {
}
return err
}
w.metrics.ObserveReclaimed(len(deliveries))
w.metrics.ObserveReclaimed(ctx, len(deliveries))
w.processBatch(ctx, deliveries)
reclaimCursor = next
if reclaimCursor == "" {
@@ -189,25 +195,48 @@ func (w *Worker) processBatch(ctx context.Context, deliveries []redisstream.Deli
go func(delivery redisstream.Delivery) {
defer group.Done()
defer func() { <-limit }()
if err := w.ProcessDelivery(ctx, delivery, FaultNone); err != nil && ctx.Err() == nil {
w.logger.Error("attachment job delivery failed", "message_id", delivery.ID, "error", err)
}
_ = w.ProcessDelivery(ctx, delivery, FaultNone)
}(delivery)
}
group.Wait()
}
func (w *Worker) ProcessDelivery(ctx context.Context, delivery redisstream.Delivery, fault FaultPoint) error {
func (w *Worker) ProcessDelivery(ctx context.Context, delivery redisstream.Delivery, fault FaultPoint) (resultErr error) {
started := w.now()
w.metrics.ObserveStarted()
result := "transient_error"
defer func() { w.metrics.ObserveFinished(result, w.now().Sub(started)) }()
if delivery.DecodeError != nil {
w.metrics.ObserveStarted(ctx)
defer func() { w.metrics.ObserveFinished(ctx, result, w.now().Sub(started)) }()
result = "invalid_job"
return delivery.ToFailed(ctx, redisstream.FailedDelivery{Code: "invalid_job", FailedAt: w.now()})
}
envelope := delivery.Envelope
ctx = telemetry.ExtractMessage(ctx, telemetry.MessageCarrier{
Traceparent: envelope.Traceparent, Tracestate: envelope.Tracestate,
})
ctx, span := otel.Tracer("messenger-worker").Start(ctx, "attachment.process", trace.WithSpanKind(trace.SpanKindConsumer),
trace.WithAttributes(
attribute.String("messaging.system", "redis"), attribute.String("messaging.destination.name", "attachment-jobs"),
attribute.String("messaging.message.id", delivery.ID), attribute.String("job.id", envelope.JobID.String()),
attribute.String("attachment.id", envelope.AttachmentID.String()), attribute.Int("job.attempt", envelope.Attempt),
attribute.Int("pipeline.version", envelope.PipelineVersion),
))
w.metrics.ObserveStarted(ctx)
defer func() {
w.metrics.ObserveFinished(ctx, result, w.now().Sub(started))
span.SetAttributes(attribute.String("job.outcome", result))
if resultErr != nil {
span.RecordError(resultErr)
span.SetStatus(codes.Error, result)
if ctx.Err() == nil {
w.logger.ErrorContext(ctx, "attachment job failed", "event", "attachment_job_failed", "message_id", delivery.ID, "job_id", envelope.JobID.String(), "job_type", envelope.Type, "attempt", envelope.Attempt, "result", result, "error", resultErr)
}
} else {
w.logger.InfoContext(ctx, "attachment job completed", "event", "attachment_job_completed", "message_id", delivery.ID, "job_id", envelope.JobID.String(), "job_type", envelope.Type, "attempt", envelope.Attempt, "result", result)
}
span.End()
}()
lease, err := w.repository.ClaimAttachmentJob(
ctx, envelope.JobID.String(), envelope.AttachmentID.String(), envelope.PipelineVersion, envelope.Attempt,
w.config.WorkerID, w.config.LeaseDuration,
@@ -246,7 +275,7 @@ func (w *Worker) ProcessDelivery(ctx context.Context, delivery redisstream.Deliv
}
if mutation == model.JobMutationApplied && strings.HasPrefix(lease.Attachment.StagingKey, "staging/") {
if err := w.objects.Delete(context.WithoutCancel(ctx), lease.Attachment.StagingKey); err != nil {
w.logger.Warn("staging cleanup failed", "attachment_id", lease.Attachment.ID, "error", err)
w.logger.WarnContext(ctx, "staging cleanup failed", "attachment_id", lease.Attachment.ID, "error", err)
}
}
if mutation != model.JobMutationApplied && mutation != model.JobMutationAlreadyComplete && mutation != model.JobMutationStale {
@@ -319,7 +348,7 @@ func (w *Worker) resolveClaim(
if job.ErrorCode != nil {
code = *job.ErrorCode
}
return "failed", delivery.ToFailed(ctx, redisstream.FailedDelivery{Code: code, FailedAt: w.now()})
return "duplicate_failed", delivery.ToFailed(ctx, redisstream.FailedDelivery{Code: code, FailedAt: w.now()})
default:
return "transient_error", fmt.Errorf("claim attachment job: %w", claimErr)
}
@@ -444,7 +473,10 @@ func (w *Worker) resolveProcessingFailure(
return "failed", delivery.ToFailed(ctx, redisstream.FailedDelivery{Code: code, FailedAt: w.now()})
}
nextAttempt := w.now().Add(w.retryDelay(lease.Job.ID, lease.Job.Attempt))
mutation, err := w.repository.RetryAttachmentJob(ctx, lease, code, nextAttempt)
traceContext := telemetry.ContextCarrier(ctx)
mutation, err := w.repository.RetryAttachmentJob(
ctx, lease, code, nextAttempt, traceContext.Traceparent, traceContext.Tracestate,
)
if err != nil {
return "transient_error", fmt.Errorf("retry attachment job: %w", err)
}
@@ -454,7 +486,7 @@ func (w *Worker) resolveProcessingFailure(
if mutation != model.JobMutationApplied {
return "transient_error", fmt.Errorf("unexpected retry result %q", mutation)
}
w.metrics.ObserveRetry()
w.metrics.ObserveRetry(ctx)
return "retry", delivery.Ack(ctx)
}
@@ -502,7 +534,7 @@ func (w *Worker) retryDelay(jobID string, attempt int) time.Duration {
type nopMetrics struct{}
func (nopMetrics) ObserveStarted() {}
func (nopMetrics) ObserveFinished(string, time.Duration) {}
func (nopMetrics) ObserveRetry() {}
func (nopMetrics) ObserveReclaimed(int) {}
func (nopMetrics) ObserveStarted(context.Context) {}
func (nopMetrics) ObserveFinished(context.Context, string, time.Duration) {}
func (nopMetrics) ObserveRetry(context.Context) {}
func (nopMetrics) ObserveReclaimed(context.Context, int) {}
@@ -9,9 +9,9 @@ import (
"github.com/stretchr/testify/require"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment/transform"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/objectstore"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment/transform"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/objectstore"
)
func TestConfigRequiresEnoughMemoryForMaximumImage(t *testing.T) {
+17 -24
View File
@@ -6,23 +6,29 @@ import (
"mime"
"net/http"
"net/url"
"regexp"
"strconv"
"strings"
"time"
"unicode"
"unicode/utf8"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/metric"
"go.opentelemetry.io/otel/trace"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/telemetry"
)
func (s *Server) uploadAttachment(w http.ResponseWriter, r *http.Request) *apiError {
started := time.Now()
result := "client_error"
defer func() {
s.metrics.attachmentUploads.WithLabelValues(result).Inc()
s.metrics.attachmentDuration.Observe(time.Since(started).Seconds())
attributes := metric.WithAttributes(attribute.String("result", result))
s.metrics.attachmentUploads.Add(r.Context(), 1, attributes)
s.metrics.attachmentDuration.Record(r.Context(), time.Since(started).Seconds(), attributes)
}()
if r.Method != http.MethodPost {
@@ -47,7 +53,7 @@ func (s *Server) uploadAttachment(w http.ResponseWriter, r *http.Request) *apiEr
uploaded, err := s.attachments.Upload(r.Context(), attachment.Upload{
ChatID: chatID, OwnerID: user.ID, Filename: filename,
ContentType: r.Header.Get("Content-Type"), ContentLength: r.ContentLength, Body: r.Body,
Traceparent: traceparent(r.Header.Get("traceparent")), Tracestate: tracestate(r.Header.Get("tracestate")),
TraceContext: telemetry.ContextCarrier(r.Context()),
})
if err != nil {
if errors.Is(err, model.ErrCapacity) {
@@ -59,7 +65,7 @@ func (s *Server) uploadAttachment(w http.ResponseWriter, r *http.Request) *apiEr
return attachmentError(err)
}
result = "success"
s.metrics.attachmentBytes.Add(float64(uploaded.SizeBytes))
s.metrics.attachmentBytes.Add(r.Context(), uploaded.SizeBytes)
writeJSON(w, http.StatusAccepted, uploaded)
return nil
}
@@ -128,7 +134,10 @@ func (s *Server) attachmentContent(w http.ResponseWriter, r *http.Request) *apiE
w.Header().Set("Content-Disposition", mime.FormatMediaType("inline", map[string]string{"filename": content.Attachment.Filename}))
w.WriteHeader(http.StatusOK)
if _, err := io.Copy(w, content.Body); err != nil {
s.logger.Warn("attachment download interrupted", "attachment_id", content.Attachment.ID, "error", err)
span := trace.SpanFromContext(r.Context())
span.RecordError(err)
span.SetStatus(codes.Error, "response body write failed")
s.logger.WarnContext(r.Context(), "attachment download interrupted", "event", "attachment_download_interrupted", "attachment_id", content.Attachment.ID, "error", err)
}
return nil
}
@@ -148,22 +157,6 @@ func attachmentVariant(query url.Values) (string, bool) {
}
}
var traceparentPattern = regexp.MustCompile(`^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$`)
func traceparent(value string) string {
if traceparentPattern.MatchString(value) {
return value
}
return ""
}
func tracestate(value string) string {
if len(value) <= 512 {
return value
}
return ""
}
func attachmentIDFromRequest(r *http.Request) (string, *apiError) {
parts := strings.Split(strings.Trim(r.URL.Path, "/"), "/")
attachmentID, valid := canonicalID(parts[3])
+1 -1
View File
@@ -6,7 +6,7 @@ import (
"net/http"
"strings"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
)
func (s *Server) createSession(w http.ResponseWriter, r *http.Request) *apiError {
+2 -2
View File
@@ -9,7 +9,7 @@ import (
"time"
"unicode/utf8"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
)
var usernamePattern = regexp.MustCompile(`^[a-z0-9_]{3,32}$`)
@@ -229,7 +229,7 @@ func (s *Server) messages(w http.ResponseWriter, r *http.Request) *apiError {
return unavailable()
}
if replay {
s.metrics.replays.Inc()
s.metrics.replays.Add(r.Context(), 1)
}
writeJSON(w, http.StatusCreated, message)
return nil
@@ -4,8 +4,8 @@ import (
"context"
"time"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
)
type UserRepository interface {
+41 -70
View File
@@ -1,14 +1,15 @@
package httpapi
import (
"context"
"net/http"
"strconv"
"strings"
"time"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/collectors"
"github.com/prometheus/client_golang/prometheus/promhttp"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/metric"
)
var (
@@ -17,80 +18,50 @@ var (
)
type metrics struct {
requests *prometheus.CounterVec
duration *prometheus.HistogramVec
inFlight prometheus.Gauge
replays prometheus.Counter
attachmentUploads *prometheus.CounterVec
attachmentDuration prometheus.Histogram
attachmentBytes prometheus.Counter
requests metric.Int64Counter
duration metric.Float64Histogram
inFlight metric.Int64UpDownCounter
replays metric.Int64Counter
attachmentUploads metric.Int64Counter
attachmentDuration metric.Float64Histogram
attachmentBytes metric.Int64Counter
handler http.Handler
}
func newMetrics() *metrics {
registry := prometheus.NewRegistry()
result := &metrics{
requests: prometheus.NewCounterVec(prometheus.CounterOpts{
Namespace: "messenger",
Subsystem: "http",
Name: "requests_total",
Help: "Completed HTTP requests.",
}, []string{"method", "route", "status_class"}),
duration: prometheus.NewHistogramVec(prometheus.HistogramOpts{
Namespace: "messenger",
Subsystem: "http",
Name: "request_duration_seconds",
Help: "HTTP request duration in seconds.",
Buckets: requestDurationBuckets,
}, []string{"method", "route"}),
inFlight: prometheus.NewGauge(prometheus.GaugeOpts{
Namespace: "messenger",
Subsystem: "http",
Name: "requests_in_flight",
Help: "HTTP requests currently being served.",
}),
replays: prometheus.NewCounter(prometheus.CounterOpts{
Namespace: "messenger",
Subsystem: "http",
Name: "idempotency_replays_total",
Help: "Message requests served from an idempotency record.",
}),
attachmentUploads: prometheus.NewCounterVec(prometheus.CounterOpts{
Namespace: "messenger",
Name: "attachment_uploads_total",
Help: "Completed attachment uploads by result.",
}, []string{"result"}),
attachmentDuration: prometheus.NewHistogram(prometheus.HistogramOpts{
Namespace: "messenger",
Name: "attachment_upload_duration_seconds",
Help: "Attachment upload duration in seconds.",
Buckets: uploadDurationBuckets,
}),
attachmentBytes: prometheus.NewCounter(prometheus.CounterOpts{
Namespace: "messenger",
Name: "attachment_upload_bytes_total",
Help: "Bytes accepted by successful attachment uploads.",
}),
func newMetrics(meter metric.Meter, handler http.Handler) *metrics {
if meter == nil {
meter = otel.Meter("messenger-api")
}
registry.MustRegister(
result.requests,
result.duration,
result.inFlight,
result.replays,
result.attachmentUploads,
result.attachmentDuration,
result.attachmentBytes,
collectors.NewGoCollector(),
collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}),
if handler == nil {
handler = http.NotFoundHandler()
}
requests, _ := meter.Int64Counter("messenger_http_requests", metric.WithDescription("Completed HTTP requests."))
duration, _ := meter.Float64Histogram("messenger_http_request_duration",
metric.WithDescription("HTTP request duration."), metric.WithUnit("s"),
metric.WithExplicitBucketBoundaries(requestDurationBuckets...),
)
result.handler = promhttp.HandlerFor(registry, promhttp.HandlerOpts{})
return result
inFlight, _ := meter.Int64UpDownCounter("messenger_http_requests_in_flight", metric.WithDescription("HTTP requests currently being served."))
replays, _ := meter.Int64Counter("messenger_http_idempotency_replays", metric.WithDescription("Message requests served from an idempotency record."))
attachmentUploads, _ := meter.Int64Counter("messenger_attachment_uploads", metric.WithDescription("Completed attachment uploads by result."))
attachmentDuration, _ := meter.Float64Histogram("messenger_attachment_upload_duration",
metric.WithDescription("Attachment upload duration."), metric.WithUnit("s"),
metric.WithExplicitBucketBoundaries(uploadDurationBuckets...),
)
attachmentBytes, _ := meter.Int64Counter("messenger_attachment_upload_bytes", metric.WithDescription("Bytes accepted by successful attachment uploads."), metric.WithUnit("By"))
return &metrics{
requests: requests, duration: duration, inFlight: inFlight, replays: replays,
attachmentUploads: attachmentUploads, attachmentDuration: attachmentDuration,
attachmentBytes: attachmentBytes, handler: handler,
}
}
func (m *metrics) observe(r *http.Request, status int, started time.Time) {
route := routeTemplate(r.URL.Path)
m.requests.WithLabelValues(r.Method, route, strconv.Itoa(status/100)+"xx").Inc()
m.duration.WithLabelValues(r.Method, route).Observe(time.Since(started).Seconds())
func (m *metrics) observe(ctx context.Context, r *http.Request, status int, started time.Time) {
attributes := metric.WithAttributes(
attribute.String("method", r.Method), attribute.String("route", routeTemplate(r.URL.Path)),
attribute.String("status_class", strconv.Itoa(status/100)+"xx"),
)
m.requests.Add(ctx, 1, attributes)
m.duration.Record(ctx, time.Since(started).Seconds(), attributes)
}
func routeTemplate(path string) string {
+1 -1
View File
@@ -1,7 +1,7 @@
package httpapi
import (
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
"github.com/google/uuid"
)
+31 -6
View File
@@ -6,6 +6,16 @@ import (
"net/http"
"sync/atomic"
"time"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/metric"
"go.opentelemetry.io/otel/propagation"
semconv "go.opentelemetry.io/otel/semconv/v1.34.0"
"go.opentelemetry.io/otel/trace"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/telemetry"
)
type Dependencies struct {
@@ -17,6 +27,8 @@ type Dependencies struct {
Passwords PasswordService
Database HealthChecker
Objects HealthChecker
Meter metric.Meter
Metrics http.Handler
}
type Config struct {
@@ -67,7 +79,8 @@ func NewServer(
attachments: dependencies.Attachments, sessions: dependencies.Sessions,
passwords: dependencies.Passwords, database: dependencies.Database, objects: dependencies.Objects,
sessionTTL: config.SessionTTL, idempotencyTTL: config.IdempotencyTTL, requestTTL: config.RequestTTL,
instanceID: config.InstanceID, logger: logger, swaggerPort: config.SwaggerPort, metrics: newMetrics(),
instanceID: config.InstanceID, logger: logger, swaggerPort: config.SwaggerPort,
metrics: newMetrics(dependencies.Meter, dependencies.Metrics),
}
s.ready.Store(true)
return s
@@ -82,27 +95,39 @@ func (s *Server) Handler() http.Handler {
}
func (s *Server) serve(w http.ResponseWriter, r *http.Request) {
requestContext := telemetry.Extract(r.Context(), propagation.HeaderCarrier(r.Header))
tracer := otel.Tracer("messenger-api/http")
route := routeTemplate(r.URL.Path)
requestContext, span := tracer.Start(requestContext, r.Method+" "+route, trace.WithSpanKind(trace.SpanKindServer),
trace.WithAttributes(semconv.HTTPRequestMethodKey.String(r.Method), semconv.HTTPRoute(route)))
defer span.End()
r = r.WithContext(requestContext)
w.Header().Set("X-Instance-Id", s.instanceID)
requestID, err := newID()
if err != nil {
http.Error(w, "server unavailable", http.StatusServiceUnavailable)
return
}
span.SetAttributes(attribute.String("request.id", requestID))
w.Header().Set("X-Request-Id", requestID)
w.Header().Set("Cache-Control", "no-store")
wrapped := &statusWriter{ResponseWriter: w, status: http.StatusOK}
started := time.Now()
s.metrics.inFlight.Inc()
s.metrics.inFlight.Add(r.Context(), 1)
defer func() {
s.metrics.inFlight.Dec()
s.metrics.inFlight.Add(r.Context(), -1)
if recovered := recover(); recovered != nil {
s.logger.Error("request panic", "request_id", requestID, "method", r.Method, "path", r.URL.Path)
s.logger.ErrorContext(r.Context(), "request panic", "event", "http_request_panic", "request_id", requestID, "method", r.Method, "route", route)
if wrapped.status == http.StatusOK {
writeError(wrapped, requestID, internalError())
}
}
s.metrics.observe(r, wrapped.status, started)
s.logger.Info("request", "request_id", requestID, "method", r.Method, "path", r.URL.Path, "status", wrapped.status, "duration_ms", time.Since(started).Milliseconds())
s.metrics.observe(r.Context(), r, wrapped.status, started)
span.SetAttributes(semconv.HTTPResponseStatusCode(wrapped.status))
if wrapped.status >= http.StatusInternalServerError {
span.SetStatus(codes.Error, http.StatusText(wrapped.status))
}
s.logger.InfoContext(r.Context(), "request", "event", "http_request_completed", "request_id", requestID, "method", r.Method, "route", route, "status", wrapped.status, "duration_ms", time.Since(started).Milliseconds())
}()
if s.allowSwaggerOrigin(wrapped, r) && r.Method == http.MethodOptions && r.Header.Get("Access-Control-Request-Method") != "" {
wrapped.Header().Set("Access-Control-Allow-Methods", "GET, POST, DELETE, OPTIONS")
+2 -2
View File
@@ -14,8 +14,8 @@ import (
"github.com/stretchr/testify/suite"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
)
var errDependency = errors.New("dependency unavailable")
+51 -2
View File
@@ -13,7 +13,12 @@ import (
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/trace"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
)
var ErrNotFound = errors.New("object not found")
@@ -85,30 +90,38 @@ func (s *Store) Ping(ctx context.Context) error {
}
func (s *Store) Put(ctx context.Context, key, contentType string, data []byte) error {
ctx, span := s.startSpan(ctx, "PUT")
defer span.End()
ctx, cancel := context.WithTimeout(ctx, s.timeout)
defer cancel()
_, err := s.client.PutObject(
ctx, s.bucket, key, bytes.NewReader(data), int64(len(data)), minio.PutObjectOptions{ContentType: contentType},
)
if err != nil {
recordSpanError(span, err)
return fmt.Errorf("put S3 object: %w", err)
}
return nil
}
func (s *Store) Get(ctx context.Context, key string) (io.ReadCloser, error) {
ctx, span := s.startSpan(ctx, "GET")
ctx, cancel := context.WithTimeout(ctx, s.timeout)
object, err := s.client.GetObject(ctx, s.bucket, key, minio.GetObjectOptions{})
if err != nil {
cancel()
recordSpanError(span, err)
span.End()
return nil, objectError("get S3 object", err)
}
if _, err := object.Stat(); err != nil {
cancel()
_ = object.Close()
recordSpanError(span, err)
span.End()
return nil, objectError("stat S3 object", err)
}
return &timeoutReader{ReadCloser: object, cancel: cancel}, nil
return &tracedReader{ReadCloser: &timeoutReader{ReadCloser: object, cancel: cancel}, span: span}, nil
}
func (s *Store) Stat(ctx context.Context, key string) (model.StoredObject, error) {
@@ -122,9 +135,12 @@ func (s *Store) Stat(ctx context.Context, key string) (model.StoredObject, error
}
func (s *Store) Delete(ctx context.Context, key string) error {
ctx, span := s.startSpan(ctx, "DELETE")
defer span.End()
ctx, cancel := context.WithTimeout(ctx, s.timeout)
defer cancel()
if err := s.client.RemoveObject(ctx, s.bucket, key, minio.RemoveObjectOptions{}); err != nil {
recordSpanError(span, err)
return objectError("delete S3 object", err)
}
return nil
@@ -161,6 +177,28 @@ type timeoutReader struct {
cancel context.CancelFunc
}
type tracedReader struct {
io.ReadCloser
span trace.Span
}
func (r *tracedReader) Read(buffer []byte) (int, error) {
read, err := r.ReadCloser.Read(buffer)
if err != nil && !errors.Is(err, io.EOF) {
recordSpanError(r.span, err)
}
return read, err
}
func (r *tracedReader) Close() error {
err := r.ReadCloser.Close()
if err != nil {
recordSpanError(r.span, err)
}
r.span.End()
return err
}
func (r *timeoutReader) Close() error {
err := r.ReadCloser.Close()
r.cancel()
@@ -174,3 +212,14 @@ func objectError(action string, err error) error {
}
return fmt.Errorf("%s: %w", action, err)
}
func (s *Store) startSpan(ctx context.Context, operation string) (context.Context, trace.Span) {
return otel.Tracer("objectstore").Start(ctx, "s3 "+operation, trace.WithSpanKind(trace.SpanKindClient),
trace.WithAttributes(attribute.String("storage.system", "s3"), attribute.String("storage.operation.name", operation),
attribute.String("s3.bucket.name", s.bucket)))
}
func recordSpanError(span trace.Span, err error) {
span.RecordError(err)
span.SetStatus(codes.Error, "S3 operation failed")
}
+43 -13
View File
@@ -6,7 +6,13 @@ import (
"fmt"
"time"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/trace"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/telemetry"
)
var errLeaseLost = errors.New("outbox lease lost")
@@ -24,9 +30,9 @@ type Publisher interface {
}
type Metrics interface {
ObservePublished()
ObservePublishFailure()
ObserveRecovered(int)
ObservePublished(context.Context)
ObservePublishFailure(context.Context)
ObserveRecovered(context.Context, int)
}
type Config struct {
@@ -156,15 +162,32 @@ func (d *Dispatcher) DispatchOnce(ctx context.Context) (int, error) {
if err := ctx.Err(); err != nil {
return published, nil
}
_, err := d.publisher.PublishConfirmed(ctx, event.Payload)
publishContext := telemetry.ExtractJSONMessage(ctx, event.Payload)
publishContext, span := otel.Tracer("messenger-dispatcher").Start(publishContext, "attachment-jobs publish",
trace.WithSpanKind(trace.SpanKindProducer), trace.WithAttributes(
attribute.String("messaging.system", "redis"),
attribute.String("messaging.destination.name", "attachment-jobs"),
attribute.String("job.id", event.JobID), attribute.Int("job.attempt", event.Attempt),
))
payload, injectErr := telemetry.InjectJSONMessage(publishContext, event.Payload)
if injectErr != nil {
span.RecordError(injectErr)
span.SetStatus(codes.Error, "inject trace context")
span.End()
return published, fmt.Errorf("inject outbox trace context: %w", injectErr)
}
_, err := d.publisher.PublishConfirmed(publishContext, payload)
if err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, "publish failed")
span.End()
if ctx.Err() != nil {
return published, nil
}
d.metrics.ObservePublishFailure()
d.metrics.ObservePublishFailure(publishContext)
nextAttemptAt := d.now().Add(d.config.RetryDelay)
released, releaseErr := d.store.ReleaseOutboxEvent(
ctx, event.ID, event.LeaseToken, "broker_publish_failed", nextAttemptAt,
publishContext, event.ID, event.LeaseToken, "broker_publish_failed", nextAttemptAt,
)
if releaseErr != nil {
return published, fmt.Errorf("release outbox event after publish failure: %w", releaseErr)
@@ -174,14 +197,21 @@ func (d *Dispatcher) DispatchOnce(ctx context.Context) (int, error) {
}
continue
}
marked, err := d.store.MarkOutboxPublished(ctx, event.ID, event.LeaseToken)
marked, err := d.store.MarkOutboxPublished(publishContext, event.ID, event.LeaseToken)
if err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, "mark published")
span.End()
return published, fmt.Errorf("mark outbox event published: %w", err)
}
if !marked {
span.SetStatus(codes.Error, "outbox lease lost")
span.End()
return published, fmt.Errorf("mark outbox event published: %w", errLeaseLost)
}
d.metrics.ObservePublished()
span.SetAttributes(attribute.String("messaging.operation.type", "publish"))
span.End()
d.metrics.ObservePublished(publishContext)
published++
}
return published, nil
@@ -192,12 +222,12 @@ func (d *Dispatcher) RecoverExpired(ctx context.Context) error {
if err != nil {
return fmt.Errorf("recover expired outbox leases: %w", err)
}
d.metrics.ObserveRecovered(recovered)
d.metrics.ObserveRecovered(ctx, recovered)
return nil
}
type nopMetrics struct{}
func (nopMetrics) ObservePublished() {}
func (nopMetrics) ObservePublishFailure() {}
func (nopMetrics) ObserveRecovered(int) {}
func (nopMetrics) ObservePublished(context.Context) {}
func (nopMetrics) ObservePublishFailure(context.Context) {}
func (nopMetrics) ObserveRecovered(context.Context, int) {}
+1 -1
View File
@@ -8,7 +8,7 @@ import (
"testing"
"time"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
"github.com/stretchr/testify/require"
)
+29 -17
View File
@@ -10,7 +10,7 @@ import (
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
)
const attachmentJobTopic = "attachment.jobs"
@@ -37,9 +37,6 @@ func (s *Store) QueueAttachmentWithJobAndOutbox(
return model.Attachment{}, model.ErrInvalidInput
}
now := time.Now().UTC()
if job.CreatedAt.IsZero() {
job.CreatedAt = now
}
if job.AvailableAt.IsZero() {
job.AvailableAt = now
}
@@ -67,8 +64,7 @@ func (s *Store) QueueAttachmentWithJobAndOutbox(
job.AttachmentID = attachmentID
job.Status = model.AttachmentJobQueued
job.UpdatedAt = job.CreatedAt
if err := insertAttachmentJob(ctx, tx, job); err != nil {
if err := insertAttachmentJob(ctx, tx, &job); err != nil {
return model.Attachment{}, err
}
if err := insertAttachmentJobOutbox(ctx, tx, job, traceparent, tracestate); err != nil {
@@ -92,7 +88,6 @@ func (s *Store) QueueAttachmentPipelineVersion(
}
job.Attempt = 1
now := time.Now().UTC()
job.CreatedAt = now
job.AvailableAt = now
tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{})
@@ -114,7 +109,6 @@ func (s *Store) QueueAttachmentPipelineVersion(
}
job.AttachmentID = attachment.ID
job.Status = model.AttachmentJobQueued
job.UpdatedAt = now
const queueAttachment = `
UPDATE attachments
SET status = 'queued', pipeline_version = $2, staging_key = object_key,
@@ -124,7 +118,7 @@ func (s *Store) QueueAttachmentPipelineVersion(
if err := scanAttachment(tx.QueryRow(ctx, queueAttachment, attachment.ID, job.PipelineVersion), &attachment); err != nil {
return model.Attachment{}, fmt.Errorf("queue attachment pipeline version: %w", err)
}
if err := insertAttachmentJob(ctx, tx, job); err != nil {
if err := insertAttachmentJob(ctx, tx, &job); err != nil {
return model.Attachment{}, err
}
if err := insertAttachmentJobOutbox(ctx, tx, job, traceparent, tracestate); err != nil {
@@ -163,15 +157,16 @@ func (s *Store) alreadyQueuedAttachment(
return publicAttachment(attachment), nil
}
func insertAttachmentJob(ctx context.Context, tx pgx.Tx, job model.AttachmentJob) error {
func insertAttachmentJob(ctx context.Context, tx pgx.Tx, job *model.AttachmentJob) error {
const query = `
INSERT INTO attachment_jobs (
id, attachment_id, pipeline_version, attempt, status, available_at, created_at, updated_at
)
VALUES ($1, $2, $3, $4, 'queued', $5, $6, $6)`
if _, err := tx.Exec(
ctx, query, job.ID, job.AttachmentID, job.PipelineVersion, job.Attempt, job.AvailableAt, job.CreatedAt,
); err != nil {
VALUES ($1, $2, $3, $4, 'queued', $5, statement_timestamp(), statement_timestamp())
RETURNING created_at, updated_at`
if err := tx.QueryRow(
ctx, query, job.ID, job.AttachmentID, job.PipelineVersion, job.Attempt, job.AvailableAt,
).Scan(&job.CreatedAt, &job.UpdatedAt); err != nil {
return fmt.Errorf("insert attachment job: %w", err)
}
return nil
@@ -417,6 +412,9 @@ func (s *Store) CompleteAttachmentJob(
if _, err := tx.Exec(ctx, completeAttachment, attachment.ID, job.PipelineVersion, originalKey); err != nil {
return "", fmt.Errorf("complete attachment: %w", err)
}
if err := insertAttachmentTerminalEvent(ctx, tx, job, model.AttachmentJobReady); err != nil {
return "", err
}
if err := tx.Commit(ctx); err != nil {
return "", fmt.Errorf("commit completed attachment job: %w", err)
}
@@ -447,6 +445,8 @@ func (s *Store) RetryAttachmentJob(
lease model.AttachmentJobLease,
errorCode string,
nextAttemptAt time.Time,
traceparent string,
tracestate string,
) (model.JobMutationResult, error) {
if !validErrorCode(errorCode) || nextAttemptAt.IsZero() {
return "", model.ErrInvalidInput
@@ -483,9 +483,7 @@ func (s *Store) RetryAttachmentJob(
if _, err := tx.Exec(ctx, queueAttachment, attachment.ID, job.PipelineVersion); err != nil {
return "", fmt.Errorf("requeue attachment: %w", err)
}
eventJob := job
eventJob.CreatedAt = time.Now().UTC()
if err := insertAttachmentJobOutbox(ctx, tx, eventJob, "", ""); err != nil {
if err := insertAttachmentJobOutbox(ctx, tx, job, traceparent, tracestate); err != nil {
return "", err
}
if err := tx.Commit(ctx); err != nil {
@@ -527,12 +525,26 @@ func (s *Store) FailAttachmentJob(
if _, err := tx.Exec(ctx, failAttachment, attachment.ID, job.PipelineVersion, errorCode); err != nil {
return "", fmt.Errorf("fail attachment: %w", err)
}
if err := insertAttachmentTerminalEvent(ctx, tx, job, model.AttachmentJobFailed); err != nil {
return "", err
}
if err := tx.Commit(ctx); err != nil {
return "", fmt.Errorf("commit failed attachment job: %w", err)
}
return model.JobMutationApplied, nil
}
func insertAttachmentTerminalEvent(ctx context.Context, tx pgx.Tx, job model.AttachmentJob, status string) error {
const query = `
INSERT INTO attachment_terminal_events (job_id, status, accepted_at)
VALUES ($1, $2, $3)
ON CONFLICT (job_id) DO NOTHING`
if _, err := tx.Exec(ctx, query, job.ID, status, job.CreatedAt); err != nil {
return fmt.Errorf("insert attachment terminal event: %w", err)
}
return nil
}
func lockJobMutation(
ctx context.Context,
tx pgx.Tx,
+1 -1
View File
@@ -5,7 +5,7 @@ import (
"fmt"
"time"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
)
func (s *Store) RenewAttachmentJobLease(
+6 -7
View File
@@ -9,7 +9,7 @@ import (
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
)
func (s *Store) RecoverExpiredAttachmentJobs(
@@ -89,9 +89,7 @@ func (s *Store) RecoverExpiredAttachmentJobs(
if _, err := tx.Exec(ctx, requeueAttachment, attachment.ID, job.PipelineVersion); err != nil {
return model.RecoveryResult{}, fmt.Errorf("requeue attachment after expired lease: %w", err)
}
eventJob := job
eventJob.CreatedAt = time.Now().UTC()
if err := insertAttachmentJobOutbox(ctx, tx, eventJob, "", ""); err != nil {
if err := insertAttachmentJobOutbox(ctx, tx, job, "", ""); err != nil {
return model.RecoveryResult{}, err
}
recovered.Requeued++
@@ -118,6 +116,9 @@ func markExpiredJobFailed(
if _, err := tx.Exec(ctx, failJob, job.ID, errorCode); err != nil {
return fmt.Errorf("fail expired attachment job: %w", err)
}
if err := insertAttachmentTerminalEvent(ctx, tx, job, model.AttachmentJobFailed); err != nil {
return err
}
if !failAttachment {
return nil
}
@@ -198,9 +199,7 @@ func (s *Store) ReplayFailedAttachment(
); err != nil {
return model.AttachmentJob{}, fmt.Errorf("audit attachment replay: %w", err)
}
eventJob := job
eventJob.CreatedAt = time.Now().UTC()
if err := insertAttachmentJobOutbox(ctx, tx, eventJob, "", ""); err != nil {
if err := insertAttachmentJobOutbox(ctx, tx, job, "", ""); err != nil {
return model.AttachmentJob{}, err
}
if err := tx.Commit(ctx); err != nil {
+1 -1
View File
@@ -8,7 +8,7 @@ import (
"github.com/jackc/pgx/v5"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
)
const attachmentColumns = `
+1 -1
View File
@@ -1,6 +1,6 @@
package postgres
import "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
import "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
var (
ErrConflict = model.ErrConflict
+1 -1
View File
@@ -12,7 +12,7 @@ import (
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
)
const expiredIdempotencyCleanupBatch = 100
+1 -1
View File
@@ -8,7 +8,7 @@ import (
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
)
func (s *Store) ClaimOutboxEvents(
+1 -1
View File
@@ -10,7 +10,7 @@ import (
"github.com/jackc/pgx/v5/pgconn"
"github.com/jackc/pgx/v5/pgxpool"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
)
type Store struct {
+218 -4
View File
@@ -12,13 +12,15 @@ import (
"time"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool"
"github.com/stretchr/testify/suite"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/database"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/postgres"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/reconcile"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/database"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/postgres"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/reconcile"
"gitea.privcloud.tech/dizi/mtusi2026lab6/migrations"
)
type PostgresSuite struct {
@@ -210,6 +212,7 @@ func (s *PostgresSuite) TestAttachmentJobLifecycleAndOutboxFencing() {
result, err := s.store.CompleteAttachmentJob(context.Background(), lease, variants)
s.Require().NoError(err)
s.Equal(model.JobMutationApplied, result)
s.assertTerminalEvent(job.ID, model.AttachmentJobReady, lease.Job.CreatedAt)
result, err = s.store.CompleteAttachmentJob(context.Background(), lease, variants)
s.Require().NoError(err)
s.Equal(model.JobMutationAlreadyComplete, result)
@@ -252,6 +255,130 @@ func (s *PostgresSuite) TestAttachmentJobLifecycleAndOutboxFencing() {
s.Equal(model.JobMutationStale, result)
}
func (s *PostgresSuite) TestAttachmentJobCreationUsesDatabaseClock() {
owner, chatID := createOwnerAndChat(s.T(), s.store)
attachmentID := uuid.NewString()
stagingKey := "staging/" + attachmentID + "/original"
pending, err := s.store.CreatePendingAttachment(context.Background(), model.Attachment{
ID: attachmentID, ChatID: chatID, OwnerID: owner.ID, Filename: "future.png",
ContentType: "image/png", SizeBytes: 128, SHA256: strings.Repeat("f", 64), ObjectKey: stagingKey,
})
s.Require().NoError(err)
future := time.Now().Add(24 * time.Hour)
job := model.AttachmentJob{
ID: uuid.NewString(), AttachmentID: attachmentID, PipelineVersion: 1,
Attempt: 1, CreatedAt: future,
}
_, err = s.store.QueueAttachmentWithJobAndOutbox(context.Background(), pending.ID, stagingKey, job, "", "")
s.Require().NoError(err)
stored, err := s.store.AttachmentJob(context.Background(), job.ID)
s.Require().NoError(err)
s.True(stored.CreatedAt.Before(future))
lease, err := s.store.ClaimAttachmentJob(
context.Background(), job.ID, attachmentID, 1, 1, "worker-clock", time.Minute,
)
s.Require().NoError(err)
result, err := s.store.CompleteAttachmentJob(context.Background(), lease, completedVariants(attachmentID, 1))
s.Require().NoError(err)
s.Equal(model.JobMutationApplied, result)
s.assertTerminalEvent(job.ID, model.AttachmentJobReady, stored.CreatedAt)
s.assertOutboxCreationTime(job.ID, 1, stored.CreatedAt)
}
func (s *PostgresSuite) TestTerminalLedgerMigrationBackfillsExistingJob() {
owner, chatID := createOwnerAndChat(s.T(), s.store)
replayedAttachment, replayedJob := createQueuedAttachment(s.T(), s.store, owner.ID, chatID)
skewedAttachment, skewedJob := createQueuedAttachment(s.T(), s.store, owner.ID, chatID)
replayedAcceptedAt := time.Now().Add(-2 * time.Minute).UTC().Truncate(time.Microsecond)
firstFailureAt := replayedAcceptedAt.Add(20 * time.Second)
replayedReadyAt := firstFailureAt.Add(time.Minute)
skewedTerminalAt := time.Now().Add(-time.Minute).UTC().Truncate(time.Microsecond)
skewedCreatedAt := skewedTerminalAt.Add(time.Minute)
tx, err := s.pool.Begin(context.Background())
s.Require().NoError(err)
defer func() { s.Require().NoError(tx.Rollback(context.Background())) }()
_, err = tx.Exec(
context.Background(),
`UPDATE attachment_jobs SET status = 'ready', attempt = 2, error_code = NULL, created_at = $2, updated_at = $3
WHERE id = $1`,
replayedJob.ID,
replayedAcceptedAt,
replayedReadyAt,
)
s.Require().NoError(err)
_, err = tx.Exec(
context.Background(),
`UPDATE attachments SET status = 'ready', error_code = NULL, ready_at = $2
WHERE id = $1`,
replayedAttachment.ID,
replayedReadyAt,
)
s.Require().NoError(err)
_, err = tx.Exec(
context.Background(),
`INSERT INTO attachment_replay_audit (
id, job_id, attachment_id, previous_attempt, new_attempt, actor, reason, created_at
) VALUES ($1, $2, $3, 1, 2, 'legacy-operator', 'legacy replay', $4)`,
uuid.NewString(),
replayedJob.ID,
replayedAttachment.ID,
firstFailureAt,
)
s.Require().NoError(err)
_, err = tx.Exec(
context.Background(),
`UPDATE attachment_jobs SET status = 'failed', error_code = 'legacy_failure', created_at = $2, updated_at = $3
WHERE id = $1`,
skewedJob.ID,
skewedCreatedAt,
skewedTerminalAt,
)
s.Require().NoError(err)
_, err = tx.Exec(
context.Background(),
`UPDATE attachments SET status = 'failed', error_code = 'legacy_failure', ready_at = NULL
WHERE id = $1`,
skewedAttachment.ID,
)
s.Require().NoError(err)
_, err = tx.Exec(context.Background(), `ALTER INDEX attachment_terminal_events_status_terminal_at_idx RENAME TO attachment_terminal_events_status_terminal_at_idx_old`)
s.Require().NoError(err)
_, err = tx.Exec(context.Background(), `ALTER TABLE attachment_terminal_events RENAME TO attachment_terminal_events_old`)
s.Require().NoError(err)
migration, err := migrations.FS.ReadFile("0008_attachment_terminal_events.sql")
s.Require().NoError(err)
_, err = tx.Exec(context.Background(), string(migration), pgx.QueryExecModeSimpleProtocol)
s.Require().NoError(err)
s.assertTerminalEventInTx(tx, replayedJob.ID, model.AttachmentJobFailed, replayedAcceptedAt, firstFailureAt)
s.assertTerminalEventInTx(tx, skewedJob.ID, model.AttachmentJobFailed, skewedTerminalAt, skewedTerminalAt)
}
func (s *PostgresSuite) assertTerminalEventInTx(
tx pgx.Tx,
jobID string,
status string,
acceptedAt time.Time,
terminalAt time.Time,
) {
s.T().Helper()
var (
actualStatus string
actualAcceptedAt time.Time
actualTerminalAt time.Time
)
err := tx.QueryRow(
context.Background(),
`SELECT status, accepted_at, terminal_at FROM attachment_terminal_events WHERE job_id = $1`,
jobID,
).Scan(&actualStatus, &actualAcceptedAt, &actualTerminalAt)
s.Require().NoError(err)
s.Equal(status, actualStatus)
s.True(acceptedAt.Equal(actualAcceptedAt))
s.True(terminalAt.Equal(actualTerminalAt))
}
func (s *PostgresSuite) TestPublishedOutboxRetentionIsBatched() {
owner, chatID := createOwnerAndChat(s.T(), s.store)
_, firstJob := createQueuedAttachment(s.T(), s.store, owner.ID, chatID)
@@ -295,6 +422,8 @@ func (s *PostgresSuite) TestPublishedOutboxRetentionIsBatched() {
func (s *PostgresSuite) TestFailedAttachmentReplayIsAudited() {
owner, chatID := createOwnerAndChat(s.T(), s.store)
attachment, job := createQueuedAttachment(s.T(), s.store, owner.ID, chatID)
original, err := s.store.AttachmentJob(context.Background(), job.ID)
s.Require().NoError(err)
lease, err := s.store.ClaimAttachmentJob(
context.Background(), job.ID, attachment.ID, job.PipelineVersion, job.Attempt, "worker-1", time.Minute,
)
@@ -302,6 +431,7 @@ func (s *PostgresSuite) TestFailedAttachmentReplayIsAudited() {
result, err := s.store.FailAttachmentJob(context.Background(), lease, "invalid_image")
s.Require().NoError(err)
s.Equal(model.JobMutationApplied, result)
firstTerminalAt := s.assertTerminalEvent(job.ID, model.AttachmentJobFailed, original.CreatedAt)
_, _, err = s.store.CreateMessage(
context.Background(), chatID, owner.ID, "failed attachment",
@@ -320,6 +450,18 @@ func (s *PostgresSuite) TestFailedAttachmentReplayIsAudited() {
s.Require().NoError(err)
s.Equal(2, replayed.Attempt)
s.Equal(model.AttachmentJobQueued, replayed.Status)
s.Equal(original.CreatedAt, replayed.CreatedAt)
s.assertOutboxCreationTime(job.ID, 2, original.CreatedAt)
replayedLease, err := s.store.ClaimAttachmentJob(
context.Background(), job.ID, attachment.ID, replayed.PipelineVersion, replayed.Attempt, "worker-2", time.Minute,
)
s.Require().NoError(err)
result, err = s.store.CompleteAttachmentJob(
context.Background(), replayedLease, completedVariants(attachment.ID, replayed.PipelineVersion),
)
s.Require().NoError(err)
s.Equal(model.JobMutationApplied, result)
s.Equal(firstTerminalAt, s.assertTerminalEvent(job.ID, model.AttachmentJobFailed, original.CreatedAt))
var auditCount int
err = s.pool.QueryRow(
@@ -336,6 +478,8 @@ func (s *PostgresSuite) TestExpiredLeaseRecoveryFencesOldWorker() {
context.Background(), job.ID, attachment.ID, job.PipelineVersion, job.Attempt, "worker-1", time.Minute,
)
s.Require().NoError(err)
original, err := s.store.AttachmentJob(context.Background(), job.ID)
s.Require().NoError(err)
_, err = s.pool.Exec(
context.Background(),
`UPDATE attachment_jobs SET lease_until = clock_timestamp() - INTERVAL '1 second' WHERE id = $1`,
@@ -351,12 +495,82 @@ func (s *PostgresSuite) TestExpiredLeaseRecoveryFencesOldWorker() {
s.Require().NoError(err)
s.Equal(2, current.Attempt)
s.Equal(model.AttachmentJobQueued, current.Status)
s.Equal(original.CreatedAt, current.CreatedAt)
s.assertOutboxCreationTime(job.ID, 2, original.CreatedAt)
result, err := s.store.CompleteAttachmentJob(context.Background(), lease, completedVariants(attachment.ID, 1))
s.Require().NoError(err)
s.Equal(model.JobMutationStale, result)
}
func (s *PostgresSuite) TestExpiredLeaseRecoveryFailsExhaustedJobOnce() {
owner, chatID := createOwnerAndChat(s.T(), s.store)
attachment, job := createQueuedAttachment(s.T(), s.store, owner.ID, chatID)
original, err := s.store.AttachmentJob(context.Background(), job.ID)
s.Require().NoError(err)
_, err = s.store.ClaimAttachmentJob(
context.Background(), job.ID, attachment.ID, job.PipelineVersion, job.Attempt, "worker-1", time.Minute,
)
s.Require().NoError(err)
_, err = s.pool.Exec(
context.Background(),
`UPDATE attachment_jobs SET lease_until = clock_timestamp() - INTERVAL '1 second' WHERE id = $1`,
job.ID,
)
s.Require().NoError(err)
recovered, err := s.store.RecoverExpiredAttachmentJobs(context.Background(), 10, 1, time.Now())
s.Require().NoError(err)
s.Zero(recovered.Requeued)
s.Equal(1, recovered.Failed)
firstTerminalAt := s.assertTerminalEvent(job.ID, model.AttachmentJobFailed, original.CreatedAt)
recovered, err = s.store.RecoverExpiredAttachmentJobs(context.Background(), 10, 1, time.Now())
s.Require().NoError(err)
s.Zero(recovered.Requeued)
s.Zero(recovered.Failed)
current, err := s.store.AttachmentJob(context.Background(), job.ID)
s.Require().NoError(err)
s.Equal(model.AttachmentJobFailed, current.Status)
s.Equal(firstTerminalAt, s.assertTerminalEvent(job.ID, model.AttachmentJobFailed, original.CreatedAt))
}
func (s *PostgresSuite) assertTerminalEvent(jobID string, status string, acceptedAt time.Time) time.Time {
s.T().Helper()
var (
actualStatus string
actualAcceptedAt time.Time
terminalAt time.Time
count int
)
err := s.pool.QueryRow(
context.Background(),
`SELECT min(status), min(accepted_at), min(terminal_at), count(*)
FROM attachment_terminal_events WHERE job_id = $1`,
jobID,
).Scan(&actualStatus, &actualAcceptedAt, &terminalAt, &count)
s.Require().NoError(err)
s.Equal(1, count)
s.Equal(status, actualStatus)
s.True(acceptedAt.Equal(actualAcceptedAt), "expected %s, got %s", acceptedAt, actualAcceptedAt)
return terminalAt
}
func (s *PostgresSuite) assertOutboxCreationTime(jobID string, attempt int, expected time.Time) {
s.T().Helper()
var payload []byte
err := s.pool.QueryRow(
context.Background(),
`SELECT payload FROM outbox_events WHERE job_id = $1 AND attempt = $2`,
jobID,
attempt,
).Scan(&payload)
s.Require().NoError(err)
var message model.AttachmentJobMessage
s.Require().NoError(json.Unmarshal(payload, &message))
s.True(expected.Equal(message.CreatedAt), "expected %s, got %s", expected, message.CreatedAt)
}
func (s *PostgresSuite) TestCompleteAttachmentJobRejectsIncompleteOrMismatchedVariants() {
owner, chatID := createOwnerAndChat(s.T(), s.store)
attachment, job := createQueuedAttachment(s.T(), s.store, owner.ID, chatID)
+1 -1
View File
@@ -6,7 +6,7 @@ import (
"fmt"
"time"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
)
type MetadataStore interface {
+1 -1
View File
@@ -8,7 +8,7 @@ import (
"github.com/stretchr/testify/require"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
)
func TestReconcilerLifecycleAndRepeat(t *testing.T) {
+1 -1
View File
@@ -14,7 +14,7 @@ import (
"github.com/valkey-io/valkey-go"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
)
const (
+1 -1
View File
@@ -11,7 +11,7 @@ import (
"github.com/google/uuid"
"github.com/stretchr/testify/suite"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/session"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/session"
)
type SessionSuite struct {
+39
View File
@@ -0,0 +1,39 @@
package telemetry
import (
"context"
"github.com/jackc/pgx/v5/pgxpool"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/metric"
)
var (
connectionStateUsed = attribute.String("state", "used")
connectionStateIdle = attribute.String("state", "idle")
connectionStateConnecting = attribute.String("state", "connecting")
)
func ObserveDatabasePool(meter metric.Meter, pool *pgxpool.Pool) error {
_, err := meter.Int64ObservableGauge("messenger_db_pool_connections",
metric.WithDescription("PostgreSQL pool connections by state."),
metric.WithInt64Callback(func(_ context.Context, observer metric.Int64Observer) error {
stats := pool.Stat()
observer.Observe(int64(stats.AcquiredConns()), metric.WithAttributes(connectionStateUsed))
observer.Observe(int64(stats.IdleConns()), metric.WithAttributes(connectionStateIdle))
observer.Observe(int64(stats.ConstructingConns()), metric.WithAttributes(connectionStateConnecting))
return nil
}),
)
if err != nil {
return err
}
_, err = meter.Int64ObservableCounter("messenger_db_pool_waited",
metric.WithDescription("Connection acquisitions that waited because the PostgreSQL pool was empty."),
metric.WithInt64Callback(func(_ context.Context, observer metric.Int64Observer) error {
observer.Observe(pool.Stat().EmptyAcquireCount())
return nil
}),
)
return err
}
+74
View File
@@ -0,0 +1,74 @@
package telemetry
import (
"context"
"log/slog"
"strings"
"go.opentelemetry.io/otel/trace"
)
type correlationHandler struct {
stdout slog.Handler
otel slog.Handler
}
func newCorrelationHandler(stdout, otel slog.Handler) slog.Handler {
return correlationHandler{stdout: stdout, otel: otel}
}
func (h correlationHandler) Enabled(ctx context.Context, level slog.Level) bool {
return h.stdout.Enabled(ctx, level) || h.otel.Enabled(ctx, level)
}
func (h correlationHandler) Handle(ctx context.Context, record slog.Record) error {
span := trace.SpanContextFromContext(ctx)
if span.IsValid() {
record.AddAttrs(slog.String("trace_id", span.TraceID().String()), slog.String("span_id", span.SpanID().String()))
}
record = sanitizedRecord(record)
if err := h.stdout.Handle(ctx, record); err != nil {
return err
}
return h.otel.Handle(ctx, record)
}
func (h correlationHandler) WithAttrs(attributes []slog.Attr) slog.Handler {
attributes = sanitizedAttributes(attributes)
return correlationHandler{stdout: h.stdout.WithAttrs(attributes), otel: h.otel.WithAttrs(attributes)}
}
func (h correlationHandler) WithGroup(name string) slog.Handler {
return correlationHandler{stdout: h.stdout.WithGroup(name), otel: h.otel.WithGroup(name)}
}
func sanitizedRecord(record slog.Record) slog.Record {
copy := slog.NewRecord(record.Time, record.Level, record.Message, record.PC)
record.Attrs(func(attribute slog.Attr) bool {
if !sensitiveKey(attribute.Key) {
copy.AddAttrs(attribute)
}
return true
})
return copy
}
func sanitizedAttributes(attributes []slog.Attr) []slog.Attr {
result := make([]slog.Attr, 0, len(attributes))
for _, attribute := range attributes {
if !sensitiveKey(attribute.Key) {
result = append(result, attribute)
}
}
return result
}
func sensitiveKey(key string) bool {
key = strings.ToLower(key)
for _, marker := range []string{"authorization", "password", "secret", "token", "cookie", "body", "payload"} {
if strings.Contains(key, marker) {
return true
}
}
return false
}
+70
View File
@@ -0,0 +1,70 @@
package telemetry
import (
"context"
"encoding/json"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/propagation"
)
type MessageCarrier struct {
Traceparent string
Tracestate string
}
func Extract(ctx context.Context, carrier propagation.TextMapCarrier) context.Context {
return otel.GetTextMapPropagator().Extract(ctx, carrier)
}
func Inject(ctx context.Context, carrier propagation.TextMapCarrier) {
otel.GetTextMapPropagator().Inject(ctx, carrier)
}
func ContextCarrier(ctx context.Context) MessageCarrier {
carrier := propagation.MapCarrier{}
Inject(ctx, carrier)
return MessageCarrier{
Traceparent: carrier.Get("traceparent"),
Tracestate: carrier.Get("tracestate"),
}
}
func ExtractMessage(ctx context.Context, carrier MessageCarrier) context.Context {
headers := propagation.MapCarrier{
"traceparent": carrier.Traceparent,
"tracestate": carrier.Tracestate,
}
return Extract(ctx, headers)
}
func ExtractJSONMessage(ctx context.Context, payload []byte) context.Context {
var carrier struct {
Traceparent string `json:"traceparent"`
Tracestate string `json:"tracestate"`
}
if json.Unmarshal(payload, &carrier) != nil {
return ctx
}
return ExtractMessage(ctx, MessageCarrier(carrier))
}
func InjectJSONMessage(ctx context.Context, payload []byte) ([]byte, error) {
var value map[string]json.RawMessage
if err := json.Unmarshal(payload, &value); err != nil {
return nil, err
}
carrier := ContextCarrier(ctx)
setJSONField(value, "traceparent", carrier.Traceparent)
setJSONField(value, "tracestate", carrier.Tracestate)
return json.Marshal(value)
}
func setJSONField(value map[string]json.RawMessage, key, field string) {
if field == "" {
delete(value, key)
return
}
encoded, _ := json.Marshal(field)
value[key] = encoded
}
+33
View File
@@ -0,0 +1,33 @@
package telemetry
import (
"context"
"encoding/json"
"testing"
"github.com/stretchr/testify/require"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/propagation"
"go.opentelemetry.io/otel/trace"
)
func TestJSONMessagePropagationPreservesTraceContext(t *testing.T) {
otel.SetTextMapPropagator(propagation.TraceContext{})
traceID, err := trace.TraceIDFromHex("4bf92f3577b34da6a3ce929d0e0e4736")
require.NoError(t, err)
spanID, err := trace.SpanIDFromHex("00f067aa0ba902b7")
require.NoError(t, err)
ctx := trace.ContextWithSpanContext(context.Background(), trace.NewSpanContext(trace.SpanContextConfig{
TraceID: traceID, SpanID: spanID, TraceFlags: trace.FlagsSampled,
}))
payload, err := InjectJSONMessage(ctx, []byte(`{"job_id":"job-1"}`))
require.NoError(t, err)
var value map[string]any
require.NoError(t, json.Unmarshal(payload, &value))
require.Equal(t, "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01", value["traceparent"])
require.NotContains(t, value, "baggage")
extracted := ExtractJSONMessage(context.Background(), payload)
require.Equal(t, traceID, trace.SpanContextFromContext(extracted).TraceID())
}
+252
View File
@@ -0,0 +1,252 @@
package telemetry
import (
"context"
"errors"
"fmt"
"log/slog"
"net/http"
"os"
"strings"
"time"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/collectors"
"github.com/prometheus/client_golang/prometheus/promhttp"
"go.opentelemetry.io/contrib/bridges/otelslog"
otelruntime "go.opentelemetry.io/contrib/instrumentation/runtime"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc"
"go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc"
"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
otelprometheus "go.opentelemetry.io/otel/exporters/prometheus"
"go.opentelemetry.io/otel/propagation"
"go.opentelemetry.io/otel/sdk/log"
"go.opentelemetry.io/otel/sdk/metric"
"go.opentelemetry.io/otel/sdk/resource"
"go.opentelemetry.io/otel/sdk/trace"
semconv "go.opentelemetry.io/otel/semconv/v1.34.0"
)
const (
defaultEnvironment = "lab"
defaultNamespace = "mtusi-highload"
)
type Config struct {
ServiceName string
Version string
InstanceID string
Endpoint string
Environment string
Namespace string
Insecure bool
}
type Runtime struct {
traces *trace.TracerProvider
metrics *metric.MeterProvider
logs *log.LoggerProvider
logger *slog.Logger
handler http.Handler
}
func ConfigFromEnvironment(serviceName, instanceID string) Config {
resourceAttributes := parseResourceAttributes(os.Getenv("OTEL_RESOURCE_ATTRIBUTES"))
return Config{
ServiceName: serviceName,
Version: firstValue(os.Getenv("SERVICE_VERSION"), resourceAttributes["service.version"], "dev"),
InstanceID: instanceID,
Endpoint: strings.TrimSpace(os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT")),
Environment: firstValue(os.Getenv("DEPLOYMENT_ENVIRONMENT"), resourceAttributes["deployment.environment.name"], defaultEnvironment),
Namespace: firstValue(os.Getenv("SERVICE_NAMESPACE"), resourceAttributes["service.namespace"], defaultNamespace),
Insecure: true,
}
}
func parseResourceAttributes(raw string) map[string]string {
result := make(map[string]string)
for _, pair := range strings.Split(raw, ",") {
key, value, found := strings.Cut(pair, "=")
if found && strings.TrimSpace(key) != "" {
result[strings.TrimSpace(key)] = strings.TrimSpace(value)
}
}
return result
}
func firstValue(values ...string) string {
for _, value := range values {
if value = strings.TrimSpace(value); value != "" {
return value
}
}
return ""
}
func New(ctx context.Context, config Config) (*Runtime, error) {
if err := config.validate(); err != nil {
return nil, err
}
res, err := newResource(ctx, config)
if err != nil {
return nil, err
}
registry := prometheus.NewRegistry()
prometheusReader, err := otelprometheus.New(otelprometheus.WithRegisterer(registry))
if err != nil {
return nil, fmt.Errorf("create Prometheus reader: %w", err)
}
traceOptions := []trace.TracerProviderOption{trace.WithResource(res)}
metricOptions := []metric.Option{metric.WithResource(res), metric.WithReader(prometheusReader)}
logOptions := []log.LoggerProviderOption{log.WithResource(res)}
if config.Endpoint != "" {
traceExporter, err := newTraceExporter(ctx, config)
if err != nil {
return nil, err
}
metricExporter, err := newMetricExporter(ctx, config)
if err != nil {
return nil, err
}
logExporter, err := newLogExporter(ctx, config)
if err != nil {
return nil, err
}
traceOptions = append(traceOptions, trace.WithBatcher(traceExporter,
trace.WithMaxQueueSize(2048), trace.WithMaxExportBatchSize(512),
trace.WithBatchTimeout(time.Second), trace.WithExportTimeout(5*time.Second),
))
metricOptions = append(metricOptions, metric.WithReader(metric.NewPeriodicReader(
metricExporter, metric.WithInterval(5*time.Second), metric.WithTimeout(5*time.Second),
)))
logOptions = append(logOptions, log.WithProcessor(log.NewBatchProcessor(logExporter,
log.WithMaxQueueSize(2048), log.WithExportMaxBatchSize(512),
log.WithExportInterval(time.Second), log.WithExportTimeout(5*time.Second),
)))
}
traces := trace.NewTracerProvider(traceOptions...)
metrics := metric.NewMeterProvider(metricOptions...)
logs := log.NewLoggerProvider(logOptions...)
propagator := propagation.NewCompositeTextMapPropagator(propagation.TraceContext{})
otel.SetTracerProvider(traces)
otel.SetMeterProvider(metrics)
otel.SetTextMapPropagator(propagator)
if err := otelruntime.Start(otelruntime.WithMeterProvider(metrics)); err != nil {
return nil, fmt.Errorf("start Go runtime metrics: %w", err)
}
stdout := slog.NewJSONHandler(os.Stdout, nil)
otelHandler := otelslog.NewHandler(config.ServiceName, otelslog.WithLoggerProvider(logs))
logger := slog.New(newCorrelationHandler(stdout, otelHandler))
registry.MustRegister(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}))
return &Runtime{
traces: traces, metrics: metrics, logs: logs, logger: logger,
handler: promhttp.HandlerFor(registry, promhttp.HandlerOpts{}),
}, nil
}
func (c Config) validate() error {
switch {
case strings.TrimSpace(c.ServiceName) == "":
return errors.New("telemetry service name is required")
case strings.TrimSpace(c.InstanceID) == "":
return errors.New("telemetry instance ID is required")
case strings.TrimSpace(c.Environment) == "":
return errors.New("telemetry environment is required")
case strings.TrimSpace(c.Namespace) == "":
return errors.New("telemetry service namespace is required")
default:
return nil
}
}
func newResource(ctx context.Context, config Config) (*resource.Resource, error) {
res, err := resource.New(ctx,
resource.WithFromEnv(), resource.WithTelemetrySDK(), resource.WithHost(),
resource.WithAttributes(
semconv.ServiceName(config.ServiceName), semconv.ServiceVersion(config.Version),
semconv.ServiceInstanceID(config.InstanceID), semconv.ServiceNamespace(config.Namespace),
semconv.DeploymentEnvironmentName(config.Environment),
),
)
if err != nil {
return nil, fmt.Errorf("create telemetry resource: %w", err)
}
return res, nil
}
func newTraceExporter(ctx context.Context, config Config) (trace.SpanExporter, error) {
var options []otlptracegrpc.Option
if strings.Contains(config.Endpoint, "://") {
options = append(options, otlptracegrpc.WithEndpointURL(config.Endpoint))
} else {
options = append(options, otlptracegrpc.WithEndpoint(config.Endpoint))
}
if config.Insecure {
options = append(options, otlptracegrpc.WithInsecure())
}
exporter, err := otlptracegrpc.New(ctx, options...)
if err != nil {
return nil, fmt.Errorf("create OTLP trace exporter: %w", err)
}
return exporter, nil
}
func newMetricExporter(ctx context.Context, config Config) (*otlpmetricgrpc.Exporter, error) {
var options []otlpmetricgrpc.Option
if strings.Contains(config.Endpoint, "://") {
options = append(options, otlpmetricgrpc.WithEndpointURL(config.Endpoint))
} else {
options = append(options, otlpmetricgrpc.WithEndpoint(config.Endpoint))
}
if config.Insecure {
options = append(options, otlpmetricgrpc.WithInsecure())
}
exporter, err := otlpmetricgrpc.New(ctx, options...)
if err != nil {
return nil, fmt.Errorf("create OTLP metric exporter: %w", err)
}
return exporter, nil
}
func newLogExporter(ctx context.Context, config Config) (*otlploggrpc.Exporter, error) {
var options []otlploggrpc.Option
if strings.Contains(config.Endpoint, "://") {
options = append(options, otlploggrpc.WithEndpointURL(config.Endpoint))
} else {
options = append(options, otlploggrpc.WithEndpoint(config.Endpoint))
}
if config.Insecure {
options = append(options, otlploggrpc.WithInsecure())
}
exporter, err := otlploggrpc.New(ctx, options...)
if err != nil {
return nil, fmt.Errorf("create OTLP log exporter: %w", err)
}
return exporter, nil
}
func (r *Runtime) Logger() *slog.Logger {
return r.logger
}
func (r *Runtime) MeterProvider() *metric.MeterProvider {
return r.metrics
}
func (r *Runtime) MetricsHandler() http.Handler {
return r.handler
}
func (r *Runtime) Shutdown(ctx context.Context) error {
var result error
for _, shutdown := range []func(context.Context) error{r.logs.Shutdown, r.metrics.Shutdown, r.traces.Shutdown} {
if err := shutdown(ctx); err != nil {
result = errors.Join(result, err)
}
}
return result
}
+59
View File
@@ -0,0 +1,59 @@
package telemetry
import (
"context"
"log/slog"
"testing"
"time"
"github.com/stretchr/testify/require"
"go.opentelemetry.io/otel/trace"
)
func TestConfigValidation(t *testing.T) {
valid := Config{ServiceName: "messenger-api", InstanceID: "api-1", Environment: "lab", Namespace: "mtusi-highload"}
require.NoError(t, valid.validate())
invalid := valid
invalid.ServiceName = ""
require.EqualError(t, invalid.validate(), "telemetry service name is required")
}
func TestCorrelationHandlerAddsTraceAndRedactsSecrets(t *testing.T) {
traceID, err := trace.TraceIDFromHex("4bf92f3577b34da6a3ce929d0e0e4736")
require.NoError(t, err)
spanID, err := trace.SpanIDFromHex("00f067aa0ba902b7")
require.NoError(t, err)
ctx := trace.ContextWithSpanContext(context.Background(), trace.NewSpanContext(trace.SpanContextConfig{
TraceID: traceID, SpanID: spanID, TraceFlags: trace.FlagsSampled,
}))
first := &recordHandler{}
second := &recordHandler{}
handler := newCorrelationHandler(first, second)
record := slog.NewRecord(time.Now(), slog.LevelInfo, "request", 0)
record.AddAttrs(slog.String("event", "http_request_completed"), slog.String("authorization", "secret"))
require.NoError(t, handler.Handle(ctx, record))
for _, captured := range []*recordHandler{first, second} {
require.Equal(t, traceID.String(), captured.attributes["trace_id"])
require.Equal(t, spanID.String(), captured.attributes["span_id"])
require.Equal(t, "http_request_completed", captured.attributes["event"])
require.NotContains(t, captured.attributes, "authorization")
}
}
type recordHandler struct {
attributes map[string]string
}
func (*recordHandler) Enabled(context.Context, slog.Level) bool { return true }
func (h *recordHandler) Handle(_ context.Context, record slog.Record) error {
h.attributes = make(map[string]string)
record.Attrs(func(attribute slog.Attr) bool {
h.attributes[attribute.Key] = attribute.Value.String()
return true
})
return nil
}
func (h *recordHandler) WithAttrs([]slog.Attr) slog.Handler { return h }
func (h *recordHandler) WithGroup(string) slog.Handler { return h }
+19 -2
View File
@@ -1,3 +1,6 @@
# Изолированное имя Compose-проекта и его volumes/networks.
COMPOSE_PROJECT_NAME=mtusi2026lab6
# Адрес и внутренний порт HTTP-сервера.
APP_HOST=0.0.0.0
APP_PORT=8080
@@ -8,6 +11,14 @@ API_BIND_ADDRESS=127.0.0.1:8080
SWAGGER_BIND_ADDRESS=127.0.0.1:8081
PROMETHEUS_BIND_ADDRESS=127.0.0.1:9090
GRAFANA_BIND_ADDRESS=127.0.0.1:3000
LOKI_BIND_ADDRESS=127.0.0.1:3100
TEMPO_BIND_ADDRESS=127.0.0.1:3200
# Ресурсные атрибуты и внутренний OTLP endpoint приложений.
DEPLOYMENT_ENVIRONMENT=lab
SERVICE_VERSION=lab6
OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4317
OTEL_EXPORTER_OTLP_PROTOCOL=grpc
# Число API-реплик за Nginx.
API_REPLICAS=2
@@ -55,8 +66,14 @@ EXPORTER_CPUS=0.25
EXPORTER_MEMORY=128m
PROMETHEUS_CPUS=1
PROMETHEUS_MEMORY=512m
GRAFANA_CPUS=0.5
GRAFANA_MEMORY=256m
GRAFANA_CPUS=1
GRAFANA_MEMORY=512m
OTEL_COLLECTOR_CPUS=0.5
OTEL_COLLECTOR_MEMORY=256m
LOKI_CPUS=1
LOKI_MEMORY=512m
TEMPO_CPUS=1
TEMPO_MEMORY=512m
POSTGRES_CPUS=2
POSTGRES_MEMORY=1g
VALKEY_CPUS=1
@@ -0,0 +1,33 @@
CREATE TABLE attachment_terminal_events (
job_id UUID PRIMARY KEY,
status TEXT NOT NULL,
accepted_at TIMESTAMPTZ NOT NULL,
terminal_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(),
CONSTRAINT attachment_terminal_events_status CHECK (status IN ('ready', 'failed')),
CONSTRAINT attachment_terminal_events_time CHECK (terminal_at >= accepted_at)
);
CREATE INDEX attachment_terminal_events_status_terminal_at_idx
ON attachment_terminal_events (status, terminal_at);
WITH first_replay AS (
SELECT job_id, MIN(created_at) AS created_at
FROM attachment_replay_audit
GROUP BY job_id
)
INSERT INTO attachment_terminal_events (job_id, status, accepted_at, terminal_at)
SELECT jobs.id, 'failed', LEAST(jobs.created_at, first_replay.created_at), first_replay.created_at
FROM attachment_jobs jobs
JOIN first_replay ON first_replay.job_id = jobs.id
ON CONFLICT (job_id) DO NOTHING;
INSERT INTO attachment_terminal_events (job_id, status, accepted_at, terminal_at)
SELECT jobs.id, jobs.status, LEAST(jobs.created_at, jobs.updated_at), jobs.updated_at
FROM attachment_jobs jobs
WHERE jobs.status IN ('ready', 'failed')
AND NOT EXISTS (
SELECT 1
FROM attachment_replay_audit replay
WHERE replay.job_id = jobs.id
)
ON CONFLICT (job_id) DO NOTHING;
@@ -67,7 +67,7 @@
},
{
"type": "timeseries",
"title": "API latency p95 by route",
"title": "API latency p50 / p95 / p99 by route",
"id": 3,
"gridPos": {
"h": 8,
@@ -77,9 +77,19 @@
},
"targets": [
{
"expr": "histogram_quantile(0.95, sum by (le, method, route) (rate(messenger_http_request_duration_seconds_bucket[5m])))",
"legendFormat": "{{method}} {{route}}",
"expr": "histogram_quantile(0.50, sum by (le, method, route) (rate(messenger_http_request_duration_seconds_bucket[5m])))",
"legendFormat": "p50 {{method}} {{route}}",
"refId": "A"
},
{
"expr": "histogram_quantile(0.95, sum by (le, method, route) (rate(messenger_http_request_duration_seconds_bucket[5m])))",
"legendFormat": "p95 {{method}} {{route}}",
"refId": "B"
},
{
"expr": "histogram_quantile(0.99, sum by (le, method, route) (rate(messenger_http_request_duration_seconds_bucket[5m])))",
"legendFormat": "p99 {{method}} {{route}}",
"refId": "C"
}
],
"fieldConfig": {
@@ -895,14 +905,563 @@
}
}
]
},
{
"type": "row",
"title": "SLO and alerts",
"id": 105,
"collapsed": true,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 109
},
"panels": [
{
"type": "timeseries",
"title": "Business API availability",
"id": 31,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 110
},
"targets": [
{
"expr": "1 - (sum(rate(messenger_http_requests_total{route!~=\"/health/.*|/metrics\",status_class=\"5xx\"}[5m])) or vector(0)) / clamp_min((sum(rate(messenger_http_requests_total{route!~=\"/health/.*|/metrics\"}[5m])) or vector(0)), 0.000001)",
"legendFormat": "availability",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"unit": "percentunit"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
}
},
{
"type": "timeseries",
"title": "API error-budget burn rate (99.5% SLO)",
"id": 32,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 110
},
"targets": [
{
"expr": "(sum(rate(messenger_http_requests_total{route!~=\"/health/.*|/metrics\",status_class=\"5xx\"}[5m])) or vector(0)) / clamp_min((sum(rate(messenger_http_requests_total{route!~=\"/health/.*|/metrics\"}[5m])) or vector(0)), 0.000001) / 0.005",
"legendFormat": "5m burn",
"refId": "A"
},
{
"expr": "(sum(rate(messenger_http_requests_total{route!~=\"/health/.*|/metrics\",status_class=\"5xx\"}[1h])) or vector(0)) / clamp_min((sum(rate(messenger_http_requests_total{route!~=\"/health/.*|/metrics\"}[1h])) or vector(0)), 0.000001) / 0.005",
"legendFormat": "1h burn",
"refId": "B"
},
{
"expr": "(sum(rate(messenger_http_requests_total{route!~=\"/health/.*|/metrics\",status_class=\"5xx\"}[6h])) or vector(0)) / clamp_min((sum(rate(messenger_http_requests_total{route!~=\"/health/.*|/metrics\"}[6h])) or vector(0)), 0.000001) / 0.005",
"legendFormat": "6h burn",
"refId": "C"
}
],
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
}
},
{
"type": "timeseries",
"title": "Attachments ready within 30 seconds",
"id": 33,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 118
},
"targets": [
{
"expr": "1 - (clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"+Inf\"}[5m])) or vector(0)) - (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"30\"}[5m])) or vector(0)), 0) + (sum(rate(messenger_attachment_terminal_transitions_total{status=\"failed\"}[5m])) or vector(0))) / clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"+Inf\"}[5m])) or vector(0)) + (sum(rate(messenger_attachment_terminal_transitions_total{status=\"failed\"}[5m])) or vector(0)), 0.000001)",
"legendFormat": "ready in 30s",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"unit": "percentunit"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
}
},
{
"type": "timeseries",
"title": "Attachment ready-time error-budget burn",
"id": 34,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 118
},
"targets": [
{
"expr": "(clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"+Inf\"}[5m])) or vector(0)) - (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"30\"}[5m])) or vector(0)), 0) + (sum(rate(messenger_attachment_terminal_transitions_total{status=\"failed\"}[5m])) or vector(0))) / clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"+Inf\"}[5m])) or vector(0)) + (sum(rate(messenger_attachment_terminal_transitions_total{status=\"failed\"}[5m])) or vector(0)), 0.000001) / 0.01",
"legendFormat": "5m burn",
"refId": "A"
},
{
"expr": "(clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"+Inf\"}[1h])) or vector(0)) - (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"30\"}[1h])) or vector(0)), 0) + (sum(rate(messenger_attachment_terminal_transitions_total{status=\"failed\"}[1h])) or vector(0))) / clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"+Inf\"}[1h])) or vector(0)) + (sum(rate(messenger_attachment_terminal_transitions_total{status=\"failed\"}[1h])) or vector(0)), 0.000001) / 0.01",
"legendFormat": "1h burn",
"refId": "B"
},
{
"expr": "(clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"+Inf\"}[6h])) or vector(0)) - (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"30\"}[6h])) or vector(0)), 0) + (sum(rate(messenger_attachment_terminal_transitions_total{status=\"failed\"}[6h])) or vector(0))) / clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"+Inf\"}[6h])) or vector(0)) + (sum(rate(messenger_attachment_terminal_transitions_total{status=\"failed\"}[6h])) or vector(0)), 0.000001) / 0.01",
"legendFormat": "6h burn",
"refId": "C"
}
],
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
}
},
{
"type": "timeseries",
"title": "SLO alert state",
"id": 35,
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 126
},
"targets": [
{
"expr": "max by (alertname, alertstate) (ALERTS{alertname=~\"MessengerAPIHighErrorBudgetBurn.*|AttachmentReadyHighErrorBudgetBurn.*|AttachmentQueueBacklogGrowing|AttachmentWorkerUnavailable\"})",
"legendFormat": "{{alertname}} {{alertstate}}",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
}
}
]
},
{
"type": "row",
"title": "Runtime resources",
"id": 107,
"collapsed": true,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 134
},
"panels": [
{
"type": "timeseries",
"title": "Process CPU by service",
"id": 38,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 135
},
"targets": [
{
"expr": "sum by (service_name) (rate(process_cpu_seconds_total[1m]))",
"legendFormat": "{{service_name}}",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"unit": "cores"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
}
},
{
"type": "timeseries",
"title": "Go heap by service",
"id": 39,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 135
},
"targets": [
{
"expr": "sum by (service_name) (go_memstats_heap_alloc_bytes)",
"legendFormat": "{{service_name}}",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"unit": "bytes"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
}
},
{
"type": "timeseries",
"title": "Database pool used and wait rate",
"id": 40,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 143
},
"targets": [
{
"expr": "sum by (service_name) (messenger_db_pool_connections{state=\"used\"})",
"legendFormat": "{{service_name}} used",
"refId": "A"
},
{
"expr": "sum by (service_name) (rate(messenger_db_pool_waited_total[5m]))",
"legendFormat": "{{service_name}} waits/s",
"refId": "B"
}
],
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
}
},
{
"type": "timeseries",
"title": "Worker saturation",
"id": 41,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 143
},
"targets": [
{
"expr": "sum(messenger_attachment_worker_jobs_in_flight)",
"legendFormat": "busy",
"refId": "A"
},
{
"expr": "sum(messenger_attachment_worker_capacity)",
"legendFormat": "capacity",
"refId": "B"
}
],
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
}
}
]
},
{
"type": "row",
"title": "Logs and traces",
"id": 106,
"collapsed": true,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 151
},
"panels": [
{
"type": "logs",
"title": "Correlated application logs",
"id": 36,
"gridPos": {
"h": 10,
"w": 12,
"x": 0,
"y": 152
},
"datasource": {
"type": "loki",
"uid": "loki"
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "loki"
},
"editorMode": "code",
"expr": "{service_name=~\"messenger-.+\"} | json",
"queryType": "range",
"refId": "A"
}
],
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"prettifyLogMessage": false,
"showCommonLabels": false,
"showLabels": false,
"showTime": true,
"sortOrder": "Descending",
"wrapLogMessage": true
}
},
{
"type": "traces",
"title": "Recent application traces (TraceQL)",
"id": 37,
"gridPos": {
"h": 10,
"w": 12,
"x": 12,
"y": 152
},
"datasource": {
"type": "tempo",
"uid": "tempo"
},
"targets": [
{
"datasource": {
"type": "tempo",
"uid": "tempo"
},
"query": "{ resource.service.name =~ \"messenger-.*\" }",
"queryType": "traceql",
"refId": "A",
"limit": 50,
"tableType": "traces"
}
],
"options": {
"showSpanFilters": true
}
},
{
"type": "timeseries",
"title": "Trace-derived service call rate",
"id": 42,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 162
},
"targets": [
{
"expr": "sum by (service, span_name) (rate(traces_spanmetrics_calls_total[5m]))",
"legendFormat": "{{service}} {{span_name}}",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"unit": "ops"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"datasource": {
"type": "prometheus",
"uid": "prometheus"
}
},
{
"type": "timeseries",
"title": "Collector delivery failures and queue pressure",
"id": 43,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 162
},
"targets": [
{
"expr": "sum by (exporter) (rate(otelcol_exporter_send_failed_spans[5m]))",
"legendFormat": "{{exporter}} spans failed/s",
"refId": "A"
},
{
"expr": "sum by (exporter) (rate(otelcol_exporter_send_failed_log_records[5m]))",
"legendFormat": "{{exporter}} logs failed/s",
"refId": "B"
},
{
"expr": "max by (exporter) (otelcol_exporter_queue_size)",
"legendFormat": "{{exporter}} queue",
"refId": "C"
},
{
"expr": "max by (exporter) (otelcol_exporter_queue_capacity)",
"legendFormat": "{{exporter}} capacity",
"refId": "D"
}
],
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"datasource": {
"type": "prometheus",
"uid": "prometheus"
}
}
]
}
],
"refresh": "5s",
"schemaVersion": 41,
"tags": [
"highload",
"lab5",
"pipeline"
"lab6",
"observability",
"slo"
],
"templating": {
"list": []
@@ -912,7 +1471,42 @@
"to": "now"
},
"timezone": "browser",
"title": "Messenger lab5 overview",
"uid": "messenger-lab5",
"version": 3
"title": "Messenger lab6 observability",
"uid": "messenger-lab6",
"version": 1,
"links": [
{
"asDropdown": false,
"icon": "external link",
"includeVars": true,
"keepTime": true,
"tags": [],
"targetBlank": false,
"title": "Explore logs and traces",
"type": "link",
"url": "/explore"
},
{
"asDropdown": false,
"icon": "external link",
"includeVars": false,
"keepTime": false,
"tags": [],
"targetBlank": true,
"title": "API runbook",
"type": "link",
"url": "https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/api-availability.md"
},
{
"asDropdown": false,
"icon": "external link",
"includeVars": false,
"keepTime": false,
"tags": [],
"targetBlank": true,
"title": "Pipeline runbook",
"type": "link",
"url": "https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/attachment-pipeline.md"
}
]
}
@@ -0,0 +1 @@
@@ -0,0 +1,76 @@
apiVersion: 1
deleteDatasources:
- name: Prometheus
orgId: 1
- name: Loki
orgId: 1
- name: Tempo
orgId: 1
datasources:
- name: Prometheus
type: prometheus
uid: prometheus
access: proxy
url: http://prometheus:9090
isDefault: true
editable: false
jsonData:
httpMethod: POST
exemplarTraceIdDestinations:
- name: trace_id
datasourceUid: tempo
- name: Loki
type: loki
uid: loki
access: proxy
url: http://loki:3100
editable: false
jsonData:
maxLines: 1000
derivedFields:
- name: TraceID
matcherType: label
matcherRegex: trace_id
datasourceUid: tempo
url: '$${__value.raw}'
urlDisplayLabel: View trace
- name: Tempo
type: tempo
uid: tempo
access: proxy
url: http://tempo:3200
editable: false
jsonData:
nodeGraph:
enabled: true
search:
hide: false
serviceMap:
datasourceUid: prometheus
tracesToLogsV2:
datasourceUid: loki
spanStartTimeShift: -5m
spanEndTimeShift: 5m
filterByTraceID: true
filterBySpanID: false
customQuery: true
query: '{service_name="$${__span.tags["service.name"]}"} | trace_id = "$${__trace.traceId}"'
tags:
- key: service.name
value: service_name
tracesToMetrics:
datasourceUid: prometheus
spanStartTimeShift: -5m
spanEndTimeShift: 5m
tags:
- key: service.name
value: service
queries:
- name: Request rate
query: 'sum(rate(traces_spanmetrics_calls_total{$$__tags}[5m]))'
- name: Error rate
query: 'sum(rate(traces_spanmetrics_calls_total{$$__tags,span_status="STATUS_CODE_ERROR"}[5m]))'
@@ -1,10 +0,0 @@
apiVersion: 1
datasources:
- name: Prometheus
type: prometheus
uid: prometheus
access: proxy
url: http://prometheus:9090
isDefault: true
editable: false
@@ -0,0 +1 @@
+58
View File
@@ -37,3 +37,61 @@ messenger_outbox:
- oldest_pending_age_seconds:
usage: GAUGE
description: Age of the oldest unpublished outbox event.
messenger_attachment_terminal_transitions:
query: |
WITH statuses(status) AS (
VALUES ('ready'), ('failed')
)
SELECT
statuses.status,
COUNT(events.job_id)::bigint AS total
FROM statuses
LEFT JOIN attachment_terminal_events events ON events.status = statuses.status
GROUP BY statuses.status
metrics:
- status:
usage: LABEL
description: First terminal status recorded for a durable attachment job.
- total:
usage: COUNTER
description: Durable attachment jobs whose first terminal outcome has this status.
messenger_attachment_time_to_ready_seconds:
query: |
WITH bounds(le, upper_bound) AS (
VALUES
('0.05', 0.05::double precision),
('0.1', 0.1::double precision),
('0.25', 0.25::double precision),
('0.5', 0.5::double precision),
('1', 1::double precision),
('2.5', 2.5::double precision),
('5', 5::double precision),
('10', 10::double precision),
('15', 15::double precision),
('20', 20::double precision),
('30', 30::double precision),
('60', 60::double precision),
('+Inf', NULL::double precision)
)
SELECT
bounds.le,
COUNT(events.job_id) FILTER (
WHERE events.status = 'ready'
AND (
bounds.upper_bound IS NULL
OR EXTRACT(EPOCH FROM (events.terminal_at - events.accepted_at)) <= bounds.upper_bound
)
)::bigint AS bucket
FROM bounds
LEFT JOIN attachment_terminal_events events ON events.status = 'ready'
GROUP BY bounds.le, bounds.upper_bound
ORDER BY bounds.upper_bound NULLS LAST
metrics:
- le:
usage: LABEL
description: Upper bound of the durable time-to-ready bucket in seconds.
- bucket:
usage: COUNTER
description: Durable attachment jobs reaching ready within the bucket bound.
+44 -2
View File
@@ -1,6 +1,11 @@
global:
scrape_interval: 1s
evaluation_interval: 1s
scrape_interval: 2s
evaluation_interval: 5s
external_labels:
deployment_environment: lab
rule_files:
- /etc/prometheus/rules/*.yml
scrape_configs:
- job_name: messenger
@@ -9,6 +14,13 @@ scrape_configs:
type: A
port: 8080
refresh_interval: 2s
relabel_configs:
- target_label: service_name
replacement: messenger-api
metric_relabel_configs:
- source_labels: [__name__]
regex: "go_.*|process_.*"
action: keep
- job_name: nginx
static_configs:
@@ -32,12 +44,42 @@ scrape_configs:
type: A
port: 9091
refresh_interval: 2s
relabel_configs:
- target_label: service_name
replacement: messenger-worker
metric_relabel_configs:
- source_labels: [__name__]
regex: "go_.*|process_.*"
action: keep
- job_name: outbox-dispatcher
static_configs:
- targets: ["dispatcher:9091"]
relabel_configs:
- target_label: service_name
replacement: messenger-dispatcher
metric_relabel_configs:
- source_labels: [__name__]
regex: "go_.*|process_.*"
action: keep
- job_name: minio
metrics_path: /minio/v2/metrics/cluster
static_configs:
- targets: ["s3:9000"]
- job_name: otel-applications
static_configs:
- targets: ["otel-collector:8889"]
- job_name: otel-collector
static_configs:
- targets: ["otel-collector:8888"]
- job_name: tempo
static_configs:
- targets: ["tempo:3200"]
- job_name: loki
static_configs:
- targets: ["loki:3100"]
+178
View File
@@ -0,0 +1,178 @@
groups:
- name: messenger-api-slo
interval: 5s
rules:
- alert: MessengerAPIBusiness5xxHigh
expr: |
(
(sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics",status_class="5xx"}[2m])) or vector(0))
/
clamp_min((sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics"}[2m])) or vector(0)), 0.000001)
) > 0.05
for: 1m
labels:
severity: warning
service: messenger-api
slo: api-availability
annotations:
summary: Business API 5xx ratio is above 5%
description: More than 5% of business requests returned 5xx for one minute. Health and metrics routes are excluded.
runbook_url: https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/api-availability.md
- alert: MessengerAPIHighErrorBudgetBurnFast
expr: |
(
(sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics",status_class="5xx"}[5m])) or vector(0))
/ clamp_min((sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics"}[5m])) or vector(0)), 0.000001)
) > 0.072
and
(
(sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics",status_class="5xx"}[1h])) or vector(0))
/ clamp_min((sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics"}[1h])) or vector(0)), 0.000001)
) > 0.072
for: 30s
labels:
severity: critical
service: messenger-api
slo: api-availability
annotations:
summary: API availability error budget is burning fast
description: The 99.5% API availability SLO burns above 14.4x in both 5m and 1h windows.
runbook_url: https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/api-availability.md
- alert: MessengerAPIHighErrorBudgetBurnSlow
expr: |
(
(sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics",status_class="5xx"}[30m])) or vector(0))
/ clamp_min((sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics"}[30m])) or vector(0)), 0.000001)
) > 0.03
and
(
(sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics",status_class="5xx"}[6h])) or vector(0))
/ clamp_min((sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics"}[6h])) or vector(0)), 0.000001)
) > 0.03
for: 1m
labels:
severity: warning
service: messenger-api
slo: api-availability
annotations:
summary: API availability error budget is burning steadily
description: The 99.5% API availability SLO burns above 6x in both 30m and 6h windows.
runbook_url: https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/api-availability.md
- name: messenger-attachment-slo
interval: 5s
rules:
- alert: AttachmentReadyHighErrorBudgetBurnFast
expr: |
(
clamp_min(
(sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="+Inf"}[5m])) or vector(0))
- (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="30"}[5m])) or vector(0)),
0
)
+ (sum(rate(messenger_attachment_terminal_transitions_total{status="failed"}[5m])) or vector(0))
)
/ clamp_min(
(sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="+Inf"}[5m])) or vector(0))
+ (sum(rate(messenger_attachment_terminal_transitions_total{status="failed"}[5m])) or vector(0)),
0.000001
) > 0.144
and
(
clamp_min(
(sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="+Inf"}[1h])) or vector(0))
- (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="30"}[1h])) or vector(0)),
0
)
+ (sum(rate(messenger_attachment_terminal_transitions_total{status="failed"}[1h])) or vector(0))
)
/ clamp_min(
(sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="+Inf"}[1h])) or vector(0))
+ (sum(rate(messenger_attachment_terminal_transitions_total{status="failed"}[1h])) or vector(0)),
0.000001
) > 0.144
for: 30s
labels:
severity: critical
service: messenger-worker
slo: attachment-ready
annotations:
summary: Attachment ready-time error budget is burning fast
description: The 99% within 30s ready-time SLO burns above 14.4x in both 5m and 1h windows.
runbook_url: https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/attachment-pipeline.md
- alert: AttachmentReadyHighErrorBudgetBurnSlow
expr: |
(
clamp_min(
(sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="+Inf"}[30m])) or vector(0))
- (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="30"}[30m])) or vector(0)),
0
)
+ (sum(rate(messenger_attachment_terminal_transitions_total{status="failed"}[30m])) or vector(0))
)
/ clamp_min(
(sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="+Inf"}[30m])) or vector(0))
+ (sum(rate(messenger_attachment_terminal_transitions_total{status="failed"}[30m])) or vector(0)),
0.000001
) > 0.06
and
(
clamp_min(
(sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="+Inf"}[6h])) or vector(0))
- (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="30"}[6h])) or vector(0)),
0
)
+ (sum(rate(messenger_attachment_terminal_transitions_total{status="failed"}[6h])) or vector(0))
)
/ clamp_min(
(sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="+Inf"}[6h])) or vector(0))
+ (sum(rate(messenger_attachment_terminal_transitions_total{status="failed"}[6h])) or vector(0)),
0.000001
) > 0.06
for: 1m
labels:
severity: warning
service: messenger-worker
slo: attachment-ready
annotations:
summary: Attachment ready-time error budget is burning steadily
description: The 99% within 30s ready-time SLO burns above 6x in both 30m and 6h windows.
runbook_url: https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/attachment-pipeline.md
- alert: AttachmentWorkerUnavailable
expr: (sum(up{job="attachment-worker"}) or vector(0)) < 1
for: 30s
labels:
severity: critical
service: messenger-worker
annotations:
summary: No attachment worker is available
description: Prometheus has not observed a healthy worker target for 30 seconds.
runbook_url: https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/attachment-pipeline.md
- alert: AttachmentQueueBacklogGrowing
expr: |
max(messenger_attachment_jobs_oldest_age_seconds{status="queued"}) > 30
and max(deriv(messenger_attachment_jobs_oldest_age_seconds{status="queued"}[2m])) > 0
for: 30s
labels:
severity: warning
service: messenger-worker
annotations:
summary: Attachment queue backlog is growing
description: The oldest queued attachment is older than 30 seconds and its age continues to increase.
runbook_url: https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/attachment-pipeline.md
- alert: AttachmentRedisStreamLagHigh
expr: max(redis_stream_group_lag{job="redis-broker",stream="attachment-jobs",group="attachment-workers"}) > 20
for: 1m
labels:
severity: warning
service: redis-broker
annotations:
summary: Redis Stream consumer lag is high
description: The attachment worker group has more than 20 messages waiting for delivery.
runbook_url: https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/attachment-pipeline.md
+72
View File
@@ -0,0 +1,72 @@
# API availability SLO
## Цель и сигналы
Объект контроля — бизнес-маршруты API за Nginx. Health endpoints исключены. Хорошими считаются ответы `2xx` и ожидаемые клиентские `4xx`; плохими — `5xx`. Цель — не менее 99,5% хороших ответов за скользящие 30 дней, error budget равен 0,5%.
Основные сигналы:
- `messenger_http_requests_total{method,route,status_class}` — число запросов;
- `messenger_http_request_duration_seconds_bucket{method,route}` — распределение server latency;
- `up{job="messenger"}` и `up{job="nginx"}` — доступность scrape targets;
- `messenger_db_pool_connections{state="used"}` и `rate(messenger_db_pool_waited_total[5m])` — использование пула и частота ожиданий свободного соединения;
- trace `messenger-api` — путь одного запроса по зависимостям;
- JSON-логи `messenger-api` с `request_id`, `trace_id` и `span_id`.
`user_id`, `chat_id`, `message_id`, `attachment_id` и `request_id` не используются как labels метрик.
## Диагностика: metrics → trace → logs
1. В Grafana открыть dashboard **Messenger Lab 6 / API** и установить общий временной диапазон инцидента.
2. Проверить error ratio, RPS, p95/p99 и `up`. Рост `5xx` при нормальном RPS подтверждает влияние на SLI; отсутствие трафика не считается успешностью.
3. Сравнить DB pool used/waiting и состояние PostgreSQL. Рост ожидания пула указывает на зависимость, а не на CPU handler.
4. В панели ошибок выбрать exemplar или перейти в Explore → Tempo. Если exemplar отсутствует, выполнить TraceQL:
```text
{ resource.service.name = "messenger-api" && span.http.route != "/health/ready" && status = error }
```
5. В trace найти первый error span. Проверить `http.route`, `http.request.method`, `http.response.status_code`, `db.system`, duration и resource attributes. Полный URL, токен и тело запроса в span отсутствуют.
6. Скопировать `trace_id`, перейти по trace-to-logs или выполнить в Explore → Loki:
```text
{service_name="messenger-api"} | json | trace_id="<trace_id>"
```
7. Сверить `request_id`, событие, severity и error class. Текст сообщений, `Authorization` и пароли в логах отсутствуют.
## Проверка alert
Состояние правил видно в Grafana Alerting и в Prometheus:
```bash
curl -s http://127.0.0.1:9090/api/v1/alerts
```
Контролируемый отказ PostgreSQL:
```bash
ENV_FILE=lab6.env.example scripts/incident-database.sh
```
Сценарий сохраняет baseline, нагрузку, состояние деградации, `alert-firing.json`, восстановление и `alert-resolved.json` в `evidence/runtime/`. Trap запускает PostgreSQL и API даже при ошибке сценария. Volumes не удаляются.
Ожидаемая последовательность:
```text
нормальный error ratio
→ PostgreSQL остановлен
→ 5xx растёт
→ MessengerAPIBusiness5xxHigh и MessengerAPIHighErrorBudgetBurnFast firing
→ PostgreSQL восстановлен
→ успешные ответы возвращаются
→ оперативный alert resolved; burn-rate хранит историю длинного окна
```
## Действия
1. Если `up{job="nginx"}=0`, восстановить ingress и проверить его логи.
2. Если API targets недоступны, восстановить реплики по одной и проверить readiness.
3. Если API работает, а DB spans завершаются ошибкой или pool waiting растёт, восстановить PostgreSQL и не увеличивать число API-реплик до устранения причины.
4. После восстановления проверить успешный smoke, отсутствие firing alerts и возвращение latency к baseline.
5. Не удалять volumes и не сбрасывать очередь ради закрытия alert.
@@ -0,0 +1,86 @@
# Attachment pipeline SLO
## Цель и сигналы
Объект контроля — первый terminal outcome каждой уникальной durable job. Цель — не менее 99% jobs впервые завершаются `ready` не позднее 30 секунд после acceptance за скользящие 30 дней. Первый `failed` и первое завершение `ready` позже 30 секунд расходуют error budget; replay не переписывает уже записанный outcome.
Основные сигналы:
- `messenger_attachment_worker_jobs_total{job_type,result}` — исходы доставок worker;
- `messenger_attachment_terminal_transitions_total{status}` — первые terminal outcomes из append-only PostgreSQL ledger;
- `messenger_attachment_worker_job_duration_seconds_bucket{job_type,result}` — время обработки;
- `messenger_attachment_time_to_ready_seconds_bucket` — end-to-end время;
- `messenger_attachment_jobs_count{status}` — состояния jobs в PostgreSQL;
- `messenger_attachment_jobs_oldest_age_seconds{status}` — возраст старейшей job;
- `messenger_attachment_worker_jobs_in_flight` и `messenger_attachment_worker_capacity` — насыщение worker;
- `messenger_outbox_pending` — задержка до публикации;
- Redis Stream lag и PEL — доставка и неподтверждённые сообщения.
Queue depth не заменяет processing latency: очередь может быть короткой, но содержать одну очень старую job.
## Диагностика: metrics → trace → logs
1. В Grafana открыть dashboard **Messenger Lab 6 / Attachment pipeline** и выбрать диапазон инцидента.
2. Определить место задержки:
- растёт `messenger_outbox_pending` — проблема до публикации в broker;
- растут stream lag и `queued` — worker не успевает или недоступен;
- растут PEL и `processing` — delivery получена, но не завершена;
- растёт `failed` — постоянные ошибки или исчерпаны retry;
- busy равен capacity — worker насыщен.
3. В Explore → Tempo найти upload по route или worker consumer span:
```text
{ resource.service.name = "messenger-api" && span.http.route = "/api/v1/chats/{chat_id}/attachments" }
```
```text
{ resource.service.name = "messenger-worker" && name = "attachment.process" }
```
4. В trace пройти цепочку `HTTP upload → outbox publish → Redis consume → S3 variants → PostgreSQL ready`. Проверить producer/consumer связь и retry spans. Технический `attachment_id` допустим только как span/log attribute синтетического стенда, но не как metric label.
5. Перейти из выбранного span в Loki или выполнить:
```text
{service_name="messenger-worker"} | json | trace_id="<trace_id>"
```
6. Сверить в логе `event`, `job_type`, `attempt`, `result`, `trace_id` и `span_id`; в span проверить `job.outcome`. Payload envelope, изображение и credentials не логируются.
## Проверка alert
Короткая деградация:
```bash
ENV_FILE=lab6.env.example scripts/incident-worker.sh short
```
Длительная деградация:
```bash
ENV_FILE=lab6.env.example scripts/incident-worker.sh long
```
Ожидаемая последовательность:
```text
worker up, backlog около нуля
→ worker остановлен
→ queued и oldest age растут
→ short: AttachmentWorkerUnavailable firing
→ long: AttachmentQueueBacklogGrowing firing
→ worker запущен
→ backlog дренируется
→ все принятые attachments достигают ready
→ alert resolved
```
Каждый сценарий восстанавливает worker через trap и сохраняет evidence без session token и raw payload.
## Действия
1. Если worker `up=0`, восстановить worker и наблюдать скорость дренирования.
2. Если worker доступен, но busy равно capacity, проверить S3/DB spans до масштабирования concurrency.
3. Если растёт outbox, проверить dispatcher и PostgreSQL lease; Redis worker здесь ещё не причина.
4. Если растёт PEL, проверить heartbeat, lease expiry и reclaim.
5. Для `failed` определить permanent/transient error по логу и trace. Replay выполнять только после устранения причины.
6. После восстановления проверить backlog, oldest age, `failed`, firing alerts и terminal state каждого синтетического attachment.
+91
View File
@@ -0,0 +1,91 @@
#!/bin/sh
set -eu
env_file=${ENV_FILE:-lab6.env.example}
base_url=${BASE_URL:-http://127.0.0.1:8080}
prometheus_url=${PROMETHEUS_URL:-http://127.0.0.1:9090}
tempo_url=${TEMPO_URL:-http://127.0.0.1:3200}
loki_url=${LOKI_URL:-http://127.0.0.1:3100}
evidence_root=${EVIDENCE_DIR:-evidence/runtime}
outage_seconds=${DB_OUTAGE_SECONDS:-75}
require_alerts=${REQUIRE_ALERT_TRANSITIONS:-1}
api_alert_name=${API_ALERT_NAME:-MessengerAPIBusiness5xxHigh}
burn_alert_name=${API_BURN_ALERT_NAME:-MessengerAPIHighErrorBudgetBurnFast}
run_id=$(date -u +%Y%m%dT%H%M%SZ)
output_dir="$evidence_root/database-$run_id"
mkdir -p "$output_dir"
load_pid=
compose() {
docker compose --env-file "$env_file" "$@"
}
restore() {
if [ -n "$load_pid" ]; then
kill "$load_pid" >/dev/null 2>&1 || true
fi
compose up -d postgres migrate api nginx >/dev/null 2>&1 || true
}
trap restore EXIT INT TERM
python3 scripts/lab6-observe.py --prometheus-url "$prometheus_url" --phase baseline --output "$output_dir/baseline.json" >/dev/null
python3 scripts/lab6-traffic.py \
--base-url "$base_url" \
--mode messages \
--duration "$((outage_seconds + 20))" \
--concurrency 4 \
--rate "${DB_LOAD_RATE:-20}" \
--started-file "$output_dir/load-started" \
--output "$output_dir/traffic.json" >/dev/null &
load_pid=$!
remaining=60
while [ ! -f "$output_dir/load-started" ] && [ "$remaining" -gt 0 ]; do
sleep 0.5
remaining=$((remaining - 1))
done
test -f "$output_dir/load-started"
sleep 2
started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)
compose stop postgres >/dev/null
sleep "$outage_seconds"
if [ "$require_alerts" = "1" ]; then
python3 scripts/lab6-alert.py --prometheus-url "$prometheus_url" --name "$api_alert_name" --state firing --timeout "${ALERT_WAIT_SECONDS:-90}" --output "$output_dir/alert-firing.json" >/dev/null
python3 scripts/lab6-alert.py --prometheus-url "$prometheus_url" --name "$burn_alert_name" --state firing --timeout "${ALERT_WAIT_SECONDS:-90}" --output "$output_dir/burn-alert-firing.json" >/dev/null
fi
python3 scripts/lab6-correlation.py --mode database --tempo-url "$tempo_url" --loki-url "$loki_url" --output "$output_dir/database-correlation.json" --require >/dev/null
python3 scripts/lab6-observe.py --prometheus-url "$prometheus_url" --phase degraded --output "$output_dir/degraded.json" >/dev/null
recovery_started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)
compose up -d --wait --wait-timeout 90 postgres migrate api nginx >/dev/null
wait "$load_pid" || true
load_pid=
if [ "$require_alerts" = "1" ]; then
python3 scripts/lab6-alert.py --prometheus-url "$prometheus_url" --name "$api_alert_name" --state resolved --timeout "${ALERT_RESOLVE_SECONDS:-180}" --output "$output_dir/alert-resolved.json" >/dev/null
fi
sleep "${RECOVERY_OBSERVATION_SECONDS:-20}"
python3 scripts/lab6-observe.py --prometheus-url "$prometheus_url" --phase recovered --output "$output_dir/recovered.json" >/dev/null
recovered_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)
python3 - "$output_dir/summary.json" "$started_at" "$recovery_started_at" "$recovered_at" "$outage_seconds" <<'PY'
import datetime
import json
import pathlib
import sys
output, started, recovery_started, recovered, duration = sys.argv[1:]
parse = lambda value: datetime.datetime.fromisoformat(value.replace("Z", "+00:00"))
report = {
"schema_version": 1,
"incident": "postgres_unavailable",
"started_at": started,
"recovery_started_at": recovery_started,
"recovered_at": recovered,
"planned_injection_seconds": int(duration),
"injected_duration_seconds": int((parse(recovery_started) - parse(started)).total_seconds()),
"recovery_observation_seconds": int((parse(recovered) - parse(recovery_started)).total_seconds()),
"artifacts": ["baseline.json", "degraded.json", "recovered.json", "traffic.json", "database-correlation.json", "alert-firing.json", "alert-resolved.json", "burn-alert-firing.json"],
}
pathlib.Path(output).write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
PY
trap - EXIT INT TERM
printf '%s\n' "$output_dir"
+73
View File
@@ -0,0 +1,73 @@
#!/bin/sh
set -eu
env_file=${ENV_FILE:-lab6.env.example}
base_url=${BASE_URL:-http://127.0.0.1:8080}
prometheus_url=${PROMETHEUS_URL:-http://127.0.0.1:9090}
evidence_root=${EVIDENCE_DIR:-evidence/runtime}
outage_seconds=${TELEMETRY_OUTAGE_SECONDS:-30}
run_id=$(date -u +%Y%m%dT%H%M%SZ)
output_dir="$evidence_root/telemetry-$run_id"
mkdir -p "$output_dir"
load_pid=
compose() {
docker compose --env-file "$env_file" "$@"
}
restore() {
if [ -n "$load_pid" ]; then
kill "$load_pid" >/dev/null 2>&1 || true
fi
compose up -d tempo loki >/dev/null 2>&1 || true
}
trap restore EXIT INT TERM
python3 scripts/lab6-observe.py --prometheus-url "$prometheus_url" --phase baseline --output "$output_dir/baseline.json" >/dev/null
started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)
compose stop tempo loki >/dev/null
python3 scripts/lab6-traffic.py \
--base-url "$base_url" \
--mode mixed \
--duration "$outage_seconds" \
--concurrency 4 \
--rate "${TELEMETRY_LOAD_RATE:-10}" \
--wait-ready \
--ready-timeout 120 \
--output "$output_dir/traffic.json" >/dev/null &
load_pid=$!
sleep "$outage_seconds"
wait "$load_pid"
load_pid=
health_code=$(curl --silent --output /dev/null --write-out '%{http_code}' "$base_url/health/ready")
python3 scripts/lab6-observe.py --prometheus-url "$prometheus_url" --phase degraded --output "$output_dir/degraded.json" >/dev/null
recovery_started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)
compose up -d --wait --wait-timeout 90 tempo loki >/dev/null
sleep "${RECOVERY_OBSERVATION_SECONDS:-20}"
python3 scripts/lab6-observe.py --prometheus-url "$prometheus_url" --phase recovered --output "$output_dir/recovered.json" >/dev/null
recovered_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)
python3 - "$output_dir/summary.json" "$started_at" "$recovery_started_at" "$recovered_at" "$outage_seconds" "$health_code" <<'PY'
import datetime
import json
import pathlib
import sys
output, started, recovery_started, recovered, duration, health_code = sys.argv[1:]
parse = lambda value: datetime.datetime.fromisoformat(value.replace("Z", "+00:00"))
report = {
"schema_version": 1,
"incident": "telemetry_backends_unavailable",
"started_at": started,
"recovery_started_at": recovery_started,
"recovered_at": recovered,
"injected_duration_seconds": int(duration),
"api_ready_http_status_during_outage": int(health_code),
"recovery_observation_seconds": int((parse(recovered) - parse(recovery_started)).total_seconds()),
"artifacts": ["baseline.json", "degraded.json", "recovered.json", "traffic.json"],
}
pathlib.Path(output).write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
PY
trap - EXIT INT TERM
printf '%s\n' "$output_dir"
+107
View File
@@ -0,0 +1,107 @@
#!/bin/sh
set -eu
mode=${1:-short}
env_file=${ENV_FILE:-lab6.env.example}
base_url=${BASE_URL:-http://127.0.0.1:8080}
prometheus_url=${PROMETHEUS_URL:-http://127.0.0.1:9090}
evidence_root=${EVIDENCE_DIR:-evidence/runtime}
require_alerts=${REQUIRE_ALERT_TRANSITIONS:-1}
case "$mode" in
short)
incident_seconds=${SHORT_WORKER_OUTAGE_SECONDS:-75}
load_seconds=${SHORT_WORKER_LOAD_SECONDS:-60}
requests=${SHORT_WORKER_REQUESTS:-300}
default_alert=AttachmentWorkerUnavailable
;;
long)
incident_seconds=${LONG_WORKER_OUTAGE_SECONDS:-180}
load_seconds=${LONG_WORKER_LOAD_SECONDS:-120}
requests=${LONG_WORKER_REQUESTS:-1000}
default_alert=AttachmentQueueBacklogGrowing
;;
*)
echo "usage: $0 [short|long]" >&2
exit 2
;;
esac
worker_alert_name=${WORKER_ALERT_NAME:-$default_alert}
run_id=$(date -u +%Y%m%dT%H%M%SZ)
output_dir="$evidence_root/worker-$mode-$run_id"
mkdir -p "$output_dir"
load_pid=
compose() {
docker compose --env-file "$env_file" "$@"
}
restore() {
if [ -n "$load_pid" ]; then
kill "$load_pid" >/dev/null 2>&1 || true
fi
compose up -d worker >/dev/null 2>&1 || true
}
trap restore EXIT INT TERM
python3 scripts/lab6-observe.py --prometheus-url "$prometheus_url" --phase baseline --output "$output_dir/baseline.json" >/dev/null
started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)
compose stop worker >/dev/null
python3 scripts/lab6-traffic.py \
--base-url "$base_url" \
--mode attachments \
--duration "$load_seconds" \
--requests "$requests" \
--concurrency 4 \
--rate "${WORKER_LOAD_RATE:-5}" \
--wait-ready \
--ready-timeout 480 \
--output "$output_dir/traffic.json" >/dev/null &
load_pid=$!
sleep "$incident_seconds"
if [ "$require_alerts" = "1" ]; then
python3 scripts/lab6-alert.py --prometheus-url "$prometheus_url" --name "$worker_alert_name" --state firing --timeout "${ALERT_WAIT_SECONDS:-90}" --output "$output_dir/alert-firing.json" >/dev/null
fi
python3 scripts/lab6-observe.py --prometheus-url "$prometheus_url" --phase degraded --output "$output_dir/degraded.json" >/dev/null
recovery_started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)
compose up -d --wait --wait-timeout 90 worker >/dev/null
wait "$load_pid"
load_pid=
if [ "$require_alerts" = "1" ]; then
python3 scripts/lab6-alert.py --prometheus-url "$prometheus_url" --name "$worker_alert_name" --state resolved --timeout "${ALERT_RESOLVE_SECONDS:-180}" --output "$output_dir/alert-resolved.json" >/dev/null
if [ "$mode" = "long" ]; then
python3 scripts/lab6-alert.py --prometheus-url "$prometheus_url" --name AttachmentReadyHighErrorBudgetBurnFast --state firing --timeout "${ALERT_WAIT_SECONDS:-90}" --output "$output_dir/burn-fast-firing.json" >/dev/null
python3 scripts/lab6-alert.py --prometheus-url "$prometheus_url" --name AttachmentReadyHighErrorBudgetBurnSlow --state firing --timeout "${ALERT_WAIT_SECONDS:-90}" --output "$output_dir/burn-slow-firing.json" >/dev/null
fi
fi
sleep "${RECOVERY_OBSERVATION_SECONDS:-20}"
python3 scripts/lab6-observe.py --prometheus-url "$prometheus_url" --phase recovered --output "$output_dir/recovered.json" >/dev/null
recovered_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)
python3 - "$output_dir/summary.json" "$mode" "$started_at" "$recovery_started_at" "$recovered_at" "$incident_seconds" <<'PY'
import datetime
import json
import pathlib
import sys
output, mode, started, recovery_started, recovered, duration = sys.argv[1:]
parse = lambda value: datetime.datetime.fromisoformat(value.replace("Z", "+00:00"))
report = {
"schema_version": 1,
"incident": "worker_unavailable",
"profile": mode,
"started_at": started,
"recovery_started_at": recovery_started,
"recovered_at": recovered,
"planned_injection_seconds": int(duration),
"injected_duration_seconds": int((parse(recovery_started) - parse(started)).total_seconds()),
"recovery_observation_seconds": int((parse(recovered) - parse(recovery_started)).total_seconds()),
"artifacts": ["baseline.json", "degraded.json", "recovered.json", "traffic.json", "alert-firing.json", "alert-resolved.json"],
}
if mode == "long":
report["artifacts"].extend(["burn-fast-firing.json", "burn-slow-firing.json"])
pathlib.Path(output).write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
PY
trap - EXIT INT TERM
printf '%s\n' "$output_dir"
+67
View File
@@ -0,0 +1,67 @@
#!/usr/bin/env python3
import argparse
import json
from pathlib import Path
import time
import urllib.error
import urllib.request
def fetch(prometheus_url):
url = prometheus_url.rstrip("/") + "/api/v1/alerts"
with urllib.request.urlopen(url, timeout=10) as response:
payload = json.loads(response.read(4 << 20).decode("utf-8"))
return payload.get("data", {}).get("alerts", [])
def matching(alerts, name):
return [alert for alert in alerts if alert.get("labels", {}).get("alertname") == name]
def main():
parser = argparse.ArgumentParser(description="Wait for a Prometheus alert transition")
parser.add_argument("--prometheus-url", default="http://127.0.0.1:9090")
parser.add_argument("--name", required=True)
parser.add_argument("--state", choices=("firing", "resolved"), required=True)
parser.add_argument("--timeout", type=float, default=120)
parser.add_argument("--output", required=True)
args = parser.parse_args()
deadline = time.monotonic() + args.timeout
last = []
error = None
observed = False
while time.monotonic() < deadline:
try:
last = matching(fetch(args.prometheus_url), args.name)
error = None
if args.state == "firing" and any(item.get("state") == "firing" for item in last):
observed = True
break
if args.state == "resolved" and not any(item.get("state") in ("pending", "firing") for item in last):
observed = True
break
except (urllib.error.URLError, json.JSONDecodeError) as current_error:
error = str(current_error)
time.sleep(2)
report = {
"schema_version": 1,
"alert": args.name,
"expected_state": args.state,
"observed": observed,
"observed_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
"last_states": sorted({item.get("state", "") for item in last}),
"error": error,
}
destination = Path(args.output)
destination.parent.mkdir(parents=True, exist_ok=True)
destination.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
print(json.dumps(report, ensure_ascii=False, indent=2))
if not observed:
raise SystemExit(1)
if __name__ == "__main__":
main()
+148
View File
@@ -0,0 +1,148 @@
#!/usr/bin/env python3
import argparse
import json
from pathlib import Path
import re
import time
import urllib.error
import urllib.parse
import urllib.request
TRACE_ID = re.compile(r'(?i)\b[0-9a-f]{32}\b')
SERVICES = ("messenger-api", "messenger-worker", "messenger-dispatcher")
def get_json(url):
with urllib.request.urlopen(url, timeout=10) as response:
return json.loads(response.read(8 << 20).decode("utf-8"))
def tempo_query(base_url, query):
url = base_url.rstrip("/") + "/api/search?" + urllib.parse.urlencode({"q": query, "limit": 100})
payload = get_json(url)
return {item.get("traceID") for item in payload.get("traces", []) if item.get("traceID")}
def extract_trace_ids(payload):
result = set()
for stream in payload.get("data", {}).get("result", []):
for value in stream.get("values", []):
candidates = [stream.get("stream", {}), value[1] if len(value) > 1 else ""]
if len(value) > 2:
candidates.append(value[2])
for candidate in candidates:
if isinstance(candidate, dict):
trace_id = candidate.get("trace_id") or candidate.get("traceid")
if trace_id:
result.add(str(trace_id))
candidate = json.dumps(candidate)
result.update(match.group(0).lower() for match in TRACE_ID.finditer(str(candidate)))
return result
def loki_logs(base_url, service, start_ns):
query = '{service_name="' + service + '"}'
url = base_url.rstrip("/") + "/loki/api/v1/query_range?" + urllib.parse.urlencode({
"query": query,
"start": str(start_ns),
"limit": 5000,
"direction": "backward",
})
payload = get_json(url)
count = sum(len(stream.get("values", [])) for stream in payload.get("data", {}).get("result", []))
return count, extract_trace_ids(payload)
def capture(tempo_url, loki_url, lookback_seconds):
start_ns = int((time.time() - lookback_seconds) * 1_000_000_000)
traces = {service: tempo_query(tempo_url, '{ resource.service.name = "' + service + '" }') for service in SERVICES}
logs = {}
log_trace_ids = {}
for service in SERVICES:
count, ids = loki_logs(loki_url, service, start_ns)
logs[service] = count
log_trace_ids[service] = ids
api_worker = traces["messenger-api"] & traces["messenger-worker"]
all_trace_ids = set().union(*traces.values())
all_log_trace_ids = set().union(*log_trace_ids.values())
return {
"traces_by_service": {service: len(ids) for service, ids in traces.items()},
"logs_by_service": logs,
"api_worker_shared_traces": len(api_worker),
"traces_with_correlated_logs": len(all_trace_ids & all_log_trace_ids),
}
def capture_database(tempo_url, loki_url, lookback_seconds):
start_ns = int((time.time() - lookback_seconds) * 1_000_000_000)
traces = tempo_query(
tempo_url,
'{ resource.service.name = "messenger-api" && span.db.system = "postgresql" && status = error }',
)
log_count, log_trace_ids = loki_logs(loki_url, "messenger-api", start_ns)
return {
"database_error_traces": len(traces),
"api_logs": log_count,
"database_error_traces_with_logs": len(traces & log_trace_ids),
}
def complete(report):
return (
report["traces_by_service"]["messenger-api"] > 0
and report["traces_by_service"]["messenger-worker"] > 0
and report["logs_by_service"]["messenger-api"] > 0
and report["logs_by_service"]["messenger-worker"] > 0
and report["api_worker_shared_traces"] > 0
and report["traces_with_correlated_logs"] > 0
)
def database_complete(report):
return report["database_error_traces"] > 0 and report["database_error_traces_with_logs"] > 0
def main():
parser = argparse.ArgumentParser(description="Verify trace and log correlation in Tempo and Loki")
parser.add_argument("--tempo-url", default="http://127.0.0.1:3200")
parser.add_argument("--loki-url", default="http://127.0.0.1:3100")
parser.add_argument("--lookback", type=float, default=900)
parser.add_argument("--wait", type=float, default=45)
parser.add_argument("--output", required=True)
parser.add_argument("--require", action="store_true")
parser.add_argument("--mode", choices=("baseline", "database"), default="baseline")
args = parser.parse_args()
deadline = time.monotonic() + args.wait
error = None
report = None
while time.monotonic() < deadline:
try:
report = capture_database(args.tempo_url, args.loki_url, args.lookback) if args.mode == "database" else capture(args.tempo_url, args.loki_url, args.lookback)
error = None
if database_complete(report) if args.mode == "database" else complete(report):
break
except (urllib.error.URLError, json.JSONDecodeError, KeyError) as current_error:
error = str(current_error)
time.sleep(2)
result = {
"schema_version": 1,
"captured_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
"lookback_seconds": args.lookback,
"signals": report,
"mode": args.mode,
"complete": bool(report and (database_complete(report) if args.mode == "database" else complete(report))),
"error": error,
}
destination = Path(args.output)
destination.parent.mkdir(parents=True, exist_ok=True)
destination.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
print(json.dumps(result, ensure_ascii=False, indent=2))
if args.require and not result["complete"]:
raise SystemExit(1)
if __name__ == "__main__":
main()
+91
View File
@@ -0,0 +1,91 @@
#!/usr/bin/env python3
import argparse
import json
from pathlib import Path
import time
import urllib.error
import urllib.parse
import urllib.request
QUERIES = {
"api_error_ratio_1m": 'sum(rate(messenger_http_requests_total{status_class="5xx",route!~"/health/.*"}[1m])) / clamp_min(sum(rate(messenger_http_requests_total{route!~"/health/.*"}[1m])), 1)',
"api_request_rate_1m": 'sum(rate(messenger_http_requests_total{route!~"/health/.*"}[1m]))',
"queued_jobs": 'sum(messenger_attachment_jobs_count{status="queued"})',
"processing_jobs": 'sum(messenger_attachment_jobs_count{status="processing"})',
"oldest_queued_job_seconds": 'max(messenger_attachment_jobs_oldest_age_seconds{status="queued"})',
"worker_jobs_in_flight": 'sum(messenger_attachment_worker_jobs_in_flight)',
"worker_capacity": 'sum(messenger_attachment_worker_capacity)',
"pending_outbox": 'sum(messenger_outbox_pending)',
"worker_up": 'min(up{job="attachment-worker"})',
"collector_queue_size": 'max by (exporter) (otelcol_exporter_queue_size)',
"collector_queue_capacity": 'max by (exporter) (otelcol_exporter_queue_capacity)',
"collector_export_failures_total": 'sum by (exporter) (otelcol_exporter_send_failed_spans) or sum by (exporter) (otelcol_exporter_send_failed_log_records) or on (exporter) (0 * max by (exporter) (otelcol_exporter_queue_capacity))',
"collector_dropped_records_total": 'sum by (exporter) (otelcol_exporter_enqueue_failed_spans) or sum by (exporter) (otelcol_exporter_enqueue_failed_log_records) or on (exporter) (0 * max by (exporter) (otelcol_exporter_queue_capacity))',
"collector_memory_bytes": 'max(otelcol_process_memory_rss) or max(process_resident_memory_bytes{job="otel-collector"})',
"collector_memory_peak_bytes": 'max(max_over_time(otelcol_process_memory_rss[2m])) or max(max_over_time(process_resident_memory_bytes{job="otel-collector"}[2m]))',
}
def get_json(url):
try:
with urllib.request.urlopen(url, timeout=10) as response:
return json.loads(response.read(4 << 20).decode("utf-8")), None
except (urllib.error.URLError, json.JSONDecodeError) as error:
return None, str(error)
def query(prometheus_url, expression):
url = prometheus_url.rstrip("/") + "/api/v1/query?" + urllib.parse.urlencode({"query": expression})
payload, error = get_json(url)
if error:
return {"error": error}
if payload.get("status") != "success":
return {"error": payload.get("error", "query failed")}
rows = payload.get("data", {}).get("result", [])
return {
"series": [
{"labels": row.get("metric", {}), "value": row.get("value", [None, None])[1]}
for row in rows
]
}
def alerts(prometheus_url):
payload, error = get_json(prometheus_url.rstrip("/") + "/api/v1/alerts")
if error:
return {"error": error, "items": []}
items = []
for alert in payload.get("data", {}).get("alerts", []):
labels = alert.get("labels", {})
items.append({
"name": labels.get("alertname", ""),
"state": alert.get("state", ""),
"severity": labels.get("severity", ""),
"active_at": alert.get("activeAt", ""),
})
return {"items": sorted(items, key=lambda item: (item["name"], item["state"]))}
def main():
parser = argparse.ArgumentParser(description="Capture a bounded Prometheus snapshot for Lab 6 evidence")
parser.add_argument("--prometheus-url", default="http://127.0.0.1:9090")
parser.add_argument("--phase", required=True)
parser.add_argument("--output", required=True)
args = parser.parse_args()
report = {
"schema_version": 1,
"captured_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
"phase": args.phase,
"queries": {name: query(args.prometheus_url, expression) for name, expression in QUERIES.items()},
"alerts": alerts(args.prometheus_url),
}
destination = Path(args.output)
destination.parent.mkdir(parents=True, exist_ok=True)
destination.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
print(json.dumps(report, ensure_ascii=False, indent=2))
if __name__ == "__main__":
main()
+261
View File
@@ -0,0 +1,261 @@
#!/usr/bin/env python3
import argparse
import base64
from concurrent.futures import ThreadPoolExecutor
import json
import math
from pathlib import Path
import secrets
import struct
import threading
import time
import urllib.error
import urllib.parse
import urllib.request
import zlib
MAX_RESPONSE_BYTES = 1 << 20
def png_fixture(width=96, height=64):
rows = []
for y in range(height):
pixels = bytearray()
for x in range(width):
pixels.extend(((x * 3) % 256, (y * 5) % 256, (x + y) % 256))
rows.append(b"\x00" + bytes(pixels))
raw = b"".join(rows)
def chunk(name, payload):
return struct.pack(">I", len(payload)) + name + payload + struct.pack(">I", zlib.crc32(name + payload) & 0xFFFFFFFF)
header = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)
return b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", header) + chunk(b"IDAT", zlib.compress(raw, 6)) + chunk(b"IEND", b"")
def request(method, url, body=None, headers=None, timeout=10):
req = urllib.request.Request(url, data=body, method=method, headers=headers or {})
try:
with urllib.request.urlopen(req, timeout=timeout) as response:
payload = response.read(MAX_RESPONSE_BYTES)
return response.status, payload
except urllib.error.HTTPError as error:
return error.code, error.read(MAX_RESPONSE_BYTES)
except urllib.error.URLError:
return 0, b""
def decode_json(payload):
return json.loads(payload.decode("utf-8"))
def json_request(method, url, value=None, token=None, headers=None):
request_headers = {"Accept": "application/json"}
body = None
if value is not None:
body = json.dumps(value, separators=(",", ":")).encode("utf-8")
request_headers["Content-Type"] = "application/json"
if token:
request_headers["Authorization"] = "Bearer " + token
if headers:
request_headers.update(headers)
status, payload = request(method, url, body, request_headers)
return status, decode_json(payload) if payload else {}
def bootstrap(base_url):
suffix = secrets.token_hex(6)
password = "Lab6-" + secrets.token_urlsafe(18)
users = []
for prefix in ("telemetry_a_", "telemetry_b_"):
username = prefix + suffix
status, user = json_request("POST", base_url + "/api/v1/users", {
"username": username,
"display_name": "Synthetic telemetry load",
"password": password,
})
if status != 201:
raise RuntimeError("register user returned HTTP %d" % status)
users.append((username, user["id"]))
credentials = base64.b64encode((users[0][0] + ":" + password).encode("utf-8")).decode("ascii")
status, payload = request("POST", base_url + "/api/v1/auth/sessions", headers={
"Authorization": "Basic " + credentials,
"Accept": "application/json",
})
if status != 201:
raise RuntimeError("create session returned HTTP %d" % status)
token = decode_json(payload)["token"]
status, chat = json_request("POST", base_url + "/api/v1/chats", {
"title": "Lab 6 synthetic telemetry",
"member_ids": [users[0][1], users[1][1]],
}, token)
if status != 201:
raise RuntimeError("create chat returned HTTP %d" % status)
return token, chat["id"]
class Results:
def __init__(self):
self.lock = threading.Lock()
self.latencies = []
self.statuses = {}
self.attachment_ids = []
def record(self, status, elapsed, attachment_id=None):
with self.lock:
self.latencies.append(elapsed)
key = str(status)
self.statuses[key] = self.statuses.get(key, 0) + 1
if attachment_id:
self.attachment_ids.append(attachment_id)
def percentile(values, quantile):
if not values:
return None
ordered = sorted(values)
index = max(0, math.ceil(quantile * len(ordered)) - 1)
return round(ordered[index] * 1000, 3)
def run_request(base_url, token, chat_id, mode, sequence, fixture):
if mode == "mixed":
mode = "attachments" if sequence % 3 == 0 else "messages"
started = time.monotonic()
if mode == "messages":
status, _ = json_request("POST", base_url + "/api/v1/chats/" + chat_id + "/messages", {
"text": "synthetic telemetry event",
"attachment_ids": [],
}, token, {"Idempotency-Key": secrets.token_hex(16)})
return status, time.monotonic() - started, None
upload_url = base_url + "/api/v1/chats/" + chat_id + "/attachments?" + urllib.parse.urlencode({
"filename": "synthetic.png",
})
status, payload = request("POST", upload_url, fixture, {
"Authorization": "Bearer " + token,
"Content-Type": "image/png",
"Accept": "application/json",
})
attachment_id = None
if status == 202:
attachment_id = decode_json(payload).get("id")
return status, time.monotonic() - started, attachment_id
def wait_for_attachments(base_url, token, attachment_ids, timeout):
remaining = set(attachment_ids)
states = {"ready": 0, "failed": 0, "timeout": 0}
deadline = time.monotonic() + timeout
while remaining and time.monotonic() < deadline:
for attachment_id in list(remaining):
status, metadata = json_request("GET", base_url + "/api/v1/attachments/" + attachment_id, token=token)
if status != 200:
continue
state = metadata.get("status")
if state in ("ready", "failed"):
states[state] += 1
remaining.remove(attachment_id)
if remaining:
time.sleep(0.25)
states["timeout"] = len(remaining)
return states
def main():
parser = argparse.ArgumentParser(description="Bounded synthetic load for Lab 6 telemetry and incidents")
parser.add_argument("--base-url", default="http://127.0.0.1:8080")
parser.add_argument("--mode", choices=("messages", "attachments", "mixed"), default="mixed")
parser.add_argument("--duration", type=float, default=30)
parser.add_argument("--requests", type=int, default=0, help="Maximum requests; zero means duration only")
parser.add_argument("--concurrency", type=int, default=4)
parser.add_argument("--rate", type=float, default=0, help="Approximate aggregate request rate; zero is unthrottled")
parser.add_argument("--wait-ready", action="store_true")
parser.add_argument("--ready-timeout", type=float, default=120)
parser.add_argument("--output")
parser.add_argument("--started-file", help="Touch after synthetic session and chat are ready")
args = parser.parse_args()
if args.duration <= 0 or args.duration > 1800:
parser.error("duration must be in (0, 1800]")
if args.requests < 0:
parser.error("requests must be non-negative")
if args.concurrency < 1 or args.concurrency > 64:
parser.error("concurrency must be in [1, 64]")
if args.rate < 0 or args.rate > 1000:
parser.error("rate must be in [0, 1000]")
base_url = args.base_url.rstrip("/")
token, chat_id = bootstrap(base_url)
if args.started_file:
started_file = Path(args.started_file)
started_file.parent.mkdir(parents=True, exist_ok=True)
started_file.touch()
fixture = png_fixture()
results = Results()
next_sequence = 0
sequence_lock = threading.Lock()
deadline = time.monotonic() + args.duration
def worker():
nonlocal next_sequence
while time.monotonic() < deadline:
with sequence_lock:
if args.requests and next_sequence >= args.requests:
return
sequence = next_sequence
next_sequence += 1
status, elapsed, attachment_id = run_request(base_url, token, chat_id, args.mode, sequence, fixture)
results.record(status, elapsed, attachment_id)
if args.rate:
delay = args.concurrency / args.rate - elapsed
if delay > 0:
time.sleep(delay)
started = time.monotonic()
with ThreadPoolExecutor(max_workers=args.concurrency) as pool:
futures = [pool.submit(worker) for _ in range(args.concurrency)]
for future in futures:
future.result()
elapsed = time.monotonic() - started
attachment_states = None
if args.wait_ready and results.attachment_ids:
attachment_states = wait_for_attachments(base_url, token, results.attachment_ids, args.ready_timeout)
total = len(results.latencies)
successes = sum(count for status, count in results.statuses.items() if status.startswith("2"))
report = {
"schema_version": 1,
"workload": args.mode,
"started_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(time.time() - elapsed)),
"duration_seconds": round(elapsed, 3),
"concurrency": args.concurrency,
"requested_rate": args.rate or None,
"requests": total,
"successful_requests": successes,
"request_rate": round(total / elapsed, 3) if elapsed else None,
"status_counts": dict(sorted(results.statuses.items())),
"latency_ms": {
"p50": percentile(results.latencies, 0.50),
"p95": percentile(results.latencies, 0.95),
"p99": percentile(results.latencies, 0.99),
},
"attachments_accepted": len(results.attachment_ids),
"attachment_terminal_states": attachment_states,
}
encoded = json.dumps(report, ensure_ascii=False, indent=2) + "\n"
if args.output:
destination = Path(args.output)
destination.parent.mkdir(parents=True, exist_ok=True)
destination.write_text(encoded, encoding="utf-8")
print(encoded, end="")
if not total:
raise SystemExit("no requests were sent")
if __name__ == "__main__":
main()
+73
View File
@@ -0,0 +1,73 @@
#!/bin/sh
set -eu
profile=${1:-full}
case "$profile" in
smoke|full) ;;
*)
echo "usage: $0 [smoke|full]" >&2
exit 2
;;
esac
env_file=${ENV_FILE:-lab6.env.example}
base_url=${BASE_URL:-http://127.0.0.1:8080}
prometheus_url=${PROMETHEUS_URL:-http://127.0.0.1:9090}
tempo_url=${TEMPO_URL:-http://127.0.0.1:3200}
loki_url=${LOKI_URL:-http://127.0.0.1:3100}
run_id=$(date -u +%Y%m%dT%H%M%SZ)
output_dir=${EVIDENCE_DIR:-evidence/runtime/lab6-$run_id}
mkdir -p "$output_dir"
compose() {
docker compose --env-file "$env_file" "$@"
}
restore() {
compose up -d postgres redis-broker s3 valkey api nginx dispatcher worker otel-collector prometheus loki tempo grafana >/dev/null 2>&1 || true
}
trap restore EXIT INT TERM
test -f "$env_file"
compose config --quiet
compose up -d --build --wait --wait-timeout 180
python3 scripts/lab6-traffic.py \
--base-url "$base_url" \
--mode mixed \
--duration "${BASELINE_LOAD_SECONDS:-15}" \
--requests "${BASELINE_REQUESTS:-120}" \
--concurrency 4 \
--rate "${BASELINE_LOAD_RATE:-10}" \
--wait-ready \
--ready-timeout 120 \
--output "$output_dir/baseline-traffic.json" >/dev/null
python3 scripts/lab6-observe.py \
--prometheus-url "$prometheus_url" \
--phase baseline \
--output "$output_dir/baseline-observability.json" >/dev/null
python3 scripts/lab6-correlation.py \
--tempo-url "$tempo_url" \
--loki-url "$loki_url" \
--output "$output_dir/baseline-correlation.json" \
--require >/dev/null
if [ "$profile" = "full" ]; then
ENV_FILE="$env_file" BASE_URL="$base_url" PROMETHEUS_URL="$prometheus_url" TEMPO_URL="$tempo_url" LOKI_URL="$loki_url" EVIDENCE_DIR="$output_dir/incidents" scripts/incident-database.sh
ENV_FILE="$env_file" BASE_URL="$base_url" PROMETHEUS_URL="$prometheus_url" EVIDENCE_DIR="$output_dir/incidents" scripts/incident-worker.sh long
ENV_FILE="$env_file" BASE_URL="$base_url" PROMETHEUS_URL="$prometheus_url" EVIDENCE_DIR="$output_dir/incidents" scripts/incident-worker.sh short
ENV_FILE="$env_file" BASE_URL="$base_url" PROMETHEUS_URL="$prometheus_url" EVIDENCE_DIR="$output_dir/incidents" scripts/incident-telemetry.sh
fi
arguments=
if [ "$profile" = "full" ]; then
arguments="--require-incidents"
if [ "${REQUIRE_ALERT_TRANSITIONS:-1}" = "1" ]; then
arguments="$arguments --require-alerts"
fi
fi
# shellcheck disable=SC2086
python3 scripts/validate-lab6-evidence.py "$output_dir" $arguments
trap - EXIT INT TERM
printf '%s\n' "$output_dir"
+202
View File
@@ -0,0 +1,202 @@
#!/usr/bin/env python3
import argparse
import json
import math
from pathlib import Path
def load(path):
return json.loads(path.read_text(encoding="utf-8"))
def check(condition, message, errors):
if not condition:
errors.append(message)
def query_scalar(snapshot, name, source, errors):
query = snapshot.get("queries", {}).get(name, {})
check("error" not in query, source + " query failed: " + name, errors)
series = query.get("series", [])
check(len(series) == 1, source + " query must return one series: " + name, errors)
if len(series) != 1:
return None
try:
value = float(series[0]["value"])
except (KeyError, TypeError, ValueError):
check(False, source + " query returned a non-numeric value: " + name, errors)
return None
check(math.isfinite(value), source + " query returned a non-finite value: " + name, errors)
return value if math.isfinite(value) else None
def query_exporters(snapshot, name, source, errors):
expected = {"otlp_grpc/tempo", "otlp_http/loki"}
query = snapshot.get("queries", {}).get(name, {})
check("error" not in query, source + " query failed: " + name, errors)
values = {}
for series in query.get("series", []):
exporter = series.get("labels", {}).get("exporter")
check(exporter in expected, source + " query returned unexpected exporter: " + name, errors)
check(exporter not in values, source + " query returned duplicate exporter: " + name, errors)
try:
value = float(series["value"])
except (KeyError, TypeError, ValueError):
check(False, source + " query returned a non-numeric value: " + name, errors)
continue
check(math.isfinite(value), source + " query returned a non-finite value: " + name, errors)
if exporter in expected and exporter not in values and math.isfinite(value):
values[exporter] = value
check(set(values) == expected, source + " query must return the exact exporter set: " + name, errors)
return values if set(values) == expected else None
def validate_telemetry_outage(directory, errors):
phase_names = ("baseline", "degraded", "recovered")
exporter_metric_names = (
"collector_queue_size",
"collector_queue_capacity",
"collector_export_failures_total",
"collector_dropped_records_total",
)
scalar_metric_names = (
"collector_memory_bytes",
"collector_memory_peak_bytes",
)
phases = {}
for phase in phase_names:
path = directory / (phase + ".json")
check(path.exists(), str(path) + " is missing", errors)
if not path.exists():
return
snapshot = load(path)
phases[phase] = {
**{name: query_exporters(snapshot, name, path.name, errors) for name in exporter_metric_names},
**{name: query_scalar(snapshot, name, path.name, errors) for name in scalar_metric_names},
}
if any(value is None for values in phases.values() for value in values.values()):
return
for phase, values in phases.items():
for exporter in ("otlp_grpc/tempo", "otlp_http/loki"):
capacity = values["collector_queue_capacity"][exporter]
size = values["collector_queue_size"][exporter]
check(capacity > 0, phase + " collector queue capacity is not positive: " + exporter, errors)
check(0 <= size <= capacity, phase + " collector queue size is outside capacity: " + exporter, errors)
check(values["collector_memory_bytes"] > 0, phase + " collector memory is not positive", errors)
check(values["collector_memory_peak_bytes"] > 0, phase + " collector peak memory is not positive", errors)
baseline = phases["baseline"]
degraded = phases["degraded"]
recovered = phases["recovered"]
for exporter in ("otlp_grpc/tempo", "otlp_http/loki"):
for name in ("collector_export_failures_total", "collector_dropped_records_total"):
check(degraded[name][exporter] >= baseline[name][exporter], name + " decreased during outage: " + exporter, errors)
check(recovered[name][exporter] >= degraded[name][exporter], name + " decreased after recovery: " + exporter, errors)
outage_observed = (
degraded["collector_queue_size"][exporter] > baseline["collector_queue_size"][exporter]
or recovered["collector_export_failures_total"][exporter] > baseline["collector_export_failures_total"][exporter]
or recovered["collector_dropped_records_total"][exporter] > baseline["collector_dropped_records_total"][exporter]
)
check(outage_observed, "telemetry outage was not observed for exporter: " + exporter, errors)
drained_limit = max(
baseline["collector_queue_size"][exporter],
recovered["collector_queue_capacity"][exporter] * 0.05,
)
check(
recovered["collector_queue_size"][exporter] <= drained_limit,
"collector queue did not drain after recovery: " + exporter,
errors,
)
memory_limit = max(baseline["collector_memory_peak_bytes"] * 1.5, baseline["collector_memory_peak_bytes"] + 64 * 1024 * 1024)
check(degraded["collector_memory_peak_bytes"] <= memory_limit, "collector peak memory grew beyond the outage bound", errors)
check(recovered["collector_memory_peak_bytes"] <= memory_limit, "collector peak memory remained above the recovery bound", errors)
def main():
parser = argparse.ArgumentParser(description="Validate generated Lab 6 incident evidence")
parser.add_argument("directory")
parser.add_argument("--require-incidents", action="store_true")
parser.add_argument("--require-alerts", action="store_true")
args = parser.parse_args()
root = Path(args.directory)
errors = []
baseline = root / "baseline-traffic.json"
check(baseline.exists(), "baseline-traffic.json is missing", errors)
if baseline.exists():
report = load(baseline)
check(report.get("requests", 0) > 0, "baseline sent no requests", errors)
check(report.get("successful_requests", 0) > 0, "baseline had no successful requests", errors)
correlation = root / "baseline-correlation.json"
check(correlation.exists(), "baseline-correlation.json is missing", errors)
if correlation.exists():
check(load(correlation).get("complete") is True, "metrics/log/trace correlation is incomplete", errors)
observability = root / "baseline-observability.json"
check(observability.exists(), "baseline-observability.json is missing", errors)
if observability.exists():
firing = [item.get("name") for item in load(observability).get("alerts", {}).get("items", []) if item.get("state") == "firing"]
check(not firing, "baseline has firing alerts: " + ", ".join(firing), errors)
summaries = list(root.rglob("summary.json"))
if args.require_incidents:
names = {load(path).get("incident") for path in summaries}
for required in ("worker_unavailable", "postgres_unavailable", "telemetry_backends_unavailable"):
check(required in names, "incident evidence is missing: " + required, errors)
for summary_path in summaries:
summary = load(summary_path)
incident = summary.get("incident")
traffic_path = summary_path.parent / "traffic.json"
check(traffic_path.exists(), str(traffic_path) + " is missing", errors)
if not traffic_path.exists():
continue
traffic = load(traffic_path)
check(traffic.get("requests", 0) > 0, incident + " sent no traffic", errors)
if incident == "worker_unavailable":
terminal = traffic.get("attachment_terminal_states") or {}
check(traffic.get("attachments_accepted", 0) > 0, incident + " accepted no attachments", errors)
check(terminal.get("ready", 0) == traffic.get("attachments_accepted", -1), incident + " did not drain every accepted attachment", errors)
check(terminal.get("failed", 0) == 0, incident + " produced failed attachments", errors)
check(terminal.get("timeout", 0) == 0, incident + " left attachment jobs non-terminal", errors)
elif incident == "postgres_unavailable":
statuses = traffic.get("status_counts", {})
failures = sum(count for status, count in statuses.items() if not status.startswith("2"))
check(failures > 0, incident + " produced no observable failed requests", errors)
correlation_path = summary_path.parent / "database-correlation.json"
check(correlation_path.exists(), str(correlation_path) + " is missing", errors)
if correlation_path.exists():
check(load(correlation_path).get("complete") is True, "database error trace/log correlation is incomplete", errors)
elif incident == "telemetry_backends_unavailable":
terminal = traffic.get("attachment_terminal_states") or {}
check(traffic.get("successful_requests", 0) > 0, incident + " had no successful business requests", errors)
check(traffic.get("attachments_accepted", 0) > 0, incident + " accepted no attachments", errors)
check(terminal.get("ready", 0) == traffic.get("attachments_accepted", -1), incident + " did not finish every attachment", errors)
check(terminal.get("failed", 0) == 0 and terminal.get("timeout", 0) == 0, incident + " left unsuccessful attachments", errors)
check(summary.get("api_ready_http_status_during_outage") == 200, incident + " broke API readiness", errors)
validate_telemetry_outage(summary_path.parent, errors)
if args.require_alerts and incident in ("worker_unavailable", "postgres_unavailable"):
for name in ("alert-firing.json", "alert-resolved.json"):
path = summary_path.parent / name
check(path.exists(), str(path) + " is missing", errors)
if path.exists():
check(load(path).get("observed") is True, str(path) + " did not observe the transition", errors)
if incident == "worker_unavailable" and summary.get("profile") == "long":
for name in ("burn-fast-firing.json", "burn-slow-firing.json"):
path = summary_path.parent / name
check(path.exists(), str(path) + " is missing", errors)
if path.exists():
check(load(path).get("observed") is True, str(path) + " did not observe attachment burn", errors)
result = {"valid": not errors, "errors": errors, "incident_summaries": len(summaries)}
print(json.dumps(result, ensure_ascii=False, indent=2))
if errors:
raise SystemExit(1)
if __name__ == "__main__":
main()