forked from mtusi2026/mtusi2026lab6
Compare commits
10
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8704718f35 | ||
|
|
8c8a0302c9 | ||
|
|
323d1e9294 | ||
|
|
c915547449 | ||
|
|
f23855f786 | ||
|
|
0b237618c0 | ||
|
|
441b752257 | ||
|
|
4bf383bd51 | ||
|
|
934dfefee7 | ||
|
|
cea50c2cd6 |
@@ -16,11 +16,13 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 45
|
||||
env:
|
||||
COMPOSE_PROJECT_NAME: mtusi2026lab5-${{ github.run_id }}
|
||||
COMPOSE_PROJECT_NAME: mtusi2026lab6-${{ github.run_id }}
|
||||
API_BIND_ADDRESS: 127.0.0.1:0
|
||||
SWAGGER_BIND_ADDRESS: 127.0.0.1:0
|
||||
PROMETHEUS_BIND_ADDRESS: 127.0.0.1:0
|
||||
GRAFANA_BIND_ADDRESS: 127.0.0.1:0
|
||||
LOKI_BIND_ADDRESS: 127.0.0.1:0
|
||||
TEMPO_BIND_ADDRESS: 127.0.0.1:0
|
||||
steps:
|
||||
- name: Checkout exact commit
|
||||
run: |
|
||||
@@ -46,7 +48,8 @@ jobs:
|
||||
- name: Check formatting and shell scripts
|
||||
run: |
|
||||
test -z "$(gofmt -l cmd internal migrations)"
|
||||
sh -n infra/minio/bootstrap.sh scripts/restart.sh scripts/rolling-update.sh scripts/stop-one.sh scripts/test-lab5.sh scripts/test-lab5-acceptance.sh
|
||||
sh -n infra/minio/bootstrap.sh scripts/restart.sh scripts/rolling-update.sh scripts/stop-one.sh scripts/test-lab5.sh scripts/test-lab5-acceptance.sh scripts/test-lab6.sh scripts/incident-worker.sh scripts/incident-database.sh scripts/incident-telemetry.sh
|
||||
python3 -m py_compile scripts/lab6-traffic.py scripts/lab6-observe.py scripts/lab6-alert.py scripts/lab6-correlation.py scripts/validate-lab6-evidence.py
|
||||
|
||||
- name: Lint
|
||||
uses: golangci/golangci-lint-action@v7
|
||||
@@ -62,11 +65,12 @@ jobs:
|
||||
|
||||
- name: Prepare local stack
|
||||
run: |
|
||||
cp lab5.env.example .env
|
||||
cp lab6.env.example .env
|
||||
docker compose config --quiet
|
||||
docker compose up -d --build --scale api=2 --wait --wait-timeout 150
|
||||
docker network connect "${COMPOSE_PROJECT_NAME}_application" "$HOSTNAME"
|
||||
docker network connect "${COMPOSE_PROJECT_NAME}_data" "$HOSTNAME"
|
||||
docker network connect "${COMPOSE_PROJECT_NAME}_monitoring" "$HOSTNAME"
|
||||
|
||||
- name: Integration checks
|
||||
run: |
|
||||
@@ -82,6 +86,7 @@ jobs:
|
||||
docker compose up -d --scale api=2 --wait --wait-timeout 90
|
||||
make test-lab5-acceptance
|
||||
make test-lab5
|
||||
ENV_FILE=.env PROMETHEUS_URL=http://prometheus:9090 TEMPO_URL=http://tempo:3200 LOKI_URL=http://loki:3100 make test-lab6-smoke BASE_URL=http://nginx:8080
|
||||
|
||||
- name: Container logs
|
||||
if: failure()
|
||||
@@ -92,4 +97,5 @@ jobs:
|
||||
run: |
|
||||
docker network disconnect "${COMPOSE_PROJECT_NAME}_application" "$HOSTNAME" 2>/dev/null || true
|
||||
docker network disconnect "${COMPOSE_PROJECT_NAME}_data" "$HOSTNAME" 2>/dev/null || true
|
||||
docker network disconnect "${COMPOSE_PROJECT_NAME}_monitoring" "$HOSTNAME" 2>/dev/null || true
|
||||
if [ -f .env ]; then docker compose --profile tests down -v --remove-orphans; fi
|
||||
|
||||
+19
-13
@@ -22,19 +22,21 @@ jobs:
|
||||
s3_image: ${{ steps.publish.outputs.s3_image }}
|
||||
s3_bootstrap_image: ${{ steps.publish.outputs.s3_bootstrap_image }}
|
||||
env:
|
||||
COMPOSE_PROJECT_NAME: mtusi2026lab5-deploy-${{ github.run_id }}
|
||||
COMPOSE_PROJECT_NAME: mtusi2026lab6-deploy-${{ github.run_id }}
|
||||
COMPOSE_FILE: compose.yaml:deploy/compose.images.yaml
|
||||
API_BIND_ADDRESS: 127.0.0.1:0
|
||||
SWAGGER_BIND_ADDRESS: 127.0.0.1:0
|
||||
PROMETHEUS_BIND_ADDRESS: 127.0.0.1:0
|
||||
GRAFANA_BIND_ADDRESS: 127.0.0.1:0
|
||||
API_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab5-api:${{ github.sha }}
|
||||
NGINX_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab5-nginx:${{ github.sha }}
|
||||
SWAGGER_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab5-swagger:${{ github.sha }}
|
||||
PROMETHEUS_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab5-prometheus:${{ github.sha }}
|
||||
GRAFANA_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab5-grafana:${{ github.sha }}
|
||||
S3_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab5-minio:${{ github.sha }}
|
||||
S3_BOOTSTRAP_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab5-minio-client:${{ github.sha }}
|
||||
LOKI_BIND_ADDRESS: 127.0.0.1:0
|
||||
TEMPO_BIND_ADDRESS: 127.0.0.1:0
|
||||
API_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab6-api:${{ github.sha }}
|
||||
NGINX_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab6-nginx:${{ github.sha }}
|
||||
SWAGGER_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab6-swagger:${{ github.sha }}
|
||||
PROMETHEUS_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab6-prometheus:${{ github.sha }}
|
||||
GRAFANA_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab6-grafana:${{ github.sha }}
|
||||
S3_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab6-minio:${{ github.sha }}
|
||||
S3_BOOTSTRAP_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab6-minio-client:${{ github.sha }}
|
||||
steps:
|
||||
- name: Checkout exact commit
|
||||
run: |
|
||||
@@ -60,7 +62,8 @@ jobs:
|
||||
- name: Check formatting and shell scripts
|
||||
run: |
|
||||
test -z "$(gofmt -l cmd internal migrations)"
|
||||
sh -n infra/minio/bootstrap.sh scripts/restart.sh scripts/rolling-update.sh scripts/stop-one.sh scripts/test-lab5.sh scripts/test-lab5-acceptance.sh
|
||||
sh -n infra/minio/bootstrap.sh scripts/restart.sh scripts/rolling-update.sh scripts/stop-one.sh scripts/test-lab5.sh scripts/test-lab5-acceptance.sh scripts/test-lab6.sh scripts/incident-worker.sh scripts/incident-database.sh scripts/incident-telemetry.sh
|
||||
python3 -m py_compile scripts/lab6-traffic.py scripts/lab6-observe.py scripts/lab6-alert.py scripts/lab6-correlation.py scripts/validate-lab6-evidence.py
|
||||
|
||||
- name: Lint
|
||||
uses: golangci/golangci-lint-action@v7
|
||||
@@ -76,11 +79,12 @@ jobs:
|
||||
|
||||
- name: Prepare local stack
|
||||
run: |
|
||||
cp lab5.env.example .env
|
||||
cp lab6.env.example .env
|
||||
docker compose config --quiet
|
||||
docker compose up -d --build --scale api=2 --wait --wait-timeout 150
|
||||
docker network connect "${COMPOSE_PROJECT_NAME}_application" "$HOSTNAME"
|
||||
docker network connect "${COMPOSE_PROJECT_NAME}_data" "$HOSTNAME"
|
||||
docker network connect "${COMPOSE_PROJECT_NAME}_monitoring" "$HOSTNAME"
|
||||
|
||||
- name: Integration checks
|
||||
run: |
|
||||
@@ -95,6 +99,7 @@ jobs:
|
||||
make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=http://nginx:8080
|
||||
make test-lab5-acceptance
|
||||
make test-lab5
|
||||
ENV_FILE=.env PROMETHEUS_URL=http://prometheus:9090 TEMPO_URL=http://tempo:3200 LOKI_URL=http://loki:3100 make test-lab6-smoke BASE_URL=http://nginx:8080
|
||||
|
||||
- name: Publish verified images
|
||||
id: publish
|
||||
@@ -132,6 +137,7 @@ jobs:
|
||||
run: |
|
||||
docker network disconnect "${COMPOSE_PROJECT_NAME}_application" "$HOSTNAME" 2>/dev/null || true
|
||||
docker network disconnect "${COMPOSE_PROJECT_NAME}_data" "$HOSTNAME" 2>/dev/null || true
|
||||
docker network disconnect "${COMPOSE_PROJECT_NAME}_monitoring" "$HOSTNAME" 2>/dev/null || true
|
||||
if [ -f .env ]; then docker compose --profile tests down -v --remove-orphans; fi
|
||||
|
||||
deploy:
|
||||
@@ -171,17 +177,17 @@ jobs:
|
||||
run: printf '%s' "$REGISTRY_PASSWORD" | ssh "$DEPLOY_USER@$DEPLOY_HOST" "docker login gitea.privcloud.tech --username '$REGISTRY_USERNAME' --password-stdin"
|
||||
|
||||
- name: Upload image manifest
|
||||
run: scp deploy-images.env "$DEPLOY_USER@$DEPLOY_HOST:/tmp/mtusi2026lab5-${{ github.run_id }}.env"
|
||||
run: scp deploy-images.env "$DEPLOY_USER@$DEPLOY_HOST:/tmp/mtusi2026lab6-${{ github.run_id }}.env"
|
||||
|
||||
- name: Deploy and verify
|
||||
run: |
|
||||
ssh "$DEPLOY_USER@$DEPLOY_HOST" "
|
||||
set -eu
|
||||
cd ~/mtusi2026lab5
|
||||
cd ~/mtusi2026lab6
|
||||
git fetch origin main
|
||||
git cat-file -e '$DEPLOY_SHA^{commit}'
|
||||
git reset --hard '$DEPLOY_SHA'
|
||||
manifest='/tmp/mtusi2026lab5-${{ github.run_id }}.env'
|
||||
manifest='/tmp/mtusi2026lab6-${{ github.run_id }}.env'
|
||||
if DEPLOY_GENERATION='${{ github.run_id }}' scripts/rolling-update.sh \"\$manifest\"; then
|
||||
rm -f \"\$manifest\"
|
||||
else
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
name: Lab 6 incidents
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
|
||||
concurrency:
|
||||
group: lab6-incidents-${{ github.repository }}-${{ github.ref }}
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
incidents:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 35
|
||||
env:
|
||||
COMPOSE_PROJECT_NAME: mtusi2026lab6-incidents-${{ github.run_id }}
|
||||
API_BIND_ADDRESS: 127.0.0.1:0
|
||||
SWAGGER_BIND_ADDRESS: 127.0.0.1:0
|
||||
PROMETHEUS_BIND_ADDRESS: 127.0.0.1:0
|
||||
GRAFANA_BIND_ADDRESS: 127.0.0.1:0
|
||||
LOKI_BIND_ADDRESS: 127.0.0.1:0
|
||||
TEMPO_BIND_ADDRESS: 127.0.0.1:0
|
||||
steps:
|
||||
- name: Checkout exact commit
|
||||
run: |
|
||||
git init .
|
||||
git remote add origin https://gitea.privcloud.tech/${{ github.repository }}.git
|
||||
git fetch --depth=1 origin ${{ github.sha }}
|
||||
git checkout --detach FETCH_HEAD
|
||||
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
cache: false
|
||||
|
||||
- name: Prepare stack
|
||||
run: |
|
||||
cp lab6.env.example .env
|
||||
docker compose config --quiet
|
||||
docker compose up -d --build --scale api=2 --wait --wait-timeout 180
|
||||
docker network connect "${COMPOSE_PROJECT_NAME}_application" "$HOSTNAME"
|
||||
docker network connect "${COMPOSE_PROJECT_NAME}_data" "$HOSTNAME"
|
||||
docker network connect "${COMPOSE_PROJECT_NAME}_monitoring" "$HOSTNAME"
|
||||
|
||||
- name: Reproduce SLO incidents
|
||||
run: |
|
||||
ENV_FILE=.env \
|
||||
BASE_URL=http://nginx:8080 \
|
||||
PROMETHEUS_URL=http://prometheus:9090 \
|
||||
TEMPO_URL=http://tempo:3200 \
|
||||
LOKI_URL=http://loki:3100 \
|
||||
EVIDENCE_DIR=evidence/runtime/ci \
|
||||
make test-lab6
|
||||
|
||||
- name: Upload sanitized evidence
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: lab6-incidents-${{ github.sha }}
|
||||
path: evidence/runtime/ci
|
||||
if-no-files-found: warn
|
||||
retention-days: 14
|
||||
|
||||
- name: Container logs
|
||||
if: failure()
|
||||
run: docker compose logs --no-color --tail=200
|
||||
|
||||
- name: Stop local stack
|
||||
if: always()
|
||||
run: |
|
||||
docker network disconnect "${COMPOSE_PROJECT_NAME}_application" "$HOSTNAME" 2>/dev/null || true
|
||||
docker network disconnect "${COMPOSE_PROJECT_NAME}_data" "$HOSTNAME" 2>/dev/null || true
|
||||
docker network disconnect "${COMPOSE_PROJECT_NAME}_monitoring" "$HOSTNAME" 2>/dev/null || true
|
||||
docker compose down -v --remove-orphans
|
||||
@@ -30,3 +30,5 @@ coverage/
|
||||
.vscode/
|
||||
# Keep small, sanitized evidence summaries in evidence/ under version control.
|
||||
# Inspect git diff --cached: ignore rules are not a secret scanner.
|
||||
|
||||
evidence/runtime/
|
||||
|
||||
+2
-1
@@ -20,8 +20,9 @@ ENV SWAGGER_JSON=/spec/openapi.json
|
||||
FROM prom/prometheus:v3.13.3 AS prometheus
|
||||
|
||||
COPY monitoring/prometheus.yml /etc/prometheus/prometheus.yml
|
||||
COPY monitoring/rules /etc/prometheus/rules
|
||||
|
||||
FROM grafana/grafana:13.1.6 AS grafana
|
||||
FROM grafana/grafana:13.2.4 AS grafana
|
||||
|
||||
COPY monitoring/grafana/provisioning /etc/grafana/provisioning
|
||||
COPY monitoring/grafana/dashboards /etc/grafana/dashboards
|
||||
|
||||
@@ -4,7 +4,7 @@ CA_FILE ?=
|
||||
ACTION_SCRIPT ?=
|
||||
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
|
||||
|
||||
.PHONY: help check validate test load test-persistence test-failover test-lab5 test-lab5-acceptance
|
||||
.PHONY: help check validate test load test-persistence test-failover test-lab5 test-lab5-acceptance lab6-load test-lab6-smoke test-lab6
|
||||
|
||||
help:
|
||||
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
|
||||
@@ -34,3 +34,12 @@ test-lab5:
|
||||
|
||||
test-lab5-acceptance:
|
||||
./scripts/test-lab5-acceptance.sh
|
||||
|
||||
lab6-load:
|
||||
$(PYTHON) scripts/lab6-traffic.py --base-url "$(BASE_URL)" --mode mixed --duration 60 --concurrency 4 --wait-ready
|
||||
|
||||
test-lab6-smoke:
|
||||
BASE_URL="$(BASE_URL)" ./scripts/test-lab6.sh smoke
|
||||
|
||||
test-lab6:
|
||||
BASE_URL="$(BASE_URL)" ./scripts/test-lab6.sh full
|
||||
|
||||
@@ -1,37 +1,260 @@
|
||||
# Отчёт — лабораторная 6
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
- Автор, группа: Пластинин Денис, БВТ2303
|
||||
- Целевая оценка: 5
|
||||
- Выбранный трек: 2 — SLO и обнаружение инцидентов
|
||||
|
||||
## Запуск из чистого clone
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
Требуются Docker Desktop с Compose v2, Go из `go.mod`, Python 3.9+ и `curl`. Все образы закреплены в Compose или Dockerfile. Основной observability stack: OpenTelemetry Collector `0.160.0`, Prometheus `3.13.3`, Loki `3.7.9`, Tempo `2.10.10`, Grafana `13.2.4`.
|
||||
|
||||
## Архитектура и решения
|
||||
```bash
|
||||
cp lab6.env.example .env
|
||||
docker compose --env-file .env config --quiet
|
||||
docker compose --env-file .env up -d --build --scale api=2 --wait --wait-timeout 180
|
||||
make test
|
||||
make test-lab5-acceptance
|
||||
make test-lab5
|
||||
make test-lab6-smoke
|
||||
```
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
Интерфейсы локального стенда:
|
||||
|
||||
- API: `http://127.0.0.1:8080`;
|
||||
- Swagger: `http://127.0.0.1:8081`;
|
||||
- Grafana: `http://127.0.0.1:3000`;
|
||||
- Prometheus: `http://127.0.0.1:9090`;
|
||||
- Loki API: `http://127.0.0.1:3100`;
|
||||
- Tempo API: `http://127.0.0.1:3200`.
|
||||
|
||||
Безопасная остановка сохраняет volumes:
|
||||
|
||||
```bash
|
||||
docker compose --env-file .env down
|
||||
```
|
||||
|
||||
Намеренное удаление всех учебных данных выполняется отдельно:
|
||||
|
||||
```bash
|
||||
docker compose --env-file .env down -v
|
||||
```
|
||||
|
||||
## Архитектура
|
||||
|
||||
Возможности Lab 5 сохранены: две API-реплики за Nginx, PostgreSQL как источник бизнес-состояния, Valkey для сессий, MinIO/S3 для изображений, transactional outbox, Redis Streams и отдельный worker. Lab 6 добавляет единый telemetry pipeline.
|
||||
|
||||
```text
|
||||
Client → Nginx → API replicas → PostgreSQL / Valkey / MinIO
|
||||
│
|
||||
PostgreSQL outbox → Dispatcher → Redis Streams → Worker → PostgreSQL / MinIO
|
||||
│ │
|
||||
└──────── OTLP ──────────┤
|
||||
API ────────────────────────────── OTLP ──────────┤
|
||||
▼
|
||||
OpenTelemetry Collector
|
||||
├─ metrics → Prometheus
|
||||
├─ logs → Loki
|
||||
└─ traces → Tempo
|
||||
│
|
||||
▼
|
||||
Grafana
|
||||
```
|
||||
|
||||
Приложения экспортируют OTLP только в Collector. Они не знают адреса Loki и Tempo. Collector выполняет `memory_limiter` до `batch`, ограничивает очереди exporter и прекращает retry через 30 секунд. Evidence проверяет queue size/capacity, failures и drops отдельно для `otlp_grpc/tempo` и `otlp_http/loki`, а память — по пиковому RSS за двухминутное окно. Недоступность telemetry backend поэтому не блокирует бизнес-запросы и не вызывает неограниченный рост памяти.
|
||||
|
||||
Prometheus, Loki, Tempo и Grafana используют отдельные volumes. Retention локальных logs/traces — 7 суток; Prometheus хранит 7 суток. Collector является транспортом и процессором, а не долговременным хранилищем.
|
||||
|
||||
### Resource attributes и среды
|
||||
|
||||
Каждый процесс объявляет:
|
||||
|
||||
```text
|
||||
service.name = messenger-api | messenger-dispatcher | messenger-worker
|
||||
service.version = значение SERVICE_VERSION
|
||||
deployment.environment.name = lab
|
||||
service.instance.id = стабильный ID процесса
|
||||
```
|
||||
|
||||
В учебном Compose одна среда `lab`. Атрибут среды всё равно передаётся: один и тот же dashboard и TraceQL остаются пригодны для `test`/`production`, если стенд будет развёрнут там позднее. Среда и service name имеют низкую кардинальность.
|
||||
|
||||
### Middleware и сквозной контекст
|
||||
|
||||
HTTP middleware создаёт server span, нормализует route template, считает request counter/histogram и добавляет в JSON-лог `request_id`, `trace_id` и `span_id`. Реальный UUID из URL не становится label.
|
||||
|
||||
При upload текущий W3C context сохраняется в outbox envelope как `traceparent`/`tracestate`. Dispatcher создаёт producer span. Worker извлекает context и создаёт consumer span. Поэтому один trace показывает цепочку:
|
||||
|
||||
```text
|
||||
HTTP upload → staging S3 → PostgreSQL/outbox → Redis publish
|
||||
→ Redis consume → image transform → S3 variants → PostgreSQL ready
|
||||
```
|
||||
|
||||
Идентификаторы пользователя, чата, сообщения, вложения, filename, raw URL и request ID запрещены в metric labels. Ограниченные технические ID допустимы в logs/spans синтетического стенда для расследования. `Authorization`, session token, пароль, тело сообщения, байты изображения, S3 credentials и signed URL не экспортируются.
|
||||
|
||||
## Хранилища и назначение сигналов
|
||||
|
||||
| Сигнал | Хранилище | На какой вопрос отвечает |
|
||||
| --- | --- | --- |
|
||||
| Метрика | Prometheus TSDB | Есть ли массовая деградация, когда началась и каков масштаб |
|
||||
| Лог | Loki | Какое событие и ошибка произошли в конкретном процессе |
|
||||
| Trace | Tempo | На каком участке сквозного запроса потрачено время или возникла ошибка |
|
||||
| Dashboard / Explore | Grafana | Единый поиск, корреляция и переходы между тремя сигналами |
|
||||
|
||||
Grafana provisioning создаёт Prometheus, Loki и Tempo без ручной настройки. Tempo datasource включает поиск TraceQL, node graph, service map, переходы trace → logs и trace → metrics. Loki derived field открывает trace по `trace_id`; Prometheus exemplars ведут из histogram в Tempo.
|
||||
|
||||
## Карта сигналов
|
||||
|
||||
| Контрактный сигнал | Реализация и единица |
|
||||
| --- | --- |
|
||||
| HTTP requests | `messenger_http_requests_total{method,route,status_class}`, requests |
|
||||
| HTTP duration | `messenger_http_request_duration_seconds_bucket`, seconds |
|
||||
| Delivery outcomes | `messenger_attachment_worker_jobs_total{job_type,result}`, deliveries |
|
||||
| Terminal job transitions | `messenger_attachment_terminal_transitions_total{status}`, first durable outcome per job |
|
||||
| Job duration | `messenger_attachment_worker_job_duration_seconds_bucket`, seconds |
|
||||
| Time to ready | `messenger_attachment_time_to_ready_seconds_bucket`, seconds |
|
||||
| Queue states | `messenger_attachment_jobs_count{status}`, jobs |
|
||||
| Oldest job | `messenger_attachment_jobs_oldest_age_seconds{status}`, seconds |
|
||||
| Worker saturation | `messenger_attachment_worker_jobs_in_flight` / `messenger_attachment_worker_capacity`, jobs |
|
||||
| Outbox | `messenger_outbox_pending`, events |
|
||||
| Logs | OTLP log records: timestamp, severity, service, event, trace/span IDs |
|
||||
| Traces | HTTP server, producer, consumer, PostgreSQL, Redis and S3 spans |
|
||||
|
||||
p50/p95/p99 вычисляются `histogram_quantile` из суммы bucket всех реплик. Перцентили реплик не усредняются. Queue depth и oldest age показываются отдельно: количество элементов не измеряет длительность ожидания.
|
||||
|
||||
## SLO
|
||||
|
||||
### Доступность API
|
||||
|
||||
SLI — доля бизнес-запросов без `5xx`. Health и `/metrics` исключены. Ожидаемые клиентские `4xx` не считаются отказом сервиса.
|
||||
|
||||
```text
|
||||
bad = business responses 5xx
|
||||
total = all business responses
|
||||
SLI = 1 - bad / total
|
||||
SLO = 99.5% за 30 дней
|
||||
error budget = 0.5%
|
||||
```
|
||||
|
||||
Цель 99,5% выбрана для учебного single-host стенда без multi-zone HA: она требует реагировать на деградацию, но не заявляет недостижимую для локального Compose доступность.
|
||||
|
||||
### Время готовности вложений
|
||||
|
||||
SLI — доля уникальных durable attachment jobs, чей первый terminal outcome равен `ready` и наступил не позднее 30 секунд после acceptance. Время acceptance назначает PostgreSQL при создании job, поэтому начало и конец интервала принадлежат одному clock domain. PostgreSQL записывает первый outcome в append-only ledger в той же транзакции, что и переход job. Миграция переносит уже завершённые jobs с `accepted_at = created_at` и `terminal_at = updated_at`. Повторная delivery и ручной replay не меняют уже учтённый результат: job, впервые завершившаяся `failed`, остаётся bad event даже после успешного replay.
|
||||
|
||||
```text
|
||||
bad = failed + ready later than 30s
|
||||
total = ready + failed
|
||||
SLI = 1 - bad / total
|
||||
SLO = 99% за 30 дней
|
||||
error budget = 1%
|
||||
```
|
||||
|
||||
`messenger_attachment_terminal_transitions_total` и cumulative buckets `messenger_attachment_time_to_ready_seconds_bucket` экспортируются из ledger. Bucket `le="+Inf"` является durable количеством первых outcomes `ready` и используется как знаменатель ready jobs.
|
||||
|
||||
30 секунд отделяют нормальную асинхронную обработку от видимой пользователю задержки и существенно превышают обычное время небольшого синтетического изображения.
|
||||
|
||||
### Multi-window burn-rate alerts
|
||||
|
||||
Для обоих SLO используются две пары окон:
|
||||
|
||||
| Alert | Burn rate | Короткое окно | Длинное окно | Назначение |
|
||||
| --- | ---: | ---: | ---: | --- |
|
||||
| Fast | 14,4× | 5 минут | 1 час | быстро обнаружить сильный инцидент |
|
||||
| Slow | 6× | 30 минут | 6 часов | обнаружить устойчивую деградацию |
|
||||
|
||||
Условие должно выполняться в обоих окнах пары. Короткое окно быстро реагирует, длинное отсекает одиночный шум. Отдельные operational alerts контролируют отсутствие worker, рост oldest queued age и Redis Stream lag.
|
||||
|
||||
## Dashboard и поиск
|
||||
|
||||
Dashboard **Messenger Lab 6 overview** содержит:
|
||||
|
||||
- API RPS, error ratio, p50/p95/p99;
|
||||
- состояния API/worker/dispatcher и DB pool;
|
||||
- accepted/ready/failed jobs, processing duration и end-to-end ready time;
|
||||
- queue depth, oldest age, Redis lag/PEL, outbox pending;
|
||||
- worker busy/capacity;
|
||||
- SLO и firing alerts;
|
||||
- Collector export failures и переходы к Explore.
|
||||
|
||||
Поиск trace выполняется в Grafana Explore → Tempo по `service.name`, span name, route, status и duration. Из trace доступен переход в Loki по тому же `trace_id`. Пошаговые процедуры:
|
||||
|
||||
- `monitoring/runbooks/api-availability.md`;
|
||||
- `monitoring/runbooks/attachment-pipeline.md`.
|
||||
|
||||
## Доказательства критериев
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| Критерий | Команда/сценарий | Проверяемый результат | Доказательство |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
| Offline contract | `make check` | структура и обязательные файлы не ослаблены | stdout CI |
|
||||
| OpenAPI | `make validate` | OpenAPI 3.1 проходит полную проверку | stdout CI |
|
||||
| Публичный smoke | `make test` | бизнес-API предыдущих лабораторных работает | stdout CI |
|
||||
| Lab 5 regression | `make test-lab5-acceptance && make test-lab5` | durable jobs, retry, replay и idempotency сохранены | stdout CI |
|
||||
| Три telemetry signal | `make test-lab6-smoke` | нагрузка создаёт metrics/logs/traces, attachments доходят до terminal state | `evidence/runtime/.../baseline-*.json` |
|
||||
| Worker incident | `scripts/incident-worker.sh short` | alert firing, после запуска worker backlog дренирован и alert resolved | `worker-short-*/` |
|
||||
| DB incident | `scripts/incident-database.sh` | видны 5xx и DB error spans, burn-rate firing, API восстанавливается, оперативный alert resolved | `database-*/` |
|
||||
| Длительная деградация | `scripts/incident-worker.sh long` | растут oldest age/lag, срабатывает backlog alert, все jobs затем ready | `worker-long-*/` |
|
||||
| Telemetry outage | `scripts/incident-telemetry.sh` | API readiness остаётся 200, Collector queue/retry ограничены, backend восстанавливается | `telemetry-*/` |
|
||||
| Полный трек 2 | `make test-lab6` | выполнены short/long incidents, firing/resolved и schema validation | `evidence/runtime/lab6-*/` |
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
Runtime evidence создаётся локально и игнорируется Git до проверки содержимого. В нём нет session tokens, cookies, passwords, payload envelope и пользовательских текстов. Для публикации выбираются только небольшие агрегированные JSON.
|
||||
|
||||
## Эксперименты и отказы
|
||||
## Методика экспериментов
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
`scripts/lab6-traffic.py` создаёт изолированных синтетических пользователей и чат, затем генерирует сообщения и PNG-вложения. Нагрузка closed-loop, concurrency и duration фиксируются в JSON; HTTP status и p50/p95/p99 рассчитываются клиентом только как вспомогательные данные. Серверные SLO используют Prometheus histograms.
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
Полный сценарий:
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
```bash
|
||||
make test-lab6
|
||||
```
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
Он выполняет:
|
||||
|
||||
1. baseline mixed load;
|
||||
2. короткую остановку worker;
|
||||
3. краткую недоступность PostgreSQL под message traffic;
|
||||
4. длительную остановку worker;
|
||||
5. остановку Loki и Tempo под mixed traffic;
|
||||
6. проверку evidence и переходов alert `firing → resolved`.
|
||||
|
||||
Каждый incident script устанавливает trap и восстанавливает затронутые сервисы. `docker compose down -v` не используется. Baseline/degraded/recovered snapshots содержат одни и те же PromQL, поэтому сравнение не зависит от ручного выбора панели.
|
||||
|
||||
Гипотезы трека:
|
||||
|
||||
- короткий сильный отказ расходует бюджет быстрее 14,4× и обнаруживается fast alert;
|
||||
- длительная деградация выявляется ростом oldest queued age и multi-window SLO;
|
||||
- после восстановления worker backlog уменьшается, все принятые jobs достигают terminal state, alerts переходят в resolved;
|
||||
- отказ Loki/Tempo не влияет на бизнес-ответы, а ограниченные очереди Collector отражают потери telemetry без неограниченного роста RAM.
|
||||
|
||||
Фактические времена обнаружения и восстановления берутся из `started_at`, `alert-firing.json`, `recovery_started_at` и `alert-resolved.json`. Отсутствие перехода считается падением сценария, а не успешным экспериментом. Ложное срабатывание проверяется baseline snapshot: до fault operational и burn-rate alerts не должны находиться в `firing`.
|
||||
|
||||
## Результаты прогона
|
||||
|
||||
Полный прогон `2026-11-04` сохранён локально в `evidence/runtime/lab6-20261104T174049Z` и прошёл schema validation без ошибок. Обезличенная сводка включена в `evidence/lab6-results.json`. Время обнаружения является верхней границей: сценарий опрашивает alert после выдержки заданной длительности отказа.
|
||||
|
||||
| Инцидент | Наблюдение | Обнаружение | Восстановление |
|
||||
| --- | --- | ---: | ---: |
|
||||
| Worker, короткий | `AttachmentWorkerUnavailable` firing, очередь затем обработана | не более 76 с | alert resolved через 13 с, полное наблюдение 33 с |
|
||||
| PostgreSQL | operational 5xx и fast burn-rate firing; сохранены 3 DB error traces, один связан с API-логом | не более 78 с | operational alert resolved через 134 с, API стабилен через 154 с |
|
||||
| Worker, длительный | oldest queued age рос; backlog, fast и slow burn alerts firing | не более 181 с | alert resolved через 20 с, backlog обработан за 91 с |
|
||||
| Loki и Tempo | очереди выросли до 15/512 и 16/512, затем дренировались до нуля; peak RSS 226.7 MB | 30 с отказа | `/health/ready` вернул 200, backends healthy через 31 с |
|
||||
|
||||
Перед первым fault список firing alerts был пуст. Burn-rate alerts после инцидента закономерно остаются активными дольше оперативных alerts: их длинные окна сохраняют уже потраченный error budget. Это не считается ложным срабатыванием следующей фазы.
|
||||
|
||||
## Ограничения
|
||||
|
||||
- Стенд single-host и не проверяет потерю всего Docker host или volume.
|
||||
- Loki и Tempo работают в single-binary режиме без репликации.
|
||||
- 30-дневный SLO определяется production-подобными окнами, но учебный эксперимент создаёт искусственно высокий error ratio и не заменяет месячное наблюдение.
|
||||
- Closed-loop generator подвержен coordinated omission; capacity conclusions по его RPS не делаются.
|
||||
- Остановка telemetry backend проверяет ограниченное поведение exporter, но не гарантирует отсутствие потерь: после заполнения bounded queue часть telemetry намеренно отбрасывается.
|
||||
- Внешняя доставка alerts не настраивается; проверяются локальные состояния Prometheus `firing` и `resolved`.
|
||||
|
||||
## Источники
|
||||
|
||||
ИИ-ассистент использовался при подготовке кода, конфигурации и текста; результат проверен тестами, CI-сценариями и воспроизводимым fault-прогоном.
|
||||
|
||||
- OpenTelemetry Collector configuration и Go SDK;
|
||||
- W3C Trace Context;
|
||||
- Prometheus histograms и multi-window burn-rate alerts;
|
||||
- Grafana provisioning, Loki OTLP ingestion и Tempo TraceQL;
|
||||
- документация Redis Streams, PostgreSQL, MinIO и Docker Compose.
|
||||
|
||||
@@ -17,7 +17,7 @@ import (
|
||||
|
||||
"github.com/google/uuid"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
|
||||
)
|
||||
|
||||
type apiState struct {
|
||||
|
||||
@@ -21,15 +21,15 @@ import (
|
||||
"github.com/google/uuid"
|
||||
"github.com/redis/go-redis/v9"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/broker/redisstream"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/config"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/database"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment/transform"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment/worker"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/objectstore"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/outbox"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/postgres"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/broker/redisstream"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/config"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/database"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment/transform"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment/worker"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/objectstore"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/outbox"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/postgres"
|
||||
)
|
||||
|
||||
const (
|
||||
|
||||
@@ -5,8 +5,8 @@ import (
|
||||
"log/slog"
|
||||
"os"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/bootstrap"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/logger"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/bootstrap"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/logger"
|
||||
)
|
||||
|
||||
func main() {
|
||||
|
||||
+94
-9
@@ -4,6 +4,11 @@ x-messenger-config-volume: &messenger-config-volume
|
||||
target: /etc/messenger/messenger.yaml
|
||||
read_only: true
|
||||
|
||||
x-telemetry-environment: &telemetry-environment
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT}
|
||||
OTEL_EXPORTER_OTLP_PROTOCOL: ${OTEL_EXPORTER_OTLP_PROTOCOL}
|
||||
OTEL_RESOURCE_ATTRIBUTES: deployment.environment.name=${DEPLOYMENT_ENVIRONMENT},service.version=${SERVICE_VERSION}
|
||||
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:17.6-alpine3.22
|
||||
@@ -118,7 +123,7 @@ services:
|
||||
networks: [data]
|
||||
|
||||
migrate:
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab5}
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab6}
|
||||
command: ["/app/messenger", "migrate"]
|
||||
environment:
|
||||
MESSENGER_CONFIG_PATH: /etc/messenger/messenger.yaml
|
||||
@@ -137,8 +142,10 @@ services:
|
||||
|
||||
api:
|
||||
build: .
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab5}
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab6}
|
||||
environment:
|
||||
<<: *telemetry-environment
|
||||
OTEL_SERVICE_NAME: messenger-api
|
||||
MESSENGER_CONFIG_PATH: /etc/messenger/messenger.yaml
|
||||
APP_HOST: ${APP_HOST}
|
||||
APP_PORT: ${APP_PORT}
|
||||
@@ -176,9 +183,11 @@ services:
|
||||
networks: [application, data]
|
||||
|
||||
dispatcher:
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab5}
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab6}
|
||||
command: ["/app/messenger", "dispatch"]
|
||||
environment:
|
||||
<<: *telemetry-environment
|
||||
OTEL_SERVICE_NAME: messenger-dispatcher
|
||||
MESSENGER_CONFIG_PATH: /etc/messenger/messenger.yaml
|
||||
DATABASE_URL: ${DATABASE_URL}
|
||||
BROKER_URL: ${BROKER_URL}
|
||||
@@ -202,9 +211,11 @@ services:
|
||||
networks: [data, monitoring]
|
||||
|
||||
worker:
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab5}
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab6}
|
||||
command: ["/app/messenger", "worker"]
|
||||
environment:
|
||||
<<: *telemetry-environment
|
||||
OTEL_SERVICE_NAME: messenger-worker
|
||||
MESSENGER_CONFIG_PATH: /etc/messenger/messenger.yaml
|
||||
DATABASE_URL: ${DATABASE_URL}
|
||||
S3_ENDPOINT: ${S3_ENDPOINT}
|
||||
@@ -239,7 +250,7 @@ services:
|
||||
build:
|
||||
context: .
|
||||
target: loadbalancer
|
||||
image: ${NGINX_IMAGE:-mtusi-nginx:lab5}
|
||||
image: ${NGINX_IMAGE:-mtusi-nginx:lab6}
|
||||
ports:
|
||||
- "${API_BIND_ADDRESS}:8080"
|
||||
depends_on:
|
||||
@@ -259,7 +270,7 @@ services:
|
||||
build:
|
||||
context: .
|
||||
target: swagger
|
||||
image: ${SWAGGER_IMAGE:-mtusi-swagger:lab5}
|
||||
image: ${SWAGGER_IMAGE:-mtusi-swagger:lab6}
|
||||
ports:
|
||||
- "${SWAGGER_BIND_ADDRESS}:8080"
|
||||
depends_on:
|
||||
@@ -289,7 +300,7 @@ services:
|
||||
|
||||
lab5-experiment:
|
||||
profiles: ["experiments"]
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab5}
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab6}
|
||||
command: ["/app/lab5experiment"]
|
||||
environment:
|
||||
MESSENGER_CONFIG_PATH: /etc/messenger/messenger.yaml
|
||||
@@ -375,11 +386,12 @@ services:
|
||||
build:
|
||||
context: .
|
||||
target: prometheus
|
||||
image: ${PROMETHEUS_IMAGE:-mtusi-prometheus:lab5}
|
||||
image: ${PROMETHEUS_IMAGE:-mtusi-prometheus:lab6}
|
||||
command:
|
||||
- --config.file=/etc/prometheus/prometheus.yml
|
||||
- --storage.tsdb.path=/prometheus
|
||||
- --storage.tsdb.retention.time=7d
|
||||
- --web.enable-remote-write-receiver
|
||||
volumes:
|
||||
- prometheus_data:/prometheus
|
||||
ports:
|
||||
@@ -398,6 +410,9 @@ services:
|
||||
- dispatcher
|
||||
- worker
|
||||
- s3
|
||||
- otel-collector
|
||||
- loki
|
||||
- tempo
|
||||
cpus: ${PROMETHEUS_CPUS}
|
||||
mem_limit: ${PROMETHEUS_MEMORY}
|
||||
restart: unless-stopped
|
||||
@@ -407,28 +422,98 @@ services:
|
||||
build:
|
||||
context: .
|
||||
target: grafana
|
||||
image: ${GRAFANA_IMAGE:-mtusi-grafana:lab5}
|
||||
image: ${GRAFANA_IMAGE:-mtusi-grafana:lab6}
|
||||
environment:
|
||||
GF_SECURITY_ADMIN_USER: admin
|
||||
GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD}
|
||||
GF_USERS_ALLOW_SIGN_UP: "false"
|
||||
GF_EXPLORE_ENABLED: "true"
|
||||
volumes:
|
||||
- grafana_data:/var/lib/grafana
|
||||
ports:
|
||||
- "${GRAFANA_BIND_ADDRESS}:3000"
|
||||
depends_on:
|
||||
- prometheus
|
||||
- loki
|
||||
- tempo
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:3000/api/health || exit 1"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
cpus: ${GRAFANA_CPUS}
|
||||
mem_limit: ${GRAFANA_MEMORY}
|
||||
restart: unless-stopped
|
||||
networks: [edge, monitoring]
|
||||
|
||||
loki:
|
||||
image: grafana/loki:3.7.9
|
||||
command: ["-config.file=/etc/loki/loki.yaml"]
|
||||
volumes:
|
||||
- ./infra/loki/loki.yaml:/etc/loki/loki.yaml:ro
|
||||
- loki_data:/loki
|
||||
ports:
|
||||
- "${LOKI_BIND_ADDRESS}:3100"
|
||||
healthcheck:
|
||||
test: ["CMD", "/usr/bin/loki", "-version"]
|
||||
interval: 10s
|
||||
timeout: 3s
|
||||
retries: 10
|
||||
cpus: ${LOKI_CPUS}
|
||||
mem_limit: ${LOKI_MEMORY}
|
||||
restart: unless-stopped
|
||||
networks: [edge, monitoring]
|
||||
|
||||
tempo:
|
||||
image: grafana/tempo:2.10.10
|
||||
command: ["-config.file=/etc/tempo/tempo.yaml"]
|
||||
volumes:
|
||||
- ./infra/tempo/tempo.yaml:/etc/tempo/tempo.yaml:ro
|
||||
- tempo_data:/var/tempo
|
||||
ports:
|
||||
- "${TEMPO_BIND_ADDRESS}:3200"
|
||||
healthcheck:
|
||||
test: ["CMD", "/tempo", "-version"]
|
||||
interval: 10s
|
||||
timeout: 3s
|
||||
retries: 10
|
||||
cpus: ${TEMPO_CPUS}
|
||||
mem_limit: ${TEMPO_MEMORY}
|
||||
restart: unless-stopped
|
||||
networks: [edge, monitoring]
|
||||
|
||||
otel-collector:
|
||||
image: otel/opentelemetry-collector-contrib:0.160.0
|
||||
command: ["--config=/etc/otelcol-contrib/collector.yaml"]
|
||||
volumes:
|
||||
- ./infra/otel/collector.yaml:/etc/otelcol-contrib/collector.yaml:ro
|
||||
expose:
|
||||
- "4317"
|
||||
- "4318"
|
||||
- "8888"
|
||||
- "8889"
|
||||
- "13133"
|
||||
depends_on:
|
||||
- loki
|
||||
- tempo
|
||||
healthcheck:
|
||||
test: ["CMD", "/otelcol-contrib", "components"]
|
||||
interval: 10s
|
||||
timeout: 3s
|
||||
retries: 10
|
||||
cpus: ${OTEL_COLLECTOR_CPUS}
|
||||
mem_limit: ${OTEL_COLLECTOR_MEMORY}
|
||||
restart: unless-stopped
|
||||
networks: [application, data, monitoring]
|
||||
|
||||
volumes:
|
||||
postgres_data:
|
||||
valkey_data:
|
||||
redis_broker_data:
|
||||
prometheus_data:
|
||||
grafana_data:
|
||||
loki_data:
|
||||
tempo_data:
|
||||
s3_data:
|
||||
|
||||
networks:
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"captured_at": "2026-11-04T17:53:52Z",
|
||||
"baseline": {
|
||||
"api_request_rate_per_second": 1.014325,
|
||||
"firing_alerts": 0,
|
||||
"traces_by_service": {
|
||||
"messenger-api": 100,
|
||||
"messenger-worker": 72,
|
||||
"messenger-dispatcher": 64
|
||||
},
|
||||
"logs_by_service": {
|
||||
"messenger-api": 207,
|
||||
"messenger-worker": 41,
|
||||
"messenger-dispatcher": 1
|
||||
},
|
||||
"api_worker_shared_traces": 22,
|
||||
"traces_with_correlated_logs": 101
|
||||
},
|
||||
"incidents": [
|
||||
{
|
||||
"name": "worker_unavailable_short",
|
||||
"injected_duration_seconds": 76,
|
||||
"alert_detection_upper_bound_seconds": 76,
|
||||
"alert_resolution_seconds": 13,
|
||||
"recovery_observation_seconds": 33
|
||||
},
|
||||
{
|
||||
"name": "postgres_unavailable",
|
||||
"injected_duration_seconds": 78,
|
||||
"alert_detection_upper_bound_seconds": 78,
|
||||
"operational_alert_resolution_seconds": 134,
|
||||
"recovery_observation_seconds": 154,
|
||||
"fast_burn_alert_observed": true,
|
||||
"database_error_traces": 3,
|
||||
"database_error_traces_with_logs": 1
|
||||
},
|
||||
{
|
||||
"name": "worker_unavailable_long",
|
||||
"injected_duration_seconds": 181,
|
||||
"alert_detection_upper_bound_seconds": 181,
|
||||
"alert_resolution_seconds": 20,
|
||||
"recovery_observation_seconds": 91,
|
||||
"fast_burn_alert_observed": true,
|
||||
"slow_burn_alert_observed": true
|
||||
},
|
||||
{
|
||||
"name": "telemetry_backends_unavailable",
|
||||
"injected_duration_seconds": 30,
|
||||
"api_ready_http_status": 200,
|
||||
"recovery_observation_seconds": 31,
|
||||
"exporters": {
|
||||
"otlp_grpc/tempo": {
|
||||
"capacity": 512,
|
||||
"degraded_queue_size": 15,
|
||||
"recovered_queue_size": 0,
|
||||
"export_failures": 16,
|
||||
"dropped_records": 0
|
||||
},
|
||||
"otlp_http/loki": {
|
||||
"capacity": 512,
|
||||
"degraded_queue_size": 16,
|
||||
"recovered_queue_size": 0,
|
||||
"export_failures": 8,
|
||||
"dropped_records": 0
|
||||
}
|
||||
},
|
||||
"collector_peak_rss_bytes": {
|
||||
"baseline": 225923072,
|
||||
"degraded": 226549760,
|
||||
"recovered": 226746368
|
||||
}
|
||||
}
|
||||
],
|
||||
"validation": {
|
||||
"valid": true,
|
||||
"errors": [],
|
||||
"incident_summaries": 4
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"$id": "https://gitea.privcloud.tech/mtusi2026/mtusi2026lab6/evidence/lab6-schema.json",
|
||||
"title": "Lab 6 incident summary",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"schema_version",
|
||||
"incident",
|
||||
"started_at",
|
||||
"recovery_started_at",
|
||||
"recovered_at",
|
||||
"injected_duration_seconds",
|
||||
"recovery_observation_seconds",
|
||||
"artifacts"
|
||||
],
|
||||
"properties": {
|
||||
"schema_version": {"const": 1},
|
||||
"incident": {
|
||||
"enum": ["worker_unavailable", "postgres_unavailable", "telemetry_backends_unavailable"]
|
||||
},
|
||||
"profile": {"enum": ["short", "long"]},
|
||||
"started_at": {"type": "string", "format": "date-time"},
|
||||
"recovery_started_at": {"type": "string", "format": "date-time"},
|
||||
"recovered_at": {"type": "string", "format": "date-time"},
|
||||
"planned_injection_seconds": {"type": "integer", "minimum": 1},
|
||||
"injected_duration_seconds": {"type": "integer", "minimum": 1},
|
||||
"recovery_observation_seconds": {"type": "integer", "minimum": 0},
|
||||
"api_ready_http_status_during_outage": {"type": "integer", "minimum": 100, "maximum": 599},
|
||||
"artifacts": {
|
||||
"type": "array",
|
||||
"items": {"type": "string", "pattern": "^[A-Za-z0-9._-]+$"},
|
||||
"minItems": 4,
|
||||
"uniqueItems": true
|
||||
}
|
||||
},
|
||||
"additionalProperties": false
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
module gitea.privcloud.tech/dizi/mtusi2026lab5
|
||||
module gitea.privcloud.tech/dizi/mtusi2026lab6
|
||||
|
||||
go 1.26.0
|
||||
|
||||
@@ -12,19 +12,36 @@ require (
|
||||
github.com/spf13/viper v1.21.0
|
||||
github.com/stretchr/testify v1.11.1
|
||||
github.com/valkey-io/valkey-go v1.0.78
|
||||
go.opentelemetry.io/contrib/bridges/otelslog v0.13.0
|
||||
go.opentelemetry.io/contrib/instrumentation/runtime v0.60.0
|
||||
go.opentelemetry.io/otel v1.38.0
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.14.0
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.38.0
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.38.0
|
||||
go.opentelemetry.io/otel/exporters/prometheus v0.60.0
|
||||
go.opentelemetry.io/otel/metric v1.38.0
|
||||
go.opentelemetry.io/otel/sdk v1.38.0
|
||||
go.opentelemetry.io/otel/sdk/log v0.14.0
|
||||
go.opentelemetry.io/otel/sdk/metric v1.38.0
|
||||
go.opentelemetry.io/otel/trace v1.38.0
|
||||
golang.org/x/crypto v0.57.0
|
||||
golang.org/x/sync v0.23.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/cenkalti/backoff/v5 v5.0.3 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/fsnotify/fsnotify v1.9.0 // indirect
|
||||
github.com/go-ini/ini v1.67.0 // indirect
|
||||
github.com/go-logr/logr v1.4.3 // indirect
|
||||
github.com/go-logr/stdr v1.2.2 // indirect
|
||||
github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
|
||||
github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc // indirect
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 // indirect
|
||||
github.com/jackc/pgpassfile v1.0.0 // indirect
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
|
||||
github.com/jackc/puddle/v2 v2.2.2 // indirect
|
||||
@@ -39,7 +56,8 @@ require (
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
github.com/prometheus/client_model v0.6.2 // indirect
|
||||
github.com/prometheus/common v0.66.1 // indirect
|
||||
github.com/prometheus/procfs v0.16.1 // indirect
|
||||
github.com/prometheus/otlptranslator v0.0.2 // indirect
|
||||
github.com/prometheus/procfs v0.17.0 // indirect
|
||||
github.com/rs/xid v1.6.0 // indirect
|
||||
github.com/sagikazarmark/locafero v0.11.0 // indirect
|
||||
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect
|
||||
@@ -48,11 +66,18 @@ require (
|
||||
github.com/spf13/pflag v1.0.10 // indirect
|
||||
github.com/subosito/gotenv v1.6.0 // indirect
|
||||
github.com/tinylib/msgp v1.6.1 // indirect
|
||||
go.opentelemetry.io/auto/sdk v1.1.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 // indirect
|
||||
go.opentelemetry.io/otel/log v0.14.0 // indirect
|
||||
go.opentelemetry.io/proto/otlp v1.7.1 // indirect
|
||||
go.yaml.in/yaml/v2 v2.4.2 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.5 // indirect
|
||||
golang.org/x/net v0.58.0 // indirect
|
||||
golang.org/x/sys v0.48.0 // indirect
|
||||
golang.org/x/text v0.42.0 // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5 // indirect
|
||||
google.golang.org/grpc v1.75.0 // indirect
|
||||
google.golang.org/protobuf v1.36.8 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
)
|
||||
|
||||
@@ -4,6 +4,8 @@ github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
|
||||
github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
|
||||
github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
|
||||
github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
|
||||
github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM=
|
||||
github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw=
|
||||
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
@@ -21,12 +23,23 @@ github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S
|
||||
github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
|
||||
github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
|
||||
github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
|
||||
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
|
||||
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
|
||||
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
|
||||
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
|
||||
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
|
||||
github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs=
|
||||
github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
|
||||
github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
|
||||
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
|
||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc h1:GN2Lv3MGO7AS6PrRoT6yV5+wkrOpcszoIsO4+4ds248=
|
||||
github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc/go.mod h1:+JKpmjMGhpgPL+rXZ5nsZieVzvarn86asRlBg4uNGnk=
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 h1:8Tjv8EJ+pM1xP8mK6egEbD1OgnVTyacbefKhmbLhIhU=
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2/go.mod h1:pkJQ2tZHJ0aFOVEEot6oZmaVEZcRme73eIFmhiVuRWs=
|
||||
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
|
||||
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
|
||||
@@ -70,12 +83,14 @@ github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNw
|
||||
github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
|
||||
github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs=
|
||||
github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA=
|
||||
github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg=
|
||||
github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is=
|
||||
github.com/prometheus/otlptranslator v0.0.2 h1:+1CdeLVrRQ6Psmhnobldo0kTp96Rj80DRXRd5OSnMEQ=
|
||||
github.com/prometheus/otlptranslator v0.0.2/go.mod h1:P8AwMgdD7XEr6QRUJ2QWLpiAZTgTE2UYgjlu3svompI=
|
||||
github.com/prometheus/procfs v0.17.0 h1:FuLQ+05u4ZI+SS/w9+BWEM2TXiHKsUQ9TADiRH7DuK0=
|
||||
github.com/prometheus/procfs v0.17.0/go.mod h1:oPQLaDAMRbA+u8H5Pbfq+dl3VDAvHxMUOVhe0wYB2zw=
|
||||
github.com/redis/go-redis/v9 v9.17.2 h1:P2EGsA4qVIM3Pp+aPocCJ7DguDHhqrXNhVcEp4ViluI=
|
||||
github.com/redis/go-redis/v9 v9.17.2/go.mod h1:u410H11HMLoB+TP67dz8rL9s6QW2j76l0//kSOd3370=
|
||||
github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
|
||||
github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
|
||||
github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII=
|
||||
github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o=
|
||||
github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
|
||||
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
|
||||
github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc=
|
||||
@@ -101,6 +116,40 @@ github.com/tinylib/msgp v1.6.1 h1:ESRv8eL3u+DNHUoSAAQRE50Hm162zqAnBoGv9PzScPY=
|
||||
github.com/tinylib/msgp v1.6.1/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA=
|
||||
github.com/valkey-io/valkey-go v1.0.78 h1:1Ce8hRaaNXpKfqmqZQRhnbVh1eW8MGR873pdXVvE4y8=
|
||||
github.com/valkey-io/valkey-go v1.0.78/go.mod h1:gvC/r2m3eW4Hbj0YnjogTzNtFdPSM/D+NCqen+5OABM=
|
||||
go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA=
|
||||
go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A=
|
||||
go.opentelemetry.io/contrib/bridges/otelslog v0.13.0 h1:bwnLpizECbPr1RrQ27waeY2SPIPeccCx/xLuoYADZ9s=
|
||||
go.opentelemetry.io/contrib/bridges/otelslog v0.13.0/go.mod h1:3nWlOiiqA9UtUnrcNk82mYasNxD8ehOspL0gOfEo6Y4=
|
||||
go.opentelemetry.io/contrib/instrumentation/runtime v0.60.0 h1:0NgN/3SYkqYJ9NBlDfl/2lzVlwos/YQLvi8sUrzJRBE=
|
||||
go.opentelemetry.io/contrib/instrumentation/runtime v0.60.0/go.mod h1:oxpUfhTkhgQaYIjtBt3T3w135dLoxq//qo3WPlPIKkE=
|
||||
go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8=
|
||||
go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.14.0 h1:OMqPldHt79PqWKOMYIAQs3CxAi7RLgPxwfFSwr4ZxtM=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.14.0/go.mod h1:1biG4qiqTxKiUCtoWDPpL3fB3KxVwCiGw81j3nKMuHE=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.38.0 h1:vl9obrcoWVKp/lwl8tRE33853I8Xru9HFbw/skNeLs8=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.38.0/go.mod h1:GAXRxmLJcVM3u22IjTg74zWBrRCKq8BnOqUVLodpcpw=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 h1:GqRJVj7UmLjCVyVJ3ZFLdPRmhDUp2zFmQe3RHIOsw24=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0/go.mod h1:ri3aaHSmCTVYu2AWv44YMauwAQc0aqI9gHKIcSbI1pU=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.38.0 h1:lwI4Dc5leUqENgGuQImwLo4WnuXFPetmPpkLi2IrX54=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.38.0/go.mod h1:Kz/oCE7z5wuyhPxsXDuaPteSWqjSBD5YaSdbxZYGbGk=
|
||||
go.opentelemetry.io/otel/exporters/prometheus v0.60.0 h1:cGtQxGvZbnrWdC2GyjZi0PDKVSLWP/Jocix3QWfXtbo=
|
||||
go.opentelemetry.io/otel/exporters/prometheus v0.60.0/go.mod h1:hkd1EekxNo69PTV4OWFGZcKQiIqg0RfuWExcPKFvepk=
|
||||
go.opentelemetry.io/otel/log v0.14.0 h1:2rzJ+pOAZ8qmZ3DDHg73NEKzSZkhkGIua9gXtxNGgrM=
|
||||
go.opentelemetry.io/otel/log v0.14.0/go.mod h1:5jRG92fEAgx0SU/vFPxmJvhIuDU9E1SUnEQrMlJpOno=
|
||||
go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA=
|
||||
go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI=
|
||||
go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E=
|
||||
go.opentelemetry.io/otel/sdk v1.38.0/go.mod h1:ghmNdGlVemJI3+ZB5iDEuk4bWA3GkTpW+DOoZMYBVVg=
|
||||
go.opentelemetry.io/otel/sdk/log v0.14.0 h1:JU/U3O7N6fsAXj0+CXz21Czg532dW2V4gG1HE/e8Zrg=
|
||||
go.opentelemetry.io/otel/sdk/log v0.14.0/go.mod h1:imQvII+0ZylXfKU7/wtOND8Hn4OpT3YUoIgqJVksUkM=
|
||||
go.opentelemetry.io/otel/sdk/log/logtest v0.14.0 h1:Ijbtz+JKXl8T2MngiwqBlPaHqc4YCaP/i13Qrow6gAM=
|
||||
go.opentelemetry.io/otel/sdk/log/logtest v0.14.0/go.mod h1:dCU8aEL6q+L9cYTqcVOk8rM9Tp8WdnHOPLiBgp0SGOA=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6qT5wthqPoM=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA=
|
||||
go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE=
|
||||
go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs=
|
||||
go.opentelemetry.io/proto/otlp v1.7.1 h1:gTOMpGDb0WTBOP8JaO72iL3auEZhVmAQg4ipjOVAtj4=
|
||||
go.opentelemetry.io/proto/otlp v1.7.1/go.mod h1:b2rVh6rfI/s2pHWNlB7ILJcRALpcNDzKhACevjI+ZnE=
|
||||
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
|
||||
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
|
||||
go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI=
|
||||
@@ -117,6 +166,14 @@ golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
|
||||
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||
golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI=
|
||||
golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E=
|
||||
gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
|
||||
gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5 h1:BIRfGDEjiHRrk0QKZe3Xv2ieMhtgRGeLcZQ0mIVn4EY=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5/go.mod h1:j3QtIyytwqGr1JUDtYXwtMXWPKsEa5LtzIFN1Wn5WvE=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5 h1:eaY8u2EuxbRv7c3NiGK0/NedzVsCcV6hDuU5qPX5EGE=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5/go.mod h1:M4/wBTSeyLxupu3W3tJtOgB14jILAS/XWPSSa3TAlJc=
|
||||
google.golang.org/grpc v1.75.0 h1:+TW+dqTd2Biwe6KKfhE5JpiYIBWq865PhKGSXiivqt4=
|
||||
google.golang.org/grpc v1.75.0/go.mod h1:JtPAzKiq4v1xcAB2hydNlWI2RnF85XXcV0mhKXr2ecQ=
|
||||
google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc=
|
||||
google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
auth_enabled: false
|
||||
|
||||
server:
|
||||
http_listen_address: 0.0.0.0
|
||||
http_listen_port: 3100
|
||||
grpc_listen_address: 0.0.0.0
|
||||
grpc_listen_port: 9096
|
||||
|
||||
common:
|
||||
path_prefix: /loki
|
||||
replication_factor: 1
|
||||
ring:
|
||||
kvstore:
|
||||
store: inmemory
|
||||
storage:
|
||||
filesystem:
|
||||
chunks_directory: /loki/chunks
|
||||
rules_directory: /loki/rules
|
||||
|
||||
schema_config:
|
||||
configs:
|
||||
- from: 2024-01-01
|
||||
store: tsdb
|
||||
object_store: filesystem
|
||||
schema: v13
|
||||
index:
|
||||
prefix: index_
|
||||
period: 24h
|
||||
|
||||
limits_config:
|
||||
allow_structured_metadata: true
|
||||
retention_period: 168h
|
||||
volume_enabled: true
|
||||
otlp_config:
|
||||
resource_attributes:
|
||||
attributes_config:
|
||||
- action: index_label
|
||||
attributes:
|
||||
- service.name
|
||||
- deployment.environment.name
|
||||
|
||||
compactor:
|
||||
working_directory: /loki/compactor
|
||||
retention_enabled: true
|
||||
delete_request_store: filesystem
|
||||
|
||||
analytics:
|
||||
reporting_enabled: false
|
||||
@@ -0,0 +1,80 @@
|
||||
extensions:
|
||||
health_check:
|
||||
endpoint: 0.0.0.0:13133
|
||||
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
grpc:
|
||||
endpoint: 0.0.0.0:4317
|
||||
http:
|
||||
endpoint: 0.0.0.0:4318
|
||||
|
||||
processors:
|
||||
memory_limiter:
|
||||
check_interval: 1s
|
||||
limit_mib: 192
|
||||
spike_limit_mib: 48
|
||||
batch:
|
||||
send_batch_size: 1024
|
||||
send_batch_max_size: 2048
|
||||
timeout: 2s
|
||||
|
||||
exporters:
|
||||
prometheus:
|
||||
endpoint: 0.0.0.0:8889
|
||||
resource_constant_labels:
|
||||
included:
|
||||
- service.name
|
||||
- service.version
|
||||
- deployment.environment.name
|
||||
otlp_grpc/tempo:
|
||||
endpoint: tempo:4317
|
||||
tls:
|
||||
insecure: true
|
||||
sending_queue:
|
||||
enabled: true
|
||||
num_consumers: 2
|
||||
queue_size: 512
|
||||
retry_on_failure:
|
||||
enabled: true
|
||||
initial_interval: 1s
|
||||
max_interval: 5s
|
||||
max_elapsed_time: 30s
|
||||
otlp_http/loki:
|
||||
endpoint: http://loki:3100/otlp
|
||||
sending_queue:
|
||||
enabled: true
|
||||
num_consumers: 2
|
||||
queue_size: 512
|
||||
retry_on_failure:
|
||||
enabled: true
|
||||
initial_interval: 1s
|
||||
max_interval: 5s
|
||||
max_elapsed_time: 30s
|
||||
|
||||
service:
|
||||
extensions: [health_check]
|
||||
telemetry:
|
||||
logs:
|
||||
level: info
|
||||
metrics:
|
||||
readers:
|
||||
- pull:
|
||||
exporter:
|
||||
prometheus:
|
||||
host: 0.0.0.0
|
||||
port: 8888
|
||||
pipelines:
|
||||
metrics:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, batch]
|
||||
exporters: [prometheus]
|
||||
logs:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, batch]
|
||||
exporters: [otlp_http/loki]
|
||||
traces:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, batch]
|
||||
exporters: [otlp_grpc/tempo]
|
||||
@@ -0,0 +1,48 @@
|
||||
server:
|
||||
http_listen_address: 0.0.0.0
|
||||
http_listen_port: 3200
|
||||
grpc_listen_address: 0.0.0.0
|
||||
grpc_listen_port: 9095
|
||||
|
||||
distributor:
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
grpc:
|
||||
endpoint: 0.0.0.0:4317
|
||||
http:
|
||||
endpoint: 0.0.0.0:4318
|
||||
|
||||
ingester:
|
||||
max_block_duration: 5m
|
||||
|
||||
compactor:
|
||||
compaction:
|
||||
block_retention: 168h
|
||||
|
||||
storage:
|
||||
trace:
|
||||
backend: local
|
||||
wal:
|
||||
path: /var/tempo/wal
|
||||
local:
|
||||
path: /var/tempo/traces
|
||||
|
||||
metrics_generator:
|
||||
registry:
|
||||
external_labels:
|
||||
source: tempo
|
||||
deployment_environment: lab
|
||||
storage:
|
||||
path: /var/tempo/generator/wal
|
||||
remote_write:
|
||||
- url: http://prometheus:9090/api/v1/write
|
||||
send_exemplars: true
|
||||
|
||||
overrides:
|
||||
defaults:
|
||||
metrics_generator:
|
||||
processors: [service-graphs, span-metrics]
|
||||
|
||||
usage_report:
|
||||
reporting_enabled: false
|
||||
@@ -3,8 +3,8 @@ package bootstrap
|
||||
import (
|
||||
"context"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/config"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/database"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/config"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/database"
|
||||
)
|
||||
|
||||
func RunMigrations() error {
|
||||
|
||||
@@ -14,15 +14,16 @@ import (
|
||||
"github.com/google/uuid"
|
||||
"github.com/redis/go-redis/v9"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/broker/redisstream"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/config"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/database"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment/pipeline"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment/transform"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment/worker"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/objectstore"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/outbox"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/postgres"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/broker/redisstream"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/config"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/database"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment/pipeline"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment/transform"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment/worker"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/objectstore"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/outbox"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/postgres"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/telemetry"
|
||||
)
|
||||
|
||||
func RunDispatcher(logger *slog.Logger) error {
|
||||
@@ -30,11 +31,23 @@ func RunDispatcher(logger *slog.Logger) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
instanceID, err := os.Hostname()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
observability, logger, err := startTelemetry("messenger-dispatcher", instanceID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer shutdownTelemetry(observability)
|
||||
pool, err := database.Open(context.Background(), dispatcherDatabaseConfig(settings))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer pool.Close()
|
||||
if err := telemetry.ObserveDatabasePool(observability.MeterProvider().Meter("messenger-dispatcher/database"), pool); err != nil {
|
||||
return err
|
||||
}
|
||||
client, err := newRedisClient(settings.BrokerURL)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -51,7 +64,7 @@ func RunDispatcher(logger *slog.Logger) error {
|
||||
if err := broker.Ensure(context.Background()); err != nil {
|
||||
return err
|
||||
}
|
||||
metrics := pipeline.NewMetrics()
|
||||
metrics := pipeline.NewMetrics(observability.MeterProvider().Meter("messenger-dispatcher"), observability.MetricsHandler(), 0)
|
||||
dispatcher, err := outbox.New(postgres.New(pool), broker, metrics, outbox.Config{
|
||||
BatchSize: settings.BatchSize, LeaseDuration: settings.LeaseDuration,
|
||||
PollInterval: settings.PollInterval, RecoveryInterval: settings.RecoveryInterval,
|
||||
@@ -70,11 +83,23 @@ func RunWorker(logger *slog.Logger) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
hostname, err := os.Hostname()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
observability, logger, err := startTelemetry("messenger-worker", hostname)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer shutdownTelemetry(observability)
|
||||
pool, err := database.Open(context.Background(), workerDatabaseConfig(settings))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer pool.Close()
|
||||
if err := telemetry.ObserveDatabasePool(observability.MeterProvider().Meter("messenger-worker/database"), pool); err != nil {
|
||||
return err
|
||||
}
|
||||
objects, err := objectstore.New(objectstore.Config{
|
||||
Endpoint: settings.S3Endpoint, Region: settings.S3Region, Bucket: settings.S3Bucket,
|
||||
AccessKey: settings.S3AccessKey, SecretKey: settings.S3SecretKey,
|
||||
@@ -88,10 +113,6 @@ func RunWorker(logger *slog.Logger) error {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = client.Close() }()
|
||||
hostname, err := os.Hostname()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
consumer := hostname + "-" + uuid.NewString()
|
||||
broker, err := redisstream.New(client, redisstream.Config{
|
||||
Stream: settings.Stream, Group: settings.ConsumerGroup, Consumer: consumer,
|
||||
@@ -108,7 +129,7 @@ func RunWorker(logger *slog.Logger) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
metrics := pipeline.NewMetrics()
|
||||
metrics := pipeline.NewMetrics(observability.MeterProvider().Meter("messenger-worker"), observability.MetricsHandler(), settings.Concurrency)
|
||||
processorWorker, err := worker.New(
|
||||
logger, postgres.New(pool), broker, objects, processor, metrics,
|
||||
worker.Config{
|
||||
|
||||
@@ -10,11 +10,11 @@ import (
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/config"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/database"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/objectstore"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/postgres"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/reconcile"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/config"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/database"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/objectstore"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/postgres"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/reconcile"
|
||||
)
|
||||
|
||||
type reconciliationOutput struct {
|
||||
|
||||
@@ -6,9 +6,9 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/config"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/database"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/postgres"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/config"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/database"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/postgres"
|
||||
)
|
||||
|
||||
func RunAttachmentReplay(arguments []string, output io.Writer) error {
|
||||
|
||||
@@ -9,14 +9,15 @@ import (
|
||||
"os/signal"
|
||||
"syscall"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/config"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/database"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/httpapi"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/objectstore"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/password"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/postgres"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/session"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/config"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/database"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/httpapi"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/objectstore"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/password"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/postgres"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/session"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/telemetry"
|
||||
)
|
||||
|
||||
func RunServer(logger *slog.Logger) error {
|
||||
@@ -28,11 +29,19 @@ func RunServer(logger *slog.Logger) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
observability, logger, err := startTelemetry("messenger-api", instanceID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer shutdownTelemetry(observability)
|
||||
pool, err := database.Open(context.Background(), databaseConfig(settings))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer pool.Close()
|
||||
if err := telemetry.ObserveDatabasePool(observability.MeterProvider().Meter("messenger-api/database"), pool); err != nil {
|
||||
return err
|
||||
}
|
||||
sessions, err := session.New(settings.CacheURL)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -62,6 +71,7 @@ func RunServer(logger *slog.Logger) error {
|
||||
app := httpapi.NewServer(logger, httpapi.Dependencies{
|
||||
Users: repository, Chats: repository, Messages: repository, Attachments: attachments,
|
||||
Sessions: sessions, Passwords: passwords, Database: repository, Objects: objects,
|
||||
Meter: observability.MeterProvider().Meter("messenger-api"), Metrics: observability.MetricsHandler(),
|
||||
}, httpapi.Config{
|
||||
SessionTTL: settings.SessionTTL, IdempotencyTTL: settings.IdempotencyTTL,
|
||||
RequestTTL: settings.RequestTimeout, InstanceID: instanceID, SwaggerPort: settings.SwaggerPort,
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
package bootstrap
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"time"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/telemetry"
|
||||
)
|
||||
|
||||
func startTelemetry(serviceName, instanceID string) (*telemetry.Runtime, *slog.Logger, error) {
|
||||
runtime, err := telemetry.New(context.Background(), telemetry.ConfigFromEnvironment(serviceName, instanceID))
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
return runtime, runtime.Logger(), nil
|
||||
}
|
||||
|
||||
func shutdownTelemetry(runtime *telemetry.Runtime) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
_ = runtime.Shutdown(ctx)
|
||||
}
|
||||
@@ -11,7 +11,12 @@ import (
|
||||
|
||||
"github.com/redis/go-redis/v9"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/outbox"
|
||||
"go.opentelemetry.io/otel"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/codes"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/outbox"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -103,6 +108,10 @@ func isBusyGroup(err error) bool {
|
||||
}
|
||||
|
||||
func (b *Broker) PublishConfirmed(ctx context.Context, payload []byte) (string, error) {
|
||||
ctx, span := otel.Tracer("redisstream").Start(ctx, "redis XADD", trace.WithSpanKind(trace.SpanKindClient),
|
||||
trace.WithAttributes(attribute.String("db.system", "redis"), attribute.String("db.operation.name", "XADD"),
|
||||
attribute.String("messaging.destination.name", b.config.Stream)))
|
||||
defer span.End()
|
||||
envelope, err := DecodeEnvelope(payload)
|
||||
if err != nil {
|
||||
return "", err
|
||||
@@ -117,12 +126,19 @@ func (b *Broker) PublishConfirmed(ctx context.Context, payload []byte) (string,
|
||||
Values: map[string]any{payloadField: string(payload)},
|
||||
}).Result()
|
||||
if err != nil {
|
||||
span.RecordError(err)
|
||||
span.SetStatus(codes.Error, "XADD failed")
|
||||
return "", fmt.Errorf("publish redis stream job: %w", err)
|
||||
}
|
||||
span.SetAttributes(attribute.String("messaging.message.id", id))
|
||||
return id, nil
|
||||
}
|
||||
|
||||
func (b *Broker) Read(ctx context.Context) ([]Delivery, error) {
|
||||
ctx, span := otel.Tracer("redisstream").Start(ctx, "redis XREADGROUP", trace.WithSpanKind(trace.SpanKindClient),
|
||||
trace.WithAttributes(attribute.String("db.system", "redis"), attribute.String("db.operation.name", "XREADGROUP"),
|
||||
attribute.String("messaging.destination.name", b.config.Stream)))
|
||||
defer span.End()
|
||||
streams, err := b.client.XReadGroup(ctx, &redis.XReadGroupArgs{
|
||||
Group: b.config.Group,
|
||||
Consumer: b.config.Consumer,
|
||||
@@ -134,6 +150,8 @@ func (b *Broker) Read(ctx context.Context) ([]Delivery, error) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
span.RecordError(err)
|
||||
span.SetStatus(codes.Error, "XREADGROUP failed")
|
||||
return nil, fmt.Errorf("read redis stream jobs: %w", err)
|
||||
}
|
||||
return b.deliveries(streams)
|
||||
|
||||
@@ -8,7 +8,7 @@ import (
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/migrations"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/migrations"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -40,6 +40,7 @@ func Open(ctx context.Context, config Config) (*pgxpool.Pool, error) {
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parse database configuration: %w", err)
|
||||
}
|
||||
poolConfig.ConnConfig.Tracer = queryTracer{}
|
||||
poolConfig.MinConns = config.MinConnections
|
||||
poolConfig.MaxConns = config.MaxConnections
|
||||
poolConfig.MaxConnLifetime = config.MaxConnectionAge
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"go.opentelemetry.io/otel"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/codes"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
)
|
||||
|
||||
type queryTracer struct{}
|
||||
|
||||
func (queryTracer) TraceQueryStart(ctx context.Context, _ *pgx.Conn, _ pgx.TraceQueryStartData) context.Context {
|
||||
ctx, _ = otel.Tracer("postgres").Start(ctx, "postgres query", trace.WithSpanKind(trace.SpanKindClient),
|
||||
trace.WithAttributes(attribute.String("db.system", "postgresql")))
|
||||
return ctx
|
||||
}
|
||||
|
||||
func (queryTracer) TraceQueryEnd(ctx context.Context, _ *pgx.Conn, data pgx.TraceQueryEndData) {
|
||||
span := trace.SpanFromContext(ctx)
|
||||
if data.Err != nil {
|
||||
span.RecordError(data.Err)
|
||||
span.SetStatus(codes.Error, "query failed")
|
||||
}
|
||||
span.End()
|
||||
}
|
||||
@@ -1,104 +1,97 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/collectors"
|
||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
"go.opentelemetry.io/otel"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/metric"
|
||||
)
|
||||
|
||||
var workerDurationBuckets = []float64{0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 15, 20, 30, 60}
|
||||
|
||||
type Metrics struct {
|
||||
workerInFlight prometheus.Gauge
|
||||
workerJobs *prometheus.CounterVec
|
||||
workerDuration *prometheus.HistogramVec
|
||||
workerRetries prometheus.Counter
|
||||
workerReclaimed prometheus.Counter
|
||||
outboxPublished prometheus.Counter
|
||||
outboxPublishFailure prometheus.Counter
|
||||
outboxRecovered prometheus.Counter
|
||||
meter metric.Meter
|
||||
workerInFlight metric.Int64UpDownCounter
|
||||
workerJobs metric.Int64Counter
|
||||
workerDuration metric.Float64Histogram
|
||||
workerRetries metric.Int64Counter
|
||||
workerReclaimed metric.Int64Counter
|
||||
outboxPublished metric.Int64Counter
|
||||
outboxPublishFailure metric.Int64Counter
|
||||
outboxRecovered metric.Int64Counter
|
||||
handler http.Handler
|
||||
}
|
||||
|
||||
func NewMetrics() *Metrics {
|
||||
registry := prometheus.NewRegistry()
|
||||
metrics := &Metrics{
|
||||
workerInFlight: prometheus.NewGauge(prometheus.GaugeOpts{
|
||||
Namespace: "messenger", Subsystem: "attachment_worker", Name: "jobs_in_flight",
|
||||
Help: "Attachment jobs currently processed by this worker.",
|
||||
}),
|
||||
workerJobs: prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Namespace: "messenger", Subsystem: "attachment_worker", Name: "jobs_total",
|
||||
Help: "Completed attachment job deliveries by result.",
|
||||
}, []string{"result"}),
|
||||
workerDuration: prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
||||
Namespace: "messenger", Subsystem: "attachment_worker", Name: "job_duration_seconds",
|
||||
Help: "Attachment job delivery processing duration.", Buckets: workerDurationBuckets,
|
||||
}, []string{"result"}),
|
||||
workerRetries: prometheus.NewCounter(prometheus.CounterOpts{
|
||||
Namespace: "messenger", Subsystem: "attachment_worker", Name: "retries_total",
|
||||
Help: "Attachment jobs scheduled for another attempt.",
|
||||
}),
|
||||
workerReclaimed: prometheus.NewCounter(prometheus.CounterOpts{
|
||||
Namespace: "messenger", Subsystem: "attachment_worker", Name: "reclaimed_deliveries_total",
|
||||
Help: "Redis Stream deliveries reclaimed after their idle timeout.",
|
||||
}),
|
||||
outboxPublished: prometheus.NewCounter(prometheus.CounterOpts{
|
||||
Namespace: "messenger", Subsystem: "outbox", Name: "published_total",
|
||||
Help: "Outbox events confirmed by Redis Streams.",
|
||||
}),
|
||||
outboxPublishFailure: prometheus.NewCounter(prometheus.CounterOpts{
|
||||
Namespace: "messenger", Subsystem: "outbox", Name: "publish_failures_total",
|
||||
Help: "Outbox events whose Redis Streams publish failed.",
|
||||
}),
|
||||
outboxRecovered: prometheus.NewCounter(prometheus.CounterOpts{
|
||||
Namespace: "messenger", Subsystem: "outbox", Name: "recovered_leases_total",
|
||||
Help: "Expired outbox leases returned for publication.",
|
||||
}),
|
||||
func NewMetrics(meter metric.Meter, handler http.Handler, workerCapacity int) *Metrics {
|
||||
if meter == nil {
|
||||
meter = otel.Meter("messenger-pipeline")
|
||||
}
|
||||
registry.MustRegister(
|
||||
metrics.workerInFlight, metrics.workerJobs, metrics.workerDuration,
|
||||
metrics.workerRetries, metrics.workerReclaimed, metrics.outboxPublished,
|
||||
metrics.outboxPublishFailure, metrics.outboxRecovered,
|
||||
collectors.NewGoCollector(), collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}),
|
||||
if handler == nil {
|
||||
handler = http.NotFoundHandler()
|
||||
}
|
||||
workerInFlight, _ := meter.Int64UpDownCounter("messenger_attachment_worker_jobs_in_flight", metric.WithDescription("Attachment jobs currently processed by this worker."))
|
||||
workerJobs, _ := meter.Int64Counter("messenger_attachment_worker_jobs", metric.WithDescription("Completed attachment job deliveries by result."))
|
||||
workerDuration, _ := meter.Float64Histogram("messenger_attachment_worker_job_duration",
|
||||
metric.WithDescription("Attachment job delivery processing duration."), metric.WithUnit("s"),
|
||||
metric.WithExplicitBucketBoundaries(workerDurationBuckets...),
|
||||
)
|
||||
metrics.handler = promhttp.HandlerFor(registry, promhttp.HandlerOpts{})
|
||||
return metrics
|
||||
workerRetries, _ := meter.Int64Counter("messenger_attachment_worker_retries", metric.WithDescription("Attachment jobs scheduled for another attempt."))
|
||||
workerReclaimed, _ := meter.Int64Counter("messenger_attachment_worker_reclaimed_deliveries", metric.WithDescription("Redis Stream deliveries reclaimed after their idle timeout."))
|
||||
outboxPublished, _ := meter.Int64Counter("messenger_outbox_published", metric.WithDescription("Outbox events confirmed by Redis Streams."))
|
||||
outboxPublishFailure, _ := meter.Int64Counter("messenger_outbox_publish_failures", metric.WithDescription("Outbox events whose Redis Streams publish failed."))
|
||||
outboxRecovered, _ := meter.Int64Counter("messenger_outbox_recovered_leases", metric.WithDescription("Expired outbox leases returned for publication."))
|
||||
if workerCapacity > 0 {
|
||||
_, _ = meter.Int64ObservableGauge("messenger_attachment_worker_capacity",
|
||||
metric.WithDescription("Maximum concurrent attachment jobs for this worker."),
|
||||
metric.WithInt64Callback(func(_ context.Context, observer metric.Int64Observer) error {
|
||||
observer.Observe(int64(workerCapacity))
|
||||
return nil
|
||||
}),
|
||||
)
|
||||
}
|
||||
return &Metrics{
|
||||
meter: meter, workerInFlight: workerInFlight, workerJobs: workerJobs,
|
||||
workerDuration: workerDuration,
|
||||
workerRetries: workerRetries, workerReclaimed: workerReclaimed,
|
||||
outboxPublished: outboxPublished, outboxPublishFailure: outboxPublishFailure,
|
||||
outboxRecovered: outboxRecovered, handler: handler,
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Metrics) Handler() http.Handler {
|
||||
return m.handler
|
||||
}
|
||||
|
||||
func (m *Metrics) ObserveStarted() {
|
||||
m.workerInFlight.Inc()
|
||||
func (m *Metrics) ObserveStarted(ctx context.Context) {
|
||||
m.workerInFlight.Add(ctx, 1)
|
||||
}
|
||||
|
||||
func (m *Metrics) ObserveFinished(result string, duration time.Duration) {
|
||||
m.workerInFlight.Dec()
|
||||
m.workerJobs.WithLabelValues(result).Inc()
|
||||
m.workerDuration.WithLabelValues(result).Observe(duration.Seconds())
|
||||
func (m *Metrics) ObserveFinished(ctx context.Context, result string, duration time.Duration) {
|
||||
attributes := metric.WithAttributes(attribute.String("job_type", "attachment.process"), attribute.String("result", result))
|
||||
m.workerInFlight.Add(ctx, -1)
|
||||
m.workerJobs.Add(ctx, 1, attributes)
|
||||
m.workerDuration.Record(ctx, duration.Seconds(), attributes)
|
||||
}
|
||||
|
||||
func (m *Metrics) ObserveRetry() {
|
||||
m.workerRetries.Inc()
|
||||
func (m *Metrics) ObserveRetry(ctx context.Context) {
|
||||
m.workerRetries.Add(ctx, 1)
|
||||
}
|
||||
|
||||
func (m *Metrics) ObserveReclaimed(count int) {
|
||||
m.workerReclaimed.Add(float64(count))
|
||||
func (m *Metrics) ObserveReclaimed(ctx context.Context, count int) {
|
||||
m.workerReclaimed.Add(ctx, int64(count))
|
||||
}
|
||||
|
||||
func (m *Metrics) ObservePublished() {
|
||||
m.outboxPublished.Inc()
|
||||
func (m *Metrics) ObservePublished(ctx context.Context) {
|
||||
m.outboxPublished.Add(ctx, 1)
|
||||
}
|
||||
|
||||
func (m *Metrics) ObservePublishFailure() {
|
||||
m.outboxPublishFailure.Inc()
|
||||
func (m *Metrics) ObservePublishFailure(ctx context.Context) {
|
||||
m.outboxPublishFailure.Add(ctx, 1)
|
||||
}
|
||||
|
||||
func (m *Metrics) ObserveRecovered(count int) {
|
||||
m.outboxRecovered.Add(float64(count))
|
||||
func (m *Metrics) ObserveRecovered(ctx context.Context, count int) {
|
||||
m.outboxRecovered.Add(ctx, int64(count))
|
||||
}
|
||||
|
||||
@@ -19,7 +19,8 @@ import (
|
||||
"github.com/google/uuid"
|
||||
"golang.org/x/sync/semaphore"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/telemetry"
|
||||
)
|
||||
|
||||
type Repository interface {
|
||||
@@ -53,8 +54,7 @@ type Upload struct {
|
||||
ContentType string
|
||||
ContentLength int64
|
||||
Body io.Reader
|
||||
Traceparent string
|
||||
Tracestate string
|
||||
TraceContext telemetry.MessageCarrier
|
||||
}
|
||||
|
||||
type Content struct {
|
||||
@@ -112,27 +112,27 @@ func (s *Service) Upload(ctx context.Context, input Upload) (model.Attachment, e
|
||||
return model.Attachment{}, mapDependencyError(err)
|
||||
}
|
||||
if err := s.objects.Put(ctx, stagingKey, contentType, data); err != nil {
|
||||
s.logger.Error("attachment staging write failed", "attachment_id", pending.ID, "error", err)
|
||||
s.deletePending(pending.ID, stagingKey)
|
||||
s.logger.ErrorContext(ctx, "attachment staging write failed", "attachment_id", pending.ID, "error", err)
|
||||
s.deletePending(ctx, pending.ID, stagingKey)
|
||||
return model.Attachment{}, model.ErrUnavailable
|
||||
}
|
||||
createdAt := time.Now().UTC()
|
||||
queued, err := s.repository.QueueAttachmentWithJobAndOutbox(ctx, pending.ID, stagingKey, model.AttachmentJob{
|
||||
ID: uuid.NewString(), AttachmentID: pending.ID, PipelineVersion: s.config.PipelineVersion,
|
||||
Attempt: 1, Status: model.AttachmentJobQueued, AvailableAt: createdAt, CreatedAt: createdAt,
|
||||
}, input.Traceparent, input.Tracestate)
|
||||
}, input.TraceContext.Traceparent, input.TraceContext.Tracestate)
|
||||
if err != nil {
|
||||
s.logger.Error("attachment queue transaction failed", "attachment_id", pending.ID, "error", err)
|
||||
s.logger.ErrorContext(ctx, "attachment queue transaction failed", "attachment_id", pending.ID, "error", err)
|
||||
return model.Attachment{}, mapDependencyError(err)
|
||||
}
|
||||
return queued, nil
|
||||
}
|
||||
|
||||
func (s *Service) deletePending(attachmentID, stagingKey string) {
|
||||
cleanupCtx, cancel := context.WithTimeout(context.Background(), s.config.CleanupTimeout)
|
||||
func (s *Service) deletePending(ctx context.Context, attachmentID, stagingKey string) {
|
||||
cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), s.config.CleanupTimeout)
|
||||
defer cancel()
|
||||
if err := s.repository.DeletePendingAttachment(cleanupCtx, attachmentID, stagingKey); err != nil {
|
||||
s.logger.Error("pending attachment cleanup failed", "attachment_id", attachmentID, "error", err)
|
||||
s.logger.ErrorContext(cleanupCtx, "pending attachment cleanup failed", "attachment_id", attachmentID, "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -15,7 +15,7 @@ import (
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
|
||||
)
|
||||
|
||||
func TestUploadLifecycle(t *testing.T) {
|
||||
|
||||
@@ -18,10 +18,16 @@ import (
|
||||
|
||||
"golang.org/x/sync/semaphore"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/broker/redisstream"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment/transform"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/objectstore"
|
||||
"go.opentelemetry.io/otel"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/codes"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/broker/redisstream"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment/transform"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/objectstore"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/telemetry"
|
||||
)
|
||||
|
||||
type Repository interface {
|
||||
@@ -29,7 +35,7 @@ type Repository interface {
|
||||
ClaimAttachmentJob(context.Context, string, string, int, int, string, time.Duration) (model.AttachmentJobLease, error)
|
||||
RenewAttachmentJobLease(context.Context, model.AttachmentJobLease, time.Duration) (bool, error)
|
||||
CompleteAttachmentJob(context.Context, model.AttachmentJobLease, []model.Variant) (model.JobMutationResult, error)
|
||||
RetryAttachmentJob(context.Context, model.AttachmentJobLease, string, time.Time) (model.JobMutationResult, error)
|
||||
RetryAttachmentJob(context.Context, model.AttachmentJobLease, string, time.Time, string, string) (model.JobMutationResult, error)
|
||||
FailAttachmentJob(context.Context, model.AttachmentJobLease, string) (model.JobMutationResult, error)
|
||||
RecoverExpiredAttachmentJobs(context.Context, int, int, time.Time) (model.RecoveryResult, error)
|
||||
}
|
||||
@@ -50,10 +56,10 @@ type ImageProcessor interface {
|
||||
}
|
||||
|
||||
type Metrics interface {
|
||||
ObserveStarted()
|
||||
ObserveFinished(string, time.Duration)
|
||||
ObserveRetry()
|
||||
ObserveReclaimed(int)
|
||||
ObserveStarted(context.Context)
|
||||
ObserveFinished(context.Context, string, time.Duration)
|
||||
ObserveRetry(context.Context)
|
||||
ObserveReclaimed(context.Context, int)
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
@@ -166,7 +172,7 @@ func (w *Worker) Run(ctx context.Context) error {
|
||||
}
|
||||
return err
|
||||
}
|
||||
w.metrics.ObserveReclaimed(len(deliveries))
|
||||
w.metrics.ObserveReclaimed(ctx, len(deliveries))
|
||||
w.processBatch(ctx, deliveries)
|
||||
reclaimCursor = next
|
||||
if reclaimCursor == "" {
|
||||
@@ -189,25 +195,48 @@ func (w *Worker) processBatch(ctx context.Context, deliveries []redisstream.Deli
|
||||
go func(delivery redisstream.Delivery) {
|
||||
defer group.Done()
|
||||
defer func() { <-limit }()
|
||||
if err := w.ProcessDelivery(ctx, delivery, FaultNone); err != nil && ctx.Err() == nil {
|
||||
w.logger.Error("attachment job delivery failed", "message_id", delivery.ID, "error", err)
|
||||
}
|
||||
_ = w.ProcessDelivery(ctx, delivery, FaultNone)
|
||||
}(delivery)
|
||||
}
|
||||
group.Wait()
|
||||
}
|
||||
|
||||
func (w *Worker) ProcessDelivery(ctx context.Context, delivery redisstream.Delivery, fault FaultPoint) error {
|
||||
func (w *Worker) ProcessDelivery(ctx context.Context, delivery redisstream.Delivery, fault FaultPoint) (resultErr error) {
|
||||
started := w.now()
|
||||
w.metrics.ObserveStarted()
|
||||
result := "transient_error"
|
||||
defer func() { w.metrics.ObserveFinished(result, w.now().Sub(started)) }()
|
||||
|
||||
if delivery.DecodeError != nil {
|
||||
w.metrics.ObserveStarted(ctx)
|
||||
defer func() { w.metrics.ObserveFinished(ctx, result, w.now().Sub(started)) }()
|
||||
result = "invalid_job"
|
||||
return delivery.ToFailed(ctx, redisstream.FailedDelivery{Code: "invalid_job", FailedAt: w.now()})
|
||||
}
|
||||
envelope := delivery.Envelope
|
||||
ctx = telemetry.ExtractMessage(ctx, telemetry.MessageCarrier{
|
||||
Traceparent: envelope.Traceparent, Tracestate: envelope.Tracestate,
|
||||
})
|
||||
ctx, span := otel.Tracer("messenger-worker").Start(ctx, "attachment.process", trace.WithSpanKind(trace.SpanKindConsumer),
|
||||
trace.WithAttributes(
|
||||
attribute.String("messaging.system", "redis"), attribute.String("messaging.destination.name", "attachment-jobs"),
|
||||
attribute.String("messaging.message.id", delivery.ID), attribute.String("job.id", envelope.JobID.String()),
|
||||
attribute.String("attachment.id", envelope.AttachmentID.String()), attribute.Int("job.attempt", envelope.Attempt),
|
||||
attribute.Int("pipeline.version", envelope.PipelineVersion),
|
||||
))
|
||||
w.metrics.ObserveStarted(ctx)
|
||||
defer func() {
|
||||
w.metrics.ObserveFinished(ctx, result, w.now().Sub(started))
|
||||
span.SetAttributes(attribute.String("job.outcome", result))
|
||||
if resultErr != nil {
|
||||
span.RecordError(resultErr)
|
||||
span.SetStatus(codes.Error, result)
|
||||
if ctx.Err() == nil {
|
||||
w.logger.ErrorContext(ctx, "attachment job failed", "event", "attachment_job_failed", "message_id", delivery.ID, "job_id", envelope.JobID.String(), "job_type", envelope.Type, "attempt", envelope.Attempt, "result", result, "error", resultErr)
|
||||
}
|
||||
} else {
|
||||
w.logger.InfoContext(ctx, "attachment job completed", "event", "attachment_job_completed", "message_id", delivery.ID, "job_id", envelope.JobID.String(), "job_type", envelope.Type, "attempt", envelope.Attempt, "result", result)
|
||||
}
|
||||
span.End()
|
||||
}()
|
||||
|
||||
lease, err := w.repository.ClaimAttachmentJob(
|
||||
ctx, envelope.JobID.String(), envelope.AttachmentID.String(), envelope.PipelineVersion, envelope.Attempt,
|
||||
w.config.WorkerID, w.config.LeaseDuration,
|
||||
@@ -246,7 +275,7 @@ func (w *Worker) ProcessDelivery(ctx context.Context, delivery redisstream.Deliv
|
||||
}
|
||||
if mutation == model.JobMutationApplied && strings.HasPrefix(lease.Attachment.StagingKey, "staging/") {
|
||||
if err := w.objects.Delete(context.WithoutCancel(ctx), lease.Attachment.StagingKey); err != nil {
|
||||
w.logger.Warn("staging cleanup failed", "attachment_id", lease.Attachment.ID, "error", err)
|
||||
w.logger.WarnContext(ctx, "staging cleanup failed", "attachment_id", lease.Attachment.ID, "error", err)
|
||||
}
|
||||
}
|
||||
if mutation != model.JobMutationApplied && mutation != model.JobMutationAlreadyComplete && mutation != model.JobMutationStale {
|
||||
@@ -319,7 +348,7 @@ func (w *Worker) resolveClaim(
|
||||
if job.ErrorCode != nil {
|
||||
code = *job.ErrorCode
|
||||
}
|
||||
return "failed", delivery.ToFailed(ctx, redisstream.FailedDelivery{Code: code, FailedAt: w.now()})
|
||||
return "duplicate_failed", delivery.ToFailed(ctx, redisstream.FailedDelivery{Code: code, FailedAt: w.now()})
|
||||
default:
|
||||
return "transient_error", fmt.Errorf("claim attachment job: %w", claimErr)
|
||||
}
|
||||
@@ -444,7 +473,10 @@ func (w *Worker) resolveProcessingFailure(
|
||||
return "failed", delivery.ToFailed(ctx, redisstream.FailedDelivery{Code: code, FailedAt: w.now()})
|
||||
}
|
||||
nextAttempt := w.now().Add(w.retryDelay(lease.Job.ID, lease.Job.Attempt))
|
||||
mutation, err := w.repository.RetryAttachmentJob(ctx, lease, code, nextAttempt)
|
||||
traceContext := telemetry.ContextCarrier(ctx)
|
||||
mutation, err := w.repository.RetryAttachmentJob(
|
||||
ctx, lease, code, nextAttempt, traceContext.Traceparent, traceContext.Tracestate,
|
||||
)
|
||||
if err != nil {
|
||||
return "transient_error", fmt.Errorf("retry attachment job: %w", err)
|
||||
}
|
||||
@@ -454,7 +486,7 @@ func (w *Worker) resolveProcessingFailure(
|
||||
if mutation != model.JobMutationApplied {
|
||||
return "transient_error", fmt.Errorf("unexpected retry result %q", mutation)
|
||||
}
|
||||
w.metrics.ObserveRetry()
|
||||
w.metrics.ObserveRetry(ctx)
|
||||
return "retry", delivery.Ack(ctx)
|
||||
}
|
||||
|
||||
@@ -502,7 +534,7 @@ func (w *Worker) retryDelay(jobID string, attempt int) time.Duration {
|
||||
|
||||
type nopMetrics struct{}
|
||||
|
||||
func (nopMetrics) ObserveStarted() {}
|
||||
func (nopMetrics) ObserveFinished(string, time.Duration) {}
|
||||
func (nopMetrics) ObserveRetry() {}
|
||||
func (nopMetrics) ObserveReclaimed(int) {}
|
||||
func (nopMetrics) ObserveStarted(context.Context) {}
|
||||
func (nopMetrics) ObserveFinished(context.Context, string, time.Duration) {}
|
||||
func (nopMetrics) ObserveRetry(context.Context) {}
|
||||
func (nopMetrics) ObserveReclaimed(context.Context, int) {}
|
||||
|
||||
@@ -9,9 +9,9 @@ import (
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment/transform"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/objectstore"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment/transform"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/objectstore"
|
||||
)
|
||||
|
||||
func TestConfigRequiresEnoughMemoryForMaximumImage(t *testing.T) {
|
||||
|
||||
@@ -6,23 +6,29 @@ import (
|
||||
"mime"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode"
|
||||
"unicode/utf8"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/codes"
|
||||
"go.opentelemetry.io/otel/metric"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/telemetry"
|
||||
)
|
||||
|
||||
func (s *Server) uploadAttachment(w http.ResponseWriter, r *http.Request) *apiError {
|
||||
started := time.Now()
|
||||
result := "client_error"
|
||||
defer func() {
|
||||
s.metrics.attachmentUploads.WithLabelValues(result).Inc()
|
||||
s.metrics.attachmentDuration.Observe(time.Since(started).Seconds())
|
||||
attributes := metric.WithAttributes(attribute.String("result", result))
|
||||
s.metrics.attachmentUploads.Add(r.Context(), 1, attributes)
|
||||
s.metrics.attachmentDuration.Record(r.Context(), time.Since(started).Seconds(), attributes)
|
||||
}()
|
||||
|
||||
if r.Method != http.MethodPost {
|
||||
@@ -47,7 +53,7 @@ func (s *Server) uploadAttachment(w http.ResponseWriter, r *http.Request) *apiEr
|
||||
uploaded, err := s.attachments.Upload(r.Context(), attachment.Upload{
|
||||
ChatID: chatID, OwnerID: user.ID, Filename: filename,
|
||||
ContentType: r.Header.Get("Content-Type"), ContentLength: r.ContentLength, Body: r.Body,
|
||||
Traceparent: traceparent(r.Header.Get("traceparent")), Tracestate: tracestate(r.Header.Get("tracestate")),
|
||||
TraceContext: telemetry.ContextCarrier(r.Context()),
|
||||
})
|
||||
if err != nil {
|
||||
if errors.Is(err, model.ErrCapacity) {
|
||||
@@ -59,7 +65,7 @@ func (s *Server) uploadAttachment(w http.ResponseWriter, r *http.Request) *apiEr
|
||||
return attachmentError(err)
|
||||
}
|
||||
result = "success"
|
||||
s.metrics.attachmentBytes.Add(float64(uploaded.SizeBytes))
|
||||
s.metrics.attachmentBytes.Add(r.Context(), uploaded.SizeBytes)
|
||||
writeJSON(w, http.StatusAccepted, uploaded)
|
||||
return nil
|
||||
}
|
||||
@@ -128,7 +134,10 @@ func (s *Server) attachmentContent(w http.ResponseWriter, r *http.Request) *apiE
|
||||
w.Header().Set("Content-Disposition", mime.FormatMediaType("inline", map[string]string{"filename": content.Attachment.Filename}))
|
||||
w.WriteHeader(http.StatusOK)
|
||||
if _, err := io.Copy(w, content.Body); err != nil {
|
||||
s.logger.Warn("attachment download interrupted", "attachment_id", content.Attachment.ID, "error", err)
|
||||
span := trace.SpanFromContext(r.Context())
|
||||
span.RecordError(err)
|
||||
span.SetStatus(codes.Error, "response body write failed")
|
||||
s.logger.WarnContext(r.Context(), "attachment download interrupted", "event", "attachment_download_interrupted", "attachment_id", content.Attachment.ID, "error", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -148,22 +157,6 @@ func attachmentVariant(query url.Values) (string, bool) {
|
||||
}
|
||||
}
|
||||
|
||||
var traceparentPattern = regexp.MustCompile(`^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$`)
|
||||
|
||||
func traceparent(value string) string {
|
||||
if traceparentPattern.MatchString(value) {
|
||||
return value
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func tracestate(value string) string {
|
||||
if len(value) <= 512 {
|
||||
return value
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func attachmentIDFromRequest(r *http.Request) (string, *apiError) {
|
||||
parts := strings.Split(strings.Trim(r.URL.Path, "/"), "/")
|
||||
attachmentID, valid := canonicalID(parts[3])
|
||||
|
||||
@@ -6,7 +6,7 @@ import (
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
|
||||
)
|
||||
|
||||
func (s *Server) createSession(w http.ResponseWriter, r *http.Request) *apiError {
|
||||
|
||||
@@ -9,7 +9,7 @@ import (
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
|
||||
)
|
||||
|
||||
var usernamePattern = regexp.MustCompile(`^[a-z0-9_]{3,32}$`)
|
||||
@@ -229,7 +229,7 @@ func (s *Server) messages(w http.ResponseWriter, r *http.Request) *apiError {
|
||||
return unavailable()
|
||||
}
|
||||
if replay {
|
||||
s.metrics.replays.Inc()
|
||||
s.metrics.replays.Add(r.Context(), 1)
|
||||
}
|
||||
writeJSON(w, http.StatusCreated, message)
|
||||
return nil
|
||||
|
||||
@@ -4,8 +4,8 @@ import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
|
||||
)
|
||||
|
||||
type UserRepository interface {
|
||||
|
||||
@@ -1,14 +1,15 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/collectors"
|
||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
"go.opentelemetry.io/otel"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/metric"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -17,80 +18,50 @@ var (
|
||||
)
|
||||
|
||||
type metrics struct {
|
||||
requests *prometheus.CounterVec
|
||||
duration *prometheus.HistogramVec
|
||||
inFlight prometheus.Gauge
|
||||
replays prometheus.Counter
|
||||
attachmentUploads *prometheus.CounterVec
|
||||
attachmentDuration prometheus.Histogram
|
||||
attachmentBytes prometheus.Counter
|
||||
requests metric.Int64Counter
|
||||
duration metric.Float64Histogram
|
||||
inFlight metric.Int64UpDownCounter
|
||||
replays metric.Int64Counter
|
||||
attachmentUploads metric.Int64Counter
|
||||
attachmentDuration metric.Float64Histogram
|
||||
attachmentBytes metric.Int64Counter
|
||||
handler http.Handler
|
||||
}
|
||||
|
||||
func newMetrics() *metrics {
|
||||
registry := prometheus.NewRegistry()
|
||||
result := &metrics{
|
||||
requests: prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Namespace: "messenger",
|
||||
Subsystem: "http",
|
||||
Name: "requests_total",
|
||||
Help: "Completed HTTP requests.",
|
||||
}, []string{"method", "route", "status_class"}),
|
||||
duration: prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
||||
Namespace: "messenger",
|
||||
Subsystem: "http",
|
||||
Name: "request_duration_seconds",
|
||||
Help: "HTTP request duration in seconds.",
|
||||
Buckets: requestDurationBuckets,
|
||||
}, []string{"method", "route"}),
|
||||
inFlight: prometheus.NewGauge(prometheus.GaugeOpts{
|
||||
Namespace: "messenger",
|
||||
Subsystem: "http",
|
||||
Name: "requests_in_flight",
|
||||
Help: "HTTP requests currently being served.",
|
||||
}),
|
||||
replays: prometheus.NewCounter(prometheus.CounterOpts{
|
||||
Namespace: "messenger",
|
||||
Subsystem: "http",
|
||||
Name: "idempotency_replays_total",
|
||||
Help: "Message requests served from an idempotency record.",
|
||||
}),
|
||||
attachmentUploads: prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Namespace: "messenger",
|
||||
Name: "attachment_uploads_total",
|
||||
Help: "Completed attachment uploads by result.",
|
||||
}, []string{"result"}),
|
||||
attachmentDuration: prometheus.NewHistogram(prometheus.HistogramOpts{
|
||||
Namespace: "messenger",
|
||||
Name: "attachment_upload_duration_seconds",
|
||||
Help: "Attachment upload duration in seconds.",
|
||||
Buckets: uploadDurationBuckets,
|
||||
}),
|
||||
attachmentBytes: prometheus.NewCounter(prometheus.CounterOpts{
|
||||
Namespace: "messenger",
|
||||
Name: "attachment_upload_bytes_total",
|
||||
Help: "Bytes accepted by successful attachment uploads.",
|
||||
}),
|
||||
func newMetrics(meter metric.Meter, handler http.Handler) *metrics {
|
||||
if meter == nil {
|
||||
meter = otel.Meter("messenger-api")
|
||||
}
|
||||
registry.MustRegister(
|
||||
result.requests,
|
||||
result.duration,
|
||||
result.inFlight,
|
||||
result.replays,
|
||||
result.attachmentUploads,
|
||||
result.attachmentDuration,
|
||||
result.attachmentBytes,
|
||||
collectors.NewGoCollector(),
|
||||
collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}),
|
||||
if handler == nil {
|
||||
handler = http.NotFoundHandler()
|
||||
}
|
||||
requests, _ := meter.Int64Counter("messenger_http_requests", metric.WithDescription("Completed HTTP requests."))
|
||||
duration, _ := meter.Float64Histogram("messenger_http_request_duration",
|
||||
metric.WithDescription("HTTP request duration."), metric.WithUnit("s"),
|
||||
metric.WithExplicitBucketBoundaries(requestDurationBuckets...),
|
||||
)
|
||||
result.handler = promhttp.HandlerFor(registry, promhttp.HandlerOpts{})
|
||||
return result
|
||||
inFlight, _ := meter.Int64UpDownCounter("messenger_http_requests_in_flight", metric.WithDescription("HTTP requests currently being served."))
|
||||
replays, _ := meter.Int64Counter("messenger_http_idempotency_replays", metric.WithDescription("Message requests served from an idempotency record."))
|
||||
attachmentUploads, _ := meter.Int64Counter("messenger_attachment_uploads", metric.WithDescription("Completed attachment uploads by result."))
|
||||
attachmentDuration, _ := meter.Float64Histogram("messenger_attachment_upload_duration",
|
||||
metric.WithDescription("Attachment upload duration."), metric.WithUnit("s"),
|
||||
metric.WithExplicitBucketBoundaries(uploadDurationBuckets...),
|
||||
)
|
||||
attachmentBytes, _ := meter.Int64Counter("messenger_attachment_upload_bytes", metric.WithDescription("Bytes accepted by successful attachment uploads."), metric.WithUnit("By"))
|
||||
return &metrics{
|
||||
requests: requests, duration: duration, inFlight: inFlight, replays: replays,
|
||||
attachmentUploads: attachmentUploads, attachmentDuration: attachmentDuration,
|
||||
attachmentBytes: attachmentBytes, handler: handler,
|
||||
}
|
||||
}
|
||||
|
||||
func (m *metrics) observe(r *http.Request, status int, started time.Time) {
|
||||
route := routeTemplate(r.URL.Path)
|
||||
m.requests.WithLabelValues(r.Method, route, strconv.Itoa(status/100)+"xx").Inc()
|
||||
m.duration.WithLabelValues(r.Method, route).Observe(time.Since(started).Seconds())
|
||||
func (m *metrics) observe(ctx context.Context, r *http.Request, status int, started time.Time) {
|
||||
attributes := metric.WithAttributes(
|
||||
attribute.String("method", r.Method), attribute.String("route", routeTemplate(r.URL.Path)),
|
||||
attribute.String("status_class", strconv.Itoa(status/100)+"xx"),
|
||||
)
|
||||
m.requests.Add(ctx, 1, attributes)
|
||||
m.duration.Record(ctx, time.Since(started).Seconds(), attributes)
|
||||
}
|
||||
|
||||
func routeTemplate(path string) string {
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
|
||||
@@ -6,6 +6,16 @@ import (
|
||||
"net/http"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"go.opentelemetry.io/otel"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/codes"
|
||||
"go.opentelemetry.io/otel/metric"
|
||||
"go.opentelemetry.io/otel/propagation"
|
||||
semconv "go.opentelemetry.io/otel/semconv/v1.34.0"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/telemetry"
|
||||
)
|
||||
|
||||
type Dependencies struct {
|
||||
@@ -17,6 +27,8 @@ type Dependencies struct {
|
||||
Passwords PasswordService
|
||||
Database HealthChecker
|
||||
Objects HealthChecker
|
||||
Meter metric.Meter
|
||||
Metrics http.Handler
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
@@ -67,7 +79,8 @@ func NewServer(
|
||||
attachments: dependencies.Attachments, sessions: dependencies.Sessions,
|
||||
passwords: dependencies.Passwords, database: dependencies.Database, objects: dependencies.Objects,
|
||||
sessionTTL: config.SessionTTL, idempotencyTTL: config.IdempotencyTTL, requestTTL: config.RequestTTL,
|
||||
instanceID: config.InstanceID, logger: logger, swaggerPort: config.SwaggerPort, metrics: newMetrics(),
|
||||
instanceID: config.InstanceID, logger: logger, swaggerPort: config.SwaggerPort,
|
||||
metrics: newMetrics(dependencies.Meter, dependencies.Metrics),
|
||||
}
|
||||
s.ready.Store(true)
|
||||
return s
|
||||
@@ -82,27 +95,39 @@ func (s *Server) Handler() http.Handler {
|
||||
}
|
||||
|
||||
func (s *Server) serve(w http.ResponseWriter, r *http.Request) {
|
||||
requestContext := telemetry.Extract(r.Context(), propagation.HeaderCarrier(r.Header))
|
||||
tracer := otel.Tracer("messenger-api/http")
|
||||
route := routeTemplate(r.URL.Path)
|
||||
requestContext, span := tracer.Start(requestContext, r.Method+" "+route, trace.WithSpanKind(trace.SpanKindServer),
|
||||
trace.WithAttributes(semconv.HTTPRequestMethodKey.String(r.Method), semconv.HTTPRoute(route)))
|
||||
defer span.End()
|
||||
r = r.WithContext(requestContext)
|
||||
w.Header().Set("X-Instance-Id", s.instanceID)
|
||||
requestID, err := newID()
|
||||
if err != nil {
|
||||
http.Error(w, "server unavailable", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
span.SetAttributes(attribute.String("request.id", requestID))
|
||||
w.Header().Set("X-Request-Id", requestID)
|
||||
w.Header().Set("Cache-Control", "no-store")
|
||||
wrapped := &statusWriter{ResponseWriter: w, status: http.StatusOK}
|
||||
started := time.Now()
|
||||
s.metrics.inFlight.Inc()
|
||||
s.metrics.inFlight.Add(r.Context(), 1)
|
||||
defer func() {
|
||||
s.metrics.inFlight.Dec()
|
||||
s.metrics.inFlight.Add(r.Context(), -1)
|
||||
if recovered := recover(); recovered != nil {
|
||||
s.logger.Error("request panic", "request_id", requestID, "method", r.Method, "path", r.URL.Path)
|
||||
s.logger.ErrorContext(r.Context(), "request panic", "event", "http_request_panic", "request_id", requestID, "method", r.Method, "route", route)
|
||||
if wrapped.status == http.StatusOK {
|
||||
writeError(wrapped, requestID, internalError())
|
||||
}
|
||||
}
|
||||
s.metrics.observe(r, wrapped.status, started)
|
||||
s.logger.Info("request", "request_id", requestID, "method", r.Method, "path", r.URL.Path, "status", wrapped.status, "duration_ms", time.Since(started).Milliseconds())
|
||||
s.metrics.observe(r.Context(), r, wrapped.status, started)
|
||||
span.SetAttributes(semconv.HTTPResponseStatusCode(wrapped.status))
|
||||
if wrapped.status >= http.StatusInternalServerError {
|
||||
span.SetStatus(codes.Error, http.StatusText(wrapped.status))
|
||||
}
|
||||
s.logger.InfoContext(r.Context(), "request", "event", "http_request_completed", "request_id", requestID, "method", r.Method, "route", route, "status", wrapped.status, "duration_ms", time.Since(started).Milliseconds())
|
||||
}()
|
||||
if s.allowSwaggerOrigin(wrapped, r) && r.Method == http.MethodOptions && r.Header.Get("Access-Control-Request-Method") != "" {
|
||||
wrapped.Header().Set("Access-Control-Allow-Methods", "GET, POST, DELETE, OPTIONS")
|
||||
|
||||
@@ -14,8 +14,8 @@ import (
|
||||
|
||||
"github.com/stretchr/testify/suite"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
|
||||
)
|
||||
|
||||
var errDependency = errors.New("dependency unavailable")
|
||||
|
||||
@@ -13,7 +13,12 @@ import (
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
|
||||
"go.opentelemetry.io/otel"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/codes"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
|
||||
)
|
||||
|
||||
var ErrNotFound = errors.New("object not found")
|
||||
@@ -85,30 +90,38 @@ func (s *Store) Ping(ctx context.Context) error {
|
||||
}
|
||||
|
||||
func (s *Store) Put(ctx context.Context, key, contentType string, data []byte) error {
|
||||
ctx, span := s.startSpan(ctx, "PUT")
|
||||
defer span.End()
|
||||
ctx, cancel := context.WithTimeout(ctx, s.timeout)
|
||||
defer cancel()
|
||||
_, err := s.client.PutObject(
|
||||
ctx, s.bucket, key, bytes.NewReader(data), int64(len(data)), minio.PutObjectOptions{ContentType: contentType},
|
||||
)
|
||||
if err != nil {
|
||||
recordSpanError(span, err)
|
||||
return fmt.Errorf("put S3 object: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) Get(ctx context.Context, key string) (io.ReadCloser, error) {
|
||||
ctx, span := s.startSpan(ctx, "GET")
|
||||
ctx, cancel := context.WithTimeout(ctx, s.timeout)
|
||||
object, err := s.client.GetObject(ctx, s.bucket, key, minio.GetObjectOptions{})
|
||||
if err != nil {
|
||||
cancel()
|
||||
recordSpanError(span, err)
|
||||
span.End()
|
||||
return nil, objectError("get S3 object", err)
|
||||
}
|
||||
if _, err := object.Stat(); err != nil {
|
||||
cancel()
|
||||
_ = object.Close()
|
||||
recordSpanError(span, err)
|
||||
span.End()
|
||||
return nil, objectError("stat S3 object", err)
|
||||
}
|
||||
return &timeoutReader{ReadCloser: object, cancel: cancel}, nil
|
||||
return &tracedReader{ReadCloser: &timeoutReader{ReadCloser: object, cancel: cancel}, span: span}, nil
|
||||
}
|
||||
|
||||
func (s *Store) Stat(ctx context.Context, key string) (model.StoredObject, error) {
|
||||
@@ -122,9 +135,12 @@ func (s *Store) Stat(ctx context.Context, key string) (model.StoredObject, error
|
||||
}
|
||||
|
||||
func (s *Store) Delete(ctx context.Context, key string) error {
|
||||
ctx, span := s.startSpan(ctx, "DELETE")
|
||||
defer span.End()
|
||||
ctx, cancel := context.WithTimeout(ctx, s.timeout)
|
||||
defer cancel()
|
||||
if err := s.client.RemoveObject(ctx, s.bucket, key, minio.RemoveObjectOptions{}); err != nil {
|
||||
recordSpanError(span, err)
|
||||
return objectError("delete S3 object", err)
|
||||
}
|
||||
return nil
|
||||
@@ -161,6 +177,28 @@ type timeoutReader struct {
|
||||
cancel context.CancelFunc
|
||||
}
|
||||
|
||||
type tracedReader struct {
|
||||
io.ReadCloser
|
||||
span trace.Span
|
||||
}
|
||||
|
||||
func (r *tracedReader) Read(buffer []byte) (int, error) {
|
||||
read, err := r.ReadCloser.Read(buffer)
|
||||
if err != nil && !errors.Is(err, io.EOF) {
|
||||
recordSpanError(r.span, err)
|
||||
}
|
||||
return read, err
|
||||
}
|
||||
|
||||
func (r *tracedReader) Close() error {
|
||||
err := r.ReadCloser.Close()
|
||||
if err != nil {
|
||||
recordSpanError(r.span, err)
|
||||
}
|
||||
r.span.End()
|
||||
return err
|
||||
}
|
||||
|
||||
func (r *timeoutReader) Close() error {
|
||||
err := r.ReadCloser.Close()
|
||||
r.cancel()
|
||||
@@ -174,3 +212,14 @@ func objectError(action string, err error) error {
|
||||
}
|
||||
return fmt.Errorf("%s: %w", action, err)
|
||||
}
|
||||
|
||||
func (s *Store) startSpan(ctx context.Context, operation string) (context.Context, trace.Span) {
|
||||
return otel.Tracer("objectstore").Start(ctx, "s3 "+operation, trace.WithSpanKind(trace.SpanKindClient),
|
||||
trace.WithAttributes(attribute.String("storage.system", "s3"), attribute.String("storage.operation.name", operation),
|
||||
attribute.String("s3.bucket.name", s.bucket)))
|
||||
}
|
||||
|
||||
func recordSpanError(span trace.Span, err error) {
|
||||
span.RecordError(err)
|
||||
span.SetStatus(codes.Error, "S3 operation failed")
|
||||
}
|
||||
|
||||
@@ -6,7 +6,13 @@ import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
|
||||
"go.opentelemetry.io/otel"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/codes"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/telemetry"
|
||||
)
|
||||
|
||||
var errLeaseLost = errors.New("outbox lease lost")
|
||||
@@ -24,9 +30,9 @@ type Publisher interface {
|
||||
}
|
||||
|
||||
type Metrics interface {
|
||||
ObservePublished()
|
||||
ObservePublishFailure()
|
||||
ObserveRecovered(int)
|
||||
ObservePublished(context.Context)
|
||||
ObservePublishFailure(context.Context)
|
||||
ObserveRecovered(context.Context, int)
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
@@ -156,15 +162,32 @@ func (d *Dispatcher) DispatchOnce(ctx context.Context) (int, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return published, nil
|
||||
}
|
||||
_, err := d.publisher.PublishConfirmed(ctx, event.Payload)
|
||||
publishContext := telemetry.ExtractJSONMessage(ctx, event.Payload)
|
||||
publishContext, span := otel.Tracer("messenger-dispatcher").Start(publishContext, "attachment-jobs publish",
|
||||
trace.WithSpanKind(trace.SpanKindProducer), trace.WithAttributes(
|
||||
attribute.String("messaging.system", "redis"),
|
||||
attribute.String("messaging.destination.name", "attachment-jobs"),
|
||||
attribute.String("job.id", event.JobID), attribute.Int("job.attempt", event.Attempt),
|
||||
))
|
||||
payload, injectErr := telemetry.InjectJSONMessage(publishContext, event.Payload)
|
||||
if injectErr != nil {
|
||||
span.RecordError(injectErr)
|
||||
span.SetStatus(codes.Error, "inject trace context")
|
||||
span.End()
|
||||
return published, fmt.Errorf("inject outbox trace context: %w", injectErr)
|
||||
}
|
||||
_, err := d.publisher.PublishConfirmed(publishContext, payload)
|
||||
if err != nil {
|
||||
span.RecordError(err)
|
||||
span.SetStatus(codes.Error, "publish failed")
|
||||
span.End()
|
||||
if ctx.Err() != nil {
|
||||
return published, nil
|
||||
}
|
||||
d.metrics.ObservePublishFailure()
|
||||
d.metrics.ObservePublishFailure(publishContext)
|
||||
nextAttemptAt := d.now().Add(d.config.RetryDelay)
|
||||
released, releaseErr := d.store.ReleaseOutboxEvent(
|
||||
ctx, event.ID, event.LeaseToken, "broker_publish_failed", nextAttemptAt,
|
||||
publishContext, event.ID, event.LeaseToken, "broker_publish_failed", nextAttemptAt,
|
||||
)
|
||||
if releaseErr != nil {
|
||||
return published, fmt.Errorf("release outbox event after publish failure: %w", releaseErr)
|
||||
@@ -174,14 +197,21 @@ func (d *Dispatcher) DispatchOnce(ctx context.Context) (int, error) {
|
||||
}
|
||||
continue
|
||||
}
|
||||
marked, err := d.store.MarkOutboxPublished(ctx, event.ID, event.LeaseToken)
|
||||
marked, err := d.store.MarkOutboxPublished(publishContext, event.ID, event.LeaseToken)
|
||||
if err != nil {
|
||||
span.RecordError(err)
|
||||
span.SetStatus(codes.Error, "mark published")
|
||||
span.End()
|
||||
return published, fmt.Errorf("mark outbox event published: %w", err)
|
||||
}
|
||||
if !marked {
|
||||
span.SetStatus(codes.Error, "outbox lease lost")
|
||||
span.End()
|
||||
return published, fmt.Errorf("mark outbox event published: %w", errLeaseLost)
|
||||
}
|
||||
d.metrics.ObservePublished()
|
||||
span.SetAttributes(attribute.String("messaging.operation.type", "publish"))
|
||||
span.End()
|
||||
d.metrics.ObservePublished(publishContext)
|
||||
published++
|
||||
}
|
||||
return published, nil
|
||||
@@ -192,12 +222,12 @@ func (d *Dispatcher) RecoverExpired(ctx context.Context) error {
|
||||
if err != nil {
|
||||
return fmt.Errorf("recover expired outbox leases: %w", err)
|
||||
}
|
||||
d.metrics.ObserveRecovered(recovered)
|
||||
d.metrics.ObserveRecovered(ctx, recovered)
|
||||
return nil
|
||||
}
|
||||
|
||||
type nopMetrics struct{}
|
||||
|
||||
func (nopMetrics) ObservePublished() {}
|
||||
func (nopMetrics) ObservePublishFailure() {}
|
||||
func (nopMetrics) ObserveRecovered(int) {}
|
||||
func (nopMetrics) ObservePublished(context.Context) {}
|
||||
func (nopMetrics) ObservePublishFailure(context.Context) {}
|
||||
func (nopMetrics) ObserveRecovered(context.Context, int) {}
|
||||
|
||||
@@ -8,7 +8,7 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
|
||||
@@ -10,7 +10,7 @@ import (
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
|
||||
)
|
||||
|
||||
const attachmentJobTopic = "attachment.jobs"
|
||||
@@ -37,9 +37,6 @@ func (s *Store) QueueAttachmentWithJobAndOutbox(
|
||||
return model.Attachment{}, model.ErrInvalidInput
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
if job.CreatedAt.IsZero() {
|
||||
job.CreatedAt = now
|
||||
}
|
||||
if job.AvailableAt.IsZero() {
|
||||
job.AvailableAt = now
|
||||
}
|
||||
@@ -67,8 +64,7 @@ func (s *Store) QueueAttachmentWithJobAndOutbox(
|
||||
|
||||
job.AttachmentID = attachmentID
|
||||
job.Status = model.AttachmentJobQueued
|
||||
job.UpdatedAt = job.CreatedAt
|
||||
if err := insertAttachmentJob(ctx, tx, job); err != nil {
|
||||
if err := insertAttachmentJob(ctx, tx, &job); err != nil {
|
||||
return model.Attachment{}, err
|
||||
}
|
||||
if err := insertAttachmentJobOutbox(ctx, tx, job, traceparent, tracestate); err != nil {
|
||||
@@ -92,7 +88,6 @@ func (s *Store) QueueAttachmentPipelineVersion(
|
||||
}
|
||||
job.Attempt = 1
|
||||
now := time.Now().UTC()
|
||||
job.CreatedAt = now
|
||||
job.AvailableAt = now
|
||||
|
||||
tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{})
|
||||
@@ -114,7 +109,6 @@ func (s *Store) QueueAttachmentPipelineVersion(
|
||||
}
|
||||
job.AttachmentID = attachment.ID
|
||||
job.Status = model.AttachmentJobQueued
|
||||
job.UpdatedAt = now
|
||||
const queueAttachment = `
|
||||
UPDATE attachments
|
||||
SET status = 'queued', pipeline_version = $2, staging_key = object_key,
|
||||
@@ -124,7 +118,7 @@ func (s *Store) QueueAttachmentPipelineVersion(
|
||||
if err := scanAttachment(tx.QueryRow(ctx, queueAttachment, attachment.ID, job.PipelineVersion), &attachment); err != nil {
|
||||
return model.Attachment{}, fmt.Errorf("queue attachment pipeline version: %w", err)
|
||||
}
|
||||
if err := insertAttachmentJob(ctx, tx, job); err != nil {
|
||||
if err := insertAttachmentJob(ctx, tx, &job); err != nil {
|
||||
return model.Attachment{}, err
|
||||
}
|
||||
if err := insertAttachmentJobOutbox(ctx, tx, job, traceparent, tracestate); err != nil {
|
||||
@@ -163,15 +157,16 @@ func (s *Store) alreadyQueuedAttachment(
|
||||
return publicAttachment(attachment), nil
|
||||
}
|
||||
|
||||
func insertAttachmentJob(ctx context.Context, tx pgx.Tx, job model.AttachmentJob) error {
|
||||
func insertAttachmentJob(ctx context.Context, tx pgx.Tx, job *model.AttachmentJob) error {
|
||||
const query = `
|
||||
INSERT INTO attachment_jobs (
|
||||
id, attachment_id, pipeline_version, attempt, status, available_at, created_at, updated_at
|
||||
)
|
||||
VALUES ($1, $2, $3, $4, 'queued', $5, $6, $6)`
|
||||
if _, err := tx.Exec(
|
||||
ctx, query, job.ID, job.AttachmentID, job.PipelineVersion, job.Attempt, job.AvailableAt, job.CreatedAt,
|
||||
); err != nil {
|
||||
VALUES ($1, $2, $3, $4, 'queued', $5, statement_timestamp(), statement_timestamp())
|
||||
RETURNING created_at, updated_at`
|
||||
if err := tx.QueryRow(
|
||||
ctx, query, job.ID, job.AttachmentID, job.PipelineVersion, job.Attempt, job.AvailableAt,
|
||||
).Scan(&job.CreatedAt, &job.UpdatedAt); err != nil {
|
||||
return fmt.Errorf("insert attachment job: %w", err)
|
||||
}
|
||||
return nil
|
||||
@@ -417,6 +412,9 @@ func (s *Store) CompleteAttachmentJob(
|
||||
if _, err := tx.Exec(ctx, completeAttachment, attachment.ID, job.PipelineVersion, originalKey); err != nil {
|
||||
return "", fmt.Errorf("complete attachment: %w", err)
|
||||
}
|
||||
if err := insertAttachmentTerminalEvent(ctx, tx, job, model.AttachmentJobReady); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return "", fmt.Errorf("commit completed attachment job: %w", err)
|
||||
}
|
||||
@@ -447,6 +445,8 @@ func (s *Store) RetryAttachmentJob(
|
||||
lease model.AttachmentJobLease,
|
||||
errorCode string,
|
||||
nextAttemptAt time.Time,
|
||||
traceparent string,
|
||||
tracestate string,
|
||||
) (model.JobMutationResult, error) {
|
||||
if !validErrorCode(errorCode) || nextAttemptAt.IsZero() {
|
||||
return "", model.ErrInvalidInput
|
||||
@@ -483,9 +483,7 @@ func (s *Store) RetryAttachmentJob(
|
||||
if _, err := tx.Exec(ctx, queueAttachment, attachment.ID, job.PipelineVersion); err != nil {
|
||||
return "", fmt.Errorf("requeue attachment: %w", err)
|
||||
}
|
||||
eventJob := job
|
||||
eventJob.CreatedAt = time.Now().UTC()
|
||||
if err := insertAttachmentJobOutbox(ctx, tx, eventJob, "", ""); err != nil {
|
||||
if err := insertAttachmentJobOutbox(ctx, tx, job, traceparent, tracestate); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
@@ -527,12 +525,26 @@ func (s *Store) FailAttachmentJob(
|
||||
if _, err := tx.Exec(ctx, failAttachment, attachment.ID, job.PipelineVersion, errorCode); err != nil {
|
||||
return "", fmt.Errorf("fail attachment: %w", err)
|
||||
}
|
||||
if err := insertAttachmentTerminalEvent(ctx, tx, job, model.AttachmentJobFailed); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return "", fmt.Errorf("commit failed attachment job: %w", err)
|
||||
}
|
||||
return model.JobMutationApplied, nil
|
||||
}
|
||||
|
||||
func insertAttachmentTerminalEvent(ctx context.Context, tx pgx.Tx, job model.AttachmentJob, status string) error {
|
||||
const query = `
|
||||
INSERT INTO attachment_terminal_events (job_id, status, accepted_at)
|
||||
VALUES ($1, $2, $3)
|
||||
ON CONFLICT (job_id) DO NOTHING`
|
||||
if _, err := tx.Exec(ctx, query, job.ID, status, job.CreatedAt); err != nil {
|
||||
return fmt.Errorf("insert attachment terminal event: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func lockJobMutation(
|
||||
ctx context.Context,
|
||||
tx pgx.Tx,
|
||||
|
||||
@@ -5,7 +5,7 @@ import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
|
||||
)
|
||||
|
||||
func (s *Store) RenewAttachmentJobLease(
|
||||
|
||||
@@ -9,7 +9,7 @@ import (
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
|
||||
)
|
||||
|
||||
func (s *Store) RecoverExpiredAttachmentJobs(
|
||||
@@ -89,9 +89,7 @@ func (s *Store) RecoverExpiredAttachmentJobs(
|
||||
if _, err := tx.Exec(ctx, requeueAttachment, attachment.ID, job.PipelineVersion); err != nil {
|
||||
return model.RecoveryResult{}, fmt.Errorf("requeue attachment after expired lease: %w", err)
|
||||
}
|
||||
eventJob := job
|
||||
eventJob.CreatedAt = time.Now().UTC()
|
||||
if err := insertAttachmentJobOutbox(ctx, tx, eventJob, "", ""); err != nil {
|
||||
if err := insertAttachmentJobOutbox(ctx, tx, job, "", ""); err != nil {
|
||||
return model.RecoveryResult{}, err
|
||||
}
|
||||
recovered.Requeued++
|
||||
@@ -118,6 +116,9 @@ func markExpiredJobFailed(
|
||||
if _, err := tx.Exec(ctx, failJob, job.ID, errorCode); err != nil {
|
||||
return fmt.Errorf("fail expired attachment job: %w", err)
|
||||
}
|
||||
if err := insertAttachmentTerminalEvent(ctx, tx, job, model.AttachmentJobFailed); err != nil {
|
||||
return err
|
||||
}
|
||||
if !failAttachment {
|
||||
return nil
|
||||
}
|
||||
@@ -198,9 +199,7 @@ func (s *Store) ReplayFailedAttachment(
|
||||
); err != nil {
|
||||
return model.AttachmentJob{}, fmt.Errorf("audit attachment replay: %w", err)
|
||||
}
|
||||
eventJob := job
|
||||
eventJob.CreatedAt = time.Now().UTC()
|
||||
if err := insertAttachmentJobOutbox(ctx, tx, eventJob, "", ""); err != nil {
|
||||
if err := insertAttachmentJobOutbox(ctx, tx, job, "", ""); err != nil {
|
||||
return model.AttachmentJob{}, err
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
|
||||
@@ -8,7 +8,7 @@ import (
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
|
||||
)
|
||||
|
||||
const attachmentColumns = `
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
package postgres
|
||||
|
||||
import "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
|
||||
import "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
|
||||
|
||||
var (
|
||||
ErrConflict = model.ErrConflict
|
||||
|
||||
@@ -12,7 +12,7 @@ import (
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
|
||||
)
|
||||
|
||||
const expiredIdempotencyCleanupBatch = 100
|
||||
|
||||
@@ -8,7 +8,7 @@ import (
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
|
||||
)
|
||||
|
||||
func (s *Store) ClaimOutboxEvents(
|
||||
|
||||
@@ -10,7 +10,7 @@ import (
|
||||
"github.com/jackc/pgx/v5/pgconn"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
|
||||
)
|
||||
|
||||
type Store struct {
|
||||
|
||||
@@ -12,13 +12,15 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
"github.com/stretchr/testify/suite"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/database"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/postgres"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/reconcile"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/database"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/postgres"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/reconcile"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/migrations"
|
||||
)
|
||||
|
||||
type PostgresSuite struct {
|
||||
@@ -210,6 +212,7 @@ func (s *PostgresSuite) TestAttachmentJobLifecycleAndOutboxFencing() {
|
||||
result, err := s.store.CompleteAttachmentJob(context.Background(), lease, variants)
|
||||
s.Require().NoError(err)
|
||||
s.Equal(model.JobMutationApplied, result)
|
||||
s.assertTerminalEvent(job.ID, model.AttachmentJobReady, lease.Job.CreatedAt)
|
||||
result, err = s.store.CompleteAttachmentJob(context.Background(), lease, variants)
|
||||
s.Require().NoError(err)
|
||||
s.Equal(model.JobMutationAlreadyComplete, result)
|
||||
@@ -252,6 +255,130 @@ func (s *PostgresSuite) TestAttachmentJobLifecycleAndOutboxFencing() {
|
||||
s.Equal(model.JobMutationStale, result)
|
||||
}
|
||||
|
||||
func (s *PostgresSuite) TestAttachmentJobCreationUsesDatabaseClock() {
|
||||
owner, chatID := createOwnerAndChat(s.T(), s.store)
|
||||
attachmentID := uuid.NewString()
|
||||
stagingKey := "staging/" + attachmentID + "/original"
|
||||
pending, err := s.store.CreatePendingAttachment(context.Background(), model.Attachment{
|
||||
ID: attachmentID, ChatID: chatID, OwnerID: owner.ID, Filename: "future.png",
|
||||
ContentType: "image/png", SizeBytes: 128, SHA256: strings.Repeat("f", 64), ObjectKey: stagingKey,
|
||||
})
|
||||
s.Require().NoError(err)
|
||||
future := time.Now().Add(24 * time.Hour)
|
||||
job := model.AttachmentJob{
|
||||
ID: uuid.NewString(), AttachmentID: attachmentID, PipelineVersion: 1,
|
||||
Attempt: 1, CreatedAt: future,
|
||||
}
|
||||
_, err = s.store.QueueAttachmentWithJobAndOutbox(context.Background(), pending.ID, stagingKey, job, "", "")
|
||||
s.Require().NoError(err)
|
||||
stored, err := s.store.AttachmentJob(context.Background(), job.ID)
|
||||
s.Require().NoError(err)
|
||||
s.True(stored.CreatedAt.Before(future))
|
||||
|
||||
lease, err := s.store.ClaimAttachmentJob(
|
||||
context.Background(), job.ID, attachmentID, 1, 1, "worker-clock", time.Minute,
|
||||
)
|
||||
s.Require().NoError(err)
|
||||
result, err := s.store.CompleteAttachmentJob(context.Background(), lease, completedVariants(attachmentID, 1))
|
||||
s.Require().NoError(err)
|
||||
s.Equal(model.JobMutationApplied, result)
|
||||
s.assertTerminalEvent(job.ID, model.AttachmentJobReady, stored.CreatedAt)
|
||||
s.assertOutboxCreationTime(job.ID, 1, stored.CreatedAt)
|
||||
}
|
||||
|
||||
func (s *PostgresSuite) TestTerminalLedgerMigrationBackfillsExistingJob() {
|
||||
owner, chatID := createOwnerAndChat(s.T(), s.store)
|
||||
replayedAttachment, replayedJob := createQueuedAttachment(s.T(), s.store, owner.ID, chatID)
|
||||
skewedAttachment, skewedJob := createQueuedAttachment(s.T(), s.store, owner.ID, chatID)
|
||||
replayedAcceptedAt := time.Now().Add(-2 * time.Minute).UTC().Truncate(time.Microsecond)
|
||||
firstFailureAt := replayedAcceptedAt.Add(20 * time.Second)
|
||||
replayedReadyAt := firstFailureAt.Add(time.Minute)
|
||||
skewedTerminalAt := time.Now().Add(-time.Minute).UTC().Truncate(time.Microsecond)
|
||||
skewedCreatedAt := skewedTerminalAt.Add(time.Minute)
|
||||
tx, err := s.pool.Begin(context.Background())
|
||||
s.Require().NoError(err)
|
||||
defer func() { s.Require().NoError(tx.Rollback(context.Background())) }()
|
||||
_, err = tx.Exec(
|
||||
context.Background(),
|
||||
`UPDATE attachment_jobs SET status = 'ready', attempt = 2, error_code = NULL, created_at = $2, updated_at = $3
|
||||
WHERE id = $1`,
|
||||
replayedJob.ID,
|
||||
replayedAcceptedAt,
|
||||
replayedReadyAt,
|
||||
)
|
||||
s.Require().NoError(err)
|
||||
_, err = tx.Exec(
|
||||
context.Background(),
|
||||
`UPDATE attachments SET status = 'ready', error_code = NULL, ready_at = $2
|
||||
WHERE id = $1`,
|
||||
replayedAttachment.ID,
|
||||
replayedReadyAt,
|
||||
)
|
||||
s.Require().NoError(err)
|
||||
_, err = tx.Exec(
|
||||
context.Background(),
|
||||
`INSERT INTO attachment_replay_audit (
|
||||
id, job_id, attachment_id, previous_attempt, new_attempt, actor, reason, created_at
|
||||
) VALUES ($1, $2, $3, 1, 2, 'legacy-operator', 'legacy replay', $4)`,
|
||||
uuid.NewString(),
|
||||
replayedJob.ID,
|
||||
replayedAttachment.ID,
|
||||
firstFailureAt,
|
||||
)
|
||||
s.Require().NoError(err)
|
||||
_, err = tx.Exec(
|
||||
context.Background(),
|
||||
`UPDATE attachment_jobs SET status = 'failed', error_code = 'legacy_failure', created_at = $2, updated_at = $3
|
||||
WHERE id = $1`,
|
||||
skewedJob.ID,
|
||||
skewedCreatedAt,
|
||||
skewedTerminalAt,
|
||||
)
|
||||
s.Require().NoError(err)
|
||||
_, err = tx.Exec(
|
||||
context.Background(),
|
||||
`UPDATE attachments SET status = 'failed', error_code = 'legacy_failure', ready_at = NULL
|
||||
WHERE id = $1`,
|
||||
skewedAttachment.ID,
|
||||
)
|
||||
s.Require().NoError(err)
|
||||
_, err = tx.Exec(context.Background(), `ALTER INDEX attachment_terminal_events_status_terminal_at_idx RENAME TO attachment_terminal_events_status_terminal_at_idx_old`)
|
||||
s.Require().NoError(err)
|
||||
_, err = tx.Exec(context.Background(), `ALTER TABLE attachment_terminal_events RENAME TO attachment_terminal_events_old`)
|
||||
s.Require().NoError(err)
|
||||
migration, err := migrations.FS.ReadFile("0008_attachment_terminal_events.sql")
|
||||
s.Require().NoError(err)
|
||||
_, err = tx.Exec(context.Background(), string(migration), pgx.QueryExecModeSimpleProtocol)
|
||||
s.Require().NoError(err)
|
||||
|
||||
s.assertTerminalEventInTx(tx, replayedJob.ID, model.AttachmentJobFailed, replayedAcceptedAt, firstFailureAt)
|
||||
s.assertTerminalEventInTx(tx, skewedJob.ID, model.AttachmentJobFailed, skewedTerminalAt, skewedTerminalAt)
|
||||
}
|
||||
|
||||
func (s *PostgresSuite) assertTerminalEventInTx(
|
||||
tx pgx.Tx,
|
||||
jobID string,
|
||||
status string,
|
||||
acceptedAt time.Time,
|
||||
terminalAt time.Time,
|
||||
) {
|
||||
s.T().Helper()
|
||||
var (
|
||||
actualStatus string
|
||||
actualAcceptedAt time.Time
|
||||
actualTerminalAt time.Time
|
||||
)
|
||||
err := tx.QueryRow(
|
||||
context.Background(),
|
||||
`SELECT status, accepted_at, terminal_at FROM attachment_terminal_events WHERE job_id = $1`,
|
||||
jobID,
|
||||
).Scan(&actualStatus, &actualAcceptedAt, &actualTerminalAt)
|
||||
s.Require().NoError(err)
|
||||
s.Equal(status, actualStatus)
|
||||
s.True(acceptedAt.Equal(actualAcceptedAt))
|
||||
s.True(terminalAt.Equal(actualTerminalAt))
|
||||
}
|
||||
|
||||
func (s *PostgresSuite) TestPublishedOutboxRetentionIsBatched() {
|
||||
owner, chatID := createOwnerAndChat(s.T(), s.store)
|
||||
_, firstJob := createQueuedAttachment(s.T(), s.store, owner.ID, chatID)
|
||||
@@ -295,6 +422,8 @@ func (s *PostgresSuite) TestPublishedOutboxRetentionIsBatched() {
|
||||
func (s *PostgresSuite) TestFailedAttachmentReplayIsAudited() {
|
||||
owner, chatID := createOwnerAndChat(s.T(), s.store)
|
||||
attachment, job := createQueuedAttachment(s.T(), s.store, owner.ID, chatID)
|
||||
original, err := s.store.AttachmentJob(context.Background(), job.ID)
|
||||
s.Require().NoError(err)
|
||||
lease, err := s.store.ClaimAttachmentJob(
|
||||
context.Background(), job.ID, attachment.ID, job.PipelineVersion, job.Attempt, "worker-1", time.Minute,
|
||||
)
|
||||
@@ -302,6 +431,7 @@ func (s *PostgresSuite) TestFailedAttachmentReplayIsAudited() {
|
||||
result, err := s.store.FailAttachmentJob(context.Background(), lease, "invalid_image")
|
||||
s.Require().NoError(err)
|
||||
s.Equal(model.JobMutationApplied, result)
|
||||
firstTerminalAt := s.assertTerminalEvent(job.ID, model.AttachmentJobFailed, original.CreatedAt)
|
||||
|
||||
_, _, err = s.store.CreateMessage(
|
||||
context.Background(), chatID, owner.ID, "failed attachment",
|
||||
@@ -320,6 +450,18 @@ func (s *PostgresSuite) TestFailedAttachmentReplayIsAudited() {
|
||||
s.Require().NoError(err)
|
||||
s.Equal(2, replayed.Attempt)
|
||||
s.Equal(model.AttachmentJobQueued, replayed.Status)
|
||||
s.Equal(original.CreatedAt, replayed.CreatedAt)
|
||||
s.assertOutboxCreationTime(job.ID, 2, original.CreatedAt)
|
||||
replayedLease, err := s.store.ClaimAttachmentJob(
|
||||
context.Background(), job.ID, attachment.ID, replayed.PipelineVersion, replayed.Attempt, "worker-2", time.Minute,
|
||||
)
|
||||
s.Require().NoError(err)
|
||||
result, err = s.store.CompleteAttachmentJob(
|
||||
context.Background(), replayedLease, completedVariants(attachment.ID, replayed.PipelineVersion),
|
||||
)
|
||||
s.Require().NoError(err)
|
||||
s.Equal(model.JobMutationApplied, result)
|
||||
s.Equal(firstTerminalAt, s.assertTerminalEvent(job.ID, model.AttachmentJobFailed, original.CreatedAt))
|
||||
|
||||
var auditCount int
|
||||
err = s.pool.QueryRow(
|
||||
@@ -336,6 +478,8 @@ func (s *PostgresSuite) TestExpiredLeaseRecoveryFencesOldWorker() {
|
||||
context.Background(), job.ID, attachment.ID, job.PipelineVersion, job.Attempt, "worker-1", time.Minute,
|
||||
)
|
||||
s.Require().NoError(err)
|
||||
original, err := s.store.AttachmentJob(context.Background(), job.ID)
|
||||
s.Require().NoError(err)
|
||||
_, err = s.pool.Exec(
|
||||
context.Background(),
|
||||
`UPDATE attachment_jobs SET lease_until = clock_timestamp() - INTERVAL '1 second' WHERE id = $1`,
|
||||
@@ -351,12 +495,82 @@ func (s *PostgresSuite) TestExpiredLeaseRecoveryFencesOldWorker() {
|
||||
s.Require().NoError(err)
|
||||
s.Equal(2, current.Attempt)
|
||||
s.Equal(model.AttachmentJobQueued, current.Status)
|
||||
s.Equal(original.CreatedAt, current.CreatedAt)
|
||||
s.assertOutboxCreationTime(job.ID, 2, original.CreatedAt)
|
||||
|
||||
result, err := s.store.CompleteAttachmentJob(context.Background(), lease, completedVariants(attachment.ID, 1))
|
||||
s.Require().NoError(err)
|
||||
s.Equal(model.JobMutationStale, result)
|
||||
}
|
||||
|
||||
func (s *PostgresSuite) TestExpiredLeaseRecoveryFailsExhaustedJobOnce() {
|
||||
owner, chatID := createOwnerAndChat(s.T(), s.store)
|
||||
attachment, job := createQueuedAttachment(s.T(), s.store, owner.ID, chatID)
|
||||
original, err := s.store.AttachmentJob(context.Background(), job.ID)
|
||||
s.Require().NoError(err)
|
||||
_, err = s.store.ClaimAttachmentJob(
|
||||
context.Background(), job.ID, attachment.ID, job.PipelineVersion, job.Attempt, "worker-1", time.Minute,
|
||||
)
|
||||
s.Require().NoError(err)
|
||||
_, err = s.pool.Exec(
|
||||
context.Background(),
|
||||
`UPDATE attachment_jobs SET lease_until = clock_timestamp() - INTERVAL '1 second' WHERE id = $1`,
|
||||
job.ID,
|
||||
)
|
||||
s.Require().NoError(err)
|
||||
|
||||
recovered, err := s.store.RecoverExpiredAttachmentJobs(context.Background(), 10, 1, time.Now())
|
||||
s.Require().NoError(err)
|
||||
s.Zero(recovered.Requeued)
|
||||
s.Equal(1, recovered.Failed)
|
||||
firstTerminalAt := s.assertTerminalEvent(job.ID, model.AttachmentJobFailed, original.CreatedAt)
|
||||
|
||||
recovered, err = s.store.RecoverExpiredAttachmentJobs(context.Background(), 10, 1, time.Now())
|
||||
s.Require().NoError(err)
|
||||
s.Zero(recovered.Requeued)
|
||||
s.Zero(recovered.Failed)
|
||||
current, err := s.store.AttachmentJob(context.Background(), job.ID)
|
||||
s.Require().NoError(err)
|
||||
s.Equal(model.AttachmentJobFailed, current.Status)
|
||||
s.Equal(firstTerminalAt, s.assertTerminalEvent(job.ID, model.AttachmentJobFailed, original.CreatedAt))
|
||||
}
|
||||
|
||||
func (s *PostgresSuite) assertTerminalEvent(jobID string, status string, acceptedAt time.Time) time.Time {
|
||||
s.T().Helper()
|
||||
var (
|
||||
actualStatus string
|
||||
actualAcceptedAt time.Time
|
||||
terminalAt time.Time
|
||||
count int
|
||||
)
|
||||
err := s.pool.QueryRow(
|
||||
context.Background(),
|
||||
`SELECT min(status), min(accepted_at), min(terminal_at), count(*)
|
||||
FROM attachment_terminal_events WHERE job_id = $1`,
|
||||
jobID,
|
||||
).Scan(&actualStatus, &actualAcceptedAt, &terminalAt, &count)
|
||||
s.Require().NoError(err)
|
||||
s.Equal(1, count)
|
||||
s.Equal(status, actualStatus)
|
||||
s.True(acceptedAt.Equal(actualAcceptedAt), "expected %s, got %s", acceptedAt, actualAcceptedAt)
|
||||
return terminalAt
|
||||
}
|
||||
|
||||
func (s *PostgresSuite) assertOutboxCreationTime(jobID string, attempt int, expected time.Time) {
|
||||
s.T().Helper()
|
||||
var payload []byte
|
||||
err := s.pool.QueryRow(
|
||||
context.Background(),
|
||||
`SELECT payload FROM outbox_events WHERE job_id = $1 AND attempt = $2`,
|
||||
jobID,
|
||||
attempt,
|
||||
).Scan(&payload)
|
||||
s.Require().NoError(err)
|
||||
var message model.AttachmentJobMessage
|
||||
s.Require().NoError(json.Unmarshal(payload, &message))
|
||||
s.True(expected.Equal(message.CreatedAt), "expected %s, got %s", expected, message.CreatedAt)
|
||||
}
|
||||
|
||||
func (s *PostgresSuite) TestCompleteAttachmentJobRejectsIncompleteOrMismatchedVariants() {
|
||||
owner, chatID := createOwnerAndChat(s.T(), s.store)
|
||||
attachment, job := createQueuedAttachment(s.T(), s.store, owner.ID, chatID)
|
||||
|
||||
@@ -6,7 +6,7 @@ import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
|
||||
)
|
||||
|
||||
type MetadataStore interface {
|
||||
|
||||
@@ -8,7 +8,7 @@ import (
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
|
||||
)
|
||||
|
||||
func TestReconcilerLifecycleAndRepeat(t *testing.T) {
|
||||
|
||||
@@ -14,7 +14,7 @@ import (
|
||||
|
||||
"github.com/valkey-io/valkey-go"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
|
||||
)
|
||||
|
||||
const (
|
||||
|
||||
@@ -11,7 +11,7 @@ import (
|
||||
"github.com/google/uuid"
|
||||
"github.com/stretchr/testify/suite"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/session"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/session"
|
||||
)
|
||||
|
||||
type SessionSuite struct {
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
package telemetry
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/metric"
|
||||
)
|
||||
|
||||
var (
|
||||
connectionStateUsed = attribute.String("state", "used")
|
||||
connectionStateIdle = attribute.String("state", "idle")
|
||||
connectionStateConnecting = attribute.String("state", "connecting")
|
||||
)
|
||||
|
||||
func ObserveDatabasePool(meter metric.Meter, pool *pgxpool.Pool) error {
|
||||
_, err := meter.Int64ObservableGauge("messenger_db_pool_connections",
|
||||
metric.WithDescription("PostgreSQL pool connections by state."),
|
||||
metric.WithInt64Callback(func(_ context.Context, observer metric.Int64Observer) error {
|
||||
stats := pool.Stat()
|
||||
observer.Observe(int64(stats.AcquiredConns()), metric.WithAttributes(connectionStateUsed))
|
||||
observer.Observe(int64(stats.IdleConns()), metric.WithAttributes(connectionStateIdle))
|
||||
observer.Observe(int64(stats.ConstructingConns()), metric.WithAttributes(connectionStateConnecting))
|
||||
return nil
|
||||
}),
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = meter.Int64ObservableCounter("messenger_db_pool_waited",
|
||||
metric.WithDescription("Connection acquisitions that waited because the PostgreSQL pool was empty."),
|
||||
metric.WithInt64Callback(func(_ context.Context, observer metric.Int64Observer) error {
|
||||
observer.Observe(pool.Stat().EmptyAcquireCount())
|
||||
return nil
|
||||
}),
|
||||
)
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
package telemetry
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"strings"
|
||||
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
)
|
||||
|
||||
type correlationHandler struct {
|
||||
stdout slog.Handler
|
||||
otel slog.Handler
|
||||
}
|
||||
|
||||
func newCorrelationHandler(stdout, otel slog.Handler) slog.Handler {
|
||||
return correlationHandler{stdout: stdout, otel: otel}
|
||||
}
|
||||
|
||||
func (h correlationHandler) Enabled(ctx context.Context, level slog.Level) bool {
|
||||
return h.stdout.Enabled(ctx, level) || h.otel.Enabled(ctx, level)
|
||||
}
|
||||
|
||||
func (h correlationHandler) Handle(ctx context.Context, record slog.Record) error {
|
||||
span := trace.SpanContextFromContext(ctx)
|
||||
if span.IsValid() {
|
||||
record.AddAttrs(slog.String("trace_id", span.TraceID().String()), slog.String("span_id", span.SpanID().String()))
|
||||
}
|
||||
record = sanitizedRecord(record)
|
||||
if err := h.stdout.Handle(ctx, record); err != nil {
|
||||
return err
|
||||
}
|
||||
return h.otel.Handle(ctx, record)
|
||||
}
|
||||
|
||||
func (h correlationHandler) WithAttrs(attributes []slog.Attr) slog.Handler {
|
||||
attributes = sanitizedAttributes(attributes)
|
||||
return correlationHandler{stdout: h.stdout.WithAttrs(attributes), otel: h.otel.WithAttrs(attributes)}
|
||||
}
|
||||
|
||||
func (h correlationHandler) WithGroup(name string) slog.Handler {
|
||||
return correlationHandler{stdout: h.stdout.WithGroup(name), otel: h.otel.WithGroup(name)}
|
||||
}
|
||||
|
||||
func sanitizedRecord(record slog.Record) slog.Record {
|
||||
copy := slog.NewRecord(record.Time, record.Level, record.Message, record.PC)
|
||||
record.Attrs(func(attribute slog.Attr) bool {
|
||||
if !sensitiveKey(attribute.Key) {
|
||||
copy.AddAttrs(attribute)
|
||||
}
|
||||
return true
|
||||
})
|
||||
return copy
|
||||
}
|
||||
|
||||
func sanitizedAttributes(attributes []slog.Attr) []slog.Attr {
|
||||
result := make([]slog.Attr, 0, len(attributes))
|
||||
for _, attribute := range attributes {
|
||||
if !sensitiveKey(attribute.Key) {
|
||||
result = append(result, attribute)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func sensitiveKey(key string) bool {
|
||||
key = strings.ToLower(key)
|
||||
for _, marker := range []string{"authorization", "password", "secret", "token", "cookie", "body", "payload"} {
|
||||
if strings.Contains(key, marker) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
package telemetry
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
|
||||
"go.opentelemetry.io/otel"
|
||||
"go.opentelemetry.io/otel/propagation"
|
||||
)
|
||||
|
||||
type MessageCarrier struct {
|
||||
Traceparent string
|
||||
Tracestate string
|
||||
}
|
||||
|
||||
func Extract(ctx context.Context, carrier propagation.TextMapCarrier) context.Context {
|
||||
return otel.GetTextMapPropagator().Extract(ctx, carrier)
|
||||
}
|
||||
|
||||
func Inject(ctx context.Context, carrier propagation.TextMapCarrier) {
|
||||
otel.GetTextMapPropagator().Inject(ctx, carrier)
|
||||
}
|
||||
|
||||
func ContextCarrier(ctx context.Context) MessageCarrier {
|
||||
carrier := propagation.MapCarrier{}
|
||||
Inject(ctx, carrier)
|
||||
return MessageCarrier{
|
||||
Traceparent: carrier.Get("traceparent"),
|
||||
Tracestate: carrier.Get("tracestate"),
|
||||
}
|
||||
}
|
||||
|
||||
func ExtractMessage(ctx context.Context, carrier MessageCarrier) context.Context {
|
||||
headers := propagation.MapCarrier{
|
||||
"traceparent": carrier.Traceparent,
|
||||
"tracestate": carrier.Tracestate,
|
||||
}
|
||||
return Extract(ctx, headers)
|
||||
}
|
||||
|
||||
func ExtractJSONMessage(ctx context.Context, payload []byte) context.Context {
|
||||
var carrier struct {
|
||||
Traceparent string `json:"traceparent"`
|
||||
Tracestate string `json:"tracestate"`
|
||||
}
|
||||
if json.Unmarshal(payload, &carrier) != nil {
|
||||
return ctx
|
||||
}
|
||||
return ExtractMessage(ctx, MessageCarrier(carrier))
|
||||
}
|
||||
|
||||
func InjectJSONMessage(ctx context.Context, payload []byte) ([]byte, error) {
|
||||
var value map[string]json.RawMessage
|
||||
if err := json.Unmarshal(payload, &value); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
carrier := ContextCarrier(ctx)
|
||||
setJSONField(value, "traceparent", carrier.Traceparent)
|
||||
setJSONField(value, "tracestate", carrier.Tracestate)
|
||||
return json.Marshal(value)
|
||||
}
|
||||
|
||||
func setJSONField(value map[string]json.RawMessage, key, field string) {
|
||||
if field == "" {
|
||||
delete(value, key)
|
||||
return
|
||||
}
|
||||
encoded, _ := json.Marshal(field)
|
||||
value[key] = encoded
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
package telemetry
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
"go.opentelemetry.io/otel"
|
||||
"go.opentelemetry.io/otel/propagation"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
)
|
||||
|
||||
func TestJSONMessagePropagationPreservesTraceContext(t *testing.T) {
|
||||
otel.SetTextMapPropagator(propagation.TraceContext{})
|
||||
traceID, err := trace.TraceIDFromHex("4bf92f3577b34da6a3ce929d0e0e4736")
|
||||
require.NoError(t, err)
|
||||
spanID, err := trace.SpanIDFromHex("00f067aa0ba902b7")
|
||||
require.NoError(t, err)
|
||||
ctx := trace.ContextWithSpanContext(context.Background(), trace.NewSpanContext(trace.SpanContextConfig{
|
||||
TraceID: traceID, SpanID: spanID, TraceFlags: trace.FlagsSampled,
|
||||
}))
|
||||
|
||||
payload, err := InjectJSONMessage(ctx, []byte(`{"job_id":"job-1"}`))
|
||||
require.NoError(t, err)
|
||||
var value map[string]any
|
||||
require.NoError(t, json.Unmarshal(payload, &value))
|
||||
require.Equal(t, "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01", value["traceparent"])
|
||||
require.NotContains(t, value, "baggage")
|
||||
|
||||
extracted := ExtractJSONMessage(context.Background(), payload)
|
||||
require.Equal(t, traceID, trace.SpanContextFromContext(extracted).TraceID())
|
||||
}
|
||||
@@ -0,0 +1,252 @@
|
||||
package telemetry
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/collectors"
|
||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
"go.opentelemetry.io/contrib/bridges/otelslog"
|
||||
otelruntime "go.opentelemetry.io/contrib/instrumentation/runtime"
|
||||
"go.opentelemetry.io/otel"
|
||||
"go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc"
|
||||
"go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc"
|
||||
"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
|
||||
otelprometheus "go.opentelemetry.io/otel/exporters/prometheus"
|
||||
"go.opentelemetry.io/otel/propagation"
|
||||
"go.opentelemetry.io/otel/sdk/log"
|
||||
"go.opentelemetry.io/otel/sdk/metric"
|
||||
"go.opentelemetry.io/otel/sdk/resource"
|
||||
"go.opentelemetry.io/otel/sdk/trace"
|
||||
semconv "go.opentelemetry.io/otel/semconv/v1.34.0"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultEnvironment = "lab"
|
||||
defaultNamespace = "mtusi-highload"
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
ServiceName string
|
||||
Version string
|
||||
InstanceID string
|
||||
Endpoint string
|
||||
Environment string
|
||||
Namespace string
|
||||
Insecure bool
|
||||
}
|
||||
|
||||
type Runtime struct {
|
||||
traces *trace.TracerProvider
|
||||
metrics *metric.MeterProvider
|
||||
logs *log.LoggerProvider
|
||||
logger *slog.Logger
|
||||
handler http.Handler
|
||||
}
|
||||
|
||||
func ConfigFromEnvironment(serviceName, instanceID string) Config {
|
||||
resourceAttributes := parseResourceAttributes(os.Getenv("OTEL_RESOURCE_ATTRIBUTES"))
|
||||
return Config{
|
||||
ServiceName: serviceName,
|
||||
Version: firstValue(os.Getenv("SERVICE_VERSION"), resourceAttributes["service.version"], "dev"),
|
||||
InstanceID: instanceID,
|
||||
Endpoint: strings.TrimSpace(os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT")),
|
||||
Environment: firstValue(os.Getenv("DEPLOYMENT_ENVIRONMENT"), resourceAttributes["deployment.environment.name"], defaultEnvironment),
|
||||
Namespace: firstValue(os.Getenv("SERVICE_NAMESPACE"), resourceAttributes["service.namespace"], defaultNamespace),
|
||||
Insecure: true,
|
||||
}
|
||||
}
|
||||
|
||||
func parseResourceAttributes(raw string) map[string]string {
|
||||
result := make(map[string]string)
|
||||
for _, pair := range strings.Split(raw, ",") {
|
||||
key, value, found := strings.Cut(pair, "=")
|
||||
if found && strings.TrimSpace(key) != "" {
|
||||
result[strings.TrimSpace(key)] = strings.TrimSpace(value)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func firstValue(values ...string) string {
|
||||
for _, value := range values {
|
||||
if value = strings.TrimSpace(value); value != "" {
|
||||
return value
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func New(ctx context.Context, config Config) (*Runtime, error) {
|
||||
if err := config.validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
res, err := newResource(ctx, config)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
registry := prometheus.NewRegistry()
|
||||
prometheusReader, err := otelprometheus.New(otelprometheus.WithRegisterer(registry))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create Prometheus reader: %w", err)
|
||||
}
|
||||
|
||||
traceOptions := []trace.TracerProviderOption{trace.WithResource(res)}
|
||||
metricOptions := []metric.Option{metric.WithResource(res), metric.WithReader(prometheusReader)}
|
||||
logOptions := []log.LoggerProviderOption{log.WithResource(res)}
|
||||
if config.Endpoint != "" {
|
||||
traceExporter, err := newTraceExporter(ctx, config)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
metricExporter, err := newMetricExporter(ctx, config)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
logExporter, err := newLogExporter(ctx, config)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
traceOptions = append(traceOptions, trace.WithBatcher(traceExporter,
|
||||
trace.WithMaxQueueSize(2048), trace.WithMaxExportBatchSize(512),
|
||||
trace.WithBatchTimeout(time.Second), trace.WithExportTimeout(5*time.Second),
|
||||
))
|
||||
metricOptions = append(metricOptions, metric.WithReader(metric.NewPeriodicReader(
|
||||
metricExporter, metric.WithInterval(5*time.Second), metric.WithTimeout(5*time.Second),
|
||||
)))
|
||||
logOptions = append(logOptions, log.WithProcessor(log.NewBatchProcessor(logExporter,
|
||||
log.WithMaxQueueSize(2048), log.WithExportMaxBatchSize(512),
|
||||
log.WithExportInterval(time.Second), log.WithExportTimeout(5*time.Second),
|
||||
)))
|
||||
}
|
||||
|
||||
traces := trace.NewTracerProvider(traceOptions...)
|
||||
metrics := metric.NewMeterProvider(metricOptions...)
|
||||
logs := log.NewLoggerProvider(logOptions...)
|
||||
propagator := propagation.NewCompositeTextMapPropagator(propagation.TraceContext{})
|
||||
otel.SetTracerProvider(traces)
|
||||
otel.SetMeterProvider(metrics)
|
||||
otel.SetTextMapPropagator(propagator)
|
||||
if err := otelruntime.Start(otelruntime.WithMeterProvider(metrics)); err != nil {
|
||||
return nil, fmt.Errorf("start Go runtime metrics: %w", err)
|
||||
}
|
||||
|
||||
stdout := slog.NewJSONHandler(os.Stdout, nil)
|
||||
otelHandler := otelslog.NewHandler(config.ServiceName, otelslog.WithLoggerProvider(logs))
|
||||
logger := slog.New(newCorrelationHandler(stdout, otelHandler))
|
||||
registry.MustRegister(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}))
|
||||
return &Runtime{
|
||||
traces: traces, metrics: metrics, logs: logs, logger: logger,
|
||||
handler: promhttp.HandlerFor(registry, promhttp.HandlerOpts{}),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (c Config) validate() error {
|
||||
switch {
|
||||
case strings.TrimSpace(c.ServiceName) == "":
|
||||
return errors.New("telemetry service name is required")
|
||||
case strings.TrimSpace(c.InstanceID) == "":
|
||||
return errors.New("telemetry instance ID is required")
|
||||
case strings.TrimSpace(c.Environment) == "":
|
||||
return errors.New("telemetry environment is required")
|
||||
case strings.TrimSpace(c.Namespace) == "":
|
||||
return errors.New("telemetry service namespace is required")
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func newResource(ctx context.Context, config Config) (*resource.Resource, error) {
|
||||
res, err := resource.New(ctx,
|
||||
resource.WithFromEnv(), resource.WithTelemetrySDK(), resource.WithHost(),
|
||||
resource.WithAttributes(
|
||||
semconv.ServiceName(config.ServiceName), semconv.ServiceVersion(config.Version),
|
||||
semconv.ServiceInstanceID(config.InstanceID), semconv.ServiceNamespace(config.Namespace),
|
||||
semconv.DeploymentEnvironmentName(config.Environment),
|
||||
),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create telemetry resource: %w", err)
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func newTraceExporter(ctx context.Context, config Config) (trace.SpanExporter, error) {
|
||||
var options []otlptracegrpc.Option
|
||||
if strings.Contains(config.Endpoint, "://") {
|
||||
options = append(options, otlptracegrpc.WithEndpointURL(config.Endpoint))
|
||||
} else {
|
||||
options = append(options, otlptracegrpc.WithEndpoint(config.Endpoint))
|
||||
}
|
||||
if config.Insecure {
|
||||
options = append(options, otlptracegrpc.WithInsecure())
|
||||
}
|
||||
exporter, err := otlptracegrpc.New(ctx, options...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create OTLP trace exporter: %w", err)
|
||||
}
|
||||
return exporter, nil
|
||||
}
|
||||
|
||||
func newMetricExporter(ctx context.Context, config Config) (*otlpmetricgrpc.Exporter, error) {
|
||||
var options []otlpmetricgrpc.Option
|
||||
if strings.Contains(config.Endpoint, "://") {
|
||||
options = append(options, otlpmetricgrpc.WithEndpointURL(config.Endpoint))
|
||||
} else {
|
||||
options = append(options, otlpmetricgrpc.WithEndpoint(config.Endpoint))
|
||||
}
|
||||
if config.Insecure {
|
||||
options = append(options, otlpmetricgrpc.WithInsecure())
|
||||
}
|
||||
exporter, err := otlpmetricgrpc.New(ctx, options...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create OTLP metric exporter: %w", err)
|
||||
}
|
||||
return exporter, nil
|
||||
}
|
||||
|
||||
func newLogExporter(ctx context.Context, config Config) (*otlploggrpc.Exporter, error) {
|
||||
var options []otlploggrpc.Option
|
||||
if strings.Contains(config.Endpoint, "://") {
|
||||
options = append(options, otlploggrpc.WithEndpointURL(config.Endpoint))
|
||||
} else {
|
||||
options = append(options, otlploggrpc.WithEndpoint(config.Endpoint))
|
||||
}
|
||||
if config.Insecure {
|
||||
options = append(options, otlploggrpc.WithInsecure())
|
||||
}
|
||||
exporter, err := otlploggrpc.New(ctx, options...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create OTLP log exporter: %w", err)
|
||||
}
|
||||
return exporter, nil
|
||||
}
|
||||
|
||||
func (r *Runtime) Logger() *slog.Logger {
|
||||
return r.logger
|
||||
}
|
||||
|
||||
func (r *Runtime) MeterProvider() *metric.MeterProvider {
|
||||
return r.metrics
|
||||
}
|
||||
|
||||
func (r *Runtime) MetricsHandler() http.Handler {
|
||||
return r.handler
|
||||
}
|
||||
|
||||
func (r *Runtime) Shutdown(ctx context.Context) error {
|
||||
var result error
|
||||
for _, shutdown := range []func(context.Context) error{r.logs.Shutdown, r.metrics.Shutdown, r.traces.Shutdown} {
|
||||
if err := shutdown(ctx); err != nil {
|
||||
result = errors.Join(result, err)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
package telemetry
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
)
|
||||
|
||||
func TestConfigValidation(t *testing.T) {
|
||||
valid := Config{ServiceName: "messenger-api", InstanceID: "api-1", Environment: "lab", Namespace: "mtusi-highload"}
|
||||
require.NoError(t, valid.validate())
|
||||
|
||||
invalid := valid
|
||||
invalid.ServiceName = ""
|
||||
require.EqualError(t, invalid.validate(), "telemetry service name is required")
|
||||
}
|
||||
|
||||
func TestCorrelationHandlerAddsTraceAndRedactsSecrets(t *testing.T) {
|
||||
traceID, err := trace.TraceIDFromHex("4bf92f3577b34da6a3ce929d0e0e4736")
|
||||
require.NoError(t, err)
|
||||
spanID, err := trace.SpanIDFromHex("00f067aa0ba902b7")
|
||||
require.NoError(t, err)
|
||||
ctx := trace.ContextWithSpanContext(context.Background(), trace.NewSpanContext(trace.SpanContextConfig{
|
||||
TraceID: traceID, SpanID: spanID, TraceFlags: trace.FlagsSampled,
|
||||
}))
|
||||
first := &recordHandler{}
|
||||
second := &recordHandler{}
|
||||
handler := newCorrelationHandler(first, second)
|
||||
record := slog.NewRecord(time.Now(), slog.LevelInfo, "request", 0)
|
||||
record.AddAttrs(slog.String("event", "http_request_completed"), slog.String("authorization", "secret"))
|
||||
|
||||
require.NoError(t, handler.Handle(ctx, record))
|
||||
for _, captured := range []*recordHandler{first, second} {
|
||||
require.Equal(t, traceID.String(), captured.attributes["trace_id"])
|
||||
require.Equal(t, spanID.String(), captured.attributes["span_id"])
|
||||
require.Equal(t, "http_request_completed", captured.attributes["event"])
|
||||
require.NotContains(t, captured.attributes, "authorization")
|
||||
}
|
||||
}
|
||||
|
||||
type recordHandler struct {
|
||||
attributes map[string]string
|
||||
}
|
||||
|
||||
func (*recordHandler) Enabled(context.Context, slog.Level) bool { return true }
|
||||
func (h *recordHandler) Handle(_ context.Context, record slog.Record) error {
|
||||
h.attributes = make(map[string]string)
|
||||
record.Attrs(func(attribute slog.Attr) bool {
|
||||
h.attributes[attribute.Key] = attribute.Value.String()
|
||||
return true
|
||||
})
|
||||
return nil
|
||||
}
|
||||
func (h *recordHandler) WithAttrs([]slog.Attr) slog.Handler { return h }
|
||||
func (h *recordHandler) WithGroup(string) slog.Handler { return h }
|
||||
@@ -1,3 +1,6 @@
|
||||
# Изолированное имя Compose-проекта и его volumes/networks.
|
||||
COMPOSE_PROJECT_NAME=mtusi2026lab6
|
||||
|
||||
# Адрес и внутренний порт HTTP-сервера.
|
||||
APP_HOST=0.0.0.0
|
||||
APP_PORT=8080
|
||||
@@ -8,6 +11,14 @@ API_BIND_ADDRESS=127.0.0.1:8080
|
||||
SWAGGER_BIND_ADDRESS=127.0.0.1:8081
|
||||
PROMETHEUS_BIND_ADDRESS=127.0.0.1:9090
|
||||
GRAFANA_BIND_ADDRESS=127.0.0.1:3000
|
||||
LOKI_BIND_ADDRESS=127.0.0.1:3100
|
||||
TEMPO_BIND_ADDRESS=127.0.0.1:3200
|
||||
|
||||
# Ресурсные атрибуты и внутренний OTLP endpoint приложений.
|
||||
DEPLOYMENT_ENVIRONMENT=lab
|
||||
SERVICE_VERSION=lab6
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4317
|
||||
OTEL_EXPORTER_OTLP_PROTOCOL=grpc
|
||||
|
||||
# Число API-реплик за Nginx.
|
||||
API_REPLICAS=2
|
||||
@@ -55,8 +66,14 @@ EXPORTER_CPUS=0.25
|
||||
EXPORTER_MEMORY=128m
|
||||
PROMETHEUS_CPUS=1
|
||||
PROMETHEUS_MEMORY=512m
|
||||
GRAFANA_CPUS=0.5
|
||||
GRAFANA_MEMORY=256m
|
||||
GRAFANA_CPUS=1
|
||||
GRAFANA_MEMORY=512m
|
||||
OTEL_COLLECTOR_CPUS=0.5
|
||||
OTEL_COLLECTOR_MEMORY=256m
|
||||
LOKI_CPUS=1
|
||||
LOKI_MEMORY=512m
|
||||
TEMPO_CPUS=1
|
||||
TEMPO_MEMORY=512m
|
||||
POSTGRES_CPUS=2
|
||||
POSTGRES_MEMORY=1g
|
||||
VALKEY_CPUS=1
|
||||
@@ -0,0 +1,33 @@
|
||||
CREATE TABLE attachment_terminal_events (
|
||||
job_id UUID PRIMARY KEY,
|
||||
status TEXT NOT NULL,
|
||||
accepted_at TIMESTAMPTZ NOT NULL,
|
||||
terminal_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(),
|
||||
CONSTRAINT attachment_terminal_events_status CHECK (status IN ('ready', 'failed')),
|
||||
CONSTRAINT attachment_terminal_events_time CHECK (terminal_at >= accepted_at)
|
||||
);
|
||||
|
||||
CREATE INDEX attachment_terminal_events_status_terminal_at_idx
|
||||
ON attachment_terminal_events (status, terminal_at);
|
||||
|
||||
WITH first_replay AS (
|
||||
SELECT job_id, MIN(created_at) AS created_at
|
||||
FROM attachment_replay_audit
|
||||
GROUP BY job_id
|
||||
)
|
||||
INSERT INTO attachment_terminal_events (job_id, status, accepted_at, terminal_at)
|
||||
SELECT jobs.id, 'failed', LEAST(jobs.created_at, first_replay.created_at), first_replay.created_at
|
||||
FROM attachment_jobs jobs
|
||||
JOIN first_replay ON first_replay.job_id = jobs.id
|
||||
ON CONFLICT (job_id) DO NOTHING;
|
||||
|
||||
INSERT INTO attachment_terminal_events (job_id, status, accepted_at, terminal_at)
|
||||
SELECT jobs.id, jobs.status, LEAST(jobs.created_at, jobs.updated_at), jobs.updated_at
|
||||
FROM attachment_jobs jobs
|
||||
WHERE jobs.status IN ('ready', 'failed')
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM attachment_replay_audit replay
|
||||
WHERE replay.job_id = jobs.id
|
||||
)
|
||||
ON CONFLICT (job_id) DO NOTHING;
|
||||
+602
-8
@@ -67,7 +67,7 @@
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "API latency p95 by route",
|
||||
"title": "API latency p50 / p95 / p99 by route",
|
||||
"id": 3,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
@@ -77,9 +77,19 @@
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "histogram_quantile(0.95, sum by (le, method, route) (rate(messenger_http_request_duration_seconds_bucket[5m])))",
|
||||
"legendFormat": "{{method}} {{route}}",
|
||||
"expr": "histogram_quantile(0.50, sum by (le, method, route) (rate(messenger_http_request_duration_seconds_bucket[5m])))",
|
||||
"legendFormat": "p50 {{method}} {{route}}",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(0.95, sum by (le, method, route) (rate(messenger_http_request_duration_seconds_bucket[5m])))",
|
||||
"legendFormat": "p95 {{method}} {{route}}",
|
||||
"refId": "B"
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(0.99, sum by (le, method, route) (rate(messenger_http_request_duration_seconds_bucket[5m])))",
|
||||
"legendFormat": "p99 {{method}} {{route}}",
|
||||
"refId": "C"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
@@ -895,14 +905,563 @@
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "row",
|
||||
"title": "SLO and alerts",
|
||||
"id": 105,
|
||||
"collapsed": true,
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 109
|
||||
},
|
||||
"panels": [
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Business API availability",
|
||||
"id": 31,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 110
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "1 - (sum(rate(messenger_http_requests_total{route!~=\"/health/.*|/metrics\",status_class=\"5xx\"}[5m])) or vector(0)) / clamp_min((sum(rate(messenger_http_requests_total{route!~=\"/health/.*|/metrics\"}[5m])) or vector(0)), 0.000001)",
|
||||
"legendFormat": "availability",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percentunit"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "API error-budget burn rate (99.5% SLO)",
|
||||
"id": 32,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 110
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "(sum(rate(messenger_http_requests_total{route!~=\"/health/.*|/metrics\",status_class=\"5xx\"}[5m])) or vector(0)) / clamp_min((sum(rate(messenger_http_requests_total{route!~=\"/health/.*|/metrics\"}[5m])) or vector(0)), 0.000001) / 0.005",
|
||||
"legendFormat": "5m burn",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "(sum(rate(messenger_http_requests_total{route!~=\"/health/.*|/metrics\",status_class=\"5xx\"}[1h])) or vector(0)) / clamp_min((sum(rate(messenger_http_requests_total{route!~=\"/health/.*|/metrics\"}[1h])) or vector(0)), 0.000001) / 0.005",
|
||||
"legendFormat": "1h burn",
|
||||
"refId": "B"
|
||||
},
|
||||
{
|
||||
"expr": "(sum(rate(messenger_http_requests_total{route!~=\"/health/.*|/metrics\",status_class=\"5xx\"}[6h])) or vector(0)) / clamp_min((sum(rate(messenger_http_requests_total{route!~=\"/health/.*|/metrics\"}[6h])) or vector(0)), 0.000001) / 0.005",
|
||||
"legendFormat": "6h burn",
|
||||
"refId": "C"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Attachments ready within 30 seconds",
|
||||
"id": 33,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 118
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "1 - (clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"+Inf\"}[5m])) or vector(0)) - (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"30\"}[5m])) or vector(0)), 0) + (sum(rate(messenger_attachment_terminal_transitions_total{status=\"failed\"}[5m])) or vector(0))) / clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"+Inf\"}[5m])) or vector(0)) + (sum(rate(messenger_attachment_terminal_transitions_total{status=\"failed\"}[5m])) or vector(0)), 0.000001)",
|
||||
"legendFormat": "ready in 30s",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percentunit"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Attachment ready-time error-budget burn",
|
||||
"id": 34,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 118
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "(clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"+Inf\"}[5m])) or vector(0)) - (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"30\"}[5m])) or vector(0)), 0) + (sum(rate(messenger_attachment_terminal_transitions_total{status=\"failed\"}[5m])) or vector(0))) / clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"+Inf\"}[5m])) or vector(0)) + (sum(rate(messenger_attachment_terminal_transitions_total{status=\"failed\"}[5m])) or vector(0)), 0.000001) / 0.01",
|
||||
"legendFormat": "5m burn",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "(clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"+Inf\"}[1h])) or vector(0)) - (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"30\"}[1h])) or vector(0)), 0) + (sum(rate(messenger_attachment_terminal_transitions_total{status=\"failed\"}[1h])) or vector(0))) / clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"+Inf\"}[1h])) or vector(0)) + (sum(rate(messenger_attachment_terminal_transitions_total{status=\"failed\"}[1h])) or vector(0)), 0.000001) / 0.01",
|
||||
"legendFormat": "1h burn",
|
||||
"refId": "B"
|
||||
},
|
||||
{
|
||||
"expr": "(clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"+Inf\"}[6h])) or vector(0)) - (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"30\"}[6h])) or vector(0)), 0) + (sum(rate(messenger_attachment_terminal_transitions_total{status=\"failed\"}[6h])) or vector(0))) / clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"+Inf\"}[6h])) or vector(0)) + (sum(rate(messenger_attachment_terminal_transitions_total{status=\"failed\"}[6h])) or vector(0)), 0.000001) / 0.01",
|
||||
"legendFormat": "6h burn",
|
||||
"refId": "C"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "SLO alert state",
|
||||
"id": 35,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 126
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "max by (alertname, alertstate) (ALERTS{alertname=~\"MessengerAPIHighErrorBudgetBurn.*|AttachmentReadyHighErrorBudgetBurn.*|AttachmentQueueBacklogGrowing|AttachmentWorkerUnavailable\"})",
|
||||
"legendFormat": "{{alertname}} {{alertstate}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "row",
|
||||
"title": "Runtime resources",
|
||||
"id": 107,
|
||||
"collapsed": true,
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 134
|
||||
},
|
||||
"panels": [
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Process CPU by service",
|
||||
"id": 38,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 135
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (service_name) (rate(process_cpu_seconds_total[1m]))",
|
||||
"legendFormat": "{{service_name}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "cores"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Go heap by service",
|
||||
"id": 39,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 135
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (service_name) (go_memstats_heap_alloc_bytes)",
|
||||
"legendFormat": "{{service_name}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "bytes"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Database pool used and wait rate",
|
||||
"id": 40,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 143
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (service_name) (messenger_db_pool_connections{state=\"used\"})",
|
||||
"legendFormat": "{{service_name}} used",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "sum by (service_name) (rate(messenger_db_pool_waited_total[5m]))",
|
||||
"legendFormat": "{{service_name}} waits/s",
|
||||
"refId": "B"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Worker saturation",
|
||||
"id": 41,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 143
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(messenger_attachment_worker_jobs_in_flight)",
|
||||
"legendFormat": "busy",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "sum(messenger_attachment_worker_capacity)",
|
||||
"legendFormat": "capacity",
|
||||
"refId": "B"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "row",
|
||||
"title": "Logs and traces",
|
||||
"id": 106,
|
||||
"collapsed": true,
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 151
|
||||
},
|
||||
"panels": [
|
||||
{
|
||||
"type": "logs",
|
||||
"title": "Correlated application logs",
|
||||
"id": 36,
|
||||
"gridPos": {
|
||||
"h": 10,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 152
|
||||
},
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "loki"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "loki"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "{service_name=~\"messenger-.+\"} | json",
|
||||
"queryType": "range",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"options": {
|
||||
"dedupStrategy": "none",
|
||||
"enableLogDetails": true,
|
||||
"prettifyLogMessage": false,
|
||||
"showCommonLabels": false,
|
||||
"showLabels": false,
|
||||
"showTime": true,
|
||||
"sortOrder": "Descending",
|
||||
"wrapLogMessage": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "traces",
|
||||
"title": "Recent application traces (TraceQL)",
|
||||
"id": 37,
|
||||
"gridPos": {
|
||||
"h": 10,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 152
|
||||
},
|
||||
"datasource": {
|
||||
"type": "tempo",
|
||||
"uid": "tempo"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "tempo",
|
||||
"uid": "tempo"
|
||||
},
|
||||
"query": "{ resource.service.name =~ \"messenger-.*\" }",
|
||||
"queryType": "traceql",
|
||||
"refId": "A",
|
||||
"limit": 50,
|
||||
"tableType": "traces"
|
||||
}
|
||||
],
|
||||
"options": {
|
||||
"showSpanFilters": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Trace-derived service call rate",
|
||||
"id": 42,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 162
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (service, span_name) (rate(traces_spanmetrics_calls_total[5m]))",
|
||||
"legendFormat": "{{service}} {{span_name}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "ops"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Collector delivery failures and queue pressure",
|
||||
"id": 43,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 162
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (exporter) (rate(otelcol_exporter_send_failed_spans[5m]))",
|
||||
"legendFormat": "{{exporter}} spans failed/s",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "sum by (exporter) (rate(otelcol_exporter_send_failed_log_records[5m]))",
|
||||
"legendFormat": "{{exporter}} logs failed/s",
|
||||
"refId": "B"
|
||||
},
|
||||
{
|
||||
"expr": "max by (exporter) (otelcol_exporter_queue_size)",
|
||||
"legendFormat": "{{exporter}} queue",
|
||||
"refId": "C"
|
||||
},
|
||||
{
|
||||
"expr": "max by (exporter) (otelcol_exporter_queue_capacity)",
|
||||
"legendFormat": "{{exporter}} capacity",
|
||||
"refId": "D"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"refresh": "5s",
|
||||
"schemaVersion": 41,
|
||||
"tags": [
|
||||
"highload",
|
||||
"lab5",
|
||||
"pipeline"
|
||||
"lab6",
|
||||
"observability",
|
||||
"slo"
|
||||
],
|
||||
"templating": {
|
||||
"list": []
|
||||
@@ -912,7 +1471,42 @@
|
||||
"to": "now"
|
||||
},
|
||||
"timezone": "browser",
|
||||
"title": "Messenger lab5 overview",
|
||||
"uid": "messenger-lab5",
|
||||
"version": 3
|
||||
"title": "Messenger lab6 observability",
|
||||
"uid": "messenger-lab6",
|
||||
"version": 1,
|
||||
"links": [
|
||||
{
|
||||
"asDropdown": false,
|
||||
"icon": "external link",
|
||||
"includeVars": true,
|
||||
"keepTime": true,
|
||||
"tags": [],
|
||||
"targetBlank": false,
|
||||
"title": "Explore logs and traces",
|
||||
"type": "link",
|
||||
"url": "/explore"
|
||||
},
|
||||
{
|
||||
"asDropdown": false,
|
||||
"icon": "external link",
|
||||
"includeVars": false,
|
||||
"keepTime": false,
|
||||
"tags": [],
|
||||
"targetBlank": true,
|
||||
"title": "API runbook",
|
||||
"type": "link",
|
||||
"url": "https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/api-availability.md"
|
||||
},
|
||||
{
|
||||
"asDropdown": false,
|
||||
"icon": "external link",
|
||||
"includeVars": false,
|
||||
"keepTime": false,
|
||||
"tags": [],
|
||||
"targetBlank": true,
|
||||
"title": "Pipeline runbook",
|
||||
"type": "link",
|
||||
"url": "https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/attachment-pipeline.md"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
apiVersion: 1
|
||||
|
||||
deleteDatasources:
|
||||
- name: Prometheus
|
||||
orgId: 1
|
||||
- name: Loki
|
||||
orgId: 1
|
||||
- name: Tempo
|
||||
orgId: 1
|
||||
|
||||
datasources:
|
||||
- name: Prometheus
|
||||
type: prometheus
|
||||
uid: prometheus
|
||||
access: proxy
|
||||
url: http://prometheus:9090
|
||||
isDefault: true
|
||||
editable: false
|
||||
jsonData:
|
||||
httpMethod: POST
|
||||
exemplarTraceIdDestinations:
|
||||
- name: trace_id
|
||||
datasourceUid: tempo
|
||||
|
||||
- name: Loki
|
||||
type: loki
|
||||
uid: loki
|
||||
access: proxy
|
||||
url: http://loki:3100
|
||||
editable: false
|
||||
jsonData:
|
||||
maxLines: 1000
|
||||
derivedFields:
|
||||
- name: TraceID
|
||||
matcherType: label
|
||||
matcherRegex: trace_id
|
||||
datasourceUid: tempo
|
||||
url: '$${__value.raw}'
|
||||
urlDisplayLabel: View trace
|
||||
|
||||
- name: Tempo
|
||||
type: tempo
|
||||
uid: tempo
|
||||
access: proxy
|
||||
url: http://tempo:3200
|
||||
editable: false
|
||||
jsonData:
|
||||
nodeGraph:
|
||||
enabled: true
|
||||
search:
|
||||
hide: false
|
||||
serviceMap:
|
||||
datasourceUid: prometheus
|
||||
tracesToLogsV2:
|
||||
datasourceUid: loki
|
||||
spanStartTimeShift: -5m
|
||||
spanEndTimeShift: 5m
|
||||
filterByTraceID: true
|
||||
filterBySpanID: false
|
||||
customQuery: true
|
||||
query: '{service_name="$${__span.tags["service.name"]}"} | trace_id = "$${__trace.traceId}"'
|
||||
tags:
|
||||
- key: service.name
|
||||
value: service_name
|
||||
tracesToMetrics:
|
||||
datasourceUid: prometheus
|
||||
spanStartTimeShift: -5m
|
||||
spanEndTimeShift: 5m
|
||||
tags:
|
||||
- key: service.name
|
||||
value: service
|
||||
queries:
|
||||
- name: Request rate
|
||||
query: 'sum(rate(traces_spanmetrics_calls_total{$$__tags}[5m]))'
|
||||
- name: Error rate
|
||||
query: 'sum(rate(traces_spanmetrics_calls_total{$$__tags,span_status="STATUS_CODE_ERROR"}[5m]))'
|
||||
@@ -1,10 +0,0 @@
|
||||
apiVersion: 1
|
||||
|
||||
datasources:
|
||||
- name: Prometheus
|
||||
type: prometheus
|
||||
uid: prometheus
|
||||
access: proxy
|
||||
url: http://prometheus:9090
|
||||
isDefault: true
|
||||
editable: false
|
||||
@@ -0,0 +1 @@
|
||||
|
||||
@@ -37,3 +37,61 @@ messenger_outbox:
|
||||
- oldest_pending_age_seconds:
|
||||
usage: GAUGE
|
||||
description: Age of the oldest unpublished outbox event.
|
||||
|
||||
messenger_attachment_terminal_transitions:
|
||||
query: |
|
||||
WITH statuses(status) AS (
|
||||
VALUES ('ready'), ('failed')
|
||||
)
|
||||
SELECT
|
||||
statuses.status,
|
||||
COUNT(events.job_id)::bigint AS total
|
||||
FROM statuses
|
||||
LEFT JOIN attachment_terminal_events events ON events.status = statuses.status
|
||||
GROUP BY statuses.status
|
||||
metrics:
|
||||
- status:
|
||||
usage: LABEL
|
||||
description: First terminal status recorded for a durable attachment job.
|
||||
- total:
|
||||
usage: COUNTER
|
||||
description: Durable attachment jobs whose first terminal outcome has this status.
|
||||
|
||||
messenger_attachment_time_to_ready_seconds:
|
||||
query: |
|
||||
WITH bounds(le, upper_bound) AS (
|
||||
VALUES
|
||||
('0.05', 0.05::double precision),
|
||||
('0.1', 0.1::double precision),
|
||||
('0.25', 0.25::double precision),
|
||||
('0.5', 0.5::double precision),
|
||||
('1', 1::double precision),
|
||||
('2.5', 2.5::double precision),
|
||||
('5', 5::double precision),
|
||||
('10', 10::double precision),
|
||||
('15', 15::double precision),
|
||||
('20', 20::double precision),
|
||||
('30', 30::double precision),
|
||||
('60', 60::double precision),
|
||||
('+Inf', NULL::double precision)
|
||||
)
|
||||
SELECT
|
||||
bounds.le,
|
||||
COUNT(events.job_id) FILTER (
|
||||
WHERE events.status = 'ready'
|
||||
AND (
|
||||
bounds.upper_bound IS NULL
|
||||
OR EXTRACT(EPOCH FROM (events.terminal_at - events.accepted_at)) <= bounds.upper_bound
|
||||
)
|
||||
)::bigint AS bucket
|
||||
FROM bounds
|
||||
LEFT JOIN attachment_terminal_events events ON events.status = 'ready'
|
||||
GROUP BY bounds.le, bounds.upper_bound
|
||||
ORDER BY bounds.upper_bound NULLS LAST
|
||||
metrics:
|
||||
- le:
|
||||
usage: LABEL
|
||||
description: Upper bound of the durable time-to-ready bucket in seconds.
|
||||
- bucket:
|
||||
usage: COUNTER
|
||||
description: Durable attachment jobs reaching ready within the bucket bound.
|
||||
|
||||
@@ -1,6 +1,11 @@
|
||||
global:
|
||||
scrape_interval: 1s
|
||||
evaluation_interval: 1s
|
||||
scrape_interval: 2s
|
||||
evaluation_interval: 5s
|
||||
external_labels:
|
||||
deployment_environment: lab
|
||||
|
||||
rule_files:
|
||||
- /etc/prometheus/rules/*.yml
|
||||
|
||||
scrape_configs:
|
||||
- job_name: messenger
|
||||
@@ -9,6 +14,13 @@ scrape_configs:
|
||||
type: A
|
||||
port: 8080
|
||||
refresh_interval: 2s
|
||||
relabel_configs:
|
||||
- target_label: service_name
|
||||
replacement: messenger-api
|
||||
metric_relabel_configs:
|
||||
- source_labels: [__name__]
|
||||
regex: "go_.*|process_.*"
|
||||
action: keep
|
||||
|
||||
- job_name: nginx
|
||||
static_configs:
|
||||
@@ -32,12 +44,42 @@ scrape_configs:
|
||||
type: A
|
||||
port: 9091
|
||||
refresh_interval: 2s
|
||||
relabel_configs:
|
||||
- target_label: service_name
|
||||
replacement: messenger-worker
|
||||
metric_relabel_configs:
|
||||
- source_labels: [__name__]
|
||||
regex: "go_.*|process_.*"
|
||||
action: keep
|
||||
|
||||
- job_name: outbox-dispatcher
|
||||
static_configs:
|
||||
- targets: ["dispatcher:9091"]
|
||||
relabel_configs:
|
||||
- target_label: service_name
|
||||
replacement: messenger-dispatcher
|
||||
metric_relabel_configs:
|
||||
- source_labels: [__name__]
|
||||
regex: "go_.*|process_.*"
|
||||
action: keep
|
||||
|
||||
- job_name: minio
|
||||
metrics_path: /minio/v2/metrics/cluster
|
||||
static_configs:
|
||||
- targets: ["s3:9000"]
|
||||
|
||||
- job_name: otel-applications
|
||||
static_configs:
|
||||
- targets: ["otel-collector:8889"]
|
||||
|
||||
- job_name: otel-collector
|
||||
static_configs:
|
||||
- targets: ["otel-collector:8888"]
|
||||
|
||||
- job_name: tempo
|
||||
static_configs:
|
||||
- targets: ["tempo:3200"]
|
||||
|
||||
- job_name: loki
|
||||
static_configs:
|
||||
- targets: ["loki:3100"]
|
||||
|
||||
@@ -0,0 +1,178 @@
|
||||
groups:
|
||||
- name: messenger-api-slo
|
||||
interval: 5s
|
||||
rules:
|
||||
- alert: MessengerAPIBusiness5xxHigh
|
||||
expr: |
|
||||
(
|
||||
(sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics",status_class="5xx"}[2m])) or vector(0))
|
||||
/
|
||||
clamp_min((sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics"}[2m])) or vector(0)), 0.000001)
|
||||
) > 0.05
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
service: messenger-api
|
||||
slo: api-availability
|
||||
annotations:
|
||||
summary: Business API 5xx ratio is above 5%
|
||||
description: More than 5% of business requests returned 5xx for one minute. Health and metrics routes are excluded.
|
||||
runbook_url: https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/api-availability.md
|
||||
|
||||
- alert: MessengerAPIHighErrorBudgetBurnFast
|
||||
expr: |
|
||||
(
|
||||
(sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics",status_class="5xx"}[5m])) or vector(0))
|
||||
/ clamp_min((sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics"}[5m])) or vector(0)), 0.000001)
|
||||
) > 0.072
|
||||
and
|
||||
(
|
||||
(sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics",status_class="5xx"}[1h])) or vector(0))
|
||||
/ clamp_min((sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics"}[1h])) or vector(0)), 0.000001)
|
||||
) > 0.072
|
||||
for: 30s
|
||||
labels:
|
||||
severity: critical
|
||||
service: messenger-api
|
||||
slo: api-availability
|
||||
annotations:
|
||||
summary: API availability error budget is burning fast
|
||||
description: The 99.5% API availability SLO burns above 14.4x in both 5m and 1h windows.
|
||||
runbook_url: https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/api-availability.md
|
||||
|
||||
- alert: MessengerAPIHighErrorBudgetBurnSlow
|
||||
expr: |
|
||||
(
|
||||
(sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics",status_class="5xx"}[30m])) or vector(0))
|
||||
/ clamp_min((sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics"}[30m])) or vector(0)), 0.000001)
|
||||
) > 0.03
|
||||
and
|
||||
(
|
||||
(sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics",status_class="5xx"}[6h])) or vector(0))
|
||||
/ clamp_min((sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics"}[6h])) or vector(0)), 0.000001)
|
||||
) > 0.03
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
service: messenger-api
|
||||
slo: api-availability
|
||||
annotations:
|
||||
summary: API availability error budget is burning steadily
|
||||
description: The 99.5% API availability SLO burns above 6x in both 30m and 6h windows.
|
||||
runbook_url: https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/api-availability.md
|
||||
|
||||
- name: messenger-attachment-slo
|
||||
interval: 5s
|
||||
rules:
|
||||
- alert: AttachmentReadyHighErrorBudgetBurnFast
|
||||
expr: |
|
||||
(
|
||||
clamp_min(
|
||||
(sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="+Inf"}[5m])) or vector(0))
|
||||
- (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="30"}[5m])) or vector(0)),
|
||||
0
|
||||
)
|
||||
+ (sum(rate(messenger_attachment_terminal_transitions_total{status="failed"}[5m])) or vector(0))
|
||||
)
|
||||
/ clamp_min(
|
||||
(sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="+Inf"}[5m])) or vector(0))
|
||||
+ (sum(rate(messenger_attachment_terminal_transitions_total{status="failed"}[5m])) or vector(0)),
|
||||
0.000001
|
||||
) > 0.144
|
||||
and
|
||||
(
|
||||
clamp_min(
|
||||
(sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="+Inf"}[1h])) or vector(0))
|
||||
- (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="30"}[1h])) or vector(0)),
|
||||
0
|
||||
)
|
||||
+ (sum(rate(messenger_attachment_terminal_transitions_total{status="failed"}[1h])) or vector(0))
|
||||
)
|
||||
/ clamp_min(
|
||||
(sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="+Inf"}[1h])) or vector(0))
|
||||
+ (sum(rate(messenger_attachment_terminal_transitions_total{status="failed"}[1h])) or vector(0)),
|
||||
0.000001
|
||||
) > 0.144
|
||||
for: 30s
|
||||
labels:
|
||||
severity: critical
|
||||
service: messenger-worker
|
||||
slo: attachment-ready
|
||||
annotations:
|
||||
summary: Attachment ready-time error budget is burning fast
|
||||
description: The 99% within 30s ready-time SLO burns above 14.4x in both 5m and 1h windows.
|
||||
runbook_url: https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/attachment-pipeline.md
|
||||
|
||||
- alert: AttachmentReadyHighErrorBudgetBurnSlow
|
||||
expr: |
|
||||
(
|
||||
clamp_min(
|
||||
(sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="+Inf"}[30m])) or vector(0))
|
||||
- (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="30"}[30m])) or vector(0)),
|
||||
0
|
||||
)
|
||||
+ (sum(rate(messenger_attachment_terminal_transitions_total{status="failed"}[30m])) or vector(0))
|
||||
)
|
||||
/ clamp_min(
|
||||
(sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="+Inf"}[30m])) or vector(0))
|
||||
+ (sum(rate(messenger_attachment_terminal_transitions_total{status="failed"}[30m])) or vector(0)),
|
||||
0.000001
|
||||
) > 0.06
|
||||
and
|
||||
(
|
||||
clamp_min(
|
||||
(sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="+Inf"}[6h])) or vector(0))
|
||||
- (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="30"}[6h])) or vector(0)),
|
||||
0
|
||||
)
|
||||
+ (sum(rate(messenger_attachment_terminal_transitions_total{status="failed"}[6h])) or vector(0))
|
||||
)
|
||||
/ clamp_min(
|
||||
(sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="+Inf"}[6h])) or vector(0))
|
||||
+ (sum(rate(messenger_attachment_terminal_transitions_total{status="failed"}[6h])) or vector(0)),
|
||||
0.000001
|
||||
) > 0.06
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
service: messenger-worker
|
||||
slo: attachment-ready
|
||||
annotations:
|
||||
summary: Attachment ready-time error budget is burning steadily
|
||||
description: The 99% within 30s ready-time SLO burns above 6x in both 30m and 6h windows.
|
||||
runbook_url: https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/attachment-pipeline.md
|
||||
|
||||
- alert: AttachmentWorkerUnavailable
|
||||
expr: (sum(up{job="attachment-worker"}) or vector(0)) < 1
|
||||
for: 30s
|
||||
labels:
|
||||
severity: critical
|
||||
service: messenger-worker
|
||||
annotations:
|
||||
summary: No attachment worker is available
|
||||
description: Prometheus has not observed a healthy worker target for 30 seconds.
|
||||
runbook_url: https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/attachment-pipeline.md
|
||||
|
||||
- alert: AttachmentQueueBacklogGrowing
|
||||
expr: |
|
||||
max(messenger_attachment_jobs_oldest_age_seconds{status="queued"}) > 30
|
||||
and max(deriv(messenger_attachment_jobs_oldest_age_seconds{status="queued"}[2m])) > 0
|
||||
for: 30s
|
||||
labels:
|
||||
severity: warning
|
||||
service: messenger-worker
|
||||
annotations:
|
||||
summary: Attachment queue backlog is growing
|
||||
description: The oldest queued attachment is older than 30 seconds and its age continues to increase.
|
||||
runbook_url: https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/attachment-pipeline.md
|
||||
|
||||
- alert: AttachmentRedisStreamLagHigh
|
||||
expr: max(redis_stream_group_lag{job="redis-broker",stream="attachment-jobs",group="attachment-workers"}) > 20
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
service: redis-broker
|
||||
annotations:
|
||||
summary: Redis Stream consumer lag is high
|
||||
description: The attachment worker group has more than 20 messages waiting for delivery.
|
||||
runbook_url: https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/attachment-pipeline.md
|
||||
@@ -0,0 +1,72 @@
|
||||
# API availability SLO
|
||||
|
||||
## Цель и сигналы
|
||||
|
||||
Объект контроля — бизнес-маршруты API за Nginx. Health endpoints исключены. Хорошими считаются ответы `2xx` и ожидаемые клиентские `4xx`; плохими — `5xx`. Цель — не менее 99,5% хороших ответов за скользящие 30 дней, error budget равен 0,5%.
|
||||
|
||||
Основные сигналы:
|
||||
|
||||
- `messenger_http_requests_total{method,route,status_class}` — число запросов;
|
||||
- `messenger_http_request_duration_seconds_bucket{method,route}` — распределение server latency;
|
||||
- `up{job="messenger"}` и `up{job="nginx"}` — доступность scrape targets;
|
||||
- `messenger_db_pool_connections{state="used"}` и `rate(messenger_db_pool_waited_total[5m])` — использование пула и частота ожиданий свободного соединения;
|
||||
- trace `messenger-api` — путь одного запроса по зависимостям;
|
||||
- JSON-логи `messenger-api` с `request_id`, `trace_id` и `span_id`.
|
||||
|
||||
`user_id`, `chat_id`, `message_id`, `attachment_id` и `request_id` не используются как labels метрик.
|
||||
|
||||
## Диагностика: metrics → trace → logs
|
||||
|
||||
1. В Grafana открыть dashboard **Messenger Lab 6 / API** и установить общий временной диапазон инцидента.
|
||||
2. Проверить error ratio, RPS, p95/p99 и `up`. Рост `5xx` при нормальном RPS подтверждает влияние на SLI; отсутствие трафика не считается успешностью.
|
||||
3. Сравнить DB pool used/waiting и состояние PostgreSQL. Рост ожидания пула указывает на зависимость, а не на CPU handler.
|
||||
4. В панели ошибок выбрать exemplar или перейти в Explore → Tempo. Если exemplar отсутствует, выполнить TraceQL:
|
||||
|
||||
```text
|
||||
{ resource.service.name = "messenger-api" && span.http.route != "/health/ready" && status = error }
|
||||
```
|
||||
|
||||
5. В trace найти первый error span. Проверить `http.route`, `http.request.method`, `http.response.status_code`, `db.system`, duration и resource attributes. Полный URL, токен и тело запроса в span отсутствуют.
|
||||
6. Скопировать `trace_id`, перейти по trace-to-logs или выполнить в Explore → Loki:
|
||||
|
||||
```text
|
||||
{service_name="messenger-api"} | json | trace_id="<trace_id>"
|
||||
```
|
||||
|
||||
7. Сверить `request_id`, событие, severity и error class. Текст сообщений, `Authorization` и пароли в логах отсутствуют.
|
||||
|
||||
## Проверка alert
|
||||
|
||||
Состояние правил видно в Grafana Alerting и в Prometheus:
|
||||
|
||||
```bash
|
||||
curl -s http://127.0.0.1:9090/api/v1/alerts
|
||||
```
|
||||
|
||||
Контролируемый отказ PostgreSQL:
|
||||
|
||||
```bash
|
||||
ENV_FILE=lab6.env.example scripts/incident-database.sh
|
||||
```
|
||||
|
||||
Сценарий сохраняет baseline, нагрузку, состояние деградации, `alert-firing.json`, восстановление и `alert-resolved.json` в `evidence/runtime/`. Trap запускает PostgreSQL и API даже при ошибке сценария. Volumes не удаляются.
|
||||
|
||||
Ожидаемая последовательность:
|
||||
|
||||
```text
|
||||
нормальный error ratio
|
||||
→ PostgreSQL остановлен
|
||||
→ 5xx растёт
|
||||
→ MessengerAPIBusiness5xxHigh и MessengerAPIHighErrorBudgetBurnFast firing
|
||||
→ PostgreSQL восстановлен
|
||||
→ успешные ответы возвращаются
|
||||
→ оперативный alert resolved; burn-rate хранит историю длинного окна
|
||||
```
|
||||
|
||||
## Действия
|
||||
|
||||
1. Если `up{job="nginx"}=0`, восстановить ingress и проверить его логи.
|
||||
2. Если API targets недоступны, восстановить реплики по одной и проверить readiness.
|
||||
3. Если API работает, а DB spans завершаются ошибкой или pool waiting растёт, восстановить PostgreSQL и не увеличивать число API-реплик до устранения причины.
|
||||
4. После восстановления проверить успешный smoke, отсутствие firing alerts и возвращение latency к baseline.
|
||||
5. Не удалять volumes и не сбрасывать очередь ради закрытия alert.
|
||||
@@ -0,0 +1,86 @@
|
||||
# Attachment pipeline SLO
|
||||
|
||||
## Цель и сигналы
|
||||
|
||||
Объект контроля — первый terminal outcome каждой уникальной durable job. Цель — не менее 99% jobs впервые завершаются `ready` не позднее 30 секунд после acceptance за скользящие 30 дней. Первый `failed` и первое завершение `ready` позже 30 секунд расходуют error budget; replay не переписывает уже записанный outcome.
|
||||
|
||||
Основные сигналы:
|
||||
|
||||
- `messenger_attachment_worker_jobs_total{job_type,result}` — исходы доставок worker;
|
||||
- `messenger_attachment_terminal_transitions_total{status}` — первые terminal outcomes из append-only PostgreSQL ledger;
|
||||
- `messenger_attachment_worker_job_duration_seconds_bucket{job_type,result}` — время обработки;
|
||||
- `messenger_attachment_time_to_ready_seconds_bucket` — end-to-end время;
|
||||
- `messenger_attachment_jobs_count{status}` — состояния jobs в PostgreSQL;
|
||||
- `messenger_attachment_jobs_oldest_age_seconds{status}` — возраст старейшей job;
|
||||
- `messenger_attachment_worker_jobs_in_flight` и `messenger_attachment_worker_capacity` — насыщение worker;
|
||||
- `messenger_outbox_pending` — задержка до публикации;
|
||||
- Redis Stream lag и PEL — доставка и неподтверждённые сообщения.
|
||||
|
||||
Queue depth не заменяет processing latency: очередь может быть короткой, но содержать одну очень старую job.
|
||||
|
||||
## Диагностика: metrics → trace → logs
|
||||
|
||||
1. В Grafana открыть dashboard **Messenger Lab 6 / Attachment pipeline** и выбрать диапазон инцидента.
|
||||
2. Определить место задержки:
|
||||
- растёт `messenger_outbox_pending` — проблема до публикации в broker;
|
||||
- растут stream lag и `queued` — worker не успевает или недоступен;
|
||||
- растут PEL и `processing` — delivery получена, но не завершена;
|
||||
- растёт `failed` — постоянные ошибки или исчерпаны retry;
|
||||
- busy равен capacity — worker насыщен.
|
||||
3. В Explore → Tempo найти upload по route или worker consumer span:
|
||||
|
||||
```text
|
||||
{ resource.service.name = "messenger-api" && span.http.route = "/api/v1/chats/{chat_id}/attachments" }
|
||||
```
|
||||
|
||||
```text
|
||||
{ resource.service.name = "messenger-worker" && name = "attachment.process" }
|
||||
```
|
||||
|
||||
4. В trace пройти цепочку `HTTP upload → outbox publish → Redis consume → S3 variants → PostgreSQL ready`. Проверить producer/consumer связь и retry spans. Технический `attachment_id` допустим только как span/log attribute синтетического стенда, но не как metric label.
|
||||
5. Перейти из выбранного span в Loki или выполнить:
|
||||
|
||||
```text
|
||||
{service_name="messenger-worker"} | json | trace_id="<trace_id>"
|
||||
```
|
||||
|
||||
6. Сверить в логе `event`, `job_type`, `attempt`, `result`, `trace_id` и `span_id`; в span проверить `job.outcome`. Payload envelope, изображение и credentials не логируются.
|
||||
|
||||
## Проверка alert
|
||||
|
||||
Короткая деградация:
|
||||
|
||||
```bash
|
||||
ENV_FILE=lab6.env.example scripts/incident-worker.sh short
|
||||
```
|
||||
|
||||
Длительная деградация:
|
||||
|
||||
```bash
|
||||
ENV_FILE=lab6.env.example scripts/incident-worker.sh long
|
||||
```
|
||||
|
||||
Ожидаемая последовательность:
|
||||
|
||||
```text
|
||||
worker up, backlog около нуля
|
||||
→ worker остановлен
|
||||
→ queued и oldest age растут
|
||||
→ short: AttachmentWorkerUnavailable firing
|
||||
→ long: AttachmentQueueBacklogGrowing firing
|
||||
→ worker запущен
|
||||
→ backlog дренируется
|
||||
→ все принятые attachments достигают ready
|
||||
→ alert resolved
|
||||
```
|
||||
|
||||
Каждый сценарий восстанавливает worker через trap и сохраняет evidence без session token и raw payload.
|
||||
|
||||
## Действия
|
||||
|
||||
1. Если worker `up=0`, восстановить worker и наблюдать скорость дренирования.
|
||||
2. Если worker доступен, но busy равно capacity, проверить S3/DB spans до масштабирования concurrency.
|
||||
3. Если растёт outbox, проверить dispatcher и PostgreSQL lease; Redis worker здесь ещё не причина.
|
||||
4. Если растёт PEL, проверить heartbeat, lease expiry и reclaim.
|
||||
5. Для `failed` определить permanent/transient error по логу и trace. Replay выполнять только после устранения причины.
|
||||
6. После восстановления проверить backlog, oldest age, `failed`, firing alerts и terminal state каждого синтетического attachment.
|
||||
Executable
+91
@@ -0,0 +1,91 @@
|
||||
#!/bin/sh
|
||||
|
||||
set -eu
|
||||
|
||||
env_file=${ENV_FILE:-lab6.env.example}
|
||||
base_url=${BASE_URL:-http://127.0.0.1:8080}
|
||||
prometheus_url=${PROMETHEUS_URL:-http://127.0.0.1:9090}
|
||||
tempo_url=${TEMPO_URL:-http://127.0.0.1:3200}
|
||||
loki_url=${LOKI_URL:-http://127.0.0.1:3100}
|
||||
evidence_root=${EVIDENCE_DIR:-evidence/runtime}
|
||||
outage_seconds=${DB_OUTAGE_SECONDS:-75}
|
||||
require_alerts=${REQUIRE_ALERT_TRANSITIONS:-1}
|
||||
api_alert_name=${API_ALERT_NAME:-MessengerAPIBusiness5xxHigh}
|
||||
burn_alert_name=${API_BURN_ALERT_NAME:-MessengerAPIHighErrorBudgetBurnFast}
|
||||
run_id=$(date -u +%Y%m%dT%H%M%SZ)
|
||||
output_dir="$evidence_root/database-$run_id"
|
||||
mkdir -p "$output_dir"
|
||||
load_pid=
|
||||
|
||||
compose() {
|
||||
docker compose --env-file "$env_file" "$@"
|
||||
}
|
||||
|
||||
restore() {
|
||||
if [ -n "$load_pid" ]; then
|
||||
kill "$load_pid" >/dev/null 2>&1 || true
|
||||
fi
|
||||
compose up -d postgres migrate api nginx >/dev/null 2>&1 || true
|
||||
}
|
||||
|
||||
trap restore EXIT INT TERM
|
||||
|
||||
python3 scripts/lab6-observe.py --prometheus-url "$prometheus_url" --phase baseline --output "$output_dir/baseline.json" >/dev/null
|
||||
python3 scripts/lab6-traffic.py \
|
||||
--base-url "$base_url" \
|
||||
--mode messages \
|
||||
--duration "$((outage_seconds + 20))" \
|
||||
--concurrency 4 \
|
||||
--rate "${DB_LOAD_RATE:-20}" \
|
||||
--started-file "$output_dir/load-started" \
|
||||
--output "$output_dir/traffic.json" >/dev/null &
|
||||
load_pid=$!
|
||||
remaining=60
|
||||
while [ ! -f "$output_dir/load-started" ] && [ "$remaining" -gt 0 ]; do
|
||||
sleep 0.5
|
||||
remaining=$((remaining - 1))
|
||||
done
|
||||
test -f "$output_dir/load-started"
|
||||
sleep 2
|
||||
started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)
|
||||
compose stop postgres >/dev/null
|
||||
sleep "$outage_seconds"
|
||||
if [ "$require_alerts" = "1" ]; then
|
||||
python3 scripts/lab6-alert.py --prometheus-url "$prometheus_url" --name "$api_alert_name" --state firing --timeout "${ALERT_WAIT_SECONDS:-90}" --output "$output_dir/alert-firing.json" >/dev/null
|
||||
python3 scripts/lab6-alert.py --prometheus-url "$prometheus_url" --name "$burn_alert_name" --state firing --timeout "${ALERT_WAIT_SECONDS:-90}" --output "$output_dir/burn-alert-firing.json" >/dev/null
|
||||
fi
|
||||
python3 scripts/lab6-correlation.py --mode database --tempo-url "$tempo_url" --loki-url "$loki_url" --output "$output_dir/database-correlation.json" --require >/dev/null
|
||||
python3 scripts/lab6-observe.py --prometheus-url "$prometheus_url" --phase degraded --output "$output_dir/degraded.json" >/dev/null
|
||||
recovery_started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)
|
||||
compose up -d --wait --wait-timeout 90 postgres migrate api nginx >/dev/null
|
||||
wait "$load_pid" || true
|
||||
load_pid=
|
||||
if [ "$require_alerts" = "1" ]; then
|
||||
python3 scripts/lab6-alert.py --prometheus-url "$prometheus_url" --name "$api_alert_name" --state resolved --timeout "${ALERT_RESOLVE_SECONDS:-180}" --output "$output_dir/alert-resolved.json" >/dev/null
|
||||
fi
|
||||
sleep "${RECOVERY_OBSERVATION_SECONDS:-20}"
|
||||
python3 scripts/lab6-observe.py --prometheus-url "$prometheus_url" --phase recovered --output "$output_dir/recovered.json" >/dev/null
|
||||
recovered_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)
|
||||
python3 - "$output_dir/summary.json" "$started_at" "$recovery_started_at" "$recovered_at" "$outage_seconds" <<'PY'
|
||||
import datetime
|
||||
import json
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
output, started, recovery_started, recovered, duration = sys.argv[1:]
|
||||
parse = lambda value: datetime.datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||
report = {
|
||||
"schema_version": 1,
|
||||
"incident": "postgres_unavailable",
|
||||
"started_at": started,
|
||||
"recovery_started_at": recovery_started,
|
||||
"recovered_at": recovered,
|
||||
"planned_injection_seconds": int(duration),
|
||||
"injected_duration_seconds": int((parse(recovery_started) - parse(started)).total_seconds()),
|
||||
"recovery_observation_seconds": int((parse(recovered) - parse(recovery_started)).total_seconds()),
|
||||
"artifacts": ["baseline.json", "degraded.json", "recovered.json", "traffic.json", "database-correlation.json", "alert-firing.json", "alert-resolved.json", "burn-alert-firing.json"],
|
||||
}
|
||||
pathlib.Path(output).write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
PY
|
||||
trap - EXIT INT TERM
|
||||
printf '%s\n' "$output_dir"
|
||||
Executable
+73
@@ -0,0 +1,73 @@
|
||||
#!/bin/sh
|
||||
|
||||
set -eu
|
||||
|
||||
env_file=${ENV_FILE:-lab6.env.example}
|
||||
base_url=${BASE_URL:-http://127.0.0.1:8080}
|
||||
prometheus_url=${PROMETHEUS_URL:-http://127.0.0.1:9090}
|
||||
evidence_root=${EVIDENCE_DIR:-evidence/runtime}
|
||||
outage_seconds=${TELEMETRY_OUTAGE_SECONDS:-30}
|
||||
run_id=$(date -u +%Y%m%dT%H%M%SZ)
|
||||
output_dir="$evidence_root/telemetry-$run_id"
|
||||
mkdir -p "$output_dir"
|
||||
load_pid=
|
||||
|
||||
compose() {
|
||||
docker compose --env-file "$env_file" "$@"
|
||||
}
|
||||
|
||||
restore() {
|
||||
if [ -n "$load_pid" ]; then
|
||||
kill "$load_pid" >/dev/null 2>&1 || true
|
||||
fi
|
||||
compose up -d tempo loki >/dev/null 2>&1 || true
|
||||
}
|
||||
|
||||
trap restore EXIT INT TERM
|
||||
|
||||
python3 scripts/lab6-observe.py --prometheus-url "$prometheus_url" --phase baseline --output "$output_dir/baseline.json" >/dev/null
|
||||
started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)
|
||||
compose stop tempo loki >/dev/null
|
||||
python3 scripts/lab6-traffic.py \
|
||||
--base-url "$base_url" \
|
||||
--mode mixed \
|
||||
--duration "$outage_seconds" \
|
||||
--concurrency 4 \
|
||||
--rate "${TELEMETRY_LOAD_RATE:-10}" \
|
||||
--wait-ready \
|
||||
--ready-timeout 120 \
|
||||
--output "$output_dir/traffic.json" >/dev/null &
|
||||
load_pid=$!
|
||||
sleep "$outage_seconds"
|
||||
wait "$load_pid"
|
||||
load_pid=
|
||||
health_code=$(curl --silent --output /dev/null --write-out '%{http_code}' "$base_url/health/ready")
|
||||
python3 scripts/lab6-observe.py --prometheus-url "$prometheus_url" --phase degraded --output "$output_dir/degraded.json" >/dev/null
|
||||
recovery_started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)
|
||||
compose up -d --wait --wait-timeout 90 tempo loki >/dev/null
|
||||
sleep "${RECOVERY_OBSERVATION_SECONDS:-20}"
|
||||
python3 scripts/lab6-observe.py --prometheus-url "$prometheus_url" --phase recovered --output "$output_dir/recovered.json" >/dev/null
|
||||
recovered_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)
|
||||
python3 - "$output_dir/summary.json" "$started_at" "$recovery_started_at" "$recovered_at" "$outage_seconds" "$health_code" <<'PY'
|
||||
import datetime
|
||||
import json
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
output, started, recovery_started, recovered, duration, health_code = sys.argv[1:]
|
||||
parse = lambda value: datetime.datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||
report = {
|
||||
"schema_version": 1,
|
||||
"incident": "telemetry_backends_unavailable",
|
||||
"started_at": started,
|
||||
"recovery_started_at": recovery_started,
|
||||
"recovered_at": recovered,
|
||||
"injected_duration_seconds": int(duration),
|
||||
"api_ready_http_status_during_outage": int(health_code),
|
||||
"recovery_observation_seconds": int((parse(recovered) - parse(recovery_started)).total_seconds()),
|
||||
"artifacts": ["baseline.json", "degraded.json", "recovered.json", "traffic.json"],
|
||||
}
|
||||
pathlib.Path(output).write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
PY
|
||||
trap - EXIT INT TERM
|
||||
printf '%s\n' "$output_dir"
|
||||
Executable
+107
@@ -0,0 +1,107 @@
|
||||
#!/bin/sh
|
||||
|
||||
set -eu
|
||||
|
||||
mode=${1:-short}
|
||||
env_file=${ENV_FILE:-lab6.env.example}
|
||||
base_url=${BASE_URL:-http://127.0.0.1:8080}
|
||||
prometheus_url=${PROMETHEUS_URL:-http://127.0.0.1:9090}
|
||||
evidence_root=${EVIDENCE_DIR:-evidence/runtime}
|
||||
require_alerts=${REQUIRE_ALERT_TRANSITIONS:-1}
|
||||
case "$mode" in
|
||||
short)
|
||||
incident_seconds=${SHORT_WORKER_OUTAGE_SECONDS:-75}
|
||||
load_seconds=${SHORT_WORKER_LOAD_SECONDS:-60}
|
||||
requests=${SHORT_WORKER_REQUESTS:-300}
|
||||
default_alert=AttachmentWorkerUnavailable
|
||||
;;
|
||||
long)
|
||||
incident_seconds=${LONG_WORKER_OUTAGE_SECONDS:-180}
|
||||
load_seconds=${LONG_WORKER_LOAD_SECONDS:-120}
|
||||
requests=${LONG_WORKER_REQUESTS:-1000}
|
||||
default_alert=AttachmentQueueBacklogGrowing
|
||||
;;
|
||||
*)
|
||||
echo "usage: $0 [short|long]" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
worker_alert_name=${WORKER_ALERT_NAME:-$default_alert}
|
||||
|
||||
run_id=$(date -u +%Y%m%dT%H%M%SZ)
|
||||
output_dir="$evidence_root/worker-$mode-$run_id"
|
||||
mkdir -p "$output_dir"
|
||||
load_pid=
|
||||
|
||||
compose() {
|
||||
docker compose --env-file "$env_file" "$@"
|
||||
}
|
||||
|
||||
restore() {
|
||||
if [ -n "$load_pid" ]; then
|
||||
kill "$load_pid" >/dev/null 2>&1 || true
|
||||
fi
|
||||
compose up -d worker >/dev/null 2>&1 || true
|
||||
}
|
||||
|
||||
trap restore EXIT INT TERM
|
||||
|
||||
python3 scripts/lab6-observe.py --prometheus-url "$prometheus_url" --phase baseline --output "$output_dir/baseline.json" >/dev/null
|
||||
started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)
|
||||
compose stop worker >/dev/null
|
||||
python3 scripts/lab6-traffic.py \
|
||||
--base-url "$base_url" \
|
||||
--mode attachments \
|
||||
--duration "$load_seconds" \
|
||||
--requests "$requests" \
|
||||
--concurrency 4 \
|
||||
--rate "${WORKER_LOAD_RATE:-5}" \
|
||||
--wait-ready \
|
||||
--ready-timeout 480 \
|
||||
--output "$output_dir/traffic.json" >/dev/null &
|
||||
load_pid=$!
|
||||
sleep "$incident_seconds"
|
||||
if [ "$require_alerts" = "1" ]; then
|
||||
python3 scripts/lab6-alert.py --prometheus-url "$prometheus_url" --name "$worker_alert_name" --state firing --timeout "${ALERT_WAIT_SECONDS:-90}" --output "$output_dir/alert-firing.json" >/dev/null
|
||||
fi
|
||||
python3 scripts/lab6-observe.py --prometheus-url "$prometheus_url" --phase degraded --output "$output_dir/degraded.json" >/dev/null
|
||||
recovery_started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)
|
||||
compose up -d --wait --wait-timeout 90 worker >/dev/null
|
||||
wait "$load_pid"
|
||||
load_pid=
|
||||
if [ "$require_alerts" = "1" ]; then
|
||||
python3 scripts/lab6-alert.py --prometheus-url "$prometheus_url" --name "$worker_alert_name" --state resolved --timeout "${ALERT_RESOLVE_SECONDS:-180}" --output "$output_dir/alert-resolved.json" >/dev/null
|
||||
if [ "$mode" = "long" ]; then
|
||||
python3 scripts/lab6-alert.py --prometheus-url "$prometheus_url" --name AttachmentReadyHighErrorBudgetBurnFast --state firing --timeout "${ALERT_WAIT_SECONDS:-90}" --output "$output_dir/burn-fast-firing.json" >/dev/null
|
||||
python3 scripts/lab6-alert.py --prometheus-url "$prometheus_url" --name AttachmentReadyHighErrorBudgetBurnSlow --state firing --timeout "${ALERT_WAIT_SECONDS:-90}" --output "$output_dir/burn-slow-firing.json" >/dev/null
|
||||
fi
|
||||
fi
|
||||
sleep "${RECOVERY_OBSERVATION_SECONDS:-20}"
|
||||
python3 scripts/lab6-observe.py --prometheus-url "$prometheus_url" --phase recovered --output "$output_dir/recovered.json" >/dev/null
|
||||
recovered_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)
|
||||
python3 - "$output_dir/summary.json" "$mode" "$started_at" "$recovery_started_at" "$recovered_at" "$incident_seconds" <<'PY'
|
||||
import datetime
|
||||
import json
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
output, mode, started, recovery_started, recovered, duration = sys.argv[1:]
|
||||
parse = lambda value: datetime.datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||
report = {
|
||||
"schema_version": 1,
|
||||
"incident": "worker_unavailable",
|
||||
"profile": mode,
|
||||
"started_at": started,
|
||||
"recovery_started_at": recovery_started,
|
||||
"recovered_at": recovered,
|
||||
"planned_injection_seconds": int(duration),
|
||||
"injected_duration_seconds": int((parse(recovery_started) - parse(started)).total_seconds()),
|
||||
"recovery_observation_seconds": int((parse(recovered) - parse(recovery_started)).total_seconds()),
|
||||
"artifacts": ["baseline.json", "degraded.json", "recovered.json", "traffic.json", "alert-firing.json", "alert-resolved.json"],
|
||||
}
|
||||
if mode == "long":
|
||||
report["artifacts"].extend(["burn-fast-firing.json", "burn-slow-firing.json"])
|
||||
pathlib.Path(output).write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
PY
|
||||
trap - EXIT INT TERM
|
||||
printf '%s\n' "$output_dir"
|
||||
Executable
+67
@@ -0,0 +1,67 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
|
||||
def fetch(prometheus_url):
|
||||
url = prometheus_url.rstrip("/") + "/api/v1/alerts"
|
||||
with urllib.request.urlopen(url, timeout=10) as response:
|
||||
payload = json.loads(response.read(4 << 20).decode("utf-8"))
|
||||
return payload.get("data", {}).get("alerts", [])
|
||||
|
||||
|
||||
def matching(alerts, name):
|
||||
return [alert for alert in alerts if alert.get("labels", {}).get("alertname") == name]
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Wait for a Prometheus alert transition")
|
||||
parser.add_argument("--prometheus-url", default="http://127.0.0.1:9090")
|
||||
parser.add_argument("--name", required=True)
|
||||
parser.add_argument("--state", choices=("firing", "resolved"), required=True)
|
||||
parser.add_argument("--timeout", type=float, default=120)
|
||||
parser.add_argument("--output", required=True)
|
||||
args = parser.parse_args()
|
||||
|
||||
deadline = time.monotonic() + args.timeout
|
||||
last = []
|
||||
error = None
|
||||
observed = False
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
last = matching(fetch(args.prometheus_url), args.name)
|
||||
error = None
|
||||
if args.state == "firing" and any(item.get("state") == "firing" for item in last):
|
||||
observed = True
|
||||
break
|
||||
if args.state == "resolved" and not any(item.get("state") in ("pending", "firing") for item in last):
|
||||
observed = True
|
||||
break
|
||||
except (urllib.error.URLError, json.JSONDecodeError) as current_error:
|
||||
error = str(current_error)
|
||||
time.sleep(2)
|
||||
|
||||
report = {
|
||||
"schema_version": 1,
|
||||
"alert": args.name,
|
||||
"expected_state": args.state,
|
||||
"observed": observed,
|
||||
"observed_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
||||
"last_states": sorted({item.get("state", "") for item in last}),
|
||||
"error": error,
|
||||
}
|
||||
destination = Path(args.output)
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
destination.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
if not observed:
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+148
@@ -0,0 +1,148 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
|
||||
TRACE_ID = re.compile(r'(?i)\b[0-9a-f]{32}\b')
|
||||
SERVICES = ("messenger-api", "messenger-worker", "messenger-dispatcher")
|
||||
|
||||
|
||||
def get_json(url):
|
||||
with urllib.request.urlopen(url, timeout=10) as response:
|
||||
return json.loads(response.read(8 << 20).decode("utf-8"))
|
||||
|
||||
|
||||
def tempo_query(base_url, query):
|
||||
url = base_url.rstrip("/") + "/api/search?" + urllib.parse.urlencode({"q": query, "limit": 100})
|
||||
payload = get_json(url)
|
||||
return {item.get("traceID") for item in payload.get("traces", []) if item.get("traceID")}
|
||||
|
||||
|
||||
def extract_trace_ids(payload):
|
||||
result = set()
|
||||
for stream in payload.get("data", {}).get("result", []):
|
||||
for value in stream.get("values", []):
|
||||
candidates = [stream.get("stream", {}), value[1] if len(value) > 1 else ""]
|
||||
if len(value) > 2:
|
||||
candidates.append(value[2])
|
||||
for candidate in candidates:
|
||||
if isinstance(candidate, dict):
|
||||
trace_id = candidate.get("trace_id") or candidate.get("traceid")
|
||||
if trace_id:
|
||||
result.add(str(trace_id))
|
||||
candidate = json.dumps(candidate)
|
||||
result.update(match.group(0).lower() for match in TRACE_ID.finditer(str(candidate)))
|
||||
return result
|
||||
|
||||
|
||||
def loki_logs(base_url, service, start_ns):
|
||||
query = '{service_name="' + service + '"}'
|
||||
url = base_url.rstrip("/") + "/loki/api/v1/query_range?" + urllib.parse.urlencode({
|
||||
"query": query,
|
||||
"start": str(start_ns),
|
||||
"limit": 5000,
|
||||
"direction": "backward",
|
||||
})
|
||||
payload = get_json(url)
|
||||
count = sum(len(stream.get("values", [])) for stream in payload.get("data", {}).get("result", []))
|
||||
return count, extract_trace_ids(payload)
|
||||
|
||||
|
||||
def capture(tempo_url, loki_url, lookback_seconds):
|
||||
start_ns = int((time.time() - lookback_seconds) * 1_000_000_000)
|
||||
traces = {service: tempo_query(tempo_url, '{ resource.service.name = "' + service + '" }') for service in SERVICES}
|
||||
logs = {}
|
||||
log_trace_ids = {}
|
||||
for service in SERVICES:
|
||||
count, ids = loki_logs(loki_url, service, start_ns)
|
||||
logs[service] = count
|
||||
log_trace_ids[service] = ids
|
||||
api_worker = traces["messenger-api"] & traces["messenger-worker"]
|
||||
all_trace_ids = set().union(*traces.values())
|
||||
all_log_trace_ids = set().union(*log_trace_ids.values())
|
||||
return {
|
||||
"traces_by_service": {service: len(ids) for service, ids in traces.items()},
|
||||
"logs_by_service": logs,
|
||||
"api_worker_shared_traces": len(api_worker),
|
||||
"traces_with_correlated_logs": len(all_trace_ids & all_log_trace_ids),
|
||||
}
|
||||
|
||||
|
||||
def capture_database(tempo_url, loki_url, lookback_seconds):
|
||||
start_ns = int((time.time() - lookback_seconds) * 1_000_000_000)
|
||||
traces = tempo_query(
|
||||
tempo_url,
|
||||
'{ resource.service.name = "messenger-api" && span.db.system = "postgresql" && status = error }',
|
||||
)
|
||||
log_count, log_trace_ids = loki_logs(loki_url, "messenger-api", start_ns)
|
||||
return {
|
||||
"database_error_traces": len(traces),
|
||||
"api_logs": log_count,
|
||||
"database_error_traces_with_logs": len(traces & log_trace_ids),
|
||||
}
|
||||
|
||||
|
||||
def complete(report):
|
||||
return (
|
||||
report["traces_by_service"]["messenger-api"] > 0
|
||||
and report["traces_by_service"]["messenger-worker"] > 0
|
||||
and report["logs_by_service"]["messenger-api"] > 0
|
||||
and report["logs_by_service"]["messenger-worker"] > 0
|
||||
and report["api_worker_shared_traces"] > 0
|
||||
and report["traces_with_correlated_logs"] > 0
|
||||
)
|
||||
|
||||
|
||||
def database_complete(report):
|
||||
return report["database_error_traces"] > 0 and report["database_error_traces_with_logs"] > 0
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Verify trace and log correlation in Tempo and Loki")
|
||||
parser.add_argument("--tempo-url", default="http://127.0.0.1:3200")
|
||||
parser.add_argument("--loki-url", default="http://127.0.0.1:3100")
|
||||
parser.add_argument("--lookback", type=float, default=900)
|
||||
parser.add_argument("--wait", type=float, default=45)
|
||||
parser.add_argument("--output", required=True)
|
||||
parser.add_argument("--require", action="store_true")
|
||||
parser.add_argument("--mode", choices=("baseline", "database"), default="baseline")
|
||||
args = parser.parse_args()
|
||||
|
||||
deadline = time.monotonic() + args.wait
|
||||
error = None
|
||||
report = None
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
report = capture_database(args.tempo_url, args.loki_url, args.lookback) if args.mode == "database" else capture(args.tempo_url, args.loki_url, args.lookback)
|
||||
error = None
|
||||
if database_complete(report) if args.mode == "database" else complete(report):
|
||||
break
|
||||
except (urllib.error.URLError, json.JSONDecodeError, KeyError) as current_error:
|
||||
error = str(current_error)
|
||||
time.sleep(2)
|
||||
|
||||
result = {
|
||||
"schema_version": 1,
|
||||
"captured_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
||||
"lookback_seconds": args.lookback,
|
||||
"signals": report,
|
||||
"mode": args.mode,
|
||||
"complete": bool(report and (database_complete(report) if args.mode == "database" else complete(report))),
|
||||
"error": error,
|
||||
}
|
||||
destination = Path(args.output)
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
destination.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
print(json.dumps(result, ensure_ascii=False, indent=2))
|
||||
if args.require and not result["complete"]:
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,91 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
|
||||
QUERIES = {
|
||||
"api_error_ratio_1m": 'sum(rate(messenger_http_requests_total{status_class="5xx",route!~"/health/.*"}[1m])) / clamp_min(sum(rate(messenger_http_requests_total{route!~"/health/.*"}[1m])), 1)',
|
||||
"api_request_rate_1m": 'sum(rate(messenger_http_requests_total{route!~"/health/.*"}[1m]))',
|
||||
"queued_jobs": 'sum(messenger_attachment_jobs_count{status="queued"})',
|
||||
"processing_jobs": 'sum(messenger_attachment_jobs_count{status="processing"})',
|
||||
"oldest_queued_job_seconds": 'max(messenger_attachment_jobs_oldest_age_seconds{status="queued"})',
|
||||
"worker_jobs_in_flight": 'sum(messenger_attachment_worker_jobs_in_flight)',
|
||||
"worker_capacity": 'sum(messenger_attachment_worker_capacity)',
|
||||
"pending_outbox": 'sum(messenger_outbox_pending)',
|
||||
"worker_up": 'min(up{job="attachment-worker"})',
|
||||
"collector_queue_size": 'max by (exporter) (otelcol_exporter_queue_size)',
|
||||
"collector_queue_capacity": 'max by (exporter) (otelcol_exporter_queue_capacity)',
|
||||
"collector_export_failures_total": 'sum by (exporter) (otelcol_exporter_send_failed_spans) or sum by (exporter) (otelcol_exporter_send_failed_log_records) or on (exporter) (0 * max by (exporter) (otelcol_exporter_queue_capacity))',
|
||||
"collector_dropped_records_total": 'sum by (exporter) (otelcol_exporter_enqueue_failed_spans) or sum by (exporter) (otelcol_exporter_enqueue_failed_log_records) or on (exporter) (0 * max by (exporter) (otelcol_exporter_queue_capacity))',
|
||||
"collector_memory_bytes": 'max(otelcol_process_memory_rss) or max(process_resident_memory_bytes{job="otel-collector"})',
|
||||
"collector_memory_peak_bytes": 'max(max_over_time(otelcol_process_memory_rss[2m])) or max(max_over_time(process_resident_memory_bytes{job="otel-collector"}[2m]))',
|
||||
}
|
||||
|
||||
|
||||
def get_json(url):
|
||||
try:
|
||||
with urllib.request.urlopen(url, timeout=10) as response:
|
||||
return json.loads(response.read(4 << 20).decode("utf-8")), None
|
||||
except (urllib.error.URLError, json.JSONDecodeError) as error:
|
||||
return None, str(error)
|
||||
|
||||
|
||||
def query(prometheus_url, expression):
|
||||
url = prometheus_url.rstrip("/") + "/api/v1/query?" + urllib.parse.urlencode({"query": expression})
|
||||
payload, error = get_json(url)
|
||||
if error:
|
||||
return {"error": error}
|
||||
if payload.get("status") != "success":
|
||||
return {"error": payload.get("error", "query failed")}
|
||||
rows = payload.get("data", {}).get("result", [])
|
||||
return {
|
||||
"series": [
|
||||
{"labels": row.get("metric", {}), "value": row.get("value", [None, None])[1]}
|
||||
for row in rows
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
def alerts(prometheus_url):
|
||||
payload, error = get_json(prometheus_url.rstrip("/") + "/api/v1/alerts")
|
||||
if error:
|
||||
return {"error": error, "items": []}
|
||||
items = []
|
||||
for alert in payload.get("data", {}).get("alerts", []):
|
||||
labels = alert.get("labels", {})
|
||||
items.append({
|
||||
"name": labels.get("alertname", ""),
|
||||
"state": alert.get("state", ""),
|
||||
"severity": labels.get("severity", ""),
|
||||
"active_at": alert.get("activeAt", ""),
|
||||
})
|
||||
return {"items": sorted(items, key=lambda item: (item["name"], item["state"]))}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Capture a bounded Prometheus snapshot for Lab 6 evidence")
|
||||
parser.add_argument("--prometheus-url", default="http://127.0.0.1:9090")
|
||||
parser.add_argument("--phase", required=True)
|
||||
parser.add_argument("--output", required=True)
|
||||
args = parser.parse_args()
|
||||
|
||||
report = {
|
||||
"schema_version": 1,
|
||||
"captured_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
||||
"phase": args.phase,
|
||||
"queries": {name: query(args.prometheus_url, expression) for name, expression in QUERIES.items()},
|
||||
"alerts": alerts(args.prometheus_url),
|
||||
}
|
||||
destination = Path(args.output)
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
destination.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+261
@@ -0,0 +1,261 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import json
|
||||
import math
|
||||
from pathlib import Path
|
||||
import secrets
|
||||
import struct
|
||||
import threading
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
import zlib
|
||||
|
||||
MAX_RESPONSE_BYTES = 1 << 20
|
||||
|
||||
|
||||
def png_fixture(width=96, height=64):
|
||||
rows = []
|
||||
for y in range(height):
|
||||
pixels = bytearray()
|
||||
for x in range(width):
|
||||
pixels.extend(((x * 3) % 256, (y * 5) % 256, (x + y) % 256))
|
||||
rows.append(b"\x00" + bytes(pixels))
|
||||
raw = b"".join(rows)
|
||||
|
||||
def chunk(name, payload):
|
||||
return struct.pack(">I", len(payload)) + name + payload + struct.pack(">I", zlib.crc32(name + payload) & 0xFFFFFFFF)
|
||||
|
||||
header = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)
|
||||
return b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", header) + chunk(b"IDAT", zlib.compress(raw, 6)) + chunk(b"IEND", b"")
|
||||
|
||||
|
||||
def request(method, url, body=None, headers=None, timeout=10):
|
||||
req = urllib.request.Request(url, data=body, method=method, headers=headers or {})
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=timeout) as response:
|
||||
payload = response.read(MAX_RESPONSE_BYTES)
|
||||
return response.status, payload
|
||||
except urllib.error.HTTPError as error:
|
||||
return error.code, error.read(MAX_RESPONSE_BYTES)
|
||||
except urllib.error.URLError:
|
||||
return 0, b""
|
||||
|
||||
|
||||
def decode_json(payload):
|
||||
return json.loads(payload.decode("utf-8"))
|
||||
|
||||
|
||||
def json_request(method, url, value=None, token=None, headers=None):
|
||||
request_headers = {"Accept": "application/json"}
|
||||
body = None
|
||||
if value is not None:
|
||||
body = json.dumps(value, separators=(",", ":")).encode("utf-8")
|
||||
request_headers["Content-Type"] = "application/json"
|
||||
if token:
|
||||
request_headers["Authorization"] = "Bearer " + token
|
||||
if headers:
|
||||
request_headers.update(headers)
|
||||
status, payload = request(method, url, body, request_headers)
|
||||
return status, decode_json(payload) if payload else {}
|
||||
|
||||
|
||||
def bootstrap(base_url):
|
||||
suffix = secrets.token_hex(6)
|
||||
password = "Lab6-" + secrets.token_urlsafe(18)
|
||||
users = []
|
||||
for prefix in ("telemetry_a_", "telemetry_b_"):
|
||||
username = prefix + suffix
|
||||
status, user = json_request("POST", base_url + "/api/v1/users", {
|
||||
"username": username,
|
||||
"display_name": "Synthetic telemetry load",
|
||||
"password": password,
|
||||
})
|
||||
if status != 201:
|
||||
raise RuntimeError("register user returned HTTP %d" % status)
|
||||
users.append((username, user["id"]))
|
||||
|
||||
credentials = base64.b64encode((users[0][0] + ":" + password).encode("utf-8")).decode("ascii")
|
||||
status, payload = request("POST", base_url + "/api/v1/auth/sessions", headers={
|
||||
"Authorization": "Basic " + credentials,
|
||||
"Accept": "application/json",
|
||||
})
|
||||
if status != 201:
|
||||
raise RuntimeError("create session returned HTTP %d" % status)
|
||||
token = decode_json(payload)["token"]
|
||||
|
||||
status, chat = json_request("POST", base_url + "/api/v1/chats", {
|
||||
"title": "Lab 6 synthetic telemetry",
|
||||
"member_ids": [users[0][1], users[1][1]],
|
||||
}, token)
|
||||
if status != 201:
|
||||
raise RuntimeError("create chat returned HTTP %d" % status)
|
||||
return token, chat["id"]
|
||||
|
||||
|
||||
class Results:
|
||||
def __init__(self):
|
||||
self.lock = threading.Lock()
|
||||
self.latencies = []
|
||||
self.statuses = {}
|
||||
self.attachment_ids = []
|
||||
|
||||
def record(self, status, elapsed, attachment_id=None):
|
||||
with self.lock:
|
||||
self.latencies.append(elapsed)
|
||||
key = str(status)
|
||||
self.statuses[key] = self.statuses.get(key, 0) + 1
|
||||
if attachment_id:
|
||||
self.attachment_ids.append(attachment_id)
|
||||
|
||||
|
||||
def percentile(values, quantile):
|
||||
if not values:
|
||||
return None
|
||||
ordered = sorted(values)
|
||||
index = max(0, math.ceil(quantile * len(ordered)) - 1)
|
||||
return round(ordered[index] * 1000, 3)
|
||||
|
||||
|
||||
def run_request(base_url, token, chat_id, mode, sequence, fixture):
|
||||
if mode == "mixed":
|
||||
mode = "attachments" if sequence % 3 == 0 else "messages"
|
||||
started = time.monotonic()
|
||||
if mode == "messages":
|
||||
status, _ = json_request("POST", base_url + "/api/v1/chats/" + chat_id + "/messages", {
|
||||
"text": "synthetic telemetry event",
|
||||
"attachment_ids": [],
|
||||
}, token, {"Idempotency-Key": secrets.token_hex(16)})
|
||||
return status, time.monotonic() - started, None
|
||||
|
||||
upload_url = base_url + "/api/v1/chats/" + chat_id + "/attachments?" + urllib.parse.urlencode({
|
||||
"filename": "synthetic.png",
|
||||
})
|
||||
status, payload = request("POST", upload_url, fixture, {
|
||||
"Authorization": "Bearer " + token,
|
||||
"Content-Type": "image/png",
|
||||
"Accept": "application/json",
|
||||
})
|
||||
attachment_id = None
|
||||
if status == 202:
|
||||
attachment_id = decode_json(payload).get("id")
|
||||
return status, time.monotonic() - started, attachment_id
|
||||
|
||||
|
||||
def wait_for_attachments(base_url, token, attachment_ids, timeout):
|
||||
remaining = set(attachment_ids)
|
||||
states = {"ready": 0, "failed": 0, "timeout": 0}
|
||||
deadline = time.monotonic() + timeout
|
||||
while remaining and time.monotonic() < deadline:
|
||||
for attachment_id in list(remaining):
|
||||
status, metadata = json_request("GET", base_url + "/api/v1/attachments/" + attachment_id, token=token)
|
||||
if status != 200:
|
||||
continue
|
||||
state = metadata.get("status")
|
||||
if state in ("ready", "failed"):
|
||||
states[state] += 1
|
||||
remaining.remove(attachment_id)
|
||||
if remaining:
|
||||
time.sleep(0.25)
|
||||
states["timeout"] = len(remaining)
|
||||
return states
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Bounded synthetic load for Lab 6 telemetry and incidents")
|
||||
parser.add_argument("--base-url", default="http://127.0.0.1:8080")
|
||||
parser.add_argument("--mode", choices=("messages", "attachments", "mixed"), default="mixed")
|
||||
parser.add_argument("--duration", type=float, default=30)
|
||||
parser.add_argument("--requests", type=int, default=0, help="Maximum requests; zero means duration only")
|
||||
parser.add_argument("--concurrency", type=int, default=4)
|
||||
parser.add_argument("--rate", type=float, default=0, help="Approximate aggregate request rate; zero is unthrottled")
|
||||
parser.add_argument("--wait-ready", action="store_true")
|
||||
parser.add_argument("--ready-timeout", type=float, default=120)
|
||||
parser.add_argument("--output")
|
||||
parser.add_argument("--started-file", help="Touch after synthetic session and chat are ready")
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.duration <= 0 or args.duration > 1800:
|
||||
parser.error("duration must be in (0, 1800]")
|
||||
if args.requests < 0:
|
||||
parser.error("requests must be non-negative")
|
||||
if args.concurrency < 1 or args.concurrency > 64:
|
||||
parser.error("concurrency must be in [1, 64]")
|
||||
if args.rate < 0 or args.rate > 1000:
|
||||
parser.error("rate must be in [0, 1000]")
|
||||
|
||||
base_url = args.base_url.rstrip("/")
|
||||
token, chat_id = bootstrap(base_url)
|
||||
if args.started_file:
|
||||
started_file = Path(args.started_file)
|
||||
started_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
started_file.touch()
|
||||
fixture = png_fixture()
|
||||
results = Results()
|
||||
next_sequence = 0
|
||||
sequence_lock = threading.Lock()
|
||||
deadline = time.monotonic() + args.duration
|
||||
|
||||
def worker():
|
||||
nonlocal next_sequence
|
||||
while time.monotonic() < deadline:
|
||||
with sequence_lock:
|
||||
if args.requests and next_sequence >= args.requests:
|
||||
return
|
||||
sequence = next_sequence
|
||||
next_sequence += 1
|
||||
status, elapsed, attachment_id = run_request(base_url, token, chat_id, args.mode, sequence, fixture)
|
||||
results.record(status, elapsed, attachment_id)
|
||||
if args.rate:
|
||||
delay = args.concurrency / args.rate - elapsed
|
||||
if delay > 0:
|
||||
time.sleep(delay)
|
||||
|
||||
started = time.monotonic()
|
||||
with ThreadPoolExecutor(max_workers=args.concurrency) as pool:
|
||||
futures = [pool.submit(worker) for _ in range(args.concurrency)]
|
||||
for future in futures:
|
||||
future.result()
|
||||
elapsed = time.monotonic() - started
|
||||
|
||||
attachment_states = None
|
||||
if args.wait_ready and results.attachment_ids:
|
||||
attachment_states = wait_for_attachments(base_url, token, results.attachment_ids, args.ready_timeout)
|
||||
|
||||
total = len(results.latencies)
|
||||
successes = sum(count for status, count in results.statuses.items() if status.startswith("2"))
|
||||
report = {
|
||||
"schema_version": 1,
|
||||
"workload": args.mode,
|
||||
"started_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(time.time() - elapsed)),
|
||||
"duration_seconds": round(elapsed, 3),
|
||||
"concurrency": args.concurrency,
|
||||
"requested_rate": args.rate or None,
|
||||
"requests": total,
|
||||
"successful_requests": successes,
|
||||
"request_rate": round(total / elapsed, 3) if elapsed else None,
|
||||
"status_counts": dict(sorted(results.statuses.items())),
|
||||
"latency_ms": {
|
||||
"p50": percentile(results.latencies, 0.50),
|
||||
"p95": percentile(results.latencies, 0.95),
|
||||
"p99": percentile(results.latencies, 0.99),
|
||||
},
|
||||
"attachments_accepted": len(results.attachment_ids),
|
||||
"attachment_terminal_states": attachment_states,
|
||||
}
|
||||
encoded = json.dumps(report, ensure_ascii=False, indent=2) + "\n"
|
||||
if args.output:
|
||||
destination = Path(args.output)
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
destination.write_text(encoded, encoding="utf-8")
|
||||
print(encoded, end="")
|
||||
if not total:
|
||||
raise SystemExit("no requests were sent")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+73
@@ -0,0 +1,73 @@
|
||||
#!/bin/sh
|
||||
|
||||
set -eu
|
||||
|
||||
profile=${1:-full}
|
||||
case "$profile" in
|
||||
smoke|full) ;;
|
||||
*)
|
||||
echo "usage: $0 [smoke|full]" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
|
||||
env_file=${ENV_FILE:-lab6.env.example}
|
||||
base_url=${BASE_URL:-http://127.0.0.1:8080}
|
||||
prometheus_url=${PROMETHEUS_URL:-http://127.0.0.1:9090}
|
||||
tempo_url=${TEMPO_URL:-http://127.0.0.1:3200}
|
||||
loki_url=${LOKI_URL:-http://127.0.0.1:3100}
|
||||
run_id=$(date -u +%Y%m%dT%H%M%SZ)
|
||||
output_dir=${EVIDENCE_DIR:-evidence/runtime/lab6-$run_id}
|
||||
mkdir -p "$output_dir"
|
||||
|
||||
compose() {
|
||||
docker compose --env-file "$env_file" "$@"
|
||||
}
|
||||
|
||||
restore() {
|
||||
compose up -d postgres redis-broker s3 valkey api nginx dispatcher worker otel-collector prometheus loki tempo grafana >/dev/null 2>&1 || true
|
||||
}
|
||||
|
||||
trap restore EXIT INT TERM
|
||||
|
||||
test -f "$env_file"
|
||||
compose config --quiet
|
||||
compose up -d --build --wait --wait-timeout 180
|
||||
python3 scripts/lab6-traffic.py \
|
||||
--base-url "$base_url" \
|
||||
--mode mixed \
|
||||
--duration "${BASELINE_LOAD_SECONDS:-15}" \
|
||||
--requests "${BASELINE_REQUESTS:-120}" \
|
||||
--concurrency 4 \
|
||||
--rate "${BASELINE_LOAD_RATE:-10}" \
|
||||
--wait-ready \
|
||||
--ready-timeout 120 \
|
||||
--output "$output_dir/baseline-traffic.json" >/dev/null
|
||||
python3 scripts/lab6-observe.py \
|
||||
--prometheus-url "$prometheus_url" \
|
||||
--phase baseline \
|
||||
--output "$output_dir/baseline-observability.json" >/dev/null
|
||||
python3 scripts/lab6-correlation.py \
|
||||
--tempo-url "$tempo_url" \
|
||||
--loki-url "$loki_url" \
|
||||
--output "$output_dir/baseline-correlation.json" \
|
||||
--require >/dev/null
|
||||
|
||||
if [ "$profile" = "full" ]; then
|
||||
ENV_FILE="$env_file" BASE_URL="$base_url" PROMETHEUS_URL="$prometheus_url" TEMPO_URL="$tempo_url" LOKI_URL="$loki_url" EVIDENCE_DIR="$output_dir/incidents" scripts/incident-database.sh
|
||||
ENV_FILE="$env_file" BASE_URL="$base_url" PROMETHEUS_URL="$prometheus_url" EVIDENCE_DIR="$output_dir/incidents" scripts/incident-worker.sh long
|
||||
ENV_FILE="$env_file" BASE_URL="$base_url" PROMETHEUS_URL="$prometheus_url" EVIDENCE_DIR="$output_dir/incidents" scripts/incident-worker.sh short
|
||||
ENV_FILE="$env_file" BASE_URL="$base_url" PROMETHEUS_URL="$prometheus_url" EVIDENCE_DIR="$output_dir/incidents" scripts/incident-telemetry.sh
|
||||
fi
|
||||
|
||||
arguments=
|
||||
if [ "$profile" = "full" ]; then
|
||||
arguments="--require-incidents"
|
||||
if [ "${REQUIRE_ALERT_TRANSITIONS:-1}" = "1" ]; then
|
||||
arguments="$arguments --require-alerts"
|
||||
fi
|
||||
fi
|
||||
# shellcheck disable=SC2086
|
||||
python3 scripts/validate-lab6-evidence.py "$output_dir" $arguments
|
||||
trap - EXIT INT TERM
|
||||
printf '%s\n' "$output_dir"
|
||||
Executable
+202
@@ -0,0 +1,202 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import math
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def load(path):
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def check(condition, message, errors):
|
||||
if not condition:
|
||||
errors.append(message)
|
||||
|
||||
|
||||
def query_scalar(snapshot, name, source, errors):
|
||||
query = snapshot.get("queries", {}).get(name, {})
|
||||
check("error" not in query, source + " query failed: " + name, errors)
|
||||
series = query.get("series", [])
|
||||
check(len(series) == 1, source + " query must return one series: " + name, errors)
|
||||
if len(series) != 1:
|
||||
return None
|
||||
try:
|
||||
value = float(series[0]["value"])
|
||||
except (KeyError, TypeError, ValueError):
|
||||
check(False, source + " query returned a non-numeric value: " + name, errors)
|
||||
return None
|
||||
check(math.isfinite(value), source + " query returned a non-finite value: " + name, errors)
|
||||
return value if math.isfinite(value) else None
|
||||
|
||||
|
||||
def query_exporters(snapshot, name, source, errors):
|
||||
expected = {"otlp_grpc/tempo", "otlp_http/loki"}
|
||||
query = snapshot.get("queries", {}).get(name, {})
|
||||
check("error" not in query, source + " query failed: " + name, errors)
|
||||
values = {}
|
||||
for series in query.get("series", []):
|
||||
exporter = series.get("labels", {}).get("exporter")
|
||||
check(exporter in expected, source + " query returned unexpected exporter: " + name, errors)
|
||||
check(exporter not in values, source + " query returned duplicate exporter: " + name, errors)
|
||||
try:
|
||||
value = float(series["value"])
|
||||
except (KeyError, TypeError, ValueError):
|
||||
check(False, source + " query returned a non-numeric value: " + name, errors)
|
||||
continue
|
||||
check(math.isfinite(value), source + " query returned a non-finite value: " + name, errors)
|
||||
if exporter in expected and exporter not in values and math.isfinite(value):
|
||||
values[exporter] = value
|
||||
check(set(values) == expected, source + " query must return the exact exporter set: " + name, errors)
|
||||
return values if set(values) == expected else None
|
||||
|
||||
|
||||
def validate_telemetry_outage(directory, errors):
|
||||
phase_names = ("baseline", "degraded", "recovered")
|
||||
exporter_metric_names = (
|
||||
"collector_queue_size",
|
||||
"collector_queue_capacity",
|
||||
"collector_export_failures_total",
|
||||
"collector_dropped_records_total",
|
||||
)
|
||||
scalar_metric_names = (
|
||||
"collector_memory_bytes",
|
||||
"collector_memory_peak_bytes",
|
||||
)
|
||||
phases = {}
|
||||
for phase in phase_names:
|
||||
path = directory / (phase + ".json")
|
||||
check(path.exists(), str(path) + " is missing", errors)
|
||||
if not path.exists():
|
||||
return
|
||||
snapshot = load(path)
|
||||
phases[phase] = {
|
||||
**{name: query_exporters(snapshot, name, path.name, errors) for name in exporter_metric_names},
|
||||
**{name: query_scalar(snapshot, name, path.name, errors) for name in scalar_metric_names},
|
||||
}
|
||||
if any(value is None for values in phases.values() for value in values.values()):
|
||||
return
|
||||
|
||||
for phase, values in phases.items():
|
||||
for exporter in ("otlp_grpc/tempo", "otlp_http/loki"):
|
||||
capacity = values["collector_queue_capacity"][exporter]
|
||||
size = values["collector_queue_size"][exporter]
|
||||
check(capacity > 0, phase + " collector queue capacity is not positive: " + exporter, errors)
|
||||
check(0 <= size <= capacity, phase + " collector queue size is outside capacity: " + exporter, errors)
|
||||
check(values["collector_memory_bytes"] > 0, phase + " collector memory is not positive", errors)
|
||||
check(values["collector_memory_peak_bytes"] > 0, phase + " collector peak memory is not positive", errors)
|
||||
|
||||
baseline = phases["baseline"]
|
||||
degraded = phases["degraded"]
|
||||
recovered = phases["recovered"]
|
||||
for exporter in ("otlp_grpc/tempo", "otlp_http/loki"):
|
||||
for name in ("collector_export_failures_total", "collector_dropped_records_total"):
|
||||
check(degraded[name][exporter] >= baseline[name][exporter], name + " decreased during outage: " + exporter, errors)
|
||||
check(recovered[name][exporter] >= degraded[name][exporter], name + " decreased after recovery: " + exporter, errors)
|
||||
outage_observed = (
|
||||
degraded["collector_queue_size"][exporter] > baseline["collector_queue_size"][exporter]
|
||||
or recovered["collector_export_failures_total"][exporter] > baseline["collector_export_failures_total"][exporter]
|
||||
or recovered["collector_dropped_records_total"][exporter] > baseline["collector_dropped_records_total"][exporter]
|
||||
)
|
||||
check(outage_observed, "telemetry outage was not observed for exporter: " + exporter, errors)
|
||||
drained_limit = max(
|
||||
baseline["collector_queue_size"][exporter],
|
||||
recovered["collector_queue_capacity"][exporter] * 0.05,
|
||||
)
|
||||
check(
|
||||
recovered["collector_queue_size"][exporter] <= drained_limit,
|
||||
"collector queue did not drain after recovery: " + exporter,
|
||||
errors,
|
||||
)
|
||||
memory_limit = max(baseline["collector_memory_peak_bytes"] * 1.5, baseline["collector_memory_peak_bytes"] + 64 * 1024 * 1024)
|
||||
check(degraded["collector_memory_peak_bytes"] <= memory_limit, "collector peak memory grew beyond the outage bound", errors)
|
||||
check(recovered["collector_memory_peak_bytes"] <= memory_limit, "collector peak memory remained above the recovery bound", errors)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Validate generated Lab 6 incident evidence")
|
||||
parser.add_argument("directory")
|
||||
parser.add_argument("--require-incidents", action="store_true")
|
||||
parser.add_argument("--require-alerts", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
root = Path(args.directory)
|
||||
errors = []
|
||||
baseline = root / "baseline-traffic.json"
|
||||
check(baseline.exists(), "baseline-traffic.json is missing", errors)
|
||||
if baseline.exists():
|
||||
report = load(baseline)
|
||||
check(report.get("requests", 0) > 0, "baseline sent no requests", errors)
|
||||
check(report.get("successful_requests", 0) > 0, "baseline had no successful requests", errors)
|
||||
|
||||
correlation = root / "baseline-correlation.json"
|
||||
check(correlation.exists(), "baseline-correlation.json is missing", errors)
|
||||
if correlation.exists():
|
||||
check(load(correlation).get("complete") is True, "metrics/log/trace correlation is incomplete", errors)
|
||||
|
||||
observability = root / "baseline-observability.json"
|
||||
check(observability.exists(), "baseline-observability.json is missing", errors)
|
||||
if observability.exists():
|
||||
firing = [item.get("name") for item in load(observability).get("alerts", {}).get("items", []) if item.get("state") == "firing"]
|
||||
check(not firing, "baseline has firing alerts: " + ", ".join(firing), errors)
|
||||
|
||||
summaries = list(root.rglob("summary.json"))
|
||||
if args.require_incidents:
|
||||
names = {load(path).get("incident") for path in summaries}
|
||||
for required in ("worker_unavailable", "postgres_unavailable", "telemetry_backends_unavailable"):
|
||||
check(required in names, "incident evidence is missing: " + required, errors)
|
||||
|
||||
for summary_path in summaries:
|
||||
summary = load(summary_path)
|
||||
incident = summary.get("incident")
|
||||
traffic_path = summary_path.parent / "traffic.json"
|
||||
check(traffic_path.exists(), str(traffic_path) + " is missing", errors)
|
||||
if not traffic_path.exists():
|
||||
continue
|
||||
traffic = load(traffic_path)
|
||||
check(traffic.get("requests", 0) > 0, incident + " sent no traffic", errors)
|
||||
if incident == "worker_unavailable":
|
||||
terminal = traffic.get("attachment_terminal_states") or {}
|
||||
check(traffic.get("attachments_accepted", 0) > 0, incident + " accepted no attachments", errors)
|
||||
check(terminal.get("ready", 0) == traffic.get("attachments_accepted", -1), incident + " did not drain every accepted attachment", errors)
|
||||
check(terminal.get("failed", 0) == 0, incident + " produced failed attachments", errors)
|
||||
check(terminal.get("timeout", 0) == 0, incident + " left attachment jobs non-terminal", errors)
|
||||
elif incident == "postgres_unavailable":
|
||||
statuses = traffic.get("status_counts", {})
|
||||
failures = sum(count for status, count in statuses.items() if not status.startswith("2"))
|
||||
check(failures > 0, incident + " produced no observable failed requests", errors)
|
||||
correlation_path = summary_path.parent / "database-correlation.json"
|
||||
check(correlation_path.exists(), str(correlation_path) + " is missing", errors)
|
||||
if correlation_path.exists():
|
||||
check(load(correlation_path).get("complete") is True, "database error trace/log correlation is incomplete", errors)
|
||||
elif incident == "telemetry_backends_unavailable":
|
||||
terminal = traffic.get("attachment_terminal_states") or {}
|
||||
check(traffic.get("successful_requests", 0) > 0, incident + " had no successful business requests", errors)
|
||||
check(traffic.get("attachments_accepted", 0) > 0, incident + " accepted no attachments", errors)
|
||||
check(terminal.get("ready", 0) == traffic.get("attachments_accepted", -1), incident + " did not finish every attachment", errors)
|
||||
check(terminal.get("failed", 0) == 0 and terminal.get("timeout", 0) == 0, incident + " left unsuccessful attachments", errors)
|
||||
check(summary.get("api_ready_http_status_during_outage") == 200, incident + " broke API readiness", errors)
|
||||
validate_telemetry_outage(summary_path.parent, errors)
|
||||
|
||||
if args.require_alerts and incident in ("worker_unavailable", "postgres_unavailable"):
|
||||
for name in ("alert-firing.json", "alert-resolved.json"):
|
||||
path = summary_path.parent / name
|
||||
check(path.exists(), str(path) + " is missing", errors)
|
||||
if path.exists():
|
||||
check(load(path).get("observed") is True, str(path) + " did not observe the transition", errors)
|
||||
if incident == "worker_unavailable" and summary.get("profile") == "long":
|
||||
for name in ("burn-fast-firing.json", "burn-slow-firing.json"):
|
||||
path = summary_path.parent / name
|
||||
check(path.exists(), str(path) + " is missing", errors)
|
||||
if path.exists():
|
||||
check(load(path).get("observed") is True, str(path) + " did not observe attachment burn", errors)
|
||||
|
||||
result = {"valid": not errors, "errors": errors, "incident_summaries": len(summaries)}
|
||||
print(json.dumps(result, ensure_ascii=False, indent=2))
|
||||
if errors:
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user