ЛР 6 — БВТ2303 — Вейнмайер — на 5 #63
@@ -18,3 +18,5 @@ __pycache__
|
||||
node_modules
|
||||
evidence
|
||||
.DS_Store
|
||||
bin
|
||||
experiments
|
||||
|
||||
+26
-2
@@ -7,7 +7,7 @@ SESSION_TTL_SECONDS=3600
|
||||
# Choose credentials locally. File paths contain no secret values.
|
||||
DATABASE_URL_FILE=/run/secrets/database_url
|
||||
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
|
||||
SESSION_STORE=postgres
|
||||
SESSION_STORE=redis
|
||||
CACHE_URL_FILE=/run/secrets/cache_url
|
||||
API_REPLICAS=2
|
||||
IDEMPOTENCY_TTL_SECONDS=86400
|
||||
@@ -20,11 +20,35 @@ MAX_UPLOAD_BYTES=10485760
|
||||
MAX_IMAGE_PIXELS=40000000
|
||||
BROKER_URL_FILE=/run/secrets/broker_url
|
||||
WORKER_CONCURRENCY=2
|
||||
# Мягкий лимит кучи Go в worker (≈75% WORKER_MEMORY)
|
||||
WORKER_GOMEMLIMIT=560MiB
|
||||
JOB_MAX_ATTEMPTS=5
|
||||
JOB_TIMEOUT_SECONDS=60
|
||||
STAGING_RETENTION_SECONDS=86400
|
||||
# Graceful shutdown: after SIGTERM readiness=503 for DRAIN_DELAY, then Shutdown up to SHUTDOWN_TIMEOUT.
|
||||
DRAIN_DELAY=3s
|
||||
SHUTDOWN_TIMEOUT=8s
|
||||
# Reconciler (PostgreSQL <-> S3): background pass interval and safe grace period.
|
||||
RECONCILE_INTERVAL=10m
|
||||
RECONCILE_GRACE=1h
|
||||
# Pipeline (lab5): published version, retry backoff (exponential with jitter), WebP quality, memory budget.
|
||||
PIPELINE_VERSION=1
|
||||
RETRY_BASE_DELAY=2s
|
||||
RETRY_MAX_DELAY=60s
|
||||
WEBP_QUALITY=80
|
||||
WORKER_PIXEL_BUDGET=60000000
|
||||
WORKER_REPLICAS=1
|
||||
# --- Наблюдаемость (lab6) ---
|
||||
OTEL_SERVICE_NAME=messenger-api
|
||||
# Worker must set a distinct service.name.
|
||||
# Worker must set a distinct service.name (compose: messenger-worker, relay: messenger-relay).
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318
|
||||
OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
|
||||
OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab
|
||||
# Трек 1: false/true выключает экспорт; sampler — стандартные переменные SDK.
|
||||
OTEL_SDK_DISABLED=false
|
||||
OTEL_TRACES_SAMPLER=parentbased_always_on
|
||||
OTEL_TRACES_SAMPLER_ARG=1.0
|
||||
# collector.yaml (все трассы) или collector-tail.yaml (tail sampling)
|
||||
COLLECTOR_CONFIG=collector.yaml
|
||||
GRAFANA_PORT=3000
|
||||
PROMETHEUS_PORT=9090
|
||||
|
||||
@@ -0,0 +1,133 @@
|
||||
# Реальный прогон Docker-стенда lab6 на GitHub-hosted runner.
|
||||
# Нужен потому, что на машине разработки нет аппаратной виртуализации для Docker Desktop.
|
||||
# Все результаты — настоящие прогоны; публикуются в ветку lab6-results.
|
||||
name: lab6 stand
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [lab6]
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: lab6-stand-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
PYTHONUTF8: "1"
|
||||
|
||||
jobs:
|
||||
# Unit-тесты и интеграционный fault-injection pipeline (трек 5) на настоящем PostgreSQL.
|
||||
go-tests:
|
||||
name: Go unit + интеграционный fault-injection
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 20
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:16.4-alpine
|
||||
env:
|
||||
POSTGRES_USER: it
|
||||
POSTGRES_PASSWORD: it
|
||||
POSTGRES_DB: postgres
|
||||
ports: ["5432:5432"]
|
||||
options: >-
|
||||
--health-cmd "pg_isready -U it" --health-interval 2s --health-timeout 2s --health-retries 30
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
- name: cwebp для теста варианта optimized
|
||||
run: sudo apt-get update -qq && sudo apt-get install -y -qq webp > /dev/null && cwebp -version
|
||||
- name: go test
|
||||
env:
|
||||
TEST_DATABASE_URL: postgres://it:it@localhost:5432/postgres?sslmode=disable
|
||||
run: |
|
||||
mkdir -p evidence/track2
|
||||
go test ./... -count=1 2>&1 | tee evidence/go_test.txt
|
||||
go test ./internal/worker/ -run Pipeline -count=1 -v 2>&1 | grep -E '^(=== RUN|--- |PASS|FAIL|ok)|FAULT INJECTION|duplicate|lease|stale|noop|attempt failed|early delivery' \
|
||||
| tee evidence/track2/pipeline_fault_injection.txt
|
||||
go test ./internal/imaging/ -run OptimizedWebP -count=1 -v 2>&1 | tee evidence/webp_test.txt
|
||||
go test ./internal/telemetry/ ./internal/transport/http/middleware/ ./internal/worker/ -run 'Telemetry|Trace|Route|Propag' -count=1 -v 2>&1 \
|
||||
| grep -E '^(=== RUN|--- |PASS|FAIL|ok)' | tee evidence/telemetry_tests.txt
|
||||
- uses: actions/upload-artifact@v4
|
||||
if: always()
|
||||
with:
|
||||
name: evidence-go
|
||||
path: evidence/
|
||||
|
||||
stand:
|
||||
name: Стенд — ${{ matrix.part }}
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 80
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
part: [base, files, pipeline, obs, track1]
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.12"
|
||||
- name: Секреты и запуск стенда
|
||||
shell: bash
|
||||
run: |
|
||||
set -o pipefail
|
||||
mkdir -p evidence
|
||||
chmod +x scripts/*.sh infra/s3/init.sh
|
||||
sh scripts/init-secrets.sh
|
||||
docker compose --env-file .env config --quiet
|
||||
for img in $(docker compose config --images | sort -u | grep -v -- '-api:local'); do
|
||||
if docker pull -q "$img" > /dev/null 2>> evidence/pull_${{ matrix.part }}.log; then echo "ok $img"; else echo "FAIL $img"; fi
|
||||
done | tee -a evidence/pull_${{ matrix.part }}.log
|
||||
docker compose --env-file .env build 2>&1 | tail -30 > evidence/build_${{ matrix.part }}.log
|
||||
# Снимок dashboard в CI делается анонимным Viewer; по умолчанию анонимный доступ выключен.
|
||||
if [ "${{ matrix.part }}" = obs ]; then export GRAFANA_ANONYMOUS=true; fi
|
||||
docker compose --env-file .env up -d --wait 2>&1 | tee evidence/up_${{ matrix.part }}.log
|
||||
- name: make check + доказательства (${{ matrix.part }})
|
||||
run: |
|
||||
make check PYTHON=python
|
||||
PYTHON=python sh scripts/collect-evidence.sh ${{ matrix.part }}
|
||||
- name: Окружение runner
|
||||
if: always()
|
||||
run: |
|
||||
mkdir -p evidence
|
||||
{ nproc; free -m; docker version --format '{{.Server.Version}}'; docker compose version; } > evidence/runner_${{ matrix.part }}.txt
|
||||
docker compose ps -a > evidence/docker_ps_final_${{ matrix.part }}.txt || true
|
||||
for c in $(docker compose ps -aq); do docker inspect -f '{{.Name}} restarts={{.RestartCount}} oom_killed={{.State.OOMKilled}} exit={{.State.ExitCode}}' "$c"; done > evidence/containers_state_${{ matrix.part }}.txt || true
|
||||
docker compose logs --no-color --tail 300 > evidence/compose_logs_${{ matrix.part }}.log 2>&1 || true
|
||||
- uses: actions/upload-artifact@v4
|
||||
if: always()
|
||||
with:
|
||||
name: evidence-${{ matrix.part }}
|
||||
path: evidence/
|
||||
|
||||
publish:
|
||||
name: Публикация результатов в ветку lab6-results
|
||||
needs: [go-tests, stand]
|
||||
if: always()
|
||||
runs-on: ubuntu-24.04
|
||||
permissions:
|
||||
contents: write
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/download-artifact@v4
|
||||
with:
|
||||
path: collected
|
||||
- name: Commit в lab6-results
|
||||
run: |
|
||||
git config user.name "github-actions[bot]"
|
||||
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
|
||||
git checkout -q --orphan lab6-results
|
||||
git rm -rf -q .
|
||||
{ echo "source_commit=${{ github.sha }}"
|
||||
echo "run=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
|
||||
echo "go_tests=${{ needs.go-tests.result }}"
|
||||
echo "stand=${{ needs.stand.result }}"; } > collected/RUN.txt
|
||||
cp -r collected/. .
|
||||
rm -rf collected
|
||||
git add -A
|
||||
git commit -q -m "results for ${{ github.sha }}"
|
||||
git push -f origin lab6-results
|
||||
+59
@@ -0,0 +1,59 @@
|
||||
# ============================================================
|
||||
# Stage 1: builder — компилируем статический бинарник
|
||||
# ============================================================
|
||||
FROM golang:1.26-alpine@sha256:27f2d3b83cfc5803c8078f8e730af0f7c431bf067afaec8f60a36fb763187cba AS builder
|
||||
|
||||
WORKDIR /src
|
||||
|
||||
# Сначала манифесты зависимостей — кэшируем слой.
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
|
||||
# Затем исходники.
|
||||
COPY . .
|
||||
|
||||
# Статический бинарник с оптимизациями:
|
||||
# CGO_ENABLED=0 — статическая линковка (не требует libc)
|
||||
# -ldflags="-s -w" — убираем отладочную информацию
|
||||
# -trimpath — убираем пути сборки из бинарника
|
||||
RUN CGO_ENABLED=0 GOOS=linux go build \
|
||||
-ldflags="-s -w -X github.com/charawein/mtusi2026lab6/internal/telemetry.Version=lab6" \
|
||||
-trimpath \
|
||||
-o /out/ \
|
||||
./cmd/api ./cmd/reconcile ./cmd/pipeline
|
||||
|
||||
# ============================================================
|
||||
# Stage 2: runtime — минимальный образ только с бинарником
|
||||
# ============================================================
|
||||
FROM alpine:3.20@sha256:d9e853e87e55526f6b2917df91a2115c36dd7c696a35be12163d44e6e2a4b6bc
|
||||
|
||||
# ca-certificates — для HTTPS в lab2+
|
||||
# tzdata — для корректной работы с временными зонами
|
||||
# libwebp-tools — cwebp для варианта optimized (WebP) в worker.
|
||||
RUN apk --no-cache add ca-certificates tzdata libwebp-tools
|
||||
|
||||
# Непривилегированный пользователь (UID/GID 1001).
|
||||
RUN addgroup -g 1001 -S appgroup && \
|
||||
adduser -u 1001 -S appuser -G appgroup
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Копируем бинарник и сразу выставляем владельца.
|
||||
COPY --from=builder --chown=appuser:appgroup /out/api /app/api
|
||||
# reconcile — сверка PostgreSQL и S3 (сервис reconciler и ручной запуск).
|
||||
COPY --from=builder --chown=appuser:appgroup /out/reconcile /app/reconcile
|
||||
# pipeline — worker, relay (outbox) и операторские команды (replay, failed, reprocess).
|
||||
COPY --from=builder --chown=appuser:appgroup /out/pipeline /app/pipeline
|
||||
|
||||
# Переключаемся на непривилегированного пользователя.
|
||||
USER 1001:1001
|
||||
|
||||
# Информативно: какой порт слушает приложение.
|
||||
EXPOSE 8080
|
||||
|
||||
# Healthcheck: readiness (БД + Redis + не в режиме drain).
|
||||
HEALTHCHECK --interval=5s --timeout=2s --start-period=5s --retries=3 \
|
||||
CMD wget --spider -q http://localhost:8080/health/ready || exit 1
|
||||
|
||||
# Exec-форма CMD — критично для доставки SIGTERM в приложение.
|
||||
CMD ["/app/api"]
|
||||
@@ -4,7 +4,7 @@ CA_FILE ?=
|
||||
ACTION_SCRIPT ?=
|
||||
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
|
||||
|
||||
.PHONY: help check validate test load test-persistence test-failover
|
||||
.PHONY: help check validate test load test-persistence test-failover up down test-files test-pipeline test-faults failed replay evidence
|
||||
|
||||
help:
|
||||
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
|
||||
@@ -28,3 +28,33 @@ test-persistence:
|
||||
test-failover:
|
||||
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
|
||||
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
# --- Студенческие цели (lab5) ---
|
||||
up:
|
||||
sh scripts/init-secrets.sh
|
||||
docker compose --env-file .env up -d --build --wait
|
||||
|
||||
down:
|
||||
docker compose --env-file .env stop
|
||||
|
||||
test-files:
|
||||
$(PYTHON) student-tests/attachments_negative.py --base-url "$(BASE_URL)"
|
||||
|
||||
# Сценарии pipeline на стенде: make test-pipeline SCENARIO=demo (список — student-tests/pipeline_ops.py)
|
||||
SCENARIO ?= demo
|
||||
test-pipeline:
|
||||
$(PYTHON) student-tests/pipeline_ops.py $(SCENARIO) --base-url "$(BASE_URL)"
|
||||
|
||||
# Интеграционный fault-injection pipeline (трек 5); нужен PostgreSQL: TEST_DATABASE_URL=postgres://...
|
||||
test-faults:
|
||||
go test ./internal/worker/ -run Pipeline -count=1 -v
|
||||
|
||||
failed:
|
||||
docker compose run --rm --no-deps -T --entrypoint /app/pipeline relay failed
|
||||
|
||||
# make replay ID=<attachment_id>
|
||||
replay:
|
||||
docker compose run --rm --no-deps -T --entrypoint /app/pipeline relay replay $(ID) --actor operator
|
||||
|
||||
evidence:
|
||||
sh scripts/collect-evidence.sh all
|
||||
|
||||
@@ -1,37 +1,195 @@
|
||||
# Отчёт — лабораторная 6
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
- Автор, группа: Вейнмайер Андрей Андреевич, БВТ2303
|
||||
- Целевая оценка: 5 (уровни 3, 4 и трек 1 «Цена наблюдаемости»)
|
||||
- Commit/PR: ветка `solution/bvt2303-weinmaier` в https://gitea.privcloud.tech/Weinmaier_Andrew/mtusi2026lab6 (PR не открывался)
|
||||
- Основа: собственная lab5 (PR mtusi2026/mtusi2026lab5#35) — RabbitMQ, outbox, worker, versioned pipeline; через неё lab3–4
|
||||
- Использованные источники:
|
||||
- Контракт: contracts/observability.md, contracts/README.md
|
||||
- Библиотеки: go.opentelemetry.io/otel v1.47 (SDK traces/metrics/logs, OTLP/HTTP exporters), contrib: otelslog, otelhttp, runtime, host; github.com/exaring/otelpgx; go-redis/extra/redisotel
|
||||
- Документация: OpenTelemetry Collector (receivers/processors/exporters, memory_limiter, exporterhelper sending_queue, tail_sampling), W3C Trace Context, Prometheus histograms, Loki OTLP ingestion, Tempo TraceQL, Grafana provisioning
|
||||
- Помощь ИИ: ИИ-ассистент использовался при реализации и отладке; все результаты ниже — реальные прогоны, ссылки на них приведены.
|
||||
|
||||
## Запуск из чистого clone
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
| Роль | Сервис | Образ | Хранение |
|
||||
| --- | --- | --- | --- |
|
||||
| Collector | `otel-collector` | otel/opentelemetry-collector-contrib:0.116.1 | — (не БД: только приём, обработка, экспорт) |
|
||||
| Метрики + alerts | `prometheus` | prom/prometheus:v3.1.0 | volume `prometheus_data`, 7 дней |
|
||||
| Логи | `loki` | grafana/loki:3.3.2 | volume `loki_data`, 7 дней |
|
||||
| Трассы | `tempo` (+ `tempo-init`) | grafana/tempo:2.6.1 | volume `tempo_data`, 7 дней |
|
||||
| Интерфейс | `grafana` | grafana/grafana:11.4.0 | volume `grafana_data` (только состояние Grafana) |
|
||||
|
||||
Остальное — как в lab5 (HAProxy, 2 × API, PostgreSQL, Redis, MinIO, RabbitMQ, relay, worker, reconciler). Digest всех образов — `evidence/images.txt`.
|
||||
|
||||
```sh
|
||||
sh scripts/init-secrets.sh # .env и secrets/ (в т.ч. пароль admin Grafana; значения не печатаются)
|
||||
docker compose --env-file .env up -d --build --wait # стенд + Collector, Prometheus, Loki, Tempo, Grafana (provisioning из Git)
|
||||
make test # публичный smoke lab6
|
||||
python3 student-tests/observability_ops.py signals # сигналы, сквозная трасса, утечки, dashboard (и retry-trace, incident-worker,
|
||||
# incident-db, backend-down)
|
||||
python3 student-tests/telemetry_cost.py --rounds 3 # трек 1
|
||||
sh scripts/collect-evidence.sh all # все доказательства в evidence/
|
||||
```
|
||||
|
||||
Grafana: `http://127.0.0.1:3000` (HTTPS — в lab7), пользователь admin, пароль — `secrets/grafana_admin_password`; dashboard «Messenger — API, worker, очередь» и три источника появляются без ручных действий. Prometheus UI — `127.0.0.1:9090`. Остановка: `docker compose stop`; удаление учебных данных вместе с телеметрией: `docker compose down -v`.
|
||||
|
||||
## Где запускался стенд
|
||||
|
||||
Docker-проверки — на GitHub-hosted runner (2 vCPU, 8 GB, Docker 28.0.4, Compose v2.38.2) workflow [.github/workflows/lab6-stand.yml](.github/workflows/lab6-stand.yml): Go-тесты и пять независимых стендов (`base`, `files`, `pipeline`, `obs`, `track1`). Итоговый прогон: https://github.com/CharaWein/2026_lab3/actions/runs/43208190207 (commit d90aef7). Все файлы `evidence/` — из него.
|
||||
|
||||
## Архитектура и решения
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
```
|
||||
api ×2 / relay / worker ── OTLP/HTTP :4318 ──► otel-collector ──► otlp/tempo ──► Tempo (traces)
|
||||
(SDK: BatchSpanProcessor, memory_limiter ──► otlphttp ───► Loki (logs, /otlp)
|
||||
PeriodicReader 5s, slog→otelslog) batch, scrub ──► prometheus :8889 ◄── Prometheus (pull 5s)
|
||||
├─ rules (SLI, alerts)
|
||||
Grafana ◄──── datasources (provisioning) ┘
|
||||
```
|
||||
|
||||
* **Почему три хранилища.** Prometheus — временные ряды (счётчики, histogram, alert rules), Loki — события (структурные логи, индекс только по меткам потока, атрибуты — structured metadata), Tempo — причинность одного запроса/задания (span-дерево по trace_id). Collector хранит только очереди в памяти; Grafana — интерфейс и связи между источниками (exemplar → трасса, лог `trace_id` → трасса, трасса → логи).
|
||||
* **SDK** ([internal/telemetry](internal/telemetry/telemetry.go)): ресурс `service.name` (messenger-api / messenger-worker / messenger-relay), `service.version`, `service.instance.id` (= X-Instance-Id), `deployment.environment.name=lab`. Логи пишутся в stdout (JSON с trace_id/span_id) и через мост otelslog в Collector. `OTEL_SDK_DISABLED=true` выключает экспорт; sampler — стандартные `OTEL_TRACES_SAMPLER(_ARG)`.
|
||||
* **HTTP** ([middleware/telemetry.go](internal/transport/http/middleware/telemetry.go)): server span `METHOD /шаблон`, метрика `http.server.request.duration` (секунды, buckets 2 ms…30 s); её `_count` — счётчик запросов (одна метрика, не две). Метки: метод, `http.route` (шаблон ServeMux), код и класс ответа. `/health/*` метрики пишет, трасс нет. Ответ несёт `traceresponse` (W3C).
|
||||
* **Зависимости**: otelpgx — spans SQL (текст с плейсхолдерами, без значений) и ожидания пула (`pool.acquire`), метрики `pgxpool_*`; otelhttp вокруг minio-go — spans `S3 GET/PUT`; redisotel — spans команд без аргументов.
|
||||
* **Очередь** (W3C через outbox и AMQP, [0006_telemetry.up.sql](migrations/0006_telemetry.up.sql)): upload пишет `traceparent` активного span в `outbox.trace_context` (и `jobs.trace_context`); relay создаёт producer span `attachment.process publish` дочерним к нему и кладёт его контекст в заголовки AMQP; worker — consumer span `attachment.process process` дочерний к producer. Повтор пишет в outbox контекст неудачной попытки — вся цепочка retry в одной трассе; sweeper (без своего span) берёт контекст исходного upload из jobs.
|
||||
* **Worker-метрики**: `messenger.jobs.processed{job.type,outcome}`, `messenger.job.duration` (histogram), `messenger.attachment.time_to_ready` (от создания job при 202 до ready), `messenger.worker.busy` / `messenger.worker.capacity` (saturation). **Очередь** (relay, по PostgreSQL — источнику истины): `messenger.jobs.backlog{state}`, `messenger.jobs.oldest_age` (s), `messenger.outbox.pending`, `messenger.outbox.oldest_age`; плюс глубина очередей RabbitMQ (`rabbitmq_queue_messages_ready`).
|
||||
* **Фоновые циклы** relay (опрос outbox каждые 300 ms, sweeper, gauges) идут под не-сэмплированным контекстом — иначе каждый опрос был бы отдельной трассой.
|
||||
|
||||
**Соответствие контракту наблюдаемости** (имена в Prometheus — после трансляции OTel → Prometheus в exporter Collector; полный список — `evidence/obs/prometheus_state.json`):
|
||||
|
||||
| Сигнал контракта | Имя | Метки |
|
||||
| --- | --- | --- |
|
||||
| HTTP requests | `http_server_request_duration_seconds_count` | `http_request_method`, `http_route`, `http_response_status_code`, `http_response_status_class`, `job`, `instance` |
|
||||
| HTTP duration | `http_server_request_duration_seconds_bucket` (s) | те же |
|
||||
| Jobs processed / failed | `messenger_jobs_processed_total` | `job_type`, `outcome` (done, retry, failed, duplicate, not_due, requeue…) |
|
||||
| Job duration | `messenger_job_duration_seconds_bucket` | `job_type`, `outcome` |
|
||||
| Логи | Loki: `{service_name=…}`, structured metadata `event`, `trace_id`, `span_id`, `severity_text`, `request_id`, `route`, `status` | — |
|
||||
| Traces | Tempo: server / producer / consumer / client spans | — |
|
||||
| Queue depth / oldest age | `messenger_jobs_backlog`, `messenger_jobs_oldest_age_seconds`, `rabbitmq_queue_messages_ready` | `state`, `queue` |
|
||||
| Pool / saturation | `pgxpool_acquired_connections`, `pgxpool_max_connections`, `pgxpool_empty_acquire_total`; `messenger_worker_busy / messenger_worker_capacity`; `process_cpu_time_seconds_total`, `go_memory_used_bytes` | `job`, `instance` |
|
||||
| Сквозная связь | `traceparent` в outbox и заголовках AMQP, parent-child | — |
|
||||
| Time to ready | `messenger_attachment_time_to_ready_seconds_bucket` | `job_type` |
|
||||
|
||||
**Error ratio:** числитель — запросы с `http_response_status_class="5xx"`, знаменатель — все запросы, кроме `/health/*`; 4xx (неверный пароль, чужой чат) — ошибки клиента, в SLO не входят. p95/p99 — `histogram_quantile` по `sum by (le)` buckets всех реплик (никакого усреднения p99 реплик).
|
||||
|
||||
**SLI/SLO** ([rules/messenger.yml](infra/prometheus/rules/messenger.yml), окно SLO — 28 дней, на стенде видны короткие окна): доступность API ≥ 99.5% бизнес-запросов без 5xx; задержка — ≥ 95% быстрее 250 ms; время до ready — ≥ 95% быстрее 30 s. **Alerts** (состояние firing/resolved — в Prometheus и на dashboard как аннотации; runbooks — [infra/RUNBOOKS.md](infra/RUNBOOKS.md)): `MessengerHighErrorRate` (5xx > 5% за 1m, 30 s), `MessengerQueueLagGrowing` (старейшая job > 60 s, 30 s), `MessengerWorkerAbsent`, `MessengerTelemetryExportFailing`.
|
||||
|
||||
**Ограниченная буферизация:** SDK — очередь BatchSpanProcessor 4096 spans, логов 4096 (переполнение — отбрасывание); Collector — `memory_limiter` 200 MiB (spike 50) первым в pipeline, `sending_queue` 20 пакетов × ≤ 1024 элементов и `retry_on_failure` ≤ 60 s на экспортёр (Tempo, Loki), лимит контейнера 256 MiB; метрики не буферизуются (Prometheus pull, `metric_expiration` 1m).
|
||||
|
||||
## Исправления, найденные прогонами
|
||||
|
||||
| Что показал прогон | Причина | Исправление |
|
||||
| --- | --- | --- |
|
||||
| API не стартовал (прогон 1) | конфликт schema URL ресурса: semconv 1.40 vs 1.43 в SDK | semconv 1.43 + unit-тест `Setup` с настоящими экспортёрами |
|
||||
| alert `MessengerTelemetryExportFailing` и панели Collector без данных | внутренние метрики Collector 0.116 без суффикса `_total` | запросы `{__name__=~"otelcol_…(_total)?"}` |
|
||||
| tail sampling терял 1 из 24 ошибок (прогон 3) | `decision_wait: 5s` ≤ длительности трассы ошибки (таймаут обработчика 5 s): решение до прихода корневого span со статусом ERROR | `decision_wait: 10s`; в итоговом прогоне потерь ошибок нет |
|
||||
| после отказа Tempo новые трассы не появлялись (прогоны 4–5) | очередь экспорта 200 пакетов × до 2048 spans упёрлась в hard limit `memory_limiter` — Collector отвергал новые данные; в Tempo single-binary querier без явного адреса frontend | очередь 20 × 1024 (помещается в soft limit), `querier.frontend_worker.frontend_address`; поиск трассы — по ID из `traceresponse` |
|
||||
| Tempo/Loki перезапускались под нагрузкой трека 1 (прогон 5) | лимит 512 MiB при ~14 тыс. spans на 1000 запросов | 1 GiB; в доказательства добавлено состояние контейнеров (`containers_state_*.txt`: restarts, OOMKilled) |
|
||||
|
||||
## Доказательства критериев
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
### Уровень 3
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
| Критерий | Сценарий | Результат | Файл |
|
||||
| --- | --- | --- | --- |
|
||||
| Минимум lab5 работает | smoke, регрессии, failover ×2, persistence (перезапуск включает стек наблюдаемости), сценарии pipeline | все PASS; failover — восстановление записи 3.535 / 3.478 s; demo: ready через 2.6 s | evidence/smoke.txt, failover_*.txt, persistence.txt, pipeline/*.json |
|
||||
| Collector, 3 хранилища, Grafana в стенде; конфиги и provisioning в Git; volumes | compose.yaml, infra/ | datasources из provisioning: loki:loki, prometheus:prometheus, tempo:tempo; dashboard «Messenger — API, worker, очередь» (30 панелей) | evidence/obs/signals.json |
|
||||
| API и worker → OTLP; RPS/errors/duration API, jobs и duration worker, логи, трассы | `observability_ops.py signals` | все сигналы контракта найдены в Prometheus (число серий в signals.json); трассы трёх сервисов; логи с trace_id | evidence/obs/signals.json, prometheus_state.json |
|
||||
| Импортируемый dashboard, не скриншот | provisioning + проверка запросов | 39 запросов панелей выполнены через Grafana без ошибок; без данных в момент проверки: 2 (окно 5m без загрузок, нет 4xx) | infra/grafana/dashboards/messenger.json, evidence/obs/dashboard.png |
|
||||
| Низкая кардинальность, trace/span ID в логах, без секретов | проверка меток и поиск утечек | запрещённых меток (user_id, chat_id, message_id, attachment_id, request_id, url, filename…) нет; `http_route` — только шаблоны (8 значений); пароль, токен, текст сообщения, заголовок Bearer: 0 вхождений в Loki, Tempo и stdout | evidence/obs/signals.json |
|
||||
| Контролируемый запрос → trace → лог; job → worker trace и лог | `signals` | запрос `obs-msg-249e829f8c1e4049` → trace `c275fd38f588c08d29aea802ac7c9c83` → лог API `event=http.request`, `route=/api/v1/chats/{chat_id}/messages`, тот же `trace_id`/`span_id`; upload → trace `cf8628e62914dd27bef25fbfca80ba11` с логами worker `job.claimed`, `job.done` | evidence/obs/signals.json |
|
||||
| make test при включённой телеметрии | smoke | PASS | evidence/smoke.txt |
|
||||
|
||||
### Уровень 4
|
||||
|
||||
| Критерий | Сценарий | Результат | Файл |
|
||||
| --- | --- | --- | --- |
|
||||
| W3C context API → broker → worker; producer/consumer; цепочка до S3/DB, включая retry | `signals`, `retry-trace` | одна трасса upload: messenger-api, messenger-relay, messenger-worker, 65 spans; связи parent-child: API server → relay producer → worker consumer → `image.decode`, `S3 PUT`, SQL (BEGIN, COMMIT, INSERT, SELECT, UPDATE…). Retry при остановленном S3: попытки 1=retry, 2=retry, 3=retry, 4=done в той же трассе, 4 producer spans | evidence/obs/signals.json, retry-trace.json |
|
||||
| p50/p95/p99, доля ошибок, queue depth/возраст, DB pool, saturation; запросы и единицы | dashboard | панели с `histogram_quantile` по buckets, единицы (s, %, req/s, bytes) в JSON | infra/grafana/dashboards/messenger.json |
|
||||
| SLI/SLO и окно; alert rules high error rate и queue lag, проверены отказом | инциденты 1 и 2 | `MessengerQueueLagGrowing` firing через 96.7 s после остановки worker, resolved через 10.6 s после запуска; `MessengerHighErrorRate` firing через 98.4 s, resolved через 18.3 s после снятия блокировки | evidence/obs/incident-*.json |
|
||||
| Collector: memory limiter/batch, ограниченный буфер; остановка backend не ломает сообщения и RAM | `backend-down` | Tempo и Loki остановлены на 155 s под нагрузкой: **0 ошибок из 100015** запросов; Collector ≤ 108.4 MiB (лимит 256); очередь экспортёров ≤ 40 из 40; потери посчитаны: 4204 spans и 2843 записей логов не доставлены, 728158 spans не приняты в полную очередь; alert `MessengerTelemetryExportFailing` через 81.0 s; трассы снова в Tempo через 20.9 s после возврата; затем Collector остановлен на 40 s — API без ошибок (SDK отбрасывает по ограниченной очереди), пик памяти API ≤ 224.5 MiB (лимит 256) | evidence/obs/backend-down.json |
|
||||
| Разбор двух инцидентов | ниже | — | evidence/obs/incident-*.json |
|
||||
|
||||
## Инциденты: поиск причины по сохранённым наблюдениям
|
||||
|
||||
**Инцидент 1 — остановлен worker.** Загружены 3 файла (202). Через 96.7 s — `MessengerQueueLagGrowing` firing. Диагностика без доступа к контейнерам (снимок в момент firing):
|
||||
1. Prometheus: `messenger_jobs_backlog{state="queued"}` = 3, возраст старейшей job 91 s, скорость обработки `rate(messenger_jobs_processed_total[1m])` = 0 — задания принимаются, но не обрабатываются; ошибок API нет.
|
||||
2. `messenger_worker_capacity` — 0 серий: ни одного живого worker (а не «worker медленный»).
|
||||
3. Tempo: после остановки 3 producer spans relay и 0 consumer spans — сообщения публикуются, но никто не потребляет (broker и relay исправны).
|
||||
4. Loki: последние события worker — job.claimed, job.done, job.retry, worker.start; после остановки событий нет.
|
||||
|
||||
Вывод — worker не запущен. После запуска backlog обработан за 2.6 s, alert resolved через 10.6 s.
|
||||
|
||||
**Инцидент 2 — БД «зависла»** (`LOCK TABLE messages IN ACCESS EXCLUSIVE MODE` на 110 s при нагрузке 4 клиентов). `MessengerHighErrorRate` firing через 98.4 s (окно 1m сначала содержит успешные запросы до блокировки — поэтому обнаружение ≈ окно + `for`). В момент firing: доля 5xx 100%, p99 9.95 s, и только у маршрута `/api/v1/chats/{chat_id}/messages`; занято соединений пула API 4; `/health/ready` без 5xx. Трассы ошибок (Tempo, `status_code >= 500`): server span ≈ 5000 ms, самый длинный дочерний span — SQL `INSERT` 4978 ms (доля 100%) — время ушло в ожидание БД, а не в обработчик. Для сравнения медленный по другой причине запрос (загрузка 4 MiB): server span 77.38 ms, самый длинный SQL 2.2 ms, S3 68.0 ms — БД не причина. Логи ошибок (88) — только маршрут сообщений. Alert resolved через 18.3 s после снятия блокировки. За инцидент клиенты получили 88 ошибок из 15036 запросов.
|
||||
|
||||
## Трек 1: цена наблюдаемости
|
||||
|
||||
### Гипотеза
|
||||
|
||||
Полная трассировка (100%) заметно дороже по CPU API и объёму ingest, чем логи и метрики; head sampling 10% снижает стоимость почти до «телеметрии нет», но теряет ~90% ошибок и редких медленных запросов; tail sampling в Collector сохраняет все ошибки и медленные запросы при объёме хранения, близком к head 10%, — ценой CPU/памяти Collector, а не API.
|
||||
|
||||
### Методика
|
||||
|
||||
[student-tests/telemetry_cost.py](student-tests/telemetry_cost.py). Одинаковая нагрузка: 4 клиента closed loop, POST сообщения, 45 s после прогрева 10 s; лимиты контейнеров неизменны (API 0.5 CPU / 256 MiB × 2, Collector 0.5 CPU / 256 MiB). В каждом прогоне два внедрённых инцидента `LOCK TABLE messages` по 6.5 s (на 12-й и 28-й секунде): запросы, попавшие в блокировку, частично завершаются 5xx по таймауту 5 s (**ошибки**), частично — успешно, но дольше 250 ms (**редкие медленные**). У каждого запроса свой `X-Request-Id`; после прогона по Tempo проверяется, сохранилась ли трасса **каждой** ошибки, **каждого** медленного запроса и случайной выборки 40 обычных. Ресурсы — cgroup контейнеров (`cpu.stat`, `memory.peak`), ingest — счётчики Collector/Tempo/Loki. 3 повтора, порядок конфигураций чередуется, приводятся медианы.
|
||||
|
||||
| Конфигурация | Что меняется |
|
||||
| --- | --- |
|
||||
| off | `OTEL_SDK_DISABLED=true` (stdout-логи остаются) |
|
||||
| full | 100% трасс, Collector пропускает всё |
|
||||
| head10 | `parentbased_traceidratio` 0.1 в SDK |
|
||||
| tail | SDK 100%, Collector `tail_sampling`: все ERROR + ≥ 250 ms + 10% остальных ([collector-tail.yaml](infra/otel/collector-tail.yaml)) |
|
||||
|
||||
### Результаты (медианы 3 прогонов; latency — вне окон инцидентов)
|
||||
|
||||
| | off | full | head10 | tail |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| Успешных RPS | 247.6 | 197.8 (-20%) | 234.3 (-5%) | 195.4 (-21%) |
|
||||
| p50 / p95 / p99, ms | 10.06 / 20.22 / 31.5 | 12.26 / 27.91 / 46.15 | 10.38 / 21.88 / 34.23 | 12.45 / 27.68 / 41.48 |
|
||||
| CPU API на запрос, ms | 1.282 | 1.875 (+46%) | 1.414 (+10%) | 1.905 (+49%) |
|
||||
| Пик памяти реплики API, MiB | 13.1 | 22.4 | 21.3 | 22.3 |
|
||||
| CPU / пик памяти Collector | 0.001 ядра / 30.1 MiB | 0.051 / 55.1 MiB | 0.014 / 55.9 MiB | 0.044 / 106.5 MiB |
|
||||
| Spans принято / сохранено на 1000 запросов | 0 / 0 | 14019 / 14019 | 1398 / 1398 | 14016 / 1785 |
|
||||
| Ingest Tempo / Loki, KiB на 1000 запросов | 0 / 0 | 4548 / 339 | 454 / 335 | 588 / 340 |
|
||||
| Трассы ошибок сохранены | 0/24 | 24/24 | **2/24** | **24/24** |
|
||||
| Трассы медленных сохранены | 0/24 | 24/24 | **2/24** | **24/24** |
|
||||
| Выборка обычных сохранена | 0/120 | 120/120 | 14/120 | 13/120 |
|
||||
|
||||
Сырые данные каждого прогона — evidence/track1/telemetry_cost.json.
|
||||
|
||||
### Выводы
|
||||
|
||||
* **Цена полной трассировки** на этом сервисе — в первую очередь CPU API: +46% CPU на запрос относительно «off» и -20% успешных RPS (closed loop: те же 4 клиента ждут более дорогие ответы — запросов за то же время меньше). Одна запись сообщения даёт ≈ 14 spans (server, pool.acquire, SQL, Redis) и ≈ 4.5 KiB в Tempo — трассы в 13 раз объёмнее логов.
|
||||
* **Head 10%** возвращает почти всю цену (CPU +10%, ingest ≈ 10%), но решение принимается в начале запроса, когда исход неизвестен: сохранились лишь 2/24 трасс ошибок и 2/24 медленных — ровно то, ради чего трассы нужны при разборе инцидента.
|
||||
* **Tail** (сбалансированная конфигурация) сохранил 24/24 ошибок и 24/24 медленных при объёме Tempo 588 KiB на 1000 запросов против 4548 у full (в 7.7 раза меньше). Цена: CPU API как у full (SDK по-прежнему создаёт и отправляет все spans), Collector держит буфер решений — пик памяти 106.5 MiB против 55.1 у full.
|
||||
* **Метрики от sampling не зависят**: доля ошибок, p95/p99 и alerts считаются в SDK до sampling — во всех конфигурациях, кроме off, они точны; sampling влияет только на то, по каким запросам можно открыть трассу.
|
||||
* **Найденная ошибка конфигурации**: при `decision_wait: 5s` tail терял трассы ошибок длительностью 5 s (решение раньше прихода корневого span, прогон https://github.com/CharaWein/2026_lab3/actions/runs/43191617128: 23/24); итоговая конфигурация — 10 s.
|
||||
|
||||
**Рекомендация:** tail sampling в Collector (ошибки + медленные + небольшая доля остальных) при `decision_wait` больше максимальной длительности запроса; если CPU API критичнее — head-sampling малой доли **плюс** метрики и логи для ошибок (логи сохраняют trace_id даже для не-сэмплированных запросов).
|
||||
|
||||
### Ограничения трека
|
||||
|
||||
* Один runner 2 vCPU, closed loop с координированным пропуском; абсолютные RPS зависят от машины, сравнение — в одинаковых условиях одного прогона.
|
||||
* «Медленные» здесь — следствие блокировки БД; естественно редкие медленные запросы распределены иначе. Ошибок и медленных по 24 на конфигурацию — доли сохранения грубые (±1 трасса).
|
||||
* Tail sampling на одном Collector; при нескольких Collector нужен балансировщик по trace_id (loadbalancing exporter) — не проверялось.
|
||||
* Объём Prometheus от sampling не зависит и в таблицу не вынесен (метрики считаются в SDK).
|
||||
|
||||
## Эксперименты и отказы
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| Воздействие | Ожидание | Наблюдение | Восстановление / потери |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
| Остановка worker | alert queue lag | firing через 96.7 s | resolved через 10.6 s после запуска; задачи не потеряны |
|
||||
| Блокировка таблицы messages 110 s | alert error rate; трасса указывает на БД | firing через 98.4 s; самый длинный span — SQL | resolved через 18.3 s |
|
||||
| S3 недоступен при обработке | retry видны в трассе | попытки 1–3 retry, 4 done в одной трассе | ready |
|
||||
| Tempo + Loki остановлены 155 s | API работает, память ограничена | 0 ошибок API, Collector ≤ 108.4 MiB | трассы через 20.9 s; трассы/логи периода отказа потеряны (счётчики) |
|
||||
| Collector остановлен 40 s | API работает | 0 ошибок, API ≤ 224.5 MiB | телеметрия периода потеряна (SDK) |
|
||||
| Остановка API-реплики под нагрузкой | 0 ошибок | 0 из 16801 | — |
|
||||
| Rolling update | 0 ошибок | 0 из 29983, p99 65.137 ms | — |
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
* Память контейнеров в `backend-down` — `memory.current` cgroup (включает page cache), поэтому у API она близка к лимиту при полной нагрузке; OOM-kill ни у одного контейнера не было (`evidence/containers_state_*.txt`).
|
||||
* Телеметрия внутри доверенной сети по HTTP без аутентификации (TLS/mTLS — lab7). Grafana на 127.0.0.1 с паролем администратора.
|
||||
* Один экземпляр Collector и каждого хранилища: их отказ — потеря телеметрии за период (посчитана, но не восстанавливается); persistent queue Collector не включалась.
|
||||
* Алерты без доставки наружу (по заданию); burn-rate по нескольким окнам — следующий шаг (трек 2 не выбирался).
|
||||
* `messenger.jobs.oldest_age` и backlog считаются запросом к PostgreSQL каждые 5 s — при больших таблицах нужен индекс (добавлен `jobs_open_idx`) или учёт в самом relay.
|
||||
|
||||
+178
@@ -0,0 +1,178 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/signal"
|
||||
"sync/atomic"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/charawein/mtusi2026lab6/internal/config"
|
||||
"github.com/charawein/mtusi2026lab6/internal/repository/postgres"
|
||||
redisrepo "github.com/charawein/mtusi2026lab6/internal/repository/redis"
|
||||
"github.com/charawein/mtusi2026lab6/internal/service"
|
||||
"github.com/charawein/mtusi2026lab6/internal/storage"
|
||||
"github.com/charawein/mtusi2026lab6/internal/telemetry"
|
||||
transporthttp "github.com/charawein/mtusi2026lab6/internal/transport/http"
|
||||
)
|
||||
|
||||
func main() {
|
||||
cfg := config.Load()
|
||||
slog.SetDefault(slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: cfg.LogLevel})))
|
||||
// OTLP → Collector: трассы, метрики, логи (service.name=messenger-api,
|
||||
// service.instance.id = X-Instance-Id реплики). Логи дублируются JSON в stdout.
|
||||
shutdownTelemetry, err := telemetry.Setup(context.Background(), "messenger-api", cfg.InstanceID, cfg.LogLevel)
|
||||
if err != nil {
|
||||
slog.Error("telemetry setup", "err", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer func() {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
_ = shutdownTelemetry(ctx)
|
||||
}()
|
||||
|
||||
if cfg.DatabaseURL == "" {
|
||||
slog.Error("DATABASE_URL_FILE is not set or empty")
|
||||
os.Exit(1)
|
||||
}
|
||||
if cfg.CacheURL == "" {
|
||||
slog.Error("CACHE_URL_FILE is not set or empty")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
pool, err := postgres.NewPool(ctx, cfg.DatabaseURL, cfg.DBPoolMax)
|
||||
cancel()
|
||||
if err != nil {
|
||||
slog.Error("connect to postgres", "err", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer pool.Close()
|
||||
|
||||
rctx, rcancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
redisClient, err := redisrepo.NewClient(rctx, cfg.CacheURL)
|
||||
rcancel()
|
||||
if err != nil {
|
||||
slog.Error("connect to redis", "err", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer redisClient.Close()
|
||||
|
||||
if cfg.S3AccessKey == "" || cfg.S3SecretKey == "" {
|
||||
slog.Error("S3_ACCESS_KEY_ID_FILE / S3_SECRET_ACCESS_KEY_FILE are not set or empty")
|
||||
os.Exit(1)
|
||||
}
|
||||
store, err := storage.NewS3(cfg.S3Endpoint, cfg.S3Region, cfg.S3Bucket, cfg.S3AccessKey, cfg.S3SecretKey)
|
||||
if err != nil {
|
||||
slog.Error("s3 client", "err", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
userRepo := postgres.NewUserRepository(pool)
|
||||
chatRepo := postgres.NewChatRepository(pool)
|
||||
msgRepo := postgres.NewMessageRepository(pool)
|
||||
sessRepo := redisrepo.NewSessionRepository(redisClient)
|
||||
|
||||
userSvc := service.NewUserService(userRepo)
|
||||
chatSvc := service.NewChatService(chatRepo, userRepo)
|
||||
|
||||
var msgCache *redisrepo.MessageCache
|
||||
if cfg.CacheEnabled {
|
||||
msgCache = redisrepo.NewMessageCache(redisClient, cfg.CacheInvalidation)
|
||||
}
|
||||
msgSvc := service.NewMessageService(msgRepo, chatRepo, msgCache, cfg.IdempotencyTTL)
|
||||
|
||||
attSvc := service.NewAttachmentService(postgres.NewAttachmentRepository(pool), chatRepo, store,
|
||||
cfg.MaxUploadBytes, cfg.MaxImagePixels, cfg.PipelineVersion, cfg.JobMaxAttempts)
|
||||
authSvc := service.NewAuthService(userRepo, sessRepo, cfg.SessionTTL())
|
||||
|
||||
// draining выставляется по SIGTERM: /health/ready начинает отвечать 503,
|
||||
// LB исключает реплику, и только затем закрывается listener.
|
||||
var draining atomic.Bool
|
||||
router := transporthttp.NewRouter(userSvc, chatSvc, msgSvc, attSvc, authSvc, pool, redisClient, cfg.InstanceID, &draining)
|
||||
|
||||
// Очистка просроченных ключей идемпотентности. Запрос идемпотентен,
|
||||
// поэтому параллельный запуск на всех репликах безопасен.
|
||||
cleanupCtx, cleanupStop := context.WithCancel(context.Background())
|
||||
defer cleanupStop()
|
||||
go func() {
|
||||
t := time.NewTicker(time.Hour)
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-cleanupCtx.Done():
|
||||
return
|
||||
case <-t.C:
|
||||
ctx, cancel := context.WithTimeout(cleanupCtx, 30*time.Second)
|
||||
if n, err := msgRepo.DeleteExpiredKeys(ctx, cfg.IdempotencyTTL); err != nil {
|
||||
slog.Warn("idempotency cleanup failed", "err", err)
|
||||
} else if n > 0 {
|
||||
slog.Info("idempotency keys expired", "deleted", n)
|
||||
}
|
||||
cancel()
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
srv := &http.Server{
|
||||
Addr: cfg.Addr(),
|
||||
Handler: router,
|
||||
ReadHeaderTimeout: 5 * time.Second,
|
||||
// Загрузка до 10 MiB должна уложиться в ReadTimeout; медленный клиент
|
||||
// дольше не держит соединение и память.
|
||||
ReadTimeout: 30 * time.Second,
|
||||
WriteTimeout: 35 * time.Second,
|
||||
IdleTimeout: 60 * time.Second,
|
||||
}
|
||||
|
||||
// Внутренний endpoint метрик для экспериментов (трек 1). Порт не публикуется
|
||||
// и не проксируется LB: читается только через docker exec внутри контейнера.
|
||||
internalSrv := &http.Server{
|
||||
Addr: cfg.InternalAddr,
|
||||
Handler: transporthttp.NewInternalStatsHandler(pool, cfg),
|
||||
ReadHeaderTimeout: 2 * time.Second,
|
||||
}
|
||||
go func() {
|
||||
if err := internalSrv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
|
||||
slog.Warn("internal stats server failed", "err", err)
|
||||
}
|
||||
}()
|
||||
defer internalSrv.Close()
|
||||
|
||||
errCh := make(chan error, 1)
|
||||
go func() {
|
||||
slog.Info("server starting", "addr", cfg.Addr(), "instance_id", cfg.InstanceID)
|
||||
if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
|
||||
errCh <- err
|
||||
}
|
||||
}()
|
||||
|
||||
sigCtx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT)
|
||||
defer stop()
|
||||
|
||||
select {
|
||||
case err := <-errCh:
|
||||
slog.Error("server failed", "err", err)
|
||||
os.Exit(1)
|
||||
case <-sigCtx.Done():
|
||||
slog.Info("shutdown signal received, draining", "delay", cfg.DrainDelay)
|
||||
}
|
||||
|
||||
// Перестаём быть ready и даём LB время заметить это (inter*fall у HAProxy),
|
||||
// продолжая обслуживать уже направленные к нам запросы.
|
||||
draining.Store(true)
|
||||
time.Sleep(cfg.DrainDelay)
|
||||
|
||||
shutdownCtx, cancel := context.WithTimeout(context.Background(), cfg.ShutdownTimeout)
|
||||
defer cancel()
|
||||
if err := srv.Shutdown(shutdownCtx); err != nil {
|
||||
slog.Error("graceful shutdown failed", "err", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
slog.Info("server stopped")
|
||||
}
|
||||
@@ -0,0 +1,371 @@
|
||||
// Command pipeline — асинхронная обработка вложений (lab5) с телеметрией (lab6).
|
||||
//
|
||||
// pipeline worker потребитель broker (сервис worker)
|
||||
// pipeline relay outbox → broker, возврат истёкших lease, очистка staging
|
||||
// pipeline failed реестр failed-заданий (JSON)
|
||||
// pipeline replay <attachment_id> [--version N] [--actor NAME] явный повтор failed
|
||||
// pipeline reprocess <attachment_id> --version N работа новой версии (трек 5)
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/charawein/mtusi2026lab6/internal/broker"
|
||||
"github.com/charawein/mtusi2026lab6/internal/config"
|
||||
"github.com/charawein/mtusi2026lab6/internal/imaging"
|
||||
"github.com/charawein/mtusi2026lab6/internal/repository/postgres"
|
||||
"github.com/charawein/mtusi2026lab6/internal/storage"
|
||||
"github.com/charawein/mtusi2026lab6/internal/telemetry"
|
||||
"github.com/charawein/mtusi2026lab6/internal/worker"
|
||||
"go.opentelemetry.io/otel"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/codes"
|
||||
"go.opentelemetry.io/otel/metric"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
)
|
||||
|
||||
func main() {
|
||||
slog.SetDefault(slog.New(slog.NewJSONHandler(os.Stdout, nil)))
|
||||
if len(os.Args) < 2 {
|
||||
fmt.Fprintln(os.Stderr, "usage: pipeline worker|relay|failed|replay|reprocess")
|
||||
os.Exit(2)
|
||||
}
|
||||
cfg := config.Load()
|
||||
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT)
|
||||
defer stop()
|
||||
|
||||
// Телеметрия только у долгоживущих процессов; служебные команды (failed,
|
||||
// replay) пишут JSON в stdout. service.name: messenger-worker / messenger-relay
|
||||
// (OTEL_SERVICE_NAME из compose имеет приоритет).
|
||||
if os.Args[1] == "worker" || os.Args[1] == "relay" {
|
||||
host, _ := os.Hostname()
|
||||
shutdown, err := telemetry.Setup(ctx, "messenger-"+os.Args[1], host, slog.LevelInfo)
|
||||
if err != nil {
|
||||
slog.Error("telemetry setup", "err", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer func() {
|
||||
sctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
_ = shutdown(sctx)
|
||||
}()
|
||||
}
|
||||
|
||||
pool, err := postgres.NewPool(ctx, cfg.DatabaseURL, int32(cfg.WorkerConcurrency+4))
|
||||
if err != nil {
|
||||
slog.Error("connect to postgres", "err", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer pool.Close()
|
||||
repo := postgres.NewPipelineRepository(pool)
|
||||
|
||||
switch os.Args[1] {
|
||||
case "worker":
|
||||
code := runWorker(ctx, cfg, repo)
|
||||
if code != 0 {
|
||||
os.Exit(code)
|
||||
}
|
||||
case "relay":
|
||||
code := runRelay(ctx, cfg, repo)
|
||||
if code != 0 {
|
||||
os.Exit(code)
|
||||
}
|
||||
case "failed":
|
||||
jobs, err := repo.FailedJobs(ctx, 200)
|
||||
if err != nil {
|
||||
slog.Error("failed jobs", "err", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
out, _ := json.MarshalIndent(jobs, "", " ")
|
||||
fmt.Println(string(out))
|
||||
case "replay", "reprocess":
|
||||
fs := flag.NewFlagSet(os.Args[1], flag.ExitOnError)
|
||||
version := fs.Int("version", cfg.PipelineVersion, "pipeline version")
|
||||
actor := fs.String("actor", "operator", "who requested the replay (audit)")
|
||||
if len(os.Args) < 3 {
|
||||
fmt.Fprintln(os.Stderr, "attachment_id required")
|
||||
os.Exit(2)
|
||||
}
|
||||
_ = fs.Parse(os.Args[3:])
|
||||
var jobID string
|
||||
if os.Args[1] == "replay" {
|
||||
jobID, err = repo.Replay(ctx, os.Args[2], *version, *actor)
|
||||
} else {
|
||||
jobID, err = repo.Reprocess(ctx, os.Args[2], *version, cfg.JobMaxAttempts)
|
||||
}
|
||||
out, _ := json.Marshal(map[string]any{"command": os.Args[1], "attachment_id": os.Args[2], "version": *version,
|
||||
"job_id": jobID, "ok": err == nil, "error": errString(err)})
|
||||
fmt.Println(string(out))
|
||||
if err != nil {
|
||||
os.Exit(3)
|
||||
}
|
||||
default:
|
||||
fmt.Fprintln(os.Stderr, "unknown command", os.Args[1])
|
||||
os.Exit(2)
|
||||
}
|
||||
}
|
||||
|
||||
func errString(err error) string {
|
||||
if err == nil {
|
||||
return ""
|
||||
}
|
||||
return err.Error()
|
||||
}
|
||||
|
||||
func faultPoints() map[string]bool {
|
||||
m := map[string]bool{}
|
||||
for _, p := range strings.Split(os.Getenv("FAULT_POINTS"), ",") {
|
||||
if p = strings.TrimSpace(p); p != "" {
|
||||
m[p] = true
|
||||
}
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
func newStore(cfg config.Config) storage.Store {
|
||||
store, err := storage.NewS3(cfg.S3Endpoint, cfg.S3Region, cfg.S3Bucket, cfg.S3AccessKey, cfg.S3SecretKey)
|
||||
if err != nil {
|
||||
slog.Error("s3 client", "err", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
return store
|
||||
}
|
||||
|
||||
func runWorker(ctx context.Context, cfg config.Config, repo *postgres.PipelineRepository) int {
|
||||
host, _ := os.Hostname()
|
||||
owner := fmt.Sprintf("%s:%d", host, os.Getpid())
|
||||
w := worker.New(repo, newStore(cfg), imaging.CWebP, worker.Config{
|
||||
Owner: owner, Lease: cfg.JobTimeout, JobTimeout: cfg.JobTimeout,
|
||||
MaxBytes: cfg.MaxUploadBytes, MaxPixels: cfg.MaxImagePixels, PixelBudget: cfg.PixelBudget,
|
||||
WebPQuality: cfg.WebPQuality, RetryBase: cfg.RetryBaseDelay, RetryMax: cfg.RetryMaxDelay,
|
||||
FaultPoints: faultPoints(), FaultDir: os.TempDir(), DeleteStaging: true,
|
||||
})
|
||||
slog.Info("worker starting", "event", "worker.start", "owner", owner, "concurrency", cfg.WorkerConcurrency, "lease", cfg.JobTimeout)
|
||||
// Ёмкость worker (saturation = messenger.worker.busy / capacity).
|
||||
capacity, _ := telemetry.Meter().Int64ObservableGauge("messenger.worker.capacity", metric.WithUnit("{job}"),
|
||||
metric.WithDescription("Concurrent job slots of this worker (prefetch)"))
|
||||
_, _ = telemetry.Meter().RegisterCallback(func(_ context.Context, o metric.Observer) error {
|
||||
o.ObserveInt64(capacity, int64(cfg.WorkerConcurrency))
|
||||
return nil
|
||||
}, capacity)
|
||||
tracer := telemetry.Tracer()
|
||||
backoff := time.Second
|
||||
for ctx.Err() == nil {
|
||||
cons, deliveries, err := broker.NewConsumer(cfg.BrokerURL, cfg.WorkerConcurrency, "worker-"+owner)
|
||||
if err != nil {
|
||||
slog.Warn("broker unavailable, retrying", "err", err, "in", backoff)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case <-time.After(backoff):
|
||||
}
|
||||
backoff = min(backoff*2, 30*time.Second)
|
||||
continue
|
||||
}
|
||||
backoff = time.Second
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < cfg.WorkerConcurrency; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for d := range deliveries {
|
||||
// Consumer span — дочерний для producer span relay (W3C traceparent
|
||||
// из заголовков AMQP), тот — для upload: одна трасса от HTTP до S3/DB.
|
||||
mctx := otel.GetTextMapPropagator().Extract(context.WithoutCancel(ctx), broker.HeaderCarrier(d.Headers))
|
||||
mctx, span := tracer.Start(mctx, broker.Queue+" process", trace.WithSpanKind(trace.SpanKindConsumer),
|
||||
trace.WithAttributes(
|
||||
attribute.String("messaging.system", "rabbitmq"),
|
||||
attribute.String("messaging.operation.type", "process"),
|
||||
attribute.String("messaging.destination.name", broker.Queue),
|
||||
attribute.String("messaging.message.id", d.MessageId),
|
||||
attribute.Bool("messaging.rabbitmq.redelivered", d.Redelivered),
|
||||
))
|
||||
decision := w.Handle(mctx, d.Body)
|
||||
span.SetAttributes(attribute.String("messenger.delivery.decision", decision.String()))
|
||||
span.End()
|
||||
switch decision {
|
||||
case worker.Ack:
|
||||
_ = d.Ack(false)
|
||||
case worker.Requeue:
|
||||
time.Sleep(time.Second)
|
||||
_ = d.Nack(false, true)
|
||||
case worker.Reject:
|
||||
_ = d.Nack(false, false) // → DLX attachment.dead
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
// Graceful: прекращаем получать новые сообщения, дорабатываем текущие.
|
||||
slog.Info("shutdown: cancel consumer, finishing in-flight jobs")
|
||||
cons.Cancel("worker-" + owner)
|
||||
done := make(chan struct{})
|
||||
go func() { wg.Wait(); close(done) }()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(cfg.JobTimeout):
|
||||
slog.Warn("shutdown timeout: unacked jobs will be redelivered")
|
||||
}
|
||||
cons.Close()
|
||||
return 0
|
||||
case <-waitAll(&wg):
|
||||
slog.Warn("broker connection lost, reconnecting")
|
||||
cons.Close()
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func waitAll(wg *sync.WaitGroup) <-chan struct{} {
|
||||
ch := make(chan struct{})
|
||||
go func() { wg.Wait(); close(ch) }()
|
||||
return ch
|
||||
}
|
||||
|
||||
func runRelay(ctx context.Context, cfg config.Config, repo *postgres.PipelineRepository) int {
|
||||
store := newStore(cfg)
|
||||
points := faultPoints()
|
||||
afterPublish := func() { worker.Fault(points, os.TempDir(), "relay.after_publish", os.Exit) }
|
||||
var pub *broker.Publisher
|
||||
defer func() {
|
||||
if pub != nil {
|
||||
pub.Close()
|
||||
}
|
||||
}()
|
||||
tPublish := time.NewTicker(300 * time.Millisecond)
|
||||
tSweep := time.NewTicker(5 * time.Second)
|
||||
tClean := time.NewTicker(time.Minute)
|
||||
defer tPublish.Stop()
|
||||
defer tSweep.Stop()
|
||||
defer tClean.Stop()
|
||||
slog.Info("relay starting", "event", "relay.start")
|
||||
registerQueueGauges(repo)
|
||||
tracer := telemetry.Tracer()
|
||||
published, _ := telemetry.Meter().Int64Counter("messenger.outbox.published", metric.WithUnit("{message}"),
|
||||
metric.WithDescription("Outbox messages confirmed by broker"))
|
||||
publishDur, _ := telemetry.Meter().Float64Histogram("messenger.outbox.publish.duration", metric.WithUnit("s"),
|
||||
metric.WithDescription("Publish + publisher confirm latency"))
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return 0
|
||||
case <-tPublish.C:
|
||||
if pub == nil || pub.Closed() {
|
||||
p, err := broker.NewPublisher(cfg.BrokerURL)
|
||||
if err != nil {
|
||||
slog.Warn("broker unavailable; outbox keeps messages", "err", err)
|
||||
continue
|
||||
}
|
||||
pub = p
|
||||
}
|
||||
for {
|
||||
// Цикл опроса outbox не трассируется (Unsampled); producer span каждого
|
||||
// сообщения — дочерний для span, записавшего его в outbox (upload/попытка).
|
||||
n, err := repo.PublishDue(telemetry.Unsampled(ctx), 100, func(c context.Context, m postgres.OutboxMessage) error {
|
||||
pctx := telemetry.ExtractMap(context.WithoutCancel(ctx), m.Trace)
|
||||
pctx, span := tracer.Start(pctx, broker.Queue+" publish", trace.WithSpanKind(trace.SpanKindProducer),
|
||||
trace.WithAttributes(
|
||||
attribute.String("messaging.system", "rabbitmq"),
|
||||
attribute.String("messaging.operation.type", "publish"),
|
||||
attribute.String("messaging.destination.name", broker.Exchange),
|
||||
attribute.String("messaging.message.id", m.JobID),
|
||||
attribute.Float64("messenger.outbox.wait_seconds", time.Since(m.Created).Seconds()),
|
||||
))
|
||||
start := time.Now()
|
||||
err := pub.Publish(pctx, m.JobID, m.Payload)
|
||||
res := "ok"
|
||||
if err != nil {
|
||||
res = "error"
|
||||
span.RecordError(err)
|
||||
span.SetStatus(codes.Error, "publish failed")
|
||||
}
|
||||
publishDur.Record(pctx, time.Since(start).Seconds(), metric.WithAttributes(attribute.String("outcome", res)))
|
||||
if err == nil {
|
||||
published.Add(pctx, 1)
|
||||
}
|
||||
span.End()
|
||||
return err
|
||||
}, afterPublish)
|
||||
if err != nil {
|
||||
slog.Warn("outbox publish failed; will retry", "err", err)
|
||||
break
|
||||
}
|
||||
if n > 0 {
|
||||
slog.Info("outbox published", "messages", n)
|
||||
}
|
||||
if n < 100 {
|
||||
break
|
||||
}
|
||||
}
|
||||
case <-tSweep.C:
|
||||
if n, err := repo.SweepExpiredLeases(telemetry.Unsampled(ctx)); err != nil {
|
||||
slog.Warn("lease sweep failed", "err", err)
|
||||
} else if n > 0 {
|
||||
slog.Warn("expired leases requeued", "jobs", n)
|
||||
}
|
||||
case <-tClean.C:
|
||||
cleanStaging(telemetry.Unsampled(ctx), cfg, repo, store)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// cleanStaging удаляет staging после безопасной публикации (ready, ≥10 мин назад —
|
||||
// fallback, если worker не удалил сразу) и у failed старше STAGING_RETENTION.
|
||||
func cleanStaging(ctx context.Context, cfg config.Config, repo *postgres.PipelineRepository, store storage.Store) {
|
||||
cands, err := repo.StagingToClean(ctx, 10*time.Minute, cfg.StagingRetention, 200)
|
||||
if err != nil {
|
||||
slog.Warn("staging cleanup query failed", "err", err)
|
||||
return
|
||||
}
|
||||
for _, c := range cands {
|
||||
if err := store.Delete(ctx, c.Key); err != nil && !errors.Is(err, storage.ErrNotFound) {
|
||||
slog.Warn("staging delete failed", "key", c.Key, "err", err)
|
||||
continue
|
||||
}
|
||||
_ = repo.ClearStaging(ctx, c.AttachmentID, c.Key)
|
||||
}
|
||||
if len(cands) > 0 {
|
||||
slog.Info("staging cleaned", "objects", len(cands))
|
||||
}
|
||||
}
|
||||
|
||||
// registerQueueGauges — глубина очереди и возраст старейшей работы по PostgreSQL
|
||||
// (источник истины; broker-метрики RabbitMQ собираются Prometheus отдельно).
|
||||
func registerQueueGauges(repo *postgres.PipelineRepository) {
|
||||
m := telemetry.Meter()
|
||||
jobs, _ := m.Int64ObservableGauge("messenger.jobs.backlog", metric.WithUnit("{job}"),
|
||||
metric.WithDescription("Jobs by state (queued, processing, failed)"))
|
||||
oldest, _ := m.Float64ObservableGauge("messenger.jobs.oldest_age", metric.WithUnit("s"),
|
||||
metric.WithDescription("Age of the oldest unfinished job (queued or processing)"))
|
||||
outbox, _ := m.Int64ObservableGauge("messenger.outbox.pending", metric.WithUnit("{message}"),
|
||||
metric.WithDescription("Due outbox messages not yet confirmed by broker"))
|
||||
outboxAge, _ := m.Float64ObservableGauge("messenger.outbox.oldest_age", metric.WithUnit("s"),
|
||||
metric.WithDescription("Age of the oldest due unpublished outbox message"))
|
||||
q := attribute.String("queue", broker.Queue)
|
||||
_, _ = m.RegisterCallback(func(ctx context.Context, o metric.Observer) error {
|
||||
qctx, cancel := context.WithTimeout(telemetry.Unsampled(ctx), 2*time.Second)
|
||||
defer cancel()
|
||||
st, err := repo.QueueStats(qctx)
|
||||
if err != nil {
|
||||
return nil // нет данных — пропуск точки, а не ошибка экспорта
|
||||
}
|
||||
o.ObserveInt64(jobs, st.Queued, metric.WithAttributes(q, attribute.String("state", "queued")))
|
||||
o.ObserveInt64(jobs, st.Processing, metric.WithAttributes(q, attribute.String("state", "processing")))
|
||||
o.ObserveInt64(jobs, st.Failed, metric.WithAttributes(q, attribute.String("state", "failed")))
|
||||
o.ObserveFloat64(oldest, st.OldestQueuedAge, metric.WithAttributes(q))
|
||||
o.ObserveInt64(outbox, st.OutboxPending)
|
||||
o.ObserveFloat64(outboxAge, st.OldestOutboxAge)
|
||||
return nil
|
||||
}, jobs, oldest, outbox, outboxAge)
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
// Command reconcile сверяет PostgreSQL и S3 и безопасно убирает расхождения.
|
||||
//
|
||||
// reconcile # dry-run, grace 1h, отчёт JSON в stdout
|
||||
// reconcile --apply # исправить
|
||||
// reconcile --apply --interval 10m # фоновый режим (сервис reconciler в compose)
|
||||
//
|
||||
// Grace period должен быть больше максимальной длительности загрузки
|
||||
// (ReadTimeout 30 s + PUT) с запасом на расхождение часов; значения < 5m
|
||||
// разрешены только с --unsafe-short-grace (для тестов).
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/charawein/mtusi2026lab6/internal/config"
|
||||
"github.com/charawein/mtusi2026lab6/internal/reconcile"
|
||||
"github.com/charawein/mtusi2026lab6/internal/repository/postgres"
|
||||
"github.com/charawein/mtusi2026lab6/internal/storage"
|
||||
)
|
||||
|
||||
func main() {
|
||||
apply := flag.Bool("apply", false, "apply fixes (default: dry-run, report only)")
|
||||
grace := flag.Duration("grace", time.Hour, "do not touch objects/intents/quarantine younger than this")
|
||||
unsafe := flag.Bool("unsafe-short-grace", false, "allow grace < 5m (tests only)")
|
||||
interval := flag.Duration("interval", 0, "repeat every interval (0 = run once)")
|
||||
flag.Parse()
|
||||
|
||||
slog.SetDefault(slog.New(slog.NewJSONHandler(os.Stderr, nil)))
|
||||
if *grace < 5*time.Minute && !*unsafe {
|
||||
fmt.Fprintln(os.Stderr, "grace < 5m is unsafe: in-flight uploads could be deleted; pass --unsafe-short-grace for tests")
|
||||
os.Exit(2)
|
||||
}
|
||||
|
||||
cfg := config.Load()
|
||||
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT)
|
||||
defer stop()
|
||||
|
||||
pool, err := postgres.NewPool(ctx, cfg.DatabaseURL, 4)
|
||||
if err != nil {
|
||||
slog.Error("connect to postgres", "err", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer pool.Close()
|
||||
store, err := storage.NewS3(cfg.S3Endpoint, cfg.S3Region, cfg.S3Bucket, cfg.S3AccessKey, cfg.S3SecretKey)
|
||||
if err != nil {
|
||||
slog.Error("s3 client", "err", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
repo := postgres.NewAttachmentRepository(pool)
|
||||
|
||||
runOnce := func() int {
|
||||
unlock, ok, err := repo.TryLock(ctx)
|
||||
if err != nil {
|
||||
slog.Error("advisory lock", "err", err)
|
||||
return 1
|
||||
}
|
||||
if !ok {
|
||||
slog.Warn("another reconciler is running, skipping this pass")
|
||||
return 3
|
||||
}
|
||||
defer unlock()
|
||||
var now time.Time
|
||||
if err := pool.QueryRow(ctx, `SELECT now()`).Scan(&now); err != nil {
|
||||
slog.Error("db time", "err", err)
|
||||
return 1
|
||||
}
|
||||
rep, err := reconcile.Run(ctx, repo, store, reconcile.Options{
|
||||
Prefix: "", Grace: *grace, Apply: *apply, Now: now, // весь bucket: a/, s/, o/, v/
|
||||
})
|
||||
out, _ := json.Marshal(rep)
|
||||
fmt.Println(string(out))
|
||||
if err != nil {
|
||||
slog.Error("reconcile pass failed", "err", err)
|
||||
return 1
|
||||
}
|
||||
slog.Info("reconcile pass done", "dry_run", rep.DryRun, "found", rep.Found, "fixed", rep.Fixed,
|
||||
"rows", rep.RowsScanned, "objects", rep.ObjectsScanned, "ms", rep.DurationMS)
|
||||
if len(rep.Errors) > 0 {
|
||||
return 4
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
if *interval <= 0 {
|
||||
os.Exit(runOnce())
|
||||
}
|
||||
t := time.NewTicker(*interval)
|
||||
defer t.Stop()
|
||||
for {
|
||||
runOnce()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-t.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
+476
@@ -0,0 +1,476 @@
|
||||
# Lab5: клиент → HAProxy → API × N → PostgreSQL (данные, метаданные, jobs, outbox), Redis (сессии, кэш),
|
||||
# приватный S3 (MinIO: staging s/, оригиналы o/, варианты v/).
|
||||
# relay: outbox → RabbitMQ (publisher confirms); worker: RabbitMQ → варианты → S3 + PostgreSQL.
|
||||
# Lab6: API / relay / worker —OTLP/HTTP→ otel-collector → Tempo (traces), Loki (logs), Prometheus (metrics,
|
||||
# pull :8889, alert rules) → Grafana (provisioning из infra/grafana). Сеть telemetry отделена от back.
|
||||
# Запуск: scripts/init-secrets.sh && docker compose --env-file .env up -d --build --wait
|
||||
name: mtusi2026lab6
|
||||
|
||||
x-api-env: &api-env
|
||||
APP_HOST: ${APP_HOST:-0.0.0.0}
|
||||
APP_PORT: ${APP_PORT:-8080}
|
||||
APP_ENV: ${APP_ENV:-lab}
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
SESSION_STORE: ${SESSION_STORE:-redis}
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
CACHE_URL_FILE: /run/secrets/cache_url
|
||||
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
|
||||
DRAIN_DELAY: ${DRAIN_DELAY:-3s}
|
||||
SHUTDOWN_TIMEOUT: ${SHUTDOWN_TIMEOUT:-8s}
|
||||
DB_POOL_MAX: ${DB_POOL_MAX:-20}
|
||||
CACHE_INVALIDATION: ${CACHE_INVALIDATION:-version}
|
||||
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000}
|
||||
S3_REGION: ${S3_REGION:-us-east-1}
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
||||
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
|
||||
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
|
||||
PIPELINE_VERSION: ${PIPELINE_VERSION:-1}
|
||||
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
||||
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
|
||||
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
|
||||
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
|
||||
WEBP_QUALITY: ${WEBP_QUALITY:-80}
|
||||
WORKER_PIXEL_BUDGET: ${WORKER_PIXEL_BUDGET:-60000000}
|
||||
RETRY_BASE_DELAY: ${RETRY_BASE_DELAY:-2s}
|
||||
RETRY_MAX_DELAY: ${RETRY_MAX_DELAY:-60s}
|
||||
# OpenTelemetry SDK (стандартные переменные; service.name задаёт каждый сервис)
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT:-http://otel-collector:4318}
|
||||
OTEL_EXPORTER_OTLP_PROTOCOL: ${OTEL_EXPORTER_OTLP_PROTOCOL:-http/protobuf}
|
||||
OTEL_RESOURCE_ATTRIBUTES: ${OTEL_RESOURCE_ATTRIBUTES:-deployment.environment.name=lab}
|
||||
OTEL_SDK_DISABLED: ${OTEL_SDK_DISABLED:-false}
|
||||
OTEL_TRACES_SAMPLER: ${OTEL_TRACES_SAMPLER:-parentbased_always_on}
|
||||
OTEL_TRACES_SAMPLER_ARG: ${OTEL_TRACES_SAMPLER_ARG:-1.0}
|
||||
|
||||
services:
|
||||
lb:
|
||||
image: haproxy:3.0.5-alpine
|
||||
ports:
|
||||
- "127.0.0.1:${LB_PORT:-8080}:8080"
|
||||
- "127.0.0.1:${LB_STATS_PORT:-8404}:8404"
|
||||
environment:
|
||||
LB_BALANCE: ${LB_BALANCE:-roundrobin}
|
||||
volumes:
|
||||
- ./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
|
||||
depends_on:
|
||||
api:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
networks: [front]
|
||||
deploy:
|
||||
resources:
|
||||
limits: { cpus: "${LB_CPUS:-0.50}", memory: 128M }
|
||||
|
||||
api:
|
||||
build: .
|
||||
image: mtusi2026lab6-api:local
|
||||
# Без container_name и без ports: реплики доступны только через lb.
|
||||
environment:
|
||||
<<: *api-env
|
||||
OTEL_SERVICE_NAME: messenger-api
|
||||
secrets: [database_url, cache_url, s3_access_key_id, s3_secret_access_key]
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
s3-init:
|
||||
condition: service_completed_successfully
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
redis:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "--spider", "-q", "http://localhost:8080/health/ready"]
|
||||
interval: 5s
|
||||
timeout: 2s
|
||||
retries: 3
|
||||
start_period: 5s
|
||||
# DRAIN_DELAY + SHUTDOWN_TIMEOUT < stop_grace_period, иначе SIGKILL оборвёт запросы.
|
||||
stop_grace_period: 15s
|
||||
restart: unless-stopped
|
||||
networks: [front, back, telemetry]
|
||||
deploy:
|
||||
replicas: ${API_REPLICAS:-2}
|
||||
resources:
|
||||
limits: { cpus: "${API_CPUS:-0.50}", memory: "${API_MEMORY:-256M}" }
|
||||
|
||||
# Однократные миграции до старта API: реплики не гоняются за схему.
|
||||
migrate:
|
||||
image: migrate/migrate:v4.17.0
|
||||
volumes:
|
||||
- ./migrations:/migrations:ro
|
||||
secrets: [database_url]
|
||||
entrypoint: ["/bin/sh", "-c", "exec migrate -path /migrations -database \"$$(cat /run/secrets/database_url)\" up"]
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
restart: "no"
|
||||
networks: [back]
|
||||
|
||||
postgres:
|
||||
image: postgres:16.4-alpine
|
||||
command: ["postgres", "-c", "max_connections=${PG_MAX_CONNECTIONS:-100}"]
|
||||
environment:
|
||||
POSTGRES_USER: messenger
|
||||
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
|
||||
POSTGRES_DB: messenger
|
||||
secrets: [postgres_password]
|
||||
volumes:
|
||||
- pg_data:/var/lib/postgresql/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 5
|
||||
restart: unless-stopped
|
||||
networks: [back]
|
||||
deploy:
|
||||
resources:
|
||||
limits: { cpus: "${PG_CPUS:-1.00}", memory: "${PG_MEMORY:-512M}" }
|
||||
|
||||
redis:
|
||||
image: redis:7.4.1-alpine
|
||||
command: ["redis-server", "--appendonly", "yes", "--appendfsync", "everysec"]
|
||||
volumes:
|
||||
- redis_data:/data
|
||||
healthcheck:
|
||||
test: ["CMD", "redis-cli", "ping"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 5
|
||||
restart: unless-stopped
|
||||
networks: [back]
|
||||
deploy:
|
||||
resources:
|
||||
limits: { cpus: "${REDIS_CPUS:-0.50}", memory: 256M }
|
||||
|
||||
# S3-совместимое хранилище байтов вложений: MinIO RELEASE.2025-04-22T22-12-26Z.
|
||||
# Официальные образы minio/minio убраны из Docker Hub и quay.io (проверено в CI),
|
||||
# поэтому используется сборка Bitnami того же релиза, закреплённая по digest.
|
||||
# Порты не публикуются, консоль выключена (MINIO_BROWSER=off): доступ только из сети back.
|
||||
s3:
|
||||
image: bitnamilegacy/minio:2025.4.22@sha256:50cec18ac4184af4671a78aedd5554942c8ae105d51a465fa82037949046da01
|
||||
entrypoint: ["/bin/bash", "-c"]
|
||||
command:
|
||||
- export MINIO_ROOT_USER="$$(cat /run/secrets/s3_root_user)" MINIO_ROOT_PASSWORD="$$(cat /run/secrets/s3_root_password)"; exec /opt/bitnami/minio/bin/minio server /bitnami/minio/data --address :9000
|
||||
environment:
|
||||
MINIO_BROWSER: "off"
|
||||
secrets: [s3_root_user, s3_root_password]
|
||||
volumes:
|
||||
- s3_data:/bitnami/minio/data
|
||||
healthcheck:
|
||||
# Без curl в образе: HTTP/1.1-запрос к /minio/health/live через /dev/tcp и read (только bash).
|
||||
test: ["CMD", "bash", "-c", "exec 3<>/dev/tcp/127.0.0.1/9000 && printf 'GET /minio/health/live HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n' >&3 && read -r line <&3 && [[ $$line == *' 200 '* ]]"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
restart: unless-stopped
|
||||
networks: [back]
|
||||
deploy:
|
||||
resources:
|
||||
limits: { cpus: "${S3_CPUS:-0.50}", memory: 512M }
|
||||
|
||||
# Идемпотентный bootstrap: приватный bucket, политика и отдельный пользователь
|
||||
# API с правами только на этот bucket. Root-учётка есть только у s3 и s3-init.
|
||||
s3-init:
|
||||
image: bitnamilegacy/minio-client:2025.4.16@sha256:8a86e441decf053093c5977d49290298295f850c7843652f93e970ab15871dd1
|
||||
entrypoint: ["/bin/sh", "/init/init.sh"]
|
||||
environment:
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
PATH: /opt/bitnami/minio-client/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
|
||||
MC_CONFIG_DIR: /tmp/.mc
|
||||
secrets: [s3_root_user, s3_root_password, s3_access_key_id, s3_secret_access_key]
|
||||
volumes:
|
||||
- ./infra/s3:/init:ro
|
||||
depends_on:
|
||||
s3:
|
||||
condition: service_healthy
|
||||
restart: "no"
|
||||
networks: [back]
|
||||
|
||||
# Сверка PostgreSQL и S3: брошенные загрузки, сироты, потерянные объекты.
|
||||
# Безопасный grace period 1h; одновременно работает один экземпляр (advisory lock).
|
||||
reconciler:
|
||||
image: mtusi2026lab6-api:local
|
||||
entrypoint: ["/app/reconcile", "--apply", "--interval", "${RECONCILE_INTERVAL:-10m}", "--grace", "${RECONCILE_GRACE:-1h}"]
|
||||
environment:
|
||||
<<: *api-env
|
||||
CACHE_URL_FILE: "" # Redis reconciler не нужен
|
||||
secrets: [database_url, s3_access_key_id, s3_secret_access_key]
|
||||
# HTTP не обслуживает, поэтому вместо HEALTHCHECK образа API — проверка, что
|
||||
# процесс reconcile жив (нужна и для docker compose up --wait).
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pgrep -x reconcile >/dev/null || pidof reconcile >/dev/null"]
|
||||
interval: 10s
|
||||
timeout: 3s
|
||||
retries: 3
|
||||
start_period: 5s
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
s3-init:
|
||||
condition: service_completed_successfully
|
||||
restart: unless-stopped
|
||||
networks: [back]
|
||||
deploy:
|
||||
resources:
|
||||
limits: { cpus: "0.25", memory: 128M }
|
||||
|
||||
# Broker: quorum-очереди на диске (volume), без management UI, порты не публикуются.
|
||||
# Пользователь и пароль — из secret rabbitmq.conf (guest отключён вне localhost).
|
||||
rabbitmq:
|
||||
image: rabbitmq:4.0.9-alpine
|
||||
hostname: rabbitmq # имя узла фиксировано: данные в volume привязаны к нему
|
||||
secrets:
|
||||
- source: rabbitmq_conf
|
||||
target: /etc/rabbitmq/conf.d/20-credentials.conf
|
||||
volumes:
|
||||
- rabbitmq_data:/var/lib/rabbitmq
|
||||
healthcheck:
|
||||
test: ["CMD", "rabbitmq-diagnostics", "-q", "check_port_connectivity"]
|
||||
interval: 5s
|
||||
timeout: 10s
|
||||
retries: 20
|
||||
start_period: 10s
|
||||
restart: unless-stopped
|
||||
# telemetry — только чтобы Prometheus забирал :15692 (метрики очередей, без AMQP-учётки).
|
||||
networks: [back, telemetry]
|
||||
deploy:
|
||||
resources:
|
||||
limits: { cpus: "0.50", memory: 512M }
|
||||
|
||||
# Relay: публикует transactional outbox с publisher confirms, возвращает в очередь
|
||||
# работу с истёкшим lease, чистит staging. Не зависит от доступности API.
|
||||
relay:
|
||||
image: mtusi2026lab6-api:local
|
||||
entrypoint: ["/app/pipeline", "relay"]
|
||||
environment:
|
||||
<<: *api-env
|
||||
CACHE_URL_FILE: ""
|
||||
BROKER_URL_FILE: /run/secrets/broker_url
|
||||
FAULT_POINTS: ${RELAY_FAULT_POINTS:-}
|
||||
OTEL_SERVICE_NAME: messenger-relay
|
||||
secrets: [database_url, s3_access_key_id, s3_secret_access_key, broker_url]
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pgrep -x pipeline >/dev/null || pidof pipeline >/dev/null"]
|
||||
interval: 10s
|
||||
timeout: 3s
|
||||
retries: 3
|
||||
start_period: 5s
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
s3-init:
|
||||
condition: service_completed_successfully
|
||||
rabbitmq:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
networks: [back, telemetry]
|
||||
deploy:
|
||||
resources:
|
||||
limits: { cpus: "0.25", memory: 128M }
|
||||
|
||||
# Worker: отдельный процесс; prefetch = WORKER_CONCURRENCY, lease = JOB_TIMEOUT_SECONDS,
|
||||
# память ограничена контейнером и пиксельным бюджетом. SIGTERM → дорабатывает текущие задания.
|
||||
worker:
|
||||
image: mtusi2026lab6-api:local
|
||||
entrypoint: ["/app/pipeline", "worker"]
|
||||
environment:
|
||||
<<: *api-env
|
||||
CACHE_URL_FILE: ""
|
||||
BROKER_URL_FILE: /run/secrets/broker_url
|
||||
FAULT_POINTS: ${WORKER_FAULT_POINTS:-}
|
||||
OTEL_SERVICE_NAME: messenger-worker
|
||||
# Мягкий лимит кучи Go ≈ 75% лимита контейнера: без него GC (GOGC=100) даёт куче
|
||||
# расти до ~2× живого объёма, и пик подходил вплотную к 768M (замер прогона 4).
|
||||
GOMEMLIMIT: ${WORKER_GOMEMLIMIT:-560MiB}
|
||||
secrets: [database_url, s3_access_key_id, s3_secret_access_key, broker_url]
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pgrep -x pipeline >/dev/null || pidof pipeline >/dev/null"]
|
||||
interval: 10s
|
||||
timeout: 3s
|
||||
retries: 3
|
||||
start_period: 5s
|
||||
stop_grace_period: 70s
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
s3-init:
|
||||
condition: service_completed_successfully
|
||||
rabbitmq:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
networks: [back, telemetry]
|
||||
deploy:
|
||||
replicas: ${WORKER_REPLICAS:-1}
|
||||
resources:
|
||||
limits: { cpus: "${WORKER_CPUS:-1.00}", memory: "${WORKER_MEMORY:-768M}" }
|
||||
|
||||
# ---------------- Наблюдаемость (lab6) ----------------
|
||||
# Collector: приём OTLP (4317 gRPC / 4318 HTTP) только из сети telemetry, порты
|
||||
# наружу не публикуются. memory_limiter + ограниченные очереди экспортёров:
|
||||
# отказ Tempo/Loki не приводит к росту памяти (infra/otel/collector.yaml).
|
||||
otel-collector:
|
||||
image: otel/opentelemetry-collector-contrib:0.116.1
|
||||
command: ["--config=/etc/otelcol/${COLLECTOR_CONFIG:-collector.yaml}"]
|
||||
volumes:
|
||||
- ./infra/otel:/etc/otelcol:ro
|
||||
restart: unless-stopped
|
||||
networks: [telemetry]
|
||||
deploy:
|
||||
resources:
|
||||
limits: { cpus: "${COLLECTOR_CPUS:-0.50}", memory: 256M }
|
||||
|
||||
# Метрики: TSDB на volume (7 дней), alert rules, exemplars → Tempo.
|
||||
# UI/API только на localhost хоста (администрирование, проверки).
|
||||
prometheus:
|
||||
image: prom/prometheus:v3.1.0
|
||||
command:
|
||||
- --config.file=/etc/prometheus/prometheus.yml
|
||||
- --storage.tsdb.path=/prometheus
|
||||
- --storage.tsdb.retention.time=7d
|
||||
- --enable-feature=exemplar-storage
|
||||
- --web.enable-lifecycle
|
||||
volumes:
|
||||
- ./infra/prometheus:/etc/prometheus:ro
|
||||
- prometheus_data:/prometheus
|
||||
ports:
|
||||
- "127.0.0.1:${PROMETHEUS_PORT:-9090}:9090"
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-q", "--spider", "http://localhost:9090/-/ready"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
restart: unless-stopped
|
||||
networks: [telemetry]
|
||||
deploy:
|
||||
resources:
|
||||
limits: { cpus: "0.50", memory: 512M }
|
||||
|
||||
# Логи: Loki (OTLP /otlp), файловое хранилище на volume, retention 7 дней.
|
||||
loki:
|
||||
image: grafana/loki:3.3.2
|
||||
command: ["-config.file=/etc/loki/loki.yaml"]
|
||||
volumes:
|
||||
- ./infra/loki/loki.yaml:/etc/loki/loki.yaml:ro
|
||||
- loki_data:/loki
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-q", "--spider", "http://localhost:3100/ready"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 30
|
||||
restart: unless-stopped
|
||||
networks: [telemetry]
|
||||
deploy:
|
||||
resources:
|
||||
limits: { cpus: "0.50", memory: 1G } # 512M: OOM под нагрузкой трека 1 (~14 тыс. spans на 1000 запросов, прогон 5)
|
||||
|
||||
# Трассы: Tempo, локальный backend на volume, retention 7 дней.
|
||||
# tempo-init один раз выдаёт volume пользователю tempo (uid 10001).
|
||||
tempo-init:
|
||||
image: &tempo-image grafana/tempo:2.6.1
|
||||
user: root
|
||||
entrypoint: ["chown", "10001:10001", "/var/tempo"]
|
||||
volumes:
|
||||
- tempo_data:/var/tempo
|
||||
restart: "no"
|
||||
networks: [telemetry]
|
||||
tempo:
|
||||
image: *tempo-image
|
||||
command: ["-config.file=/etc/tempo/tempo.yaml"]
|
||||
volumes:
|
||||
- ./infra/tempo/tempo.yaml:/etc/tempo/tempo.yaml:ro
|
||||
- tempo_data:/var/tempo
|
||||
depends_on:
|
||||
tempo-init:
|
||||
condition: service_completed_successfully
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-q", "--spider", "http://localhost:3200/ready"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 30
|
||||
start_period: 10s
|
||||
restart: unless-stopped
|
||||
networks: [telemetry]
|
||||
deploy:
|
||||
resources:
|
||||
limits: { cpus: "0.50", memory: 1G } # 512M: OOM под нагрузкой трека 1 (~14 тыс. spans на 1000 запросов, прогон 5)
|
||||
|
||||
# Grafana: только интерфейс и запросы; datasources и dashboard — provisioning из Git.
|
||||
# Пароль admin — secret; порт только на localhost.
|
||||
grafana:
|
||||
image: grafana/grafana:11.4.0
|
||||
environment:
|
||||
GF_SECURITY_ADMIN_USER: admin
|
||||
GF_SECURITY_ADMIN_PASSWORD__FILE: /run/secrets/grafana_admin_password
|
||||
GF_USERS_ALLOW_SIGN_UP: "false"
|
||||
GF_ANALYTICS_REPORTING_ENABLED: "false"
|
||||
GF_ANALYTICS_CHECK_FOR_UPDATES: "false"
|
||||
GF_FEATURE_TOGGLES_ENABLE: traceqlEditor
|
||||
# Только для снимка dashboard в CI (GRAFANA_ANONYMOUS=true): анонимный Viewer.
|
||||
GF_AUTH_ANONYMOUS_ENABLED: ${GRAFANA_ANONYMOUS:-false}
|
||||
GF_AUTH_ANONYMOUS_ORG_ROLE: Viewer
|
||||
secrets: [grafana_admin_password]
|
||||
volumes:
|
||||
- ./infra/grafana/provisioning:/etc/grafana/provisioning:ro
|
||||
- ./infra/grafana/dashboards:/var/lib/grafana/dashboards:ro
|
||||
- grafana_data:/var/lib/grafana
|
||||
ports:
|
||||
- "127.0.0.1:${GRAFANA_PORT:-3000}:3000"
|
||||
depends_on:
|
||||
prometheus:
|
||||
condition: service_healthy
|
||||
loki:
|
||||
condition: service_healthy
|
||||
tempo:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-q", "--spider", "http://localhost:3000/api/health"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 30
|
||||
restart: unless-stopped
|
||||
networks: [telemetry]
|
||||
deploy:
|
||||
resources:
|
||||
limits: { cpus: "0.50", memory: 256M }
|
||||
|
||||
networks:
|
||||
# front: lb ↔ api; back: api ↔ хранилища и broker. LB не видит БД, S3 и broker.
|
||||
# telemetry: процессы приложения → Collector → хранилища сигналов → Grafana;
|
||||
# backend-ы телеметрии не видят БД, S3 и broker (кроме порта метрик RabbitMQ).
|
||||
front:
|
||||
back:
|
||||
telemetry:
|
||||
|
||||
secrets:
|
||||
postgres_password:
|
||||
file: ./secrets/postgres_password
|
||||
database_url:
|
||||
file: ./secrets/database_url
|
||||
cache_url:
|
||||
file: ./secrets/cache_url
|
||||
s3_root_user:
|
||||
file: ./secrets/s3_root_user
|
||||
s3_root_password:
|
||||
file: ./secrets/s3_root_password
|
||||
s3_access_key_id:
|
||||
file: ./secrets/s3_access_key_id
|
||||
s3_secret_access_key:
|
||||
file: ./secrets/s3_secret_access_key
|
||||
rabbitmq_conf:
|
||||
file: ./secrets/rabbitmq.conf
|
||||
broker_url:
|
||||
file: ./secrets/broker_url
|
||||
grafana_admin_password:
|
||||
file: ./secrets/grafana_admin_password
|
||||
|
||||
volumes:
|
||||
prometheus_data:
|
||||
loki_data:
|
||||
tempo_data:
|
||||
grafana_data:
|
||||
rabbitmq_data:
|
||||
s3_data:
|
||||
pg_data:
|
||||
redis_data:
|
||||
@@ -0,0 +1,4 @@
|
||||
source_commit=d90aef716adeab32daa81ea5fce789ea6b627aaf
|
||||
run=https://github.com/CharaWein/2026_lab3/actions/runs/43208190207
|
||||
go_tests=success
|
||||
stand=success
|
||||
@@ -0,0 +1,18 @@
|
||||
/mtusi2026lab6-api-3 restarts=0 oom_killed=false exit=0
|
||||
/mtusi2026lab6-api-4 restarts=0 oom_killed=false exit=0
|
||||
/mtusi2026lab6-grafana-1 restarts=0 oom_killed=false exit=0
|
||||
/mtusi2026lab6-lb-1 restarts=0 oom_killed=false exit=0
|
||||
/mtusi2026lab6-loki-1 restarts=0 oom_killed=false exit=0
|
||||
/mtusi2026lab6-migrate-1 restarts=0 oom_killed=false exit=0
|
||||
/mtusi2026lab6-otel-collector-1 restarts=0 oom_killed=false exit=0
|
||||
/mtusi2026lab6-postgres-1 restarts=0 oom_killed=false exit=0
|
||||
/mtusi2026lab6-prometheus-1 restarts=0 oom_killed=false exit=0
|
||||
/mtusi2026lab6-rabbitmq-1 restarts=0 oom_killed=false exit=0
|
||||
/mtusi2026lab6-reconciler-1 restarts=0 oom_killed=false exit=0
|
||||
/mtusi2026lab6-redis-1 restarts=0 oom_killed=false exit=0
|
||||
/mtusi2026lab6-relay-1 restarts=0 oom_killed=false exit=0
|
||||
/mtusi2026lab6-s3-1 restarts=0 oom_killed=false exit=0
|
||||
/mtusi2026lab6-s3-init-1 restarts=0 oom_killed=false exit=0
|
||||
/mtusi2026lab6-tempo-1 restarts=0 oom_killed=false exit=0
|
||||
/mtusi2026lab6-tempo-init-1 restarts=0 oom_killed=false exit=0
|
||||
/mtusi2026lab6-worker-1 restarts=0 oom_killed=false exit=0
|
||||
@@ -0,0 +1,18 @@
|
||||
/mtusi2026lab6-api-1 restarts=0 oom_killed=false exit=0
|
||||
/mtusi2026lab6-api-2 restarts=0 oom_killed=false exit=0
|
||||
/mtusi2026lab6-grafana-1 restarts=0 oom_killed=false exit=0
|
||||
/mtusi2026lab6-lb-1 restarts=0 oom_killed=false exit=0
|
||||
/mtusi2026lab6-loki-1 restarts=0 oom_killed=false exit=0
|
||||
/mtusi2026lab6-migrate-1 restarts=0 oom_killed=false exit=0
|
||||
/mtusi2026lab6-otel-collector-1 restarts=0 oom_killed=false exit=0
|
||||
/mtusi2026lab6-postgres-1 restarts=0 oom_killed=false exit=0
|
||||
/mtusi2026lab6-prometheus-1 restarts=0 oom_killed=false exit=0
|
||||
/mtusi2026lab6-rabbitmq-1 restarts=0 oom_killed=false exit=0
|
||||
/mtusi2026lab6-reconciler-1 restarts=0 oom_killed=false exit=0
|
||||
/mtusi2026lab6-redis-1 restarts=0 oom_killed=false exit=0
|
||||
/mtusi2026lab6-relay-1 restarts=0 oom_killed=false exit=0
|
||||
/mtusi2026lab6-s3-1 restarts=0 oom_killed=false exit=0
|
||||
/mtusi2026lab6-s3-init-1 restarts=0 oom_killed=false exit=0
|
||||
/mtusi2026lab6-tempo-1 restarts=0 oom_killed=false exit=0
|
||||
/mtusi2026lab6-tempo-init-1 restarts=0 oom_killed=false exit=0
|
||||
/mtusi2026lab6-worker-1 restarts=0 oom_killed=false exit=0
|
||||
@@ -0,0 +1,18 @@
|
||||
/mtusi2026lab6-api-1 restarts=0 oom_killed=false exit=0
|
||||
/mtusi2026lab6-api-2 restarts=0 oom_killed=false exit=0
|
||||
/mtusi2026lab6-grafana-1 restarts=0 oom_killed=false exit=0
|
||||
/mtusi2026lab6-lb-1 restarts=0 oom_killed=false exit=0
|
||||
/mtusi2026lab6-loki-1 restarts=0 oom_killed=false exit=0
|
||||
/mtusi2026lab6-migrate-1 restarts=0 oom_killed=false exit=0
|
||||
/mtusi2026lab6-otel-collector-1 restarts=0 oom_killed=false exit=0
|
||||
/mtusi2026lab6-postgres-1 restarts=0 oom_killed=false exit=0
|
||||
/mtusi2026lab6-prometheus-1 restarts=0 oom_killed=false exit=0
|
||||
/mtusi2026lab6-rabbitmq-1 restarts=0 oom_killed=false exit=0
|
||||
/mtusi2026lab6-reconciler-1 restarts=0 oom_killed=false exit=0
|
||||
/mtusi2026lab6-redis-1 restarts=0 oom_killed=false exit=0
|
||||
/mtusi2026lab6-relay-1 restarts=0 oom_killed=false exit=0
|
||||
/mtusi2026lab6-s3-1 restarts=0 oom_killed=false exit=0
|
||||
/mtusi2026lab6-s3-init-1 restarts=0 oom_killed=false exit=0
|
||||
/mtusi2026lab6-tempo-1 restarts=0 oom_killed=false exit=0
|
||||
/mtusi2026lab6-tempo-init-1 restarts=0 oom_killed=false exit=0
|
||||
/mtusi2026lab6-worker-1 restarts=0 oom_killed=false exit=0
|
||||
@@ -0,0 +1,18 @@
|
||||
/mtusi2026lab6-api-1 restarts=0 oom_killed=false exit=0
|
||||
/mtusi2026lab6-api-2 restarts=0 oom_killed=false exit=0
|
||||
/mtusi2026lab6-grafana-1 restarts=0 oom_killed=false exit=0
|
||||
/mtusi2026lab6-lb-1 restarts=0 oom_killed=false exit=0
|
||||
/mtusi2026lab6-loki-1 restarts=0 oom_killed=false exit=0
|
||||
/mtusi2026lab6-migrate-1 restarts=0 oom_killed=false exit=0
|
||||
/mtusi2026lab6-otel-collector-1 restarts=0 oom_killed=false exit=0
|
||||
/mtusi2026lab6-postgres-1 restarts=0 oom_killed=false exit=0
|
||||
/mtusi2026lab6-prometheus-1 restarts=0 oom_killed=false exit=0
|
||||
/mtusi2026lab6-rabbitmq-1 restarts=0 oom_killed=false exit=0
|
||||
/mtusi2026lab6-reconciler-1 restarts=0 oom_killed=false exit=0
|
||||
/mtusi2026lab6-redis-1 restarts=0 oom_killed=false exit=0
|
||||
/mtusi2026lab6-relay-1 restarts=0 oom_killed=false exit=0
|
||||
/mtusi2026lab6-s3-1 restarts=0 oom_killed=false exit=0
|
||||
/mtusi2026lab6-s3-init-1 restarts=0 oom_killed=false exit=0
|
||||
/mtusi2026lab6-tempo-1 restarts=0 oom_killed=false exit=0
|
||||
/mtusi2026lab6-tempo-init-1 restarts=0 oom_killed=false exit=0
|
||||
/mtusi2026lab6-worker-1 restarts=0 oom_killed=false exit=0
|
||||
@@ -0,0 +1,18 @@
|
||||
/mtusi2026lab6-api-1 restarts=0 oom_killed=false exit=0
|
||||
/mtusi2026lab6-api-2 restarts=0 oom_killed=false exit=0
|
||||
/mtusi2026lab6-grafana-1 restarts=0 oom_killed=false exit=0
|
||||
/mtusi2026lab6-lb-1 restarts=0 oom_killed=false exit=0
|
||||
/mtusi2026lab6-loki-1 restarts=0 oom_killed=false exit=0
|
||||
/mtusi2026lab6-migrate-1 restarts=0 oom_killed=false exit=0
|
||||
/mtusi2026lab6-otel-collector-1 restarts=0 oom_killed=false exit=0
|
||||
/mtusi2026lab6-postgres-1 restarts=0 oom_killed=false exit=0
|
||||
/mtusi2026lab6-prometheus-1 restarts=0 oom_killed=false exit=0
|
||||
/mtusi2026lab6-rabbitmq-1 restarts=0 oom_killed=false exit=0
|
||||
/mtusi2026lab6-reconciler-1 restarts=0 oom_killed=false exit=0
|
||||
/mtusi2026lab6-redis-1 restarts=0 oom_killed=false exit=0
|
||||
/mtusi2026lab6-relay-1 restarts=0 oom_killed=false exit=0
|
||||
/mtusi2026lab6-s3-1 restarts=0 oom_killed=false exit=0
|
||||
/mtusi2026lab6-s3-init-1 restarts=0 oom_killed=false exit=0
|
||||
/mtusi2026lab6-tempo-1 restarts=0 oom_killed=false exit=0
|
||||
/mtusi2026lab6-tempo-init-1 restarts=0 oom_killed=false exit=0
|
||||
/mtusi2026lab6-worker-1 restarts=0 oom_killed=false exit=0
|
||||
@@ -0,0 +1,18 @@
|
||||
2026-11-12T11:23:35Z
|
||||
Docker Compose version v2.38.2
|
||||
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
|
||||
mtusi2026lab6-api-1 mtusi2026lab6-api:local "/app/api" api 28 seconds ago Up 19 seconds (healthy) 8080/tcp
|
||||
mtusi2026lab6-api-2 mtusi2026lab6-api:local "/app/api" api 28 seconds ago Up 19 seconds (healthy) 8080/tcp
|
||||
mtusi2026lab6-grafana-1 grafana/grafana:11.4.0 "/run.sh" grafana 28 seconds ago Up 6 seconds (healthy) 127.0.0.1:3000->3000/tcp
|
||||
mtusi2026lab6-lb-1 haproxy:3.0.5-alpine "docker-entrypoint.s…" lb 28 seconds ago Up 13 seconds 127.0.0.1:8080->8080/tcp, 127.0.0.1:8404->8404/tcp
|
||||
mtusi2026lab6-loki-1 grafana/loki:3.3.2 "/usr/bin/loki -conf…" loki 28 seconds ago Up 27 seconds (healthy) 3100/tcp
|
||||
mtusi2026lab6-otel-collector-1 otel/opentelemetry-collector-contrib:0.116.1 "/otelcol-contrib --…" otel-collector 28 seconds ago Up 27 seconds 4317-4318/tcp, 55678-55679/tcp
|
||||
mtusi2026lab6-postgres-1 postgres:16.4-alpine "docker-entrypoint.s…" postgres 28 seconds ago Up 27 seconds (healthy) 5432/tcp
|
||||
mtusi2026lab6-prometheus-1 prom/prometheus:v3.1.0 "/bin/prometheus --c…" prometheus 28 seconds ago Up 27 seconds (healthy) 127.0.0.1:9090->9090/tcp
|
||||
mtusi2026lab6-rabbitmq-1 rabbitmq:4.0.9-alpine "docker-entrypoint.s…" rabbitmq 28 seconds ago Up 27 seconds (healthy) 4369/tcp, 5671-5672/tcp, 15691-15692/tcp, 25672/tcp
|
||||
mtusi2026lab6-reconciler-1 mtusi2026lab6-api:local "/app/reconcile --ap…" reconciler 28 seconds ago Up 19 seconds (healthy) 8080/tcp
|
||||
mtusi2026lab6-redis-1 redis:7.4.1-alpine "docker-entrypoint.s…" redis 28 seconds ago Up 27 seconds (healthy) 6379/tcp
|
||||
mtusi2026lab6-relay-1 mtusi2026lab6-api:local "/app/pipeline relay" relay 28 seconds ago Up 13 seconds (healthy) 8080/tcp
|
||||
mtusi2026lab6-s3-1 bitnamilegacy/minio:2025.4.22@sha256:50cec18ac4184af4671a78aedd5554942c8ae105d51a465fa82037949046da01 "/bin/bash -c 'expor…" s3 28 seconds ago Up 27 seconds (healthy) 9000-9001/tcp
|
||||
mtusi2026lab6-tempo-1 grafana/tempo:2.6.1 "/tempo -config.file…" tempo 28 seconds ago Up 26 seconds (healthy)
|
||||
mtusi2026lab6-worker-1 mtusi2026lab6-api:local "/app/pipeline worker" worker 28 seconds ago Up 13 seconds (healthy) 8080/tcp
|
||||
@@ -0,0 +1,19 @@
|
||||
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
|
||||
mtusi2026lab6-api-3 mtusi2026lab6-api:local "/app/api" api 55 seconds ago Up 55 seconds (healthy) 8080/tcp
|
||||
mtusi2026lab6-api-4 mtusi2026lab6-api:local "/app/api" api 41 seconds ago Up 41 seconds (healthy) 8080/tcp
|
||||
mtusi2026lab6-grafana-1 grafana/grafana:11.4.0 "/run.sh" grafana 6 minutes ago Up 4 minutes (healthy) 127.0.0.1:3000->3000/tcp
|
||||
mtusi2026lab6-lb-1 haproxy:3.0.5-alpine "docker-entrypoint.s…" lb 6 minutes ago Up 4 minutes 127.0.0.1:8080->8080/tcp, 127.0.0.1:8404->8404/tcp
|
||||
mtusi2026lab6-loki-1 grafana/loki:3.3.2 "/usr/bin/loki -conf…" loki 6 minutes ago Up 4 minutes (healthy) 3100/tcp
|
||||
mtusi2026lab6-migrate-1 migrate/migrate:v4.17.0 "/bin/sh -c 'exec mi…" migrate 6 minutes ago Exited (0) 4 minutes ago
|
||||
mtusi2026lab6-otel-collector-1 otel/opentelemetry-collector-contrib:0.116.1 "/otelcol-contrib --…" otel-collector 6 minutes ago Up 4 minutes 4317-4318/tcp, 55678-55679/tcp
|
||||
mtusi2026lab6-postgres-1 postgres:16.4-alpine "docker-entrypoint.s…" postgres 6 minutes ago Up 4 minutes (healthy) 5432/tcp
|
||||
mtusi2026lab6-prometheus-1 prom/prometheus:v3.1.0 "/bin/prometheus --c…" prometheus 6 minutes ago Up 4 minutes (healthy) 127.0.0.1:9090->9090/tcp
|
||||
mtusi2026lab6-rabbitmq-1 rabbitmq:4.0.9-alpine "docker-entrypoint.s…" rabbitmq 6 minutes ago Up 4 minutes (healthy) 4369/tcp, 5671-5672/tcp, 15691-15692/tcp, 25672/tcp
|
||||
mtusi2026lab6-reconciler-1 mtusi2026lab6-api:local "/app/reconcile --ap…" reconciler 6 minutes ago Up 4 minutes (healthy) 8080/tcp
|
||||
mtusi2026lab6-redis-1 redis:7.4.1-alpine "docker-entrypoint.s…" redis 6 minutes ago Up 4 minutes (healthy) 6379/tcp
|
||||
mtusi2026lab6-relay-1 mtusi2026lab6-api:local "/app/pipeline relay" relay 6 minutes ago Up 4 minutes (healthy) 8080/tcp
|
||||
mtusi2026lab6-s3-1 bitnamilegacy/minio:2025.4.22@sha256:50cec18ac4184af4671a78aedd5554942c8ae105d51a465fa82037949046da01 "/bin/bash -c 'expor…" s3 6 minutes ago Up 4 minutes (healthy) 9000-9001/tcp
|
||||
mtusi2026lab6-s3-init-1 bitnamilegacy/minio-client:2025.4.16@sha256:8a86e441decf053093c5977d49290298295f850c7843652f93e970ab15871dd1 "/bin/sh /init/init.…" s3-init 6 minutes ago Exited (0) 4 minutes ago
|
||||
mtusi2026lab6-tempo-1 grafana/tempo:2.6.1 "/tempo -config.file…" tempo 6 minutes ago Up 4 minutes (healthy)
|
||||
mtusi2026lab6-tempo-init-1 grafana/tempo:2.6.1 "chown 10001:10001 /…" tempo-init 6 minutes ago Exited (0) 6 minutes ago
|
||||
mtusi2026lab6-worker-1 mtusi2026lab6-api:local "/app/pipeline worker" worker 6 minutes ago Up 4 minutes (healthy) 8080/tcp
|
||||
@@ -0,0 +1,19 @@
|
||||
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
|
||||
mtusi2026lab6-api-1 mtusi2026lab6-api:local "/app/api" api 3 minutes ago Up 12 seconds (healthy) 8080/tcp
|
||||
mtusi2026lab6-api-2 mtusi2026lab6-api:local "/app/api" api 3 minutes ago Up 12 seconds (healthy) 8080/tcp
|
||||
mtusi2026lab6-grafana-1 grafana/grafana:11.4.0 "/run.sh" grafana 3 minutes ago Up 2 minutes (healthy) 127.0.0.1:3000->3000/tcp
|
||||
mtusi2026lab6-lb-1 haproxy:3.0.5-alpine "docker-entrypoint.s…" lb 3 minutes ago Up 2 minutes 127.0.0.1:8080->8080/tcp, 127.0.0.1:8404->8404/tcp
|
||||
mtusi2026lab6-loki-1 grafana/loki:3.3.2 "/usr/bin/loki -conf…" loki 3 minutes ago Up 3 minutes (healthy) 3100/tcp
|
||||
mtusi2026lab6-migrate-1 migrate/migrate:v4.17.0 "/bin/sh -c 'exec mi…" migrate 3 minutes ago Exited (0) 10 seconds ago
|
||||
mtusi2026lab6-otel-collector-1 otel/opentelemetry-collector-contrib:0.116.1 "/otelcol-contrib --…" otel-collector 3 minutes ago Up 3 minutes 4317-4318/tcp, 55678-55679/tcp
|
||||
mtusi2026lab6-postgres-1 postgres:16.4-alpine "docker-entrypoint.s…" postgres 3 minutes ago Up 3 minutes (healthy) 5432/tcp
|
||||
mtusi2026lab6-prometheus-1 prom/prometheus:v3.1.0 "/bin/prometheus --c…" prometheus 3 minutes ago Up 3 minutes (healthy) 127.0.0.1:9090->9090/tcp
|
||||
mtusi2026lab6-rabbitmq-1 rabbitmq:4.0.9-alpine "docker-entrypoint.s…" rabbitmq 3 minutes ago Up 3 minutes (healthy) 4369/tcp, 5671-5672/tcp, 15691-15692/tcp, 25672/tcp
|
||||
mtusi2026lab6-reconciler-1 mtusi2026lab6-api:local "/app/reconcile --ap…" reconciler 3 minutes ago Up 2 minutes (healthy) 8080/tcp
|
||||
mtusi2026lab6-redis-1 redis:7.4.1-alpine "docker-entrypoint.s…" redis 3 minutes ago Up 3 minutes (healthy) 6379/tcp
|
||||
mtusi2026lab6-relay-1 mtusi2026lab6-api:local "/app/pipeline relay" relay 3 minutes ago Up 2 minutes (healthy) 8080/tcp
|
||||
mtusi2026lab6-s3-1 bitnamilegacy/minio:2025.4.22@sha256:50cec18ac4184af4671a78aedd5554942c8ae105d51a465fa82037949046da01 "/bin/bash -c 'expor…" s3 3 minutes ago Up About a minute (healthy) 9000-9001/tcp
|
||||
mtusi2026lab6-s3-init-1 bitnamilegacy/minio-client:2025.4.16@sha256:8a86e441decf053093c5977d49290298295f850c7843652f93e970ab15871dd1 "/bin/sh /init/init.…" s3-init 3 minutes ago Exited (0) 9 seconds ago
|
||||
mtusi2026lab6-tempo-1 grafana/tempo:2.6.1 "/tempo -config.file…" tempo 3 minutes ago Up 3 minutes (healthy)
|
||||
mtusi2026lab6-tempo-init-1 grafana/tempo:2.6.1 "chown 10001:10001 /…" tempo-init 3 minutes ago Exited (0) 3 minutes ago
|
||||
mtusi2026lab6-worker-1 mtusi2026lab6-api:local "/app/pipeline worker" worker 3 minutes ago Up 2 minutes (healthy) 8080/tcp
|
||||
@@ -0,0 +1,19 @@
|
||||
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
|
||||
mtusi2026lab6-api-1 mtusi2026lab6-api:local "/app/api" api 10 minutes ago Up 10 minutes (healthy) 8080/tcp
|
||||
mtusi2026lab6-api-2 mtusi2026lab6-api:local "/app/api" api 10 minutes ago Up 10 minutes (healthy) 8080/tcp
|
||||
mtusi2026lab6-grafana-1 grafana/grafana:11.4.0 "/run.sh" grafana 10 minutes ago Up 9 minutes (healthy) 127.0.0.1:3000->3000/tcp
|
||||
mtusi2026lab6-lb-1 haproxy:3.0.5-alpine "docker-entrypoint.s…" lb 10 minutes ago Up 10 minutes 127.0.0.1:8080->8080/tcp, 127.0.0.1:8404->8404/tcp
|
||||
mtusi2026lab6-loki-1 grafana/loki:3.3.2 "/usr/bin/loki -conf…" loki 10 minutes ago Up 2 minutes (healthy) 3100/tcp
|
||||
mtusi2026lab6-migrate-1 migrate/migrate:v4.17.0 "/bin/sh -c 'exec mi…" migrate 10 minutes ago Exited (0) 7 minutes ago
|
||||
mtusi2026lab6-otel-collector-1 otel/opentelemetry-collector-contrib:0.116.1 "/otelcol-contrib --…" otel-collector 10 minutes ago Up 39 seconds 4317-4318/tcp, 55678-55679/tcp
|
||||
mtusi2026lab6-postgres-1 postgres:16.4-alpine "docker-entrypoint.s…" postgres 10 minutes ago Up 10 minutes (healthy) 5432/tcp
|
||||
mtusi2026lab6-prometheus-1 prom/prometheus:v3.1.0 "/bin/prometheus --c…" prometheus 10 minutes ago Up 10 minutes (healthy) 127.0.0.1:9090->9090/tcp
|
||||
mtusi2026lab6-rabbitmq-1 rabbitmq:4.0.9-alpine "docker-entrypoint.s…" rabbitmq 10 minutes ago Up 10 minutes (healthy) 4369/tcp, 5671-5672/tcp, 15691-15692/tcp, 25672/tcp
|
||||
mtusi2026lab6-reconciler-1 mtusi2026lab6-api:local "/app/reconcile --ap…" reconciler 10 minutes ago Up 10 minutes (healthy) 8080/tcp
|
||||
mtusi2026lab6-redis-1 redis:7.4.1-alpine "docker-entrypoint.s…" redis 10 minutes ago Up 10 minutes (healthy) 6379/tcp
|
||||
mtusi2026lab6-relay-1 mtusi2026lab6-api:local "/app/pipeline relay" relay 10 minutes ago Up 10 minutes (healthy) 8080/tcp
|
||||
mtusi2026lab6-s3-1 bitnamilegacy/minio:2025.4.22@sha256:50cec18ac4184af4671a78aedd5554942c8ae105d51a465fa82037949046da01 "/bin/bash -c 'expor…" s3 10 minutes ago Up 9 minutes (healthy) 9000-9001/tcp
|
||||
mtusi2026lab6-s3-init-1 bitnamilegacy/minio-client:2025.4.16@sha256:8a86e441decf053093c5977d49290298295f850c7843652f93e970ab15871dd1 "/bin/sh /init/init.…" s3-init 10 minutes ago Exited (0) 7 minutes ago
|
||||
mtusi2026lab6-tempo-1 grafana/tempo:2.6.1 "/tempo -config.file…" tempo 10 minutes ago Up 2 minutes (healthy)
|
||||
mtusi2026lab6-tempo-init-1 grafana/tempo:2.6.1 "chown 10001:10001 /…" tempo-init 10 minutes ago Exited (0) 2 minutes ago
|
||||
mtusi2026lab6-worker-1 mtusi2026lab6-api:local "/app/pipeline worker" worker 10 minutes ago Up 7 minutes (healthy) 8080/tcp
|
||||
@@ -0,0 +1,19 @@
|
||||
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
|
||||
mtusi2026lab6-api-1 mtusi2026lab6-api:local "/app/api" api 4 minutes ago Up 2 minutes (healthy) 8080/tcp
|
||||
mtusi2026lab6-api-2 mtusi2026lab6-api:local "/app/api" api 4 minutes ago Up 2 minutes (healthy) 8080/tcp
|
||||
mtusi2026lab6-grafana-1 grafana/grafana:11.4.0 "/run.sh" grafana 4 minutes ago Up 3 minutes (healthy) 127.0.0.1:3000->3000/tcp
|
||||
mtusi2026lab6-lb-1 haproxy:3.0.5-alpine "docker-entrypoint.s…" lb 4 minutes ago Up 3 minutes 127.0.0.1:8080->8080/tcp, 127.0.0.1:8404->8404/tcp
|
||||
mtusi2026lab6-loki-1 grafana/loki:3.3.2 "/usr/bin/loki -conf…" loki 4 minutes ago Up 4 minutes (healthy) 3100/tcp
|
||||
mtusi2026lab6-migrate-1 migrate/migrate:v4.17.0 "/bin/sh -c 'exec mi…" migrate 4 minutes ago Exited (0) 36 seconds ago
|
||||
mtusi2026lab6-otel-collector-1 otel/opentelemetry-collector-contrib:0.116.1 "/otelcol-contrib --…" otel-collector 4 minutes ago Up 4 minutes 4317-4318/tcp, 55678-55679/tcp
|
||||
mtusi2026lab6-postgres-1 postgres:16.4-alpine "docker-entrypoint.s…" postgres 4 minutes ago Up 4 minutes (healthy) 5432/tcp
|
||||
mtusi2026lab6-prometheus-1 prom/prometheus:v3.1.0 "/bin/prometheus --c…" prometheus 4 minutes ago Up 4 minutes (healthy) 127.0.0.1:9090->9090/tcp
|
||||
mtusi2026lab6-rabbitmq-1 rabbitmq:4.0.9-alpine "docker-entrypoint.s…" rabbitmq 4 minutes ago Up About a minute (healthy) 4369/tcp, 5671-5672/tcp, 15691-15692/tcp, 25672/tcp
|
||||
mtusi2026lab6-reconciler-1 mtusi2026lab6-api:local "/app/reconcile --ap…" reconciler 4 minutes ago Up 3 minutes (healthy) 8080/tcp
|
||||
mtusi2026lab6-redis-1 redis:7.4.1-alpine "docker-entrypoint.s…" redis 4 minutes ago Up 4 minutes (healthy) 6379/tcp
|
||||
mtusi2026lab6-relay-1 mtusi2026lab6-api:local "/app/pipeline relay" relay 2 minutes ago Up 2 minutes (healthy) 8080/tcp
|
||||
mtusi2026lab6-s3-1 bitnamilegacy/minio:2025.4.22@sha256:50cec18ac4184af4671a78aedd5554942c8ae105d51a465fa82037949046da01 "/bin/bash -c 'expor…" s3 4 minutes ago Up 46 seconds (healthy) 9000-9001/tcp
|
||||
mtusi2026lab6-s3-init-1 bitnamilegacy/minio-client:2025.4.16@sha256:8a86e441decf053093c5977d49290298295f850c7843652f93e970ab15871dd1 "/bin/sh /init/init.…" s3-init 4 minutes ago Exited (0) 35 seconds ago
|
||||
mtusi2026lab6-tempo-1 grafana/tempo:2.6.1 "/tempo -config.file…" tempo 4 minutes ago Up 3 minutes (healthy)
|
||||
mtusi2026lab6-tempo-init-1 grafana/tempo:2.6.1 "chown 10001:10001 /…" tempo-init 4 minutes ago Exited (0) 4 minutes ago
|
||||
mtusi2026lab6-worker-1 mtusi2026lab6-api:local "/app/pipeline worker" worker About a minute ago Up 20 seconds (healthy) 8080/tcp
|
||||
@@ -0,0 +1,19 @@
|
||||
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
|
||||
mtusi2026lab6-api-1 mtusi2026lab6-api:local "/app/api" api About a minute ago Up About a minute (healthy) 8080/tcp
|
||||
mtusi2026lab6-api-2 mtusi2026lab6-api:local "/app/api" api About a minute ago Up About a minute (healthy) 8080/tcp
|
||||
mtusi2026lab6-grafana-1 grafana/grafana:11.4.0 "/run.sh" grafana 19 minutes ago Up 19 minutes (healthy) 127.0.0.1:3000->3000/tcp
|
||||
mtusi2026lab6-lb-1 haproxy:3.0.5-alpine "docker-entrypoint.s…" lb 19 minutes ago Up 19 minutes 127.0.0.1:8080->8080/tcp, 127.0.0.1:8404->8404/tcp
|
||||
mtusi2026lab6-loki-1 grafana/loki:3.3.2 "/usr/bin/loki -conf…" loki 19 minutes ago Up 19 minutes (healthy) 3100/tcp
|
||||
mtusi2026lab6-migrate-1 migrate/migrate:v4.17.0 "/bin/sh -c 'exec mi…" migrate 19 minutes ago Exited (0) 19 minutes ago
|
||||
mtusi2026lab6-otel-collector-1 otel/opentelemetry-collector-contrib:0.116.1 "/otelcol-contrib --…" otel-collector 3 minutes ago Up 3 minutes 4317-4318/tcp, 55678-55679/tcp
|
||||
mtusi2026lab6-postgres-1 postgres:16.4-alpine "docker-entrypoint.s…" postgres 19 minutes ago Up 19 minutes (healthy) 5432/tcp
|
||||
mtusi2026lab6-prometheus-1 prom/prometheus:v3.1.0 "/bin/prometheus --c…" prometheus 19 minutes ago Up 19 minutes (healthy) 127.0.0.1:9090->9090/tcp
|
||||
mtusi2026lab6-rabbitmq-1 rabbitmq:4.0.9-alpine "docker-entrypoint.s…" rabbitmq 19 minutes ago Up 19 minutes (healthy) 4369/tcp, 5671-5672/tcp, 15691-15692/tcp, 25672/tcp
|
||||
mtusi2026lab6-reconciler-1 mtusi2026lab6-api:local "/app/reconcile --ap…" reconciler 19 minutes ago Up 19 minutes (healthy) 8080/tcp
|
||||
mtusi2026lab6-redis-1 redis:7.4.1-alpine "docker-entrypoint.s…" redis 19 minutes ago Up 19 minutes (healthy) 6379/tcp
|
||||
mtusi2026lab6-relay-1 mtusi2026lab6-api:local "/app/pipeline relay" relay About a minute ago Up About a minute (healthy) 8080/tcp
|
||||
mtusi2026lab6-s3-1 bitnamilegacy/minio:2025.4.22@sha256:50cec18ac4184af4671a78aedd5554942c8ae105d51a465fa82037949046da01 "/bin/bash -c 'expor…" s3 19 minutes ago Up 19 minutes (healthy) 9000-9001/tcp
|
||||
mtusi2026lab6-s3-init-1 bitnamilegacy/minio-client:2025.4.16@sha256:8a86e441decf053093c5977d49290298295f850c7843652f93e970ab15871dd1 "/bin/sh /init/init.…" s3-init 19 minutes ago Exited (0) 19 minutes ago
|
||||
mtusi2026lab6-tempo-1 grafana/tempo:2.6.1 "/tempo -config.file…" tempo 19 minutes ago Up 19 minutes (healthy)
|
||||
mtusi2026lab6-tempo-init-1 grafana/tempo:2.6.1 "chown 10001:10001 /…" tempo-init 19 minutes ago Exited (0) 19 minutes ago
|
||||
mtusi2026lab6-worker-1 mtusi2026lab6-api:local "/app/pipeline worker" worker About a minute ago Up About a minute (healthy) 8080/tcp
|
||||
@@ -0,0 +1,15 @@
|
||||
mtusi2026lab6-api-4 0.33% 15.89MiB / 256MiB
|
||||
mtusi2026lab6-api-3 0.19% 14.84MiB / 256MiB
|
||||
mtusi2026lab6-lb-1 0.08% 8.844MiB / 128MiB
|
||||
mtusi2026lab6-grafana-1 0.14% 51.9MiB / 256MiB
|
||||
mtusi2026lab6-relay-1 0.14% 11.45MiB / 128MiB
|
||||
mtusi2026lab6-worker-1 0.02% 11.4MiB / 768MiB
|
||||
mtusi2026lab6-reconciler-1 0.00% 4.32MiB / 128MiB
|
||||
mtusi2026lab6-tempo-1 2.87% 156.6MiB / 1GiB
|
||||
mtusi2026lab6-postgres-1 21.75% 79.28MiB / 512MiB
|
||||
mtusi2026lab6-s3-1 0.00% 224MiB / 512MiB
|
||||
mtusi2026lab6-otel-collector-1 1.22% 50.91MiB / 256MiB
|
||||
mtusi2026lab6-rabbitmq-1 0.11% 69.58MiB / 512MiB
|
||||
mtusi2026lab6-prometheus-1 0.67% 39.98MiB / 512MiB
|
||||
mtusi2026lab6-redis-1 0.34% 3.977MiB / 256MiB
|
||||
mtusi2026lab6-loki-1 3.34% 122.5MiB / 1GiB
|
||||
@@ -0,0 +1,15 @@
|
||||
mtusi2026lab6-lb-1 0.09% 8.566MiB / 128MiB
|
||||
mtusi2026lab6-reconciler-1 0.00% 4.137MiB / 128MiB
|
||||
mtusi2026lab6-relay-1 0.28% 12.17MiB / 128MiB
|
||||
mtusi2026lab6-api-2 3.71% 18.97MiB / 256MiB
|
||||
mtusi2026lab6-api-1 3.16% 18.38MiB / 256MiB
|
||||
mtusi2026lab6-worker-1 99.82% 153.3MiB / 768MiB
|
||||
mtusi2026lab6-grafana-1 3.18% 57.65MiB / 256MiB
|
||||
mtusi2026lab6-tempo-1 1.94% 55.03MiB / 1GiB
|
||||
mtusi2026lab6-loki-1 0.45% 62.09MiB / 1GiB
|
||||
mtusi2026lab6-s3-1 7.44% 447.9MiB / 512MiB
|
||||
mtusi2026lab6-rabbitmq-1 22.57% 85.34MiB / 512MiB
|
||||
mtusi2026lab6-prometheus-1 0.98% 39.7MiB / 512MiB
|
||||
mtusi2026lab6-otel-collector-1 0.99% 48.46MiB / 256MiB
|
||||
mtusi2026lab6-postgres-1 4.18% 84.61MiB / 512MiB
|
||||
mtusi2026lab6-redis-1 0.47% 3.414MiB / 256MiB
|
||||
@@ -0,0 +1,15 @@
|
||||
mtusi2026lab6-lb-1 0.08% 8.562MiB / 128MiB
|
||||
mtusi2026lab6-grafana-1 0.03% 74.72MiB / 256MiB
|
||||
mtusi2026lab6-api-1 0.12% 21.26MiB / 256MiB
|
||||
mtusi2026lab6-reconciler-1 2.26% 4.445MiB / 128MiB
|
||||
mtusi2026lab6-relay-1 0.16% 12.93MiB / 128MiB
|
||||
mtusi2026lab6-api-2 0.12% 18.84MiB / 256MiB
|
||||
mtusi2026lab6-worker-1 0.01% 12.86MiB / 768MiB
|
||||
mtusi2026lab6-tempo-1 0.03% 145.2MiB / 1GiB
|
||||
mtusi2026lab6-rabbitmq-1 0.13% 86.12MiB / 512MiB
|
||||
mtusi2026lab6-otel-collector-1 0.32% 54.26MiB / 256MiB
|
||||
mtusi2026lab6-prometheus-1 0.18% 57.23MiB / 512MiB
|
||||
mtusi2026lab6-loki-1 0.37% 130.3MiB / 1GiB
|
||||
mtusi2026lab6-postgres-1 0.19% 132MiB / 512MiB
|
||||
mtusi2026lab6-s3-1 2.28% 231.6MiB / 512MiB
|
||||
mtusi2026lab6-redis-1 0.34% 3.406MiB / 256MiB
|
||||
@@ -0,0 +1,15 @@
|
||||
mtusi2026lab6-worker-1 0.01% 528.9MiB / 768MiB
|
||||
mtusi2026lab6-relay-1 0.20% 11.69MiB / 128MiB
|
||||
mtusi2026lab6-lb-1 0.10% 8.027MiB / 128MiB
|
||||
mtusi2026lab6-reconciler-1 0.00% 4.328MiB / 128MiB
|
||||
mtusi2026lab6-api-1 0.30% 203.4MiB / 256MiB
|
||||
mtusi2026lab6-api-2 0.24% 142.4MiB / 256MiB
|
||||
mtusi2026lab6-grafana-1 2.77% 66.38MiB / 256MiB
|
||||
mtusi2026lab6-tempo-1 2.86% 43.02MiB / 1GiB
|
||||
mtusi2026lab6-rabbitmq-1 51.00% 123.6MiB / 512MiB
|
||||
mtusi2026lab6-postgres-1 2.13% 57.27MiB / 512MiB
|
||||
mtusi2026lab6-redis-1 0.52% 3.598MiB / 256MiB
|
||||
mtusi2026lab6-s3-1 2.85% 342MiB / 512MiB
|
||||
mtusi2026lab6-prometheus-1 0.95% 42.73MiB / 512MiB
|
||||
mtusi2026lab6-otel-collector-1 0.21% 47.2MiB / 256MiB
|
||||
mtusi2026lab6-loki-1 0.46% 63.41MiB / 1GiB
|
||||
@@ -0,0 +1,15 @@
|
||||
mtusi2026lab6-api-1 0.13% 16.12MiB / 256MiB
|
||||
mtusi2026lab6-relay-1 0.30% 10.56MiB / 128MiB
|
||||
mtusi2026lab6-api-2 2.94% 16.2MiB / 256MiB
|
||||
mtusi2026lab6-worker-1 0.01% 9.699MiB / 768MiB
|
||||
mtusi2026lab6-otel-collector-1 0.11% 50.07MiB / 256MiB
|
||||
mtusi2026lab6-lb-1 0.10% 8.715MiB / 128MiB
|
||||
mtusi2026lab6-grafana-1 0.04% 65.7MiB / 256MiB
|
||||
mtusi2026lab6-reconciler-1 0.00% 4.344MiB / 128MiB
|
||||
mtusi2026lab6-tempo-1 0.47% 91.59MiB / 1GiB
|
||||
mtusi2026lab6-postgres-1 3.74% 152.8MiB / 512MiB
|
||||
mtusi2026lab6-rabbitmq-1 22.76% 119.6MiB / 512MiB
|
||||
mtusi2026lab6-prometheus-1 0.15% 57.73MiB / 512MiB
|
||||
mtusi2026lab6-loki-1 0.34% 170.3MiB / 1GiB
|
||||
mtusi2026lab6-s3-1 0.07% 224.1MiB / 512MiB
|
||||
mtusi2026lab6-redis-1 0.45% 3.66MiB / 256MiB
|
||||
@@ -0,0 +1,8 @@
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 143, "failed_attempts": 0, "first_recovery_seconds": 3.535}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
-- контейнеры api после stop-one:
|
||||
mtusi2026lab6-api-1 84c136cad07b Up 43 seconds (healthy)
|
||||
mtusi2026lab6-api-2 1354d20e1722 Exited (0) 12 seconds ago
|
||||
mtusi2026lab6-api-1 Up 51 seconds (healthy)
|
||||
mtusi2026lab6-api-2 Up 6 seconds (healthy)
|
||||
@@ -0,0 +1,8 @@
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 144, "failed_attempts": 0, "first_recovery_seconds": 3.478}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
-- контейнеры api после stop-one:
|
||||
mtusi2026lab6-api-1 84c136cad07b Up About a minute (healthy)
|
||||
mtusi2026lab6-api-2 1354d20e1722 Exited (0) 12 seconds ago
|
||||
mtusi2026lab6-api-1 Up About a minute (healthy)
|
||||
mtusi2026lab6-api-2 Up 6 seconds (healthy)
|
||||
@@ -0,0 +1,64 @@
|
||||
{
|
||||
"concurrency": 4,
|
||||
"replicas": 2,
|
||||
"total": {
|
||||
"seconds": 53.16,
|
||||
"ok": 16801,
|
||||
"errors": 0,
|
||||
"ok_rps": 316.1
|
||||
},
|
||||
"confirmed_writes": 8401,
|
||||
"stored_messages": 8404,
|
||||
"duplicates": 0,
|
||||
"lost_confirmed": 0,
|
||||
"per_replica": [
|
||||
{
|
||||
"target_instance": "84c136cad07b",
|
||||
"docker_stop_seconds": 4.03,
|
||||
"before": {
|
||||
"seconds": 5.0,
|
||||
"ok": 1759,
|
||||
"errors": 0,
|
||||
"ok_rps": 351.8
|
||||
},
|
||||
"during_stop_and_observe": {
|
||||
"seconds": 15.0,
|
||||
"ok": 5183,
|
||||
"errors": 0,
|
||||
"ok_rps": 345.5
|
||||
},
|
||||
"errors_after_stop": 0,
|
||||
"recovery_seconds": 0.0,
|
||||
"target_last_served_after_stop_seconds": 1.15,
|
||||
"other_instances_after": [
|
||||
"1354d20e1722"
|
||||
],
|
||||
"restore_healthy_seconds": 5.57,
|
||||
"restore_serving_via_lb_seconds": 5.57
|
||||
},
|
||||
{
|
||||
"target_instance": "1354d20e1722",
|
||||
"docker_stop_seconds": 4.85,
|
||||
"before": {
|
||||
"seconds": 5.0,
|
||||
"ok": 1717,
|
||||
"errors": 0,
|
||||
"ok_rps": 343.4
|
||||
},
|
||||
"during_stop_and_observe": {
|
||||
"seconds": 15.0,
|
||||
"ok": 4302,
|
||||
"errors": 0,
|
||||
"ok_rps": 286.8
|
||||
},
|
||||
"errors_after_stop": 0,
|
||||
"recovery_seconds": 0.0,
|
||||
"target_last_served_after_stop_seconds": 1.5,
|
||||
"other_instances_after": [
|
||||
"84c136cad07b"
|
||||
],
|
||||
"restore_healthy_seconds": 6.61,
|
||||
"restore_serving_via_lb_seconds": 6.61
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
ok.png image/png 106 -> 202
|
||||
ok.jpg image/jpeg 900 -> 202
|
||||
%D1%84%D0%BE%D1%82%D0%BE%20%F0%9F%98%80. image/png 106 -> 202
|
||||
params.png image/png; charset=binary 106 -> 202
|
||||
near-limit.png image/png 10288524 -> 202
|
||||
too-big.png (Content-Length) image/png 10485761 -> 413
|
||||
too-big-chunked.png (no Content-Length) image/png 11534336 -> 413
|
||||
bomb.png image/png 66 -> 413
|
||||
over-pixels.png image/png 66 -> 413
|
||||
truncated.png image/png 53 -> 415
|
||||
corrupt.png image/png 106 -> 415
|
||||
truncated.jpg image/jpeg 898 -> 415
|
||||
mime-swap.png image/jpeg 106 -> 415
|
||||
mime-swap.jpg image/png 900 -> 415
|
||||
text.png image/png 23 -> 415
|
||||
gif.gif image/gif 14 -> 415
|
||||
ctype-missing.png application/octet-stream 106 -> 415
|
||||
empty.png image/png 0 -> 415
|
||||
short-data.png image/png 68 -> 415
|
||||
..%2Fetc%2Fpasswd image/png 106 -> 400
|
||||
a%5Cb.png image/png 106 -> 400
|
||||
nul%00.png image/png 106 -> 400
|
||||
bell%07.png image/png 106 -> 400
|
||||
%20%20%20 image/png 106 -> 400
|
||||
xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx image/png 106 -> 400
|
||||
image/png 106 -> 400
|
||||
eve.png image/png 106 -> 404
|
||||
foreign.png image/png 106 -> 202
|
||||
PASS: attachment negative suite (28 upload cases, no 5xx)
|
||||
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"scenario": "db-gap",
|
||||
"result": {
|
||||
"client_status": 504,
|
||||
"seconds": 54.2,
|
||||
"after_gap": {
|
||||
"row_status": "uploading",
|
||||
"staging_objects": 1
|
||||
},
|
||||
"final": {
|
||||
"row_status": "none"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"scenario": "failover",
|
||||
"result": {
|
||||
"uploaded_by": "e8856700a5dd",
|
||||
"downloaded_by": [
|
||||
"23f43d87eef5"
|
||||
],
|
||||
"sha256": "9cd8ae086706327908419e4e9782e8ed003310802f9381e4f8776a81079460e5",
|
||||
"stopped_container": "e8856700a5dd"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
{
|
||||
"scenario": "interrupt",
|
||||
"result": {
|
||||
"sent_bytes": 1048576,
|
||||
"declared_bytes": 4192910,
|
||||
"rows_after": 0,
|
||||
"objects_after": 0
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
{
|
||||
"scenario": "memory",
|
||||
"result": {
|
||||
"file_bytes": 10187085,
|
||||
"runs": [
|
||||
{
|
||||
"parallel": 10,
|
||||
"ok": 10,
|
||||
"failed": [],
|
||||
"max_seconds": 1.21,
|
||||
"baseline_peak_mib": {
|
||||
"e8856700a5dd": 12.7,
|
||||
"23f43d87eef5": 13.4
|
||||
},
|
||||
"peak_mib": {
|
||||
"e8856700a5dd": 20.7,
|
||||
"23f43d87eef5": 29.5
|
||||
},
|
||||
"payload_total_mib": 97.2,
|
||||
"max_peak_growth_mib": 16.1
|
||||
},
|
||||
{
|
||||
"parallel": 20,
|
||||
"ok": 20,
|
||||
"failed": [],
|
||||
"max_seconds": 2.12,
|
||||
"baseline_peak_mib": {
|
||||
"e8856700a5dd": 12.6,
|
||||
"23f43d87eef5": 13.0
|
||||
},
|
||||
"peak_mib": {
|
||||
"e8856700a5dd": 41.0,
|
||||
"23f43d87eef5": 38.4
|
||||
},
|
||||
"payload_total_mib": 194.3,
|
||||
"max_peak_growth_mib": 28.5
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"scenario": "s3-outage",
|
||||
"result": {
|
||||
"upload_status_during_outage": 503,
|
||||
"upload_error_seconds": 0.32,
|
||||
"download_status_during_outage": 503,
|
||||
"metadata_during_outage": "ready",
|
||||
"text_message_during_outage": 201,
|
||||
"readiness_during_outage": 200,
|
||||
"intents_left": 0,
|
||||
"upload_after_recovery": "queued"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"scenario": "security",
|
||||
"result": {
|
||||
"anonymous_get": "403",
|
||||
"anonymous_list": "403",
|
||||
"api_key_own_bucket": "allowed",
|
||||
"api_key_make_bucket": "denied",
|
||||
"api_key_admin": "denied",
|
||||
"api_key_visible_buckets": [
|
||||
"messenger-attachments"
|
||||
],
|
||||
"bootstrap_runs": [
|
||||
0,
|
||||
0
|
||||
],
|
||||
"objects_before_after": [
|
||||
40,
|
||||
40
|
||||
],
|
||||
"s3_published_ports": "9000-9001/tcp"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
? github.com/charawein/mtusi2026lab6/cmd/api [no test files]
|
||||
? github.com/charawein/mtusi2026lab6/cmd/pipeline [no test files]
|
||||
? github.com/charawein/mtusi2026lab6/cmd/reconcile [no test files]
|
||||
? github.com/charawein/mtusi2026lab6/internal/broker [no test files]
|
||||
? github.com/charawein/mtusi2026lab6/internal/config [no test files]
|
||||
? github.com/charawein/mtusi2026lab6/internal/domain [no test files]
|
||||
ok github.com/charawein/mtusi2026lab6/internal/imagecheck 0.045s
|
||||
ok github.com/charawein/mtusi2026lab6/internal/imaging 0.690s
|
||||
ok github.com/charawein/mtusi2026lab6/internal/reconcile 0.005s
|
||||
? github.com/charawein/mtusi2026lab6/internal/repository [no test files]
|
||||
? github.com/charawein/mtusi2026lab6/internal/repository/postgres [no test files]
|
||||
? github.com/charawein/mtusi2026lab6/internal/repository/redis [no test files]
|
||||
ok github.com/charawein/mtusi2026lab6/internal/service 0.008s
|
||||
? github.com/charawein/mtusi2026lab6/internal/storage [no test files]
|
||||
ok github.com/charawein/mtusi2026lab6/internal/telemetry 0.017s
|
||||
? github.com/charawein/mtusi2026lab6/internal/transport/http [no test files]
|
||||
ok github.com/charawein/mtusi2026lab6/internal/transport/http/middleware 0.009s
|
||||
? github.com/charawein/mtusi2026lab6/internal/transport/httputil [no test files]
|
||||
ok github.com/charawein/mtusi2026lab6/internal/worker 7.709s
|
||||
? github.com/charawein/mtusi2026lab6/student-tests/genfixtures [no test files]
|
||||
@@ -0,0 +1,88 @@
|
||||
{
|
||||
"rounds": 10,
|
||||
"parallel_per_round": 16,
|
||||
"distinct_instances_seen": 2,
|
||||
"duplicates_found": 0,
|
||||
"per_round": [
|
||||
{
|
||||
"round": 0,
|
||||
"requests": 16,
|
||||
"status_201": 16,
|
||||
"distinct_messages": 1,
|
||||
"stored": 1,
|
||||
"instances": 2
|
||||
},
|
||||
{
|
||||
"round": 1,
|
||||
"requests": 16,
|
||||
"status_201": 16,
|
||||
"distinct_messages": 1,
|
||||
"stored": 1,
|
||||
"instances": 2
|
||||
},
|
||||
{
|
||||
"round": 2,
|
||||
"requests": 16,
|
||||
"status_201": 16,
|
||||
"distinct_messages": 1,
|
||||
"stored": 1,
|
||||
"instances": 2
|
||||
},
|
||||
{
|
||||
"round": 3,
|
||||
"requests": 16,
|
||||
"status_201": 16,
|
||||
"distinct_messages": 1,
|
||||
"stored": 1,
|
||||
"instances": 2
|
||||
},
|
||||
{
|
||||
"round": 4,
|
||||
"requests": 16,
|
||||
"status_201": 16,
|
||||
"distinct_messages": 1,
|
||||
"stored": 1,
|
||||
"instances": 2
|
||||
},
|
||||
{
|
||||
"round": 5,
|
||||
"requests": 16,
|
||||
"status_201": 16,
|
||||
"distinct_messages": 1,
|
||||
"stored": 1,
|
||||
"instances": 2
|
||||
},
|
||||
{
|
||||
"round": 6,
|
||||
"requests": 16,
|
||||
"status_201": 16,
|
||||
"distinct_messages": 1,
|
||||
"stored": 1,
|
||||
"instances": 2
|
||||
},
|
||||
{
|
||||
"round": 7,
|
||||
"requests": 16,
|
||||
"status_201": 16,
|
||||
"distinct_messages": 1,
|
||||
"stored": 1,
|
||||
"instances": 2
|
||||
},
|
||||
{
|
||||
"round": 8,
|
||||
"requests": 16,
|
||||
"status_201": 16,
|
||||
"distinct_messages": 1,
|
||||
"stored": 1,
|
||||
"instances": 2
|
||||
},
|
||||
{
|
||||
"round": 9,
|
||||
"requests": 16,
|
||||
"status_201": 16,
|
||||
"distinct_messages": 1,
|
||||
"stored": 1,
|
||||
"instances": 2
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
bitnamilegacy/minio-client:2025.4.16@sha256:8a86e441decf053093c5977d49290298295f850c7843652f93e970ab15871dd1 bitnamilegacy/minio-client@sha256:8a86e441decf053093c5977d49290298295f850c7843652f93e970ab15871dd1
|
||||
bitnamilegacy/minio:2025.4.22@sha256:50cec18ac4184af4671a78aedd5554942c8ae105d51a465fa82037949046da01 bitnamilegacy/minio@sha256:50cec18ac4184af4671a78aedd5554942c8ae105d51a465fa82037949046da01
|
||||
grafana/grafana:11.4.0 grafana/grafana@sha256:d8ea37798ccc41061a62ab080f2676dda6bf7815558499f901bdb0f533a456fb
|
||||
grafana/loki:3.3.2 grafana/loki@sha256:8af2de1abbdd7aa92b27c9bcc96f0f4140c9096b507c77921ffddf1c6ad6c48f
|
||||
grafana/tempo:2.6.1 grafana/tempo@sha256:ef4384fce6e8ad22b95b243d8fc165628cda655376fd50e7850536ad89d71d50
|
||||
haproxy:3.0.5-alpine haproxy@sha256:43a2591a6a62a41dd22edb4452880e9c70de489fbd7c54ffb52d035cccbfe949
|
||||
migrate/migrate:v4.17.0 migrate/migrate@sha256:4d017c6fb5997127093648cab09e63d377997125c3d3dcca18e5d1c847da49fa
|
||||
mtusi2026lab6-api:local
|
||||
local-build
|
||||
otel/opentelemetry-collector-contrib:0.116.1 otel/opentelemetry-collector-contrib@sha256:d0ebf65280da2e1b1491d1b93648281afd353d4b9ea19160090303cec9a233bd
|
||||
postgres:16.4-alpine postgres@sha256:5660c2cbfea50c7a9127d17dc4e48543eedd3d7a41a595a2dfa572471e37e64c
|
||||
prom/prometheus:v3.1.0 prom/prometheus@sha256:6559acbd5d770b15bb3c954629ce190ac3cbbdb2b7f1c30f0385c4e05104e218
|
||||
rabbitmq:4.0.9-alpine rabbitmq@sha256:0a09ddfd00a1ef2e58f497d731e9b2814ef3bebb3668069185c54d085352d2b9
|
||||
redis:7.4.1-alpine redis@sha256:c1e88455c85225310bbea54816e9c3f4b5295815e6dbf80c34d40afc6df28275
|
||||
@@ -0,0 +1,2 @@
|
||||
1354d20e1722 10
|
||||
84c136cad07b 10
|
||||
@@ -0,0 +1,420 @@
|
||||
{
|
||||
"scenario": "backend-down",
|
||||
"result": {
|
||||
"load": {
|
||||
"requests": 100015,
|
||||
"ok": 100015,
|
||||
"errors": 0,
|
||||
"error_ratio": 0.0,
|
||||
"ok_rps": 406.0,
|
||||
"seconds": 246.3,
|
||||
"latency_ms_all_ok": {
|
||||
"p50": 8.07,
|
||||
"p95": 19.0,
|
||||
"p99": 33.38
|
||||
},
|
||||
"latency_ms_outside_faults": {
|
||||
"p50": 8.07,
|
||||
"p95": 19.0,
|
||||
"p99": 33.38
|
||||
}
|
||||
},
|
||||
"collector_limit_mib": 256,
|
||||
"collector_max_mib": 108.4,
|
||||
"api_max_mib": 224.5,
|
||||
"api_limit_mib": 256,
|
||||
"phases": {
|
||||
"export_failing_alert_seconds": 81.0,
|
||||
"during": {
|
||||
"exporter_queue_size_max": 40.0,
|
||||
"exporter_queue_capacity": 40.0,
|
||||
"send_failed_spans": 4203.596068571429,
|
||||
"send_failed_log_records": 2843.3654285714283,
|
||||
"enqueue_failed_spans": 728158.464,
|
||||
"refused_spans_memory_limiter": 0.0,
|
||||
"metrics_still_scraped": 430.16363636363633
|
||||
},
|
||||
"tempo_spans_exported_after_up": 37888.0,
|
||||
"refused_spans_total": 0.0,
|
||||
"new_trace_after_recovery": true,
|
||||
"traces_visible_seconds_after_tempo_up": 20.9,
|
||||
"probe_requests_after_up": 5
|
||||
},
|
||||
"timeline": {
|
||||
"tempo_loki_down_s": 18,
|
||||
"tempo_loki_up_s": 173,
|
||||
"collector_down_s": 195,
|
||||
"collector_down_for_s": 40
|
||||
},
|
||||
"memory_samples": [
|
||||
{
|
||||
"t": 0.0,
|
||||
"collector_mib": 52.1,
|
||||
"api_mib": [
|
||||
224.5,
|
||||
224.3
|
||||
]
|
||||
},
|
||||
{
|
||||
"t": 5.6,
|
||||
"collector_mib": 52.4,
|
||||
"api_mib": [
|
||||
21.0,
|
||||
19.4
|
||||
]
|
||||
},
|
||||
{
|
||||
"t": 10.9,
|
||||
"collector_mib": 54.3,
|
||||
"api_mib": [
|
||||
20.8,
|
||||
17.3
|
||||
]
|
||||
},
|
||||
{
|
||||
"t": 16.1,
|
||||
"collector_mib": 52.3,
|
||||
"api_mib": [
|
||||
20.6,
|
||||
18.2
|
||||
]
|
||||
},
|
||||
{
|
||||
"t": 21.8,
|
||||
"collector_mib": 73.3,
|
||||
"api_mib": [
|
||||
20.9,
|
||||
19.9
|
||||
]
|
||||
},
|
||||
{
|
||||
"t": 27.1,
|
||||
"collector_mib": 82.3,
|
||||
"api_mib": [
|
||||
20.3,
|
||||
17.4
|
||||
]
|
||||
},
|
||||
{
|
||||
"t": 32.4,
|
||||
"collector_mib": 84.0,
|
||||
"api_mib": [
|
||||
19.8,
|
||||
20.6
|
||||
]
|
||||
},
|
||||
{
|
||||
"t": 37.8,
|
||||
"collector_mib": 87.8,
|
||||
"api_mib": [
|
||||
17.9,
|
||||
18.9
|
||||
]
|
||||
},
|
||||
{
|
||||
"t": 43.2,
|
||||
"collector_mib": 90.8,
|
||||
"api_mib": [
|
||||
18.0,
|
||||
20.1
|
||||
]
|
||||
},
|
||||
{
|
||||
"t": 49.0,
|
||||
"collector_mib": 95.5,
|
||||
"api_mib": [
|
||||
20.3,
|
||||
17.6
|
||||
]
|
||||
},
|
||||
{
|
||||
"t": 54.3,
|
||||
"collector_mib": 96.0,
|
||||
"api_mib": [
|
||||
21.0,
|
||||
19.7
|
||||
]
|
||||
},
|
||||
{
|
||||
"t": 59.6,
|
||||
"collector_mib": 96.4,
|
||||
"api_mib": [
|
||||
21.4,
|
||||
20.0
|
||||
]
|
||||
},
|
||||
{
|
||||
"t": 64.9,
|
||||
"collector_mib": 103.6,
|
||||
"api_mib": [
|
||||
19.6,
|
||||
19.1
|
||||
]
|
||||
},
|
||||
{
|
||||
"t": 70.3,
|
||||
"collector_mib": 105.3,
|
||||
"api_mib": [
|
||||
21.0,
|
||||
19.0
|
||||
]
|
||||
},
|
||||
{
|
||||
"t": 75.7,
|
||||
"collector_mib": 106.6,
|
||||
"api_mib": [
|
||||
20.3,
|
||||
19.0
|
||||
]
|
||||
},
|
||||
{
|
||||
"t": 81.2,
|
||||
"collector_mib": 108.4,
|
||||
"api_mib": [
|
||||
20.5,
|
||||
17.2
|
||||
]
|
||||
},
|
||||
{
|
||||
"t": 86.6,
|
||||
"collector_mib": 108.3,
|
||||
"api_mib": [
|
||||
18.5,
|
||||
20.0
|
||||
]
|
||||
},
|
||||
{
|
||||
"t": 91.9,
|
||||
"collector_mib": 108.4,
|
||||
"api_mib": [
|
||||
19.7,
|
||||
18.6
|
||||
]
|
||||
},
|
||||
{
|
||||
"t": 97.2,
|
||||
"collector_mib": 108.4,
|
||||
"api_mib": [
|
||||
20.2,
|
||||
18.8
|
||||
]
|
||||
},
|
||||
{
|
||||
"t": 102.7,
|
||||
"collector_mib": 105.1,
|
||||
"api_mib": [
|
||||
18.3,
|
||||
21.1
|
||||
]
|
||||
},
|
||||
{
|
||||
"t": 108.2,
|
||||
"collector_mib": 104.8,
|
||||
"api_mib": [
|
||||
19.2,
|
||||
17.5
|
||||
]
|
||||
},
|
||||
{
|
||||
"t": 113.4,
|
||||
"collector_mib": 104.8,
|
||||
"api_mib": [
|
||||
21.7,
|
||||
18.2
|
||||
]
|
||||
},
|
||||
{
|
||||
"t": 118.7,
|
||||
"collector_mib": 104.8,
|
||||
"api_mib": [
|
||||
20.2,
|
||||
17.8
|
||||
]
|
||||
},
|
||||
{
|
||||
"t": 124.1,
|
||||
"collector_mib": 107.8,
|
||||
"api_mib": [
|
||||
19.9,
|
||||
19.0
|
||||
]
|
||||
},
|
||||
{
|
||||
"t": 129.4,
|
||||
"collector_mib": 107.8,
|
||||
"api_mib": [
|
||||
19.4,
|
||||
19.5
|
||||
]
|
||||
},
|
||||
{
|
||||
"t": 135.0,
|
||||
"collector_mib": 108.1,
|
||||
"api_mib": [
|
||||
20.1,
|
||||
18.2
|
||||
]
|
||||
},
|
||||
{
|
||||
"t": 140.4,
|
||||
"collector_mib": 106.0,
|
||||
"api_mib": [
|
||||
20.1,
|
||||
19.0
|
||||
]
|
||||
},
|
||||
{
|
||||
"t": 145.8,
|
||||
"collector_mib": 106.4,
|
||||
"api_mib": [
|
||||
19.2,
|
||||
20.4
|
||||
]
|
||||
},
|
||||
{
|
||||
"t": 151.2,
|
||||
"collector_mib": 106.8,
|
||||
"api_mib": [
|
||||
20.3,
|
||||
17.9
|
||||
]
|
||||
},
|
||||
{
|
||||
"t": 156.8,
|
||||
"collector_mib": 103.8,
|
||||
"api_mib": [
|
||||
21.5,
|
||||
19.1
|
||||
]
|
||||
},
|
||||
{
|
||||
"t": 163.0,
|
||||
"collector_mib": 85.3,
|
||||
"api_mib": [
|
||||
17.7,
|
||||
17.0
|
||||
]
|
||||
},
|
||||
{
|
||||
"t": 168.4,
|
||||
"collector_mib": 82.5,
|
||||
"api_mib": [
|
||||
20.1,
|
||||
18.7
|
||||
]
|
||||
},
|
||||
{
|
||||
"t": 173.7,
|
||||
"collector_mib": 83.8,
|
||||
"api_mib": [
|
||||
19.8,
|
||||
19.8
|
||||
]
|
||||
},
|
||||
{
|
||||
"t": 179.2,
|
||||
"collector_mib": 83.8,
|
||||
"api_mib": [
|
||||
20.6,
|
||||
19.2
|
||||
]
|
||||
},
|
||||
{
|
||||
"t": 184.6,
|
||||
"collector_mib": 83.9,
|
||||
"api_mib": [
|
||||
20.8,
|
||||
17.2
|
||||
]
|
||||
},
|
||||
{
|
||||
"t": 189.9,
|
||||
"collector_mib": 54.8,
|
||||
"api_mib": [
|
||||
19.7,
|
||||
19.1
|
||||
]
|
||||
},
|
||||
{
|
||||
"t": 195.4,
|
||||
"collector_mib": 0.0,
|
||||
"api_mib": [
|
||||
24.2,
|
||||
22.9
|
||||
]
|
||||
},
|
||||
{
|
||||
"t": 200.7,
|
||||
"collector_mib": 0.0,
|
||||
"api_mib": [
|
||||
29.1,
|
||||
31.2
|
||||
]
|
||||
},
|
||||
{
|
||||
"t": 206.1,
|
||||
"collector_mib": 0.0,
|
||||
"api_mib": [
|
||||
33.3,
|
||||
34.8
|
||||
]
|
||||
},
|
||||
{
|
||||
"t": 211.6,
|
||||
"collector_mib": 0.0,
|
||||
"api_mib": [
|
||||
33.3,
|
||||
32.8
|
||||
]
|
||||
},
|
||||
{
|
||||
"t": 217.2,
|
||||
"collector_mib": 0.0,
|
||||
"api_mib": [
|
||||
32.5,
|
||||
32.9
|
||||
]
|
||||
},
|
||||
{
|
||||
"t": 222.8,
|
||||
"collector_mib": 0.0,
|
||||
"api_mib": [
|
||||
34.3,
|
||||
33.7
|
||||
]
|
||||
},
|
||||
{
|
||||
"t": 228.2,
|
||||
"collector_mib": 0.0,
|
||||
"api_mib": [
|
||||
35.5,
|
||||
32.1
|
||||
]
|
||||
},
|
||||
{
|
||||
"t": 233.6,
|
||||
"collector_mib": 0.0,
|
||||
"api_mib": [
|
||||
34.0,
|
||||
32.4
|
||||
]
|
||||
},
|
||||
{
|
||||
"t": 239.0,
|
||||
"collector_mib": 58.3,
|
||||
"api_mib": [
|
||||
30.3,
|
||||
23.8
|
||||
]
|
||||
},
|
||||
{
|
||||
"t": 244.4,
|
||||
"collector_mib": 54.7,
|
||||
"api_mib": [
|
||||
22.0,
|
||||
18.7
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 485 KiB |
@@ -0,0 +1,155 @@
|
||||
{
|
||||
"scenario": "incident-db",
|
||||
"result": {
|
||||
"fault": "LOCK TABLE messages IN ACCESS EXCLUSIVE MODE for 110 s under load (4 clients)",
|
||||
"detection_seconds": 98.4,
|
||||
"resolved_seconds_after_unlock": 18.3,
|
||||
"metrics_at_firing": {
|
||||
"error_ratio_1m": 1.0,
|
||||
"p99_s": 9.95,
|
||||
"p99_by_route": {
|
||||
"/api/v1/chats/{chat_id}/attachments": NaN,
|
||||
"/api/v1/auth/sessions": NaN,
|
||||
"/api/v1/users": NaN,
|
||||
"/api/v1/attachments/{attachment_id}": NaN,
|
||||
"/api/v1/chats/{chat_id}/messages": 9.95,
|
||||
"/api/v1/chats": NaN
|
||||
},
|
||||
"db_pool_acquired": 4.0,
|
||||
"health_ready_5xx_rate": 0.0
|
||||
},
|
||||
"error_trace_breakdown": [
|
||||
{
|
||||
"server_span": "POST /api/v1/chats/{chat_id}/messages",
|
||||
"server_ms": 5000.48,
|
||||
"status": "500",
|
||||
"longest_db_span_ms": 4977.9,
|
||||
"longest_s3_span_ms": 0.0,
|
||||
"db_share": 0.995,
|
||||
"db_spans_longest": [
|
||||
[
|
||||
"INSERT",
|
||||
4977.87
|
||||
],
|
||||
[
|
||||
"INSERT",
|
||||
4977.85
|
||||
],
|
||||
[
|
||||
"pool.acquire",
|
||||
12.77
|
||||
],
|
||||
[
|
||||
"connect",
|
||||
12.72
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"server_span": "POST /api/v1/chats/{chat_id}/messages",
|
||||
"server_ms": 5000.53,
|
||||
"status": "500",
|
||||
"longest_db_span_ms": 4973.6,
|
||||
"longest_s3_span_ms": 0.0,
|
||||
"db_share": 0.995,
|
||||
"db_spans_longest": [
|
||||
[
|
||||
"INSERT",
|
||||
4973.55
|
||||
],
|
||||
[
|
||||
"INSERT",
|
||||
4973.51
|
||||
],
|
||||
[
|
||||
"SELECT",
|
||||
10.33
|
||||
],
|
||||
[
|
||||
"SELECT",
|
||||
6.0
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"server_span": "POST /api/v1/chats/{chat_id}/messages",
|
||||
"server_ms": 5001.06,
|
||||
"status": "500",
|
||||
"longest_db_span_ms": 4982.3,
|
||||
"longest_s3_span_ms": 0.0,
|
||||
"db_share": 0.996,
|
||||
"db_spans_longest": [
|
||||
[
|
||||
"INSERT",
|
||||
4982.27
|
||||
],
|
||||
[
|
||||
"INSERT",
|
||||
4982.25
|
||||
],
|
||||
[
|
||||
"DELETE",
|
||||
6.95
|
||||
],
|
||||
[
|
||||
"DELETE",
|
||||
5.49
|
||||
]
|
||||
]
|
||||
}
|
||||
],
|
||||
"slow_handler_comparison": {
|
||||
"server_span": "POST /api/v1/chats/{chat_id}/attachments",
|
||||
"server_ms": 77.38,
|
||||
"status": "202",
|
||||
"longest_db_span_ms": 2.2,
|
||||
"longest_s3_span_ms": 68.0,
|
||||
"db_share": 0.028,
|
||||
"db_spans_longest": [
|
||||
[
|
||||
"INSERT",
|
||||
2.18
|
||||
],
|
||||
[
|
||||
"INSERT",
|
||||
1.33
|
||||
],
|
||||
[
|
||||
"INSERT",
|
||||
1.31
|
||||
],
|
||||
[
|
||||
"INSERT",
|
||||
1.22
|
||||
]
|
||||
]
|
||||
},
|
||||
"error_logs": {
|
||||
"count": 88,
|
||||
"events": [
|
||||
"http.request"
|
||||
],
|
||||
"routes": [
|
||||
"/api/v1/chats/{chat_id}/messages"
|
||||
]
|
||||
},
|
||||
"load": {
|
||||
"requests": 15036,
|
||||
"ok": 14948,
|
||||
"errors": 88,
|
||||
"error_ratio": 0.00585,
|
||||
"ok_rps": 100.5,
|
||||
"seconds": 148.8,
|
||||
"latency_ms_all_ok": {
|
||||
"p50": 8.55,
|
||||
"p95": 20.86,
|
||||
"p99": 35.95
|
||||
},
|
||||
"latency_ms_outside_faults": {
|
||||
"p50": 8.53,
|
||||
"p95": 20.73,
|
||||
"p99": 35.35
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
{
|
||||
"scenario": "incident-worker",
|
||||
"result": {
|
||||
"detection_seconds_from_worker_stop": 96.7,
|
||||
"diagnosis": {
|
||||
"alerts_firing": [
|
||||
"MessengerQueueLagGrowing"
|
||||
],
|
||||
"backlog_queued": 3.0,
|
||||
"oldest_age_s": 91.144665,
|
||||
"jobs_processed_rate_1m": 0.0,
|
||||
"worker_capacity_series": 0,
|
||||
"rabbitmq_ready": null,
|
||||
"api_5xx_ratio": NaN,
|
||||
"producer_spans_since_stop": 3,
|
||||
"consumer_spans_since_stop": 0,
|
||||
"last_worker_log_events": [
|
||||
"",
|
||||
"job.claimed",
|
||||
"job.done",
|
||||
"job.retry",
|
||||
"worker.start"
|
||||
]
|
||||
},
|
||||
"backlog_ready_seconds_after_start": 2.6,
|
||||
"resolved_seconds_after_worker_start": 10.6,
|
||||
"oldest_age_series": [
|
||||
[
|
||||
-30,
|
||||
0.0
|
||||
],
|
||||
[
|
||||
-20,
|
||||
0.0
|
||||
],
|
||||
[
|
||||
-10,
|
||||
7.4
|
||||
],
|
||||
[
|
||||
0,
|
||||
0.0
|
||||
],
|
||||
[
|
||||
10,
|
||||
6.1
|
||||
],
|
||||
[
|
||||
20,
|
||||
16.1
|
||||
],
|
||||
[
|
||||
30,
|
||||
26.1
|
||||
],
|
||||
[
|
||||
40,
|
||||
36.1
|
||||
],
|
||||
[
|
||||
50,
|
||||
46.1
|
||||
],
|
||||
[
|
||||
60,
|
||||
56.1
|
||||
],
|
||||
[
|
||||
70,
|
||||
66.1
|
||||
],
|
||||
[
|
||||
80,
|
||||
76.1
|
||||
],
|
||||
[
|
||||
90,
|
||||
86.1
|
||||
],
|
||||
[
|
||||
100,
|
||||
96.1
|
||||
]
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,310 @@
|
||||
{
|
||||
"metric_names": [
|
||||
"go_cgo_go_to_c_calls_calls_total",
|
||||
"go_config_gogc_percent",
|
||||
"go_cpu_classes_gc_mark_assist_cpu_seconds_total",
|
||||
"go_cpu_classes_gc_mark_dedicated_cpu_seconds_total",
|
||||
"go_cpu_classes_gc_mark_idle_cpu_seconds_total",
|
||||
"go_cpu_classes_gc_pause_cpu_seconds_total",
|
||||
"go_cpu_classes_gc_total_cpu_seconds_total",
|
||||
"go_cpu_classes_idle_cpu_seconds_total",
|
||||
"go_cpu_classes_scavenge_assist_cpu_seconds_total",
|
||||
"go_cpu_classes_scavenge_background_cpu_seconds_total",
|
||||
"go_cpu_classes_scavenge_total_cpu_seconds_total",
|
||||
"go_cpu_classes_total_cpu_seconds_total",
|
||||
"go_cpu_classes_user_cpu_seconds_total",
|
||||
"go_gc_cycles_automatic_gc_cycles_total",
|
||||
"go_gc_cycles_forced_gc_cycles_total",
|
||||
"go_gc_cycles_total_gc_cycles_total",
|
||||
"go_gc_duration_seconds",
|
||||
"go_gc_duration_seconds_count",
|
||||
"go_gc_duration_seconds_sum",
|
||||
"go_gc_gogc_percent",
|
||||
"go_gc_gomemlimit_bytes",
|
||||
"go_gc_heap_allocs_by_size_bytes_bucket",
|
||||
"go_gc_heap_allocs_by_size_bytes_count",
|
||||
"go_gc_heap_allocs_by_size_bytes_sum",
|
||||
"go_gc_heap_allocs_bytes_total",
|
||||
"go_gc_heap_allocs_objects_total",
|
||||
"go_gc_heap_frees_by_size_bytes_bucket",
|
||||
"go_gc_heap_frees_by_size_bytes_count",
|
||||
"go_gc_heap_frees_by_size_bytes_sum",
|
||||
"go_gc_heap_frees_bytes_total",
|
||||
"go_gc_heap_frees_objects_total",
|
||||
"go_gc_heap_goal_bytes",
|
||||
"go_gc_heap_live_bytes",
|
||||
"go_gc_heap_objects_objects",
|
||||
"go_gc_heap_tiny_allocs_objects_total",
|
||||
"go_gc_limiter_last_enabled_gc_cycle",
|
||||
"go_gc_pauses_seconds_bucket",
|
||||
"go_gc_pauses_seconds_count",
|
||||
"go_gc_pauses_seconds_sum",
|
||||
"go_gc_scan_globals_bytes",
|
||||
"go_gc_scan_heap_bytes",
|
||||
"go_gc_scan_stack_bytes",
|
||||
"go_gc_scan_total_bytes",
|
||||
"go_gc_stack_starting_size_bytes",
|
||||
"go_godebug_non_default_behavior_asynctimerchan_events_total",
|
||||
"go_godebug_non_default_behavior_execerrdot_events_total",
|
||||
"go_godebug_non_default_behavior_gocachehash_events_total",
|
||||
"go_godebug_non_default_behavior_gocachetest_events_total",
|
||||
"go_godebug_non_default_behavior_gocacheverify_events_total",
|
||||
"go_godebug_non_default_behavior_gotypesalias_events_total",
|
||||
"go_godebug_non_default_behavior_http2client_events_total",
|
||||
"go_godebug_non_default_behavior_http2server_events_total",
|
||||
"go_godebug_non_default_behavior_httplaxcontentlength_events_total",
|
||||
"go_godebug_non_default_behavior_httpmuxgo121_events_total",
|
||||
"go_godebug_non_default_behavior_httpservecontentkeepheaders_events_total",
|
||||
"go_godebug_non_default_behavior_installgoroot_events_total",
|
||||
"go_godebug_non_default_behavior_multipartmaxheaders_events_total",
|
||||
"go_godebug_non_default_behavior_multipartmaxparts_events_total",
|
||||
"go_godebug_non_default_behavior_multipathtcp_events_total",
|
||||
"go_godebug_non_default_behavior_netedns0_events_total",
|
||||
"go_godebug_non_default_behavior_panicnil_events_total",
|
||||
"go_godebug_non_default_behavior_randautoseed_events_total",
|
||||
"go_godebug_non_default_behavior_tarinsecurepath_events_total",
|
||||
"go_godebug_non_default_behavior_tls10server_events_total",
|
||||
"go_godebug_non_default_behavior_tls3des_events_total",
|
||||
"go_godebug_non_default_behavior_tlsmaxrsasize_events_total",
|
||||
"go_godebug_non_default_behavior_tlsrsakex_events_total",
|
||||
"go_godebug_non_default_behavior_tlsunsafeekm_events_total",
|
||||
"go_godebug_non_default_behavior_winreadlinkvolume_events_total",
|
||||
"go_godebug_non_default_behavior_winsymlink_events_total",
|
||||
"go_godebug_non_default_behavior_x509keypairleaf_events_total",
|
||||
"go_godebug_non_default_behavior_x509negativeserial_events_total",
|
||||
"go_godebug_non_default_behavior_x509sha1_events_total",
|
||||
"go_godebug_non_default_behavior_x509usefallbackroots_events_total",
|
||||
"go_godebug_non_default_behavior_x509usepolicies_events_total",
|
||||
"go_godebug_non_default_behavior_zipinsecurepath_events_total",
|
||||
"go_goroutine_count",
|
||||
"go_goroutines",
|
||||
"go_info",
|
||||
"go_memory_allocated_bytes_total",
|
||||
"go_memory_allocations_total",
|
||||
"go_memory_classes_heap_free_bytes",
|
||||
"go_memory_classes_heap_objects_bytes",
|
||||
"go_memory_classes_heap_released_bytes",
|
||||
"go_memory_classes_heap_stacks_bytes",
|
||||
"go_memory_classes_heap_unused_bytes",
|
||||
"go_memory_classes_metadata_mcache_free_bytes",
|
||||
"go_memory_classes_metadata_mcache_inuse_bytes",
|
||||
"go_memory_classes_metadata_mspan_free_bytes",
|
||||
"go_memory_classes_metadata_mspan_inuse_bytes",
|
||||
"go_memory_classes_metadata_other_bytes",
|
||||
"go_memory_classes_os_stacks_bytes",
|
||||
"go_memory_classes_other_bytes",
|
||||
"go_memory_classes_profiling_buckets_bytes",
|
||||
"go_memory_classes_total_bytes",
|
||||
"go_memory_gc_goal_bytes",
|
||||
"go_memory_limit_bytes",
|
||||
"go_memory_used_bytes",
|
||||
"go_memstats_alloc_bytes",
|
||||
"go_memstats_alloc_bytes_total",
|
||||
"go_memstats_buck_hash_sys_bytes",
|
||||
"go_memstats_frees_total",
|
||||
"go_memstats_gc_sys_bytes",
|
||||
"go_memstats_heap_alloc_bytes",
|
||||
"go_memstats_heap_idle_bytes",
|
||||
"go_memstats_heap_inuse_bytes",
|
||||
"go_memstats_heap_objects",
|
||||
"go_memstats_heap_released_bytes",
|
||||
"go_memstats_heap_sys_bytes",
|
||||
"go_memstats_last_gc_time_seconds",
|
||||
"go_memstats_lookups_total",
|
||||
"go_memstats_mallocs_total",
|
||||
"go_memstats_mcache_inuse_bytes",
|
||||
"go_memstats_mcache_sys_bytes",
|
||||
"go_memstats_mspan_inuse_bytes",
|
||||
"go_memstats_mspan_sys_bytes",
|
||||
"go_memstats_next_gc_bytes",
|
||||
"go_memstats_other_sys_bytes",
|
||||
"go_memstats_stack_inuse_bytes",
|
||||
"go_memstats_stack_sys_bytes",
|
||||
"go_memstats_sys_bytes",
|
||||
"go_processor_limit",
|
||||
"go_sched_gomaxprocs_threads",
|
||||
"go_sched_goroutines_goroutines",
|
||||
"go_sched_latencies_seconds_bucket",
|
||||
"go_sched_latencies_seconds_count",
|
||||
"go_sched_latencies_seconds_sum",
|
||||
"go_sched_pauses_stopping_gc_seconds_bucket",
|
||||
"go_sched_pauses_stopping_gc_seconds_count",
|
||||
"go_sched_pauses_stopping_gc_seconds_sum",
|
||||
"go_sched_pauses_stopping_other_seconds_bucket",
|
||||
"go_sched_pauses_stopping_other_seconds_count",
|
||||
"go_sched_pauses_stopping_other_seconds_sum",
|
||||
"go_sched_pauses_total_gc_seconds_bucket",
|
||||
"go_sched_pauses_total_gc_seconds_count",
|
||||
"go_sched_pauses_total_gc_seconds_sum",
|
||||
"go_sched_pauses_total_other_seconds_bucket",
|
||||
"go_sched_pauses_total_other_seconds_count",
|
||||
"go_sched_pauses_total_other_seconds_sum",
|
||||
"go_sync_mutex_wait_total_seconds_total",
|
||||
"go_threads",
|
||||
"http_client_duration_bucket",
|
||||
"http_client_duration_count",
|
||||
"http_client_duration_sum",
|
||||
"http_client_request_body_size_bytes_bucket",
|
||||
"http_client_request_body_size_bytes_count",
|
||||
"http_client_request_body_size_bytes_sum",
|
||||
"http_client_request_duration_seconds_bucket",
|
||||
"http_client_request_duration_seconds_count",
|
||||
"http_client_request_duration_seconds_sum",
|
||||
"http_client_request_size",
|
||||
"http_client_response_size",
|
||||
"http_server_active_requests",
|
||||
"http_server_duration_bucket",
|
||||
"http_server_duration_count",
|
||||
"http_server_duration_sum",
|
||||
"http_server_request_duration_seconds_bucket",
|
||||
"http_server_request_duration_seconds_count",
|
||||
"http_server_request_duration_seconds_sum",
|
||||
"http_server_request_size",
|
||||
"http_server_response_size",
|
||||
"messenger:http_error_ratio:rate1m",
|
||||
"messenger:http_errors:rate1m",
|
||||
"messenger:http_fast_ratio:rate5m",
|
||||
"messenger:http_requests:rate1m",
|
||||
"messenger_attachment_time_to_ready_seconds_bucket",
|
||||
"messenger_attachment_time_to_ready_seconds_count",
|
||||
"messenger_attachment_time_to_ready_seconds_sum",
|
||||
"messenger_job_duration_seconds_bucket",
|
||||
"messenger_job_duration_seconds_count",
|
||||
"messenger_job_duration_seconds_sum",
|
||||
"messenger_jobs_backlog",
|
||||
"messenger_jobs_oldest_age_seconds",
|
||||
"messenger_jobs_processed_total",
|
||||
"messenger_outbox_oldest_age_seconds",
|
||||
"messenger_outbox_pending",
|
||||
"messenger_outbox_publish_duration_seconds_bucket",
|
||||
"messenger_outbox_publish_duration_seconds_count",
|
||||
"messenger_outbox_publish_duration_seconds_sum",
|
||||
"messenger_outbox_published_total",
|
||||
"messenger_worker_busy",
|
||||
"messenger_worker_capacity",
|
||||
"otelcol_exporter_enqueue_failed_log_records",
|
||||
"otelcol_exporter_enqueue_failed_spans",
|
||||
"otelcol_exporter_queue_capacity",
|
||||
"otelcol_exporter_queue_size",
|
||||
"otelcol_exporter_send_failed_log_records",
|
||||
"otelcol_exporter_send_failed_metric_points",
|
||||
"otelcol_exporter_send_failed_spans",
|
||||
"otelcol_exporter_sent_log_records",
|
||||
"otelcol_exporter_sent_metric_points",
|
||||
"otelcol_exporter_sent_spans",
|
||||
"otelcol_receiver_accepted_log_records",
|
||||
"otelcol_receiver_accepted_metric_points",
|
||||
"otelcol_receiver_accepted_spans",
|
||||
"otelcol_receiver_refused_log_records",
|
||||
"otelcol_receiver_refused_metric_points",
|
||||
"otelcol_receiver_refused_spans",
|
||||
"pgxpool_acquire_duration_nanoseconds_total",
|
||||
"pgxpool_acquired_connections",
|
||||
"pgxpool_acquires_total",
|
||||
"pgxpool_canceled_acquires_total",
|
||||
"pgxpool_constructing_connections",
|
||||
"pgxpool_empty_acquire_total",
|
||||
"pgxpool_empty_acquire_wait_time_nanoseconds_total",
|
||||
"pgxpool_idle_connections",
|
||||
"pgxpool_max_connections",
|
||||
"pgxpool_max_idle_destroys_total",
|
||||
"pgxpool_max_lifetime_destroys_total",
|
||||
"pgxpool_new_connections_total",
|
||||
"pgxpool_total_connections",
|
||||
"process_cpu_seconds_total",
|
||||
"process_cpu_time_seconds_total",
|
||||
"process_max_fds",
|
||||
"process_network_receive_bytes_total",
|
||||
"process_network_transmit_bytes_total",
|
||||
"process_open_fds",
|
||||
"process_resident_memory_bytes",
|
||||
"process_start_time_seconds",
|
||||
"process_virtual_memory_bytes",
|
||||
"process_virtual_memory_max_bytes",
|
||||
"target_info"
|
||||
],
|
||||
"rules": [
|
||||
{
|
||||
"group": "messenger-alerts",
|
||||
"name": "MessengerHighErrorRate",
|
||||
"type": "alerting",
|
||||
"health": "ok",
|
||||
"state": "inactive",
|
||||
"query": "messenger:http_error_ratio:rate1m > 0.05 and messenger:http_requests:rate1m > 0.5",
|
||||
"duration": 30
|
||||
},
|
||||
{
|
||||
"group": "messenger-alerts",
|
||||
"name": "MessengerQueueLagGrowing",
|
||||
"type": "alerting",
|
||||
"health": "ok",
|
||||
"state": "inactive",
|
||||
"query": "max(messenger_jobs_oldest_age_seconds) > 60",
|
||||
"duration": 30
|
||||
},
|
||||
{
|
||||
"group": "messenger-alerts",
|
||||
"name": "MessengerWorkerAbsent",
|
||||
"type": "alerting",
|
||||
"health": "ok",
|
||||
"state": "inactive",
|
||||
"query": "absent(messenger_worker_capacity)",
|
||||
"duration": 60
|
||||
},
|
||||
{
|
||||
"group": "messenger-alerts",
|
||||
"name": "MessengerTelemetryExportFailing",
|
||||
"type": "alerting",
|
||||
"health": "ok",
|
||||
"state": "inactive",
|
||||
"query": "(sum(rate({__name__=~\"otelcol_exporter_send_failed_spans(_total)?\"}[1m])) or vector(0)) + (sum(rate({__name__=~\"otelcol_exporter_send_failed_log_records(_total)?\"}[1m])) or vector(0)) > 0",
|
||||
"duration": 30
|
||||
},
|
||||
{
|
||||
"group": "messenger-sli",
|
||||
"name": "messenger:http_requests:rate1m",
|
||||
"type": "recording",
|
||||
"health": "ok",
|
||||
"state": null,
|
||||
"query": "sum(rate(http_server_request_duration_seconds_count{http_route!~\"/health/.*\",job=\"messenger-api\"}[1m]))",
|
||||
"duration": null
|
||||
},
|
||||
{
|
||||
"group": "messenger-sli",
|
||||
"name": "messenger:http_errors:rate1m",
|
||||
"type": "recording",
|
||||
"health": "ok",
|
||||
"state": null,
|
||||
"query": "sum(rate(http_server_request_duration_seconds_count{http_response_status_class=\"5xx\",http_route!~\"/health/.*\",job=\"messenger-api\"}[1m])) or vector(0)",
|
||||
"duration": null
|
||||
},
|
||||
{
|
||||
"group": "messenger-sli",
|
||||
"name": "messenger:http_error_ratio:rate1m",
|
||||
"type": "recording",
|
||||
"health": "ok",
|
||||
"state": null,
|
||||
"query": "messenger:http_errors:rate1m / messenger:http_requests:rate1m",
|
||||
"duration": null
|
||||
},
|
||||
{
|
||||
"group": "messenger-sli",
|
||||
"name": "messenger:http_fast_ratio:rate5m",
|
||||
"type": "recording",
|
||||
"health": "ok",
|
||||
"state": null,
|
||||
"query": "sum(rate(http_server_request_duration_seconds_bucket{http_route!~\"/health/.*\",job=\"messenger-api\",le=\"0.25\"}[5m])) / sum(rate(http_server_request_duration_seconds_count{http_route!~\"/health/.*\",job=\"messenger-api\"}[5m]))",
|
||||
"duration": null
|
||||
},
|
||||
{
|
||||
"group": "messenger-sli",
|
||||
"name": "messenger:time_to_ready_fast_ratio:rate5m",
|
||||
"type": "recording",
|
||||
"health": "ok",
|
||||
"state": null,
|
||||
"query": "sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"30\"}[5m])) / sum(rate(messenger_attachment_time_to_ready_seconds_count[5m]))",
|
||||
"duration": null
|
||||
}
|
||||
],
|
||||
"alerts_now": {}
|
||||
}
|
||||
@@ -0,0 +1,124 @@
|
||||
{
|
||||
"scenario": "retry-trace",
|
||||
"result": {
|
||||
"request_id": "obs-retry-a9fc5acc8add4f72",
|
||||
"trace_id": "48230bc7d9dc9e469f654518739a6c8a",
|
||||
"attempts_in_trace": [
|
||||
[
|
||||
"1",
|
||||
"retry",
|
||||
"STATUS_CODE_ERROR"
|
||||
],
|
||||
[
|
||||
"2",
|
||||
"retry",
|
||||
"STATUS_CODE_ERROR"
|
||||
],
|
||||
[
|
||||
"3",
|
||||
"retry",
|
||||
"STATUS_CODE_ERROR"
|
||||
],
|
||||
[
|
||||
"4",
|
||||
"done",
|
||||
""
|
||||
]
|
||||
],
|
||||
"publish_spans": 4,
|
||||
"job_events": "enqueued,claimed,retry_scheduled,claimed,retry_scheduled,claimed,retry_scheduled,claimed,done",
|
||||
"final_status": "ready",
|
||||
"tree": [
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "POST /api/v1/chats/{chat_id}/attachments",
|
||||
"kind": "SPAN_KIND_SERVER",
|
||||
"ms": 17.78,
|
||||
"parent": null,
|
||||
"outcome": null,
|
||||
"attempt": null,
|
||||
"status": ""
|
||||
},
|
||||
{
|
||||
"service": "messenger-relay",
|
||||
"name": "attachment.process publish",
|
||||
"kind": "SPAN_KIND_PRODUCER",
|
||||
"ms": 45.89,
|
||||
"parent": "messenger-api: POST /api/v1/chats/{chat_id}/attachments",
|
||||
"outcome": null,
|
||||
"attempt": null,
|
||||
"status": ""
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "attachment.process process",
|
||||
"kind": "SPAN_KIND_CONSUMER",
|
||||
"ms": 384.22,
|
||||
"parent": "messenger-relay: attachment.process publish",
|
||||
"outcome": "retry",
|
||||
"attempt": "1",
|
||||
"status": "STATUS_CODE_ERROR"
|
||||
},
|
||||
{
|
||||
"service": "messenger-relay",
|
||||
"name": "attachment.process publish",
|
||||
"kind": "SPAN_KIND_PRODUCER",
|
||||
"ms": 0.99,
|
||||
"parent": "messenger-worker: attachment.process process",
|
||||
"outcome": null,
|
||||
"attempt": null,
|
||||
"status": ""
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "attachment.process process",
|
||||
"kind": "SPAN_KIND_CONSUMER",
|
||||
"ms": 245.3,
|
||||
"parent": "messenger-relay: attachment.process publish",
|
||||
"outcome": "retry",
|
||||
"attempt": "2",
|
||||
"status": "STATUS_CODE_ERROR"
|
||||
},
|
||||
{
|
||||
"service": "messenger-relay",
|
||||
"name": "attachment.process publish",
|
||||
"kind": "SPAN_KIND_PRODUCER",
|
||||
"ms": 44.48,
|
||||
"parent": "messenger-worker: attachment.process process",
|
||||
"outcome": null,
|
||||
"attempt": null,
|
||||
"status": ""
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "attachment.process process",
|
||||
"kind": "SPAN_KIND_CONSUMER",
|
||||
"ms": 271.24,
|
||||
"parent": "messenger-relay: attachment.process publish",
|
||||
"outcome": "retry",
|
||||
"attempt": "3",
|
||||
"status": "STATUS_CODE_ERROR"
|
||||
},
|
||||
{
|
||||
"service": "messenger-relay",
|
||||
"name": "attachment.process publish",
|
||||
"kind": "SPAN_KIND_PRODUCER",
|
||||
"ms": 2.29,
|
||||
"parent": "messenger-worker: attachment.process process",
|
||||
"outcome": null,
|
||||
"attempt": null,
|
||||
"status": ""
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "attachment.process process",
|
||||
"kind": "SPAN_KIND_CONSUMER",
|
||||
"ms": 34.25,
|
||||
"parent": "messenger-relay: attachment.process publish",
|
||||
"outcome": "done",
|
||||
"attempt": "4",
|
||||
"status": ""
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,929 @@
|
||||
{
|
||||
"scenario": "signals",
|
||||
"result": {
|
||||
"request": {
|
||||
"request_id": "obs-msg-249e829f8c1e4049",
|
||||
"trace_id": "c275fd38f588c08d29aea802ac7c9c83",
|
||||
"spans": [
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "POST /api/v1/chats/{chat_id}/messages",
|
||||
"kind": "SPAN_KIND_SERVER",
|
||||
"ms": 6.55,
|
||||
"parent": null,
|
||||
"outcome": null,
|
||||
"attempt": null,
|
||||
"status": ""
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "get",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"ms": 0.11,
|
||||
"parent": "messenger-api: POST /api/v1/chats/{chat_id}/messages",
|
||||
"outcome": null,
|
||||
"attempt": null,
|
||||
"status": ""
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "ttl",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"ms": 0.06,
|
||||
"parent": "messenger-api: POST /api/v1/chats/{chat_id}/messages",
|
||||
"outcome": null,
|
||||
"attempt": null,
|
||||
"status": ""
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "pool.acquire",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"ms": 0.01,
|
||||
"parent": "messenger-api: POST /api/v1/chats/{chat_id}/messages",
|
||||
"outcome": null,
|
||||
"attempt": null,
|
||||
"status": ""
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "SELECT",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"ms": 0.38,
|
||||
"parent": "messenger-api: POST /api/v1/chats/{chat_id}/messages",
|
||||
"outcome": null,
|
||||
"attempt": null,
|
||||
"status": ""
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "SELECT",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"ms": 0.18,
|
||||
"parent": "messenger-api: SELECT",
|
||||
"outcome": null,
|
||||
"attempt": null,
|
||||
"status": ""
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "pool.acquire",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"ms": 0.0,
|
||||
"parent": "messenger-api: POST /api/v1/chats/{chat_id}/messages",
|
||||
"outcome": null,
|
||||
"attempt": null,
|
||||
"status": ""
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "SELECT",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"ms": 0.79,
|
||||
"parent": "messenger-api: POST /api/v1/chats/{chat_id}/messages",
|
||||
"outcome": null,
|
||||
"attempt": null,
|
||||
"status": ""
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "SELECT",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"ms": 0.29,
|
||||
"parent": "messenger-api: SELECT",
|
||||
"outcome": null,
|
||||
"attempt": null,
|
||||
"status": ""
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "pool.acquire",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"ms": 0.0,
|
||||
"parent": "messenger-api: POST /api/v1/chats/{chat_id}/messages",
|
||||
"outcome": null,
|
||||
"attempt": null,
|
||||
"status": ""
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "BEGIN",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"ms": 0.18,
|
||||
"parent": "messenger-api: POST /api/v1/chats/{chat_id}/messages",
|
||||
"outcome": null,
|
||||
"attempt": null,
|
||||
"status": ""
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "DELETE",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"ms": 1.54,
|
||||
"parent": "messenger-api: POST /api/v1/chats/{chat_id}/messages",
|
||||
"outcome": null,
|
||||
"attempt": null,
|
||||
"status": ""
|
||||
}
|
||||
],
|
||||
"log": {
|
||||
"line": "http request",
|
||||
"labels": {
|
||||
"detected_level": "info",
|
||||
"event": "http.request",
|
||||
"request_id": "obs-msg-249e829f8c1e4049",
|
||||
"route": "/api/v1/chats/{chat_id}/messages",
|
||||
"service_name": "messenger-api",
|
||||
"severity_text": "INFO",
|
||||
"span_id": "ce40a5e763ec9dd8",
|
||||
"status": "201",
|
||||
"trace_id": "c275fd38f588c08d29aea802ac7c9c83"
|
||||
}
|
||||
}
|
||||
},
|
||||
"upload": {
|
||||
"request_id": "obs-upload-e0c3f49f30f5446d",
|
||||
"trace_id": "cf8628e62914dd27bef25fbfca80ba11",
|
||||
"services": [
|
||||
"messenger-api",
|
||||
"messenger-relay",
|
||||
"messenger-worker"
|
||||
],
|
||||
"links": {
|
||||
"api server → relay producer": true,
|
||||
"relay producer → worker consumer": true,
|
||||
"worker consumer → S3 PUT": true,
|
||||
"worker consumer → image.decode": true
|
||||
},
|
||||
"span_count": 65,
|
||||
"worker_db_spans": [
|
||||
"BEGIN",
|
||||
"COMMIT",
|
||||
"INSERT",
|
||||
"SELECT",
|
||||
"UPDATE",
|
||||
"batch start",
|
||||
"pool.acquire"
|
||||
],
|
||||
"tree": [
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "POST /api/v1/chats/{chat_id}/attachments",
|
||||
"kind": "SPAN_KIND_SERVER",
|
||||
"ms": 14.68,
|
||||
"parent": null,
|
||||
"outcome": null,
|
||||
"attempt": null,
|
||||
"status": ""
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "get",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"ms": 0.07,
|
||||
"parent": "messenger-api: POST /api/v1/chats/{chat_id}/attachments",
|
||||
"outcome": null,
|
||||
"attempt": null,
|
||||
"status": ""
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "ttl",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"ms": 0.06,
|
||||
"parent": "messenger-api: POST /api/v1/chats/{chat_id}/attachments",
|
||||
"outcome": null,
|
||||
"attempt": null,
|
||||
"status": ""
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "pool.acquire",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"ms": 0.01,
|
||||
"parent": "messenger-api: POST /api/v1/chats/{chat_id}/attachments",
|
||||
"outcome": null,
|
||||
"attempt": null,
|
||||
"status": ""
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "SELECT",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"ms": 0.2,
|
||||
"parent": "messenger-api: POST /api/v1/chats/{chat_id}/attachments",
|
||||
"outcome": null,
|
||||
"attempt": null,
|
||||
"status": ""
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "pool.acquire",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"ms": 0.0,
|
||||
"parent": "messenger-api: POST /api/v1/chats/{chat_id}/attachments",
|
||||
"outcome": null,
|
||||
"attempt": null,
|
||||
"status": ""
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "SELECT",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"ms": 0.37,
|
||||
"parent": "messenger-api: POST /api/v1/chats/{chat_id}/attachments",
|
||||
"outcome": null,
|
||||
"attempt": null,
|
||||
"status": ""
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "SELECT",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"ms": 0.1,
|
||||
"parent": "messenger-api: SELECT",
|
||||
"outcome": null,
|
||||
"attempt": null,
|
||||
"status": ""
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "pool.acquire",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"ms": 0.0,
|
||||
"parent": "messenger-api: POST /api/v1/chats/{chat_id}/attachments",
|
||||
"outcome": null,
|
||||
"attempt": null,
|
||||
"status": ""
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "INSERT",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"ms": 1.69,
|
||||
"parent": "messenger-api: POST /api/v1/chats/{chat_id}/attachments",
|
||||
"outcome": null,
|
||||
"attempt": null,
|
||||
"status": ""
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "INSERT",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"ms": 0.46,
|
||||
"parent": "messenger-api: INSERT",
|
||||
"outcome": null,
|
||||
"attempt": null,
|
||||
"status": ""
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "S3 PUT",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"ms": 4.93,
|
||||
"parent": "messenger-api: POST /api/v1/chats/{chat_id}/attachments",
|
||||
"outcome": null,
|
||||
"attempt": null,
|
||||
"status": ""
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "pool.acquire",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"ms": 0.01,
|
||||
"parent": "messenger-api: POST /api/v1/chats/{chat_id}/attachments",
|
||||
"outcome": null,
|
||||
"attempt": null,
|
||||
"status": ""
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "BEGIN",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"ms": 0.13,
|
||||
"parent": "messenger-api: POST /api/v1/chats/{chat_id}/attachments",
|
||||
"outcome": null,
|
||||
"attempt": null,
|
||||
"status": ""
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "UPDATE",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"ms": 1.34,
|
||||
"parent": "messenger-api: POST /api/v1/chats/{chat_id}/attachments",
|
||||
"outcome": null,
|
||||
"attempt": null,
|
||||
"status": ""
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "UPDATE",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"ms": 0.26,
|
||||
"parent": "messenger-api: UPDATE",
|
||||
"outcome": null,
|
||||
"attempt": null,
|
||||
"status": ""
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "INSERT",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"ms": 1.56,
|
||||
"parent": "messenger-api: POST /api/v1/chats/{chat_id}/attachments",
|
||||
"outcome": null,
|
||||
"attempt": null,
|
||||
"status": ""
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "INSERT",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"ms": 0.44,
|
||||
"parent": "messenger-api: INSERT",
|
||||
"outcome": null,
|
||||
"attempt": null,
|
||||
"status": ""
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "INSERT",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"ms": 1.48,
|
||||
"parent": "messenger-api: POST /api/v1/chats/{chat_id}/attachments",
|
||||
"outcome": null,
|
||||
"attempt": null,
|
||||
"status": ""
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "INSERT",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"ms": 0.29,
|
||||
"parent": "messenger-api: INSERT",
|
||||
"outcome": null,
|
||||
"attempt": null,
|
||||
"status": ""
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "INSERT",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"ms": 1.82,
|
||||
"parent": "messenger-api: POST /api/v1/chats/{chat_id}/attachments",
|
||||
"outcome": null,
|
||||
"attempt": null,
|
||||
"status": ""
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "INSERT",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"ms": 1.19,
|
||||
"parent": "messenger-api: INSERT",
|
||||
"outcome": null,
|
||||
"attempt": null,
|
||||
"status": ""
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "COMMIT",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"ms": 0.23,
|
||||
"parent": "messenger-api: POST /api/v1/chats/{chat_id}/attachments",
|
||||
"outcome": null,
|
||||
"attempt": null,
|
||||
"status": ""
|
||||
},
|
||||
{
|
||||
"service": "messenger-relay",
|
||||
"name": "attachment.process publish",
|
||||
"kind": "SPAN_KIND_PRODUCER",
|
||||
"ms": 1.79,
|
||||
"parent": "messenger-api: POST /api/v1/chats/{chat_id}/attachments",
|
||||
"outcome": null,
|
||||
"attempt": null,
|
||||
"status": ""
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "attachment.process process",
|
||||
"kind": "SPAN_KIND_CONSUMER",
|
||||
"ms": 39.52,
|
||||
"parent": "messenger-relay: attachment.process publish",
|
||||
"outcome": "done",
|
||||
"attempt": "1",
|
||||
"status": ""
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "pool.acquire",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"ms": 0.16,
|
||||
"parent": "messenger-worker: attachment.process process",
|
||||
"outcome": null,
|
||||
"attempt": null,
|
||||
"status": ""
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "BEGIN",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"ms": 0.12,
|
||||
"parent": "messenger-worker: attachment.process process",
|
||||
"outcome": null,
|
||||
"attempt": null,
|
||||
"status": ""
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "SELECT",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"ms": 3.24,
|
||||
"parent": "messenger-worker: attachment.process process",
|
||||
"outcome": null,
|
||||
"attempt": null,
|
||||
"status": ""
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "SELECT",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"ms": 1.66,
|
||||
"parent": "messenger-worker: SELECT",
|
||||
"outcome": null,
|
||||
"attempt": null,
|
||||
"status": ""
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "SELECT",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"ms": 0.33,
|
||||
"parent": "messenger-worker: attachment.process process",
|
||||
"outcome": null,
|
||||
"attempt": null,
|
||||
"status": ""
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "SELECT",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"ms": 0.2,
|
||||
"parent": "messenger-worker: SELECT",
|
||||
"outcome": null,
|
||||
"attempt": null,
|
||||
"status": ""
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "SELECT",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"ms": 1.33,
|
||||
"parent": "messenger-worker: attachment.process process",
|
||||
"outcome": null,
|
||||
"attempt": null,
|
||||
"status": ""
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "SELECT",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"ms": 0.43,
|
||||
"parent": "messenger-worker: SELECT",
|
||||
"outcome": null,
|
||||
"attempt": null,
|
||||
"status": ""
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "UPDATE",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"ms": 0.65,
|
||||
"parent": "messenger-worker: attachment.process process",
|
||||
"outcome": null,
|
||||
"attempt": null,
|
||||
"status": ""
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "UPDATE",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"ms": 0.13,
|
||||
"parent": "messenger-worker: UPDATE",
|
||||
"outcome": null,
|
||||
"attempt": null,
|
||||
"status": ""
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "UPDATE",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"ms": 0.66,
|
||||
"parent": "messenger-worker: attachment.process process",
|
||||
"outcome": null,
|
||||
"attempt": null,
|
||||
"status": ""
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "UPDATE",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"ms": 0.11,
|
||||
"parent": "messenger-worker: UPDATE",
|
||||
"outcome": null,
|
||||
"attempt": null,
|
||||
"status": ""
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "INSERT",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"ms": 1.52,
|
||||
"parent": "messenger-worker: attachment.process process",
|
||||
"outcome": null,
|
||||
"attempt": null,
|
||||
"status": ""
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "INSERT",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"ms": 1.03,
|
||||
"parent": "messenger-worker: INSERT",
|
||||
"outcome": null,
|
||||
"attempt": null,
|
||||
"status": ""
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "COMMIT",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"ms": 0.27,
|
||||
"parent": "messenger-worker: attachment.process process",
|
||||
"outcome": null,
|
||||
"attempt": null,
|
||||
"status": ""
|
||||
}
|
||||
],
|
||||
"worker_log_events": [
|
||||
"job.claimed",
|
||||
"job.done"
|
||||
]
|
||||
},
|
||||
"metrics": {
|
||||
"found": {
|
||||
"http requests (counter = _count)": {
|
||||
"series": 11,
|
||||
"names": [
|
||||
"http_server_request_duration_seconds_count"
|
||||
]
|
||||
},
|
||||
"http duration histogram (s)": {
|
||||
"series": 154,
|
||||
"names": [
|
||||
"http_server_request_duration_seconds_bucket"
|
||||
]
|
||||
},
|
||||
"jobs processed by outcome": {
|
||||
"series": 1,
|
||||
"names": [
|
||||
"messenger_jobs_processed_total"
|
||||
]
|
||||
},
|
||||
"job duration histogram (s)": {
|
||||
"series": 14,
|
||||
"names": [
|
||||
"messenger_job_duration_seconds_bucket"
|
||||
]
|
||||
},
|
||||
"time to ready histogram (s)": {
|
||||
"series": 14,
|
||||
"names": [
|
||||
"messenger_attachment_time_to_ready_seconds_bucket"
|
||||
]
|
||||
},
|
||||
"queue backlog by state": {
|
||||
"series": 3,
|
||||
"names": [
|
||||
"messenger_jobs_backlog"
|
||||
]
|
||||
},
|
||||
"oldest job age (s)": {
|
||||
"series": 1,
|
||||
"names": [
|
||||
"messenger_jobs_oldest_age_seconds"
|
||||
]
|
||||
},
|
||||
"outbox pending": {
|
||||
"series": 1,
|
||||
"names": [
|
||||
"messenger_outbox_pending"
|
||||
]
|
||||
},
|
||||
"rabbitmq queue messages": {
|
||||
"series": 2,
|
||||
"names": [
|
||||
"rabbitmq_queue_messages_ready"
|
||||
]
|
||||
},
|
||||
"worker busy": {
|
||||
"series": 1,
|
||||
"names": [
|
||||
"messenger_worker_busy"
|
||||
]
|
||||
},
|
||||
"worker capacity": {
|
||||
"series": 1,
|
||||
"names": [
|
||||
"messenger_worker_capacity"
|
||||
]
|
||||
},
|
||||
"db pool acquired": {
|
||||
"series": 4,
|
||||
"names": [
|
||||
"pgxpool_acquired_connections"
|
||||
]
|
||||
},
|
||||
"db pool waits": {
|
||||
"series": 8,
|
||||
"names": [
|
||||
"pgxpool_empty_acquire_total",
|
||||
"pgxpool_empty_acquire_wait_time_nanoseconds_total"
|
||||
]
|
||||
},
|
||||
"process cpu": {
|
||||
"series": 8,
|
||||
"names": [
|
||||
"process_cpu_time_seconds_total"
|
||||
]
|
||||
},
|
||||
"go memory": {
|
||||
"series": 8,
|
||||
"names": [
|
||||
"go_memory_used_bytes"
|
||||
]
|
||||
}
|
||||
},
|
||||
"labels_total": 131,
|
||||
"forbidden_labels": [],
|
||||
"http_route_values": [
|
||||
"/api/v1/attachments/{attachment_id}",
|
||||
"/api/v1/auth/sessions",
|
||||
"/api/v1/chats",
|
||||
"/api/v1/chats/{chat_id}/attachments",
|
||||
"/api/v1/chats/{chat_id}/messages",
|
||||
"/api/v1/users",
|
||||
"/health/live",
|
||||
"/health/ready"
|
||||
],
|
||||
"example_series_labels": [
|
||||
"__name__",
|
||||
"http_request_method",
|
||||
"http_response_status_class",
|
||||
"http_response_status_code",
|
||||
"http_route",
|
||||
"instance",
|
||||
"job"
|
||||
]
|
||||
},
|
||||
"leak_check": {
|
||||
"loki:password": 0,
|
||||
"loki:session_token": 0,
|
||||
"loki:message_text": 0,
|
||||
"loki:bearer_header": 0,
|
||||
"tempo:password": 0,
|
||||
"tempo:session_token": 0,
|
||||
"tempo:message_text": 0,
|
||||
"tempo:bearer_header": 0,
|
||||
"stdout:password": 0,
|
||||
"stdout:session_token": 0,
|
||||
"stdout:message_text": 0,
|
||||
"stdout:bearer_header": 0
|
||||
},
|
||||
"grafana": {
|
||||
"datasources": [
|
||||
"loki:loki",
|
||||
"prometheus:prometheus",
|
||||
"tempo:tempo"
|
||||
],
|
||||
"dashboard": "Messenger — API, worker, очередь",
|
||||
"panels": 30,
|
||||
"prometheus_targets": 39,
|
||||
"targets_without_data_now": [
|
||||
{
|
||||
"panel": "Ready быстрее 30 s (5m)",
|
||||
"legend": "",
|
||||
"series": 0
|
||||
},
|
||||
{
|
||||
"panel": "Доля ошибок 5xx и 4xx",
|
||||
"legend": "4xx / все (ошибки клиента)",
|
||||
"series": 0
|
||||
}
|
||||
],
|
||||
"targets": [
|
||||
{
|
||||
"panel": "RPS API (бизнес)",
|
||||
"legend": "",
|
||||
"series": 1
|
||||
},
|
||||
{
|
||||
"panel": "Доля 5xx (1m)",
|
||||
"legend": "",
|
||||
"series": 1
|
||||
},
|
||||
{
|
||||
"panel": "Быстрее 250 ms (5m)",
|
||||
"legend": "",
|
||||
"series": 1
|
||||
},
|
||||
{
|
||||
"panel": "Ready быстрее 30 s (5m)",
|
||||
"legend": "",
|
||||
"series": 0
|
||||
},
|
||||
{
|
||||
"panel": "Старейшая job",
|
||||
"legend": "",
|
||||
"series": 1
|
||||
},
|
||||
{
|
||||
"panel": "Alerts firing",
|
||||
"legend": "",
|
||||
"series": 1
|
||||
},
|
||||
{
|
||||
"panel": "RPS по маршруту и классу ответа",
|
||||
"legend": "{{http_route}} {{http_response_status_class}}",
|
||||
"series": 6
|
||||
},
|
||||
{
|
||||
"panel": "Задержка API p50 / p95 / p99 (все реплики)",
|
||||
"legend": "p50",
|
||||
"series": 1
|
||||
},
|
||||
{
|
||||
"panel": "Задержка API p50 / p95 / p99 (все реплики)",
|
||||
"legend": "p95",
|
||||
"series": 1
|
||||
},
|
||||
{
|
||||
"panel": "Задержка API p50 / p95 / p99 (все реплики)",
|
||||
"legend": "p99",
|
||||
"series": 1
|
||||
},
|
||||
{
|
||||
"panel": "Доля ошибок 5xx и 4xx",
|
||||
"legend": "5xx / все",
|
||||
"series": 1
|
||||
},
|
||||
{
|
||||
"panel": "Доля ошибок 5xx и 4xx",
|
||||
"legend": "4xx / все (ошибки клиента)",
|
||||
"series": 0
|
||||
},
|
||||
{
|
||||
"panel": "p95 по маршруту",
|
||||
"legend": "{{http_route}}",
|
||||
"series": 6
|
||||
},
|
||||
{
|
||||
"panel": "Запросы в обработке (in-flight) по реплике",
|
||||
"legend": "{{instance}}",
|
||||
"series": 2
|
||||
},
|
||||
{
|
||||
"panel": "Jobs по исходу (rate)",
|
||||
"legend": "{{outcome}}",
|
||||
"series": 1
|
||||
},
|
||||
{
|
||||
"panel": "Длительность обработки job p50 / p95 / p99",
|
||||
"legend": "p50",
|
||||
"series": 1
|
||||
},
|
||||
{
|
||||
"panel": "Длительность обработки job p50 / p95 / p99",
|
||||
"legend": "p95",
|
||||
"series": 1
|
||||
},
|
||||
{
|
||||
"panel": "Длительность обработки job p50 / p95 / p99",
|
||||
"legend": "p99",
|
||||
"series": 1
|
||||
},
|
||||
{
|
||||
"panel": "Время до ready (202 → ready) p50 / p95",
|
||||
"legend": "p50",
|
||||
"series": 1
|
||||
},
|
||||
{
|
||||
"panel": "Время до ready (202 → ready) p50 / p95",
|
||||
"legend": "p95",
|
||||
"series": 1
|
||||
},
|
||||
{
|
||||
"panel": "Очередь: jobs по состоянию (PostgreSQL)",
|
||||
"legend": "{{state}}",
|
||||
"series": 3
|
||||
},
|
||||
{
|
||||
"panel": "Очередь: jobs по состоянию (PostgreSQL)",
|
||||
"legend": "outbox не опубликовано",
|
||||
"series": 1
|
||||
},
|
||||
{
|
||||
"panel": "Возраст старейшей job / outbox",
|
||||
"legend": "job (queued/processing)",
|
||||
"series": 1
|
||||
},
|
||||
{
|
||||
"panel": "Возраст старейшей job / outbox",
|
||||
"legend": "outbox",
|
||||
"series": 1
|
||||
},
|
||||
{
|
||||
"panel": "RabbitMQ: сообщения в очередях",
|
||||
"legend": "ready {{queue}}",
|
||||
"series": 2
|
||||
},
|
||||
{
|
||||
"panel": "RabbitMQ: сообщения в очередях",
|
||||
"legend": "unacked {{queue}}",
|
||||
"series": 2
|
||||
},
|
||||
{
|
||||
"panel": "Saturation worker (busy / capacity)",
|
||||
"legend": "занято слотов",
|
||||
"series": 1
|
||||
},
|
||||
{
|
||||
"panel": "Publish outbox → broker (confirm) p95",
|
||||
"legend": "p95",
|
||||
"series": 1
|
||||
},
|
||||
{
|
||||
"panel": "Publish outbox → broker (confirm) p95",
|
||||
"legend": "сообщений/с",
|
||||
"series": 1
|
||||
},
|
||||
{
|
||||
"panel": "Пул БД: занято / ожидания",
|
||||
"legend": "занято {{job}}",
|
||||
"series": 3
|
||||
},
|
||||
{
|
||||
"panel": "Пул БД: занято / ожидания",
|
||||
"legend": "max {{job}}",
|
||||
"series": 3
|
||||
},
|
||||
{
|
||||
"panel": "Пул БД: занято / ожидания",
|
||||
"legend": "ожиданий/с {{job}}",
|
||||
"series": 3
|
||||
},
|
||||
{
|
||||
"panel": "CPU процессов",
|
||||
"legend": "{{job}} {{instance}}",
|
||||
"series": 4
|
||||
},
|
||||
{
|
||||
"panel": "Память Go (used)",
|
||||
"legend": "{{job}} {{instance}}",
|
||||
"series": 4
|
||||
},
|
||||
{
|
||||
"panel": "Collector: принято / отправлено / потеряно spans",
|
||||
"legend": "принято",
|
||||
"series": 1
|
||||
},
|
||||
{
|
||||
"panel": "Collector: принято / отправлено / потеряно spans",
|
||||
"legend": "отправлено",
|
||||
"series": 1
|
||||
},
|
||||
{
|
||||
"panel": "Collector: принято / отправлено / потеряно spans",
|
||||
"legend": "не доставлено",
|
||||
"series": 1
|
||||
},
|
||||
{
|
||||
"panel": "Collector: принято / отправлено / потеряно spans",
|
||||
"legend": "отклонено (memory_limiter)",
|
||||
"series": 1
|
||||
},
|
||||
{
|
||||
"panel": "Collector: принято / отправлено / потеряно spans",
|
||||
"legend": "очередь экспортёров",
|
||||
"series": 1
|
||||
}
|
||||
]
|
||||
},
|
||||
"attachment": {
|
||||
"id": "ec322ba6-ae46-4757-b0c8-0d79d3ba2b42",
|
||||
"status": "ready"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
-- до:
|
||||
mtusi2026lab6-api-1 Up About a minute (healthy)
|
||||
mtusi2026lab6-api-2 Up 6 seconds (healthy)
|
||||
mtusi2026lab6-grafana-1 Up About a minute (healthy)
|
||||
mtusi2026lab6-lb-1 Up About a minute
|
||||
mtusi2026lab6-loki-1 Up About a minute (healthy)
|
||||
mtusi2026lab6-otel-collector-1 Up About a minute
|
||||
mtusi2026lab6-postgres-1 Up About a minute (healthy)
|
||||
mtusi2026lab6-prometheus-1 Up About a minute (healthy)
|
||||
mtusi2026lab6-rabbitmq-1 Up About a minute (healthy)
|
||||
mtusi2026lab6-reconciler-1 Up About a minute (healthy)
|
||||
mtusi2026lab6-redis-1 Up About a minute (healthy)
|
||||
mtusi2026lab6-relay-1 Up About a minute (healthy)
|
||||
mtusi2026lab6-s3-1 Up About a minute (healthy)
|
||||
mtusi2026lab6-tempo-1 Up About a minute (healthy)
|
||||
mtusi2026lab6-worker-1 Up About a minute (healthy)
|
||||
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
|
||||
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
-- после:
|
||||
mtusi2026lab6-api-1 Up 6 seconds (healthy)
|
||||
mtusi2026lab6-api-2 Up 6 seconds (healthy)
|
||||
mtusi2026lab6-grafana-1 Up 18 seconds (healthy)
|
||||
mtusi2026lab6-lb-1 Up Less than a second
|
||||
mtusi2026lab6-loki-1 Up 17 seconds (health: starting)
|
||||
mtusi2026lab6-otel-collector-1 Up 18 seconds
|
||||
mtusi2026lab6-postgres-1 Up 18 seconds (healthy)
|
||||
mtusi2026lab6-prometheus-1 Up 18 seconds (healthy)
|
||||
mtusi2026lab6-rabbitmq-1 Up 17 seconds (healthy)
|
||||
mtusi2026lab6-reconciler-1 Up 6 seconds (healthy)
|
||||
mtusi2026lab6-redis-1 Up 18 seconds (healthy)
|
||||
mtusi2026lab6-relay-1 Up 6 seconds (healthy)
|
||||
mtusi2026lab6-s3-1 Up 18 seconds (healthy)
|
||||
mtusi2026lab6-tempo-1 Up 17 seconds (health: starting)
|
||||
mtusi2026lab6-worker-1 Up 6 seconds (healthy)
|
||||
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"scenario": "broker-restart",
|
||||
"result": {
|
||||
"attachment_id": "3b4fbbcd-d9e2-4fd8-940f-c8fb30f617ef",
|
||||
"queue_before_restart": {
|
||||
"attachment.dead": 0,
|
||||
"attachment.process": 0
|
||||
},
|
||||
"queue_after_restart": {
|
||||
"attachment.dead": 0,
|
||||
"attachment.process": 1
|
||||
},
|
||||
"outbox_publish_attempts": 1,
|
||||
"invariants": {
|
||||
"status": "ready",
|
||||
"published_version": 1,
|
||||
"objects_per_version": {
|
||||
"1": 3
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"scenario": "crash-commit",
|
||||
"result": {
|
||||
"attachment_id": "577b04d0-15fc-440d-be43-c8302d17d1d8",
|
||||
"job_id": "22dba878-3bc7-46b4-9000-62f4e732243c",
|
||||
"worker_restarts": 1,
|
||||
"attempt": 1,
|
||||
"redelivery_acked_as_noop": true,
|
||||
"invariants": {
|
||||
"status": "ready",
|
||||
"published_version": 1,
|
||||
"objects_per_version": {
|
||||
"1": 3
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
{
|
||||
"scenario": "crash-effect",
|
||||
"result": {
|
||||
"attachment_id": "8ed34b9a-8cd6-43da-a713-3b38d1f0c8ef",
|
||||
"job_id": "0e6f1a68-ecf9-457c-8d34-c131cb2fb98e",
|
||||
"worker_restarts": 1,
|
||||
"attempt": 2,
|
||||
"seconds_to_ready": 17.7,
|
||||
"events": "enqueued,claimed,lease_expired_requeued,claimed,done",
|
||||
"invariants": {
|
||||
"status": "ready",
|
||||
"published_version": 1,
|
||||
"objects_per_version": {
|
||||
"1": 3
|
||||
}
|
||||
},
|
||||
"variants": {
|
||||
"thumbnail": [
|
||||
256,
|
||||
160
|
||||
],
|
||||
"crop": [
|
||||
128,
|
||||
128
|
||||
],
|
||||
"optimized": [
|
||||
320,
|
||||
200
|
||||
],
|
||||
"optimized_bytes": 2462,
|
||||
"original_bytes": 111817
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
{
|
||||
"scenario": "demo",
|
||||
"result": {
|
||||
"attachment_id": "40ed2137-afda-4551-a46e-e26af011d234",
|
||||
"job_id": "56db4cee-5fa9-46a7-9e87-6004bde53d25",
|
||||
"job_state": "done",
|
||||
"attempt": 1,
|
||||
"timeline": [
|
||||
"worker stopped",
|
||||
"upload 202 queued id=40ed2137-afda-4551-a46e-e26af011d234",
|
||||
"message 201 with queued attachment",
|
||||
"content 409, other messages OK",
|
||||
"api and rabbitmq restarted (volumes kept), outbox published=1",
|
||||
"rabbitmq queues after restart: {'attachment.dead': 0, 'attachment.process': 1}",
|
||||
"worker started → ready in 2.6 s"
|
||||
],
|
||||
"variants": {
|
||||
"thumbnail": [
|
||||
256,
|
||||
160
|
||||
],
|
||||
"crop": [
|
||||
128,
|
||||
128
|
||||
],
|
||||
"optimized": [
|
||||
320,
|
||||
200
|
||||
],
|
||||
"optimized_bytes": 2462,
|
||||
"original_bytes": 111817
|
||||
},
|
||||
"invariants": {
|
||||
"status": "ready",
|
||||
"published_version": 1,
|
||||
"objects_per_version": {
|
||||
"1": 3
|
||||
}
|
||||
},
|
||||
"events": "enqueued,claimed,done"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
{
|
||||
"scenario": "failed-replay",
|
||||
"result": {
|
||||
"attachment_id": "8619fa76-2bf8-46a4-a757-07591ca939f3",
|
||||
"failed_error_code": "checksum_mismatch",
|
||||
"content_while_failed": 409,
|
||||
"attach_failed_to_message": 409,
|
||||
"registry_listed": true,
|
||||
"replay": {
|
||||
"attachment_id": "8619fa76-2bf8-46a4-a757-07591ca939f3",
|
||||
"command": "replay",
|
||||
"error": "",
|
||||
"job_id": "7e26e051-dabe-4b1d-b511-85d5f9e2c743",
|
||||
"ok": true,
|
||||
"version": 1
|
||||
},
|
||||
"replay_after_success": {
|
||||
"attachment_id": "8619fa76-2bf8-46a4-a757-07591ca939f3",
|
||||
"command": "replay",
|
||||
"error": "conflict: job is done, only failed jobs can be replayed",
|
||||
"job_id": "7e26e051-dabe-4b1d-b511-85d5f9e2c743",
|
||||
"ok": false,
|
||||
"version": 1
|
||||
},
|
||||
"final_status": "ready",
|
||||
"events": "enqueued,claimed,failed,replay,claimed,done,replay_refused",
|
||||
"dlq_messages": 1,
|
||||
"invariants": {
|
||||
"status": "ready",
|
||||
"published_version": 1,
|
||||
"objects_per_version": {
|
||||
"1": 3
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
{
|
||||
"scenario": "limits",
|
||||
"result": {
|
||||
"image": "6000x6000 PNG (119755 bytes)",
|
||||
"parallel_uploads": 3,
|
||||
"seconds_all_ready": 12.1,
|
||||
"worker_anon_memory_max_mib": 574.8,
|
||||
"worker_memory_peak_incl_page_cache_mib": 580.9,
|
||||
"oom_kill_events": 0,
|
||||
"go_memlimit": "560MiB",
|
||||
"oom_events": 0,
|
||||
"worker_memory_limit_mib": 768.0,
|
||||
"pixel_budget": 60000000,
|
||||
"decoded_rgba_per_image_mib": 137.3,
|
||||
"variant_sizes": {
|
||||
"thumbnail": [
|
||||
256,
|
||||
256
|
||||
],
|
||||
"crop": [
|
||||
128,
|
||||
128
|
||||
],
|
||||
"optimized": [
|
||||
1280,
|
||||
1280
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
{
|
||||
"scenario": "outbox-gap",
|
||||
"result": {
|
||||
"relay_down_then_up": {
|
||||
"attachment_id": "0766e424-4356-474b-a409-282566dbbd81",
|
||||
"unpublished_while_relay_down": 1,
|
||||
"then": "ready"
|
||||
},
|
||||
"relay_crash_after_confirm": {
|
||||
"attachment_id": "5bb073a8-f9d1-4e64-8226-4ac24792e214",
|
||||
"relay_restarts": 1,
|
||||
"outbox_publish_attempts": 1,
|
||||
"worker_saw_message_twice": true,
|
||||
"job_attempt": 1,
|
||||
"invariants": {
|
||||
"status": "ready",
|
||||
"published_version": 1,
|
||||
"objects_per_version": {
|
||||
"1": 3
|
||||
}
|
||||
}
|
||||
},
|
||||
"events": "enqueued,claimed,done"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"scenario": "transient",
|
||||
"result": {
|
||||
"attachment_id": "c5d86b25-e26e-42ca-af2e-bed3ee98b538",
|
||||
"attempts": 3,
|
||||
"final_status": "ready",
|
||||
"retry_details": "processing_failed; next_attempt=2 in 1.609s | processing_failed; next_attempt=3 in 2.66s",
|
||||
"events": "enqueued,claimed,retry_scheduled,claimed,retry_scheduled,claimed,done",
|
||||
"invariants": {
|
||||
"status": "ready",
|
||||
"published_version": 1,
|
||||
"objects_per_version": {
|
||||
"1": 3
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
worker-1 | {"time":"2026-11-12T11:25:53.786929702Z","level":"INFO","msg":"worker starting","event":"worker.start","owner":"d392ccfc07f1:1","concurrency":2,"lease":60000000000,"service.name":"messenger-worker"}
|
||||
relay-1 | {"time":"2026-11-12T11:25:26.41695347Z","level":"WARN","msg":"expired leases requeued","jobs":1,"service.name":"messenger-relay"}
|
||||
worker-1 | {"time":"2026-11-12T11:26:27.839005278Z","level":"INFO","msg":"worker starting","event":"worker.start","owner":"d392ccfc07f1:1","concurrency":2,"lease":60000000000,"service.name":"messenger-worker"}
|
||||
worker-1 | {"time":"2026-11-12T11:26:36.585881442Z","level":"INFO","msg":"worker starting","event":"worker.start","owner":"d392ccfc07f1:1","concurrency":2,"lease":60000000000,"service.name":"messenger-worker"}
|
||||
worker-1 | {"time":"2026-11-12T11:26:37.150745975Z","level":"WARN","msg":"attempt failed","job_id":"b47271a0-7ba3-4716-b2fe-9f6f03b36194","attachment_id":"c5d86b25-e26e-42ca-af2e-bed3ee98b538","version":1,"event":"job.retry","attempt":1,"code":"processing_failed","final":false,"backoff":1608723537,"trace_id":"a57663e672319236293d4ab274186eae","span_id":"d1c251107bb9c10f","service.name":"messenger-worker"}
|
||||
worker-1 | {"time":"2026-11-12T11:26:39.012900329Z","level":"WARN","msg":"attempt failed","job_id":"b47271a0-7ba3-4716-b2fe-9f6f03b36194","attachment_id":"c5d86b25-e26e-42ca-af2e-bed3ee98b538","version":1,"event":"job.retry","attempt":2,"code":"processing_failed","final":false,"backoff":2659621162,"trace_id":"a57663e672319236293d4ab274186eae","span_id":"ae8a58278600849a","service.name":"messenger-worker"}
|
||||
worker-1 | {"time":"2026-11-12T11:26:51.026383949Z","level":"INFO","msg":"worker starting","event":"worker.start","owner":"d392ccfc07f1:1","concurrency":2,"lease":60000000000,"service.name":"messenger-worker"}
|
||||
worker-1 | {"time":"2026-11-12T11:26:51.087077531Z","level":"WARN","msg":"attempt failed","job_id":"7e26e051-dabe-4b1d-b511-85d5f9e2c743","attachment_id":"8619fa76-2bf8-46a4-a757-07591ca939f3","version":1,"event":"job.failed","attempt":1,"code":"checksum_mismatch","final":true,"backoff":1116187727,"trace_id":"70012050ef241cf92d8f9b19f088cf41","span_id":"191f5c2578ce4819","service.name":"messenger-worker"}
|
||||
worker-1 | {"time":"2026-11-12T11:26:54.713069545Z","level":"ERROR","msg":"invalid job message, dead-lettering","event":"job.rejected","err":null,"trace_id":"47832279361ecd6b346277a7df057eee","span_id":"ea86222bd1cd28d0","service.name":"messenger-worker"}
|
||||
worker-1 | {"time":"2026-11-12T11:27:06.108177234Z","level":"INFO","msg":"worker starting","event":"worker.start","owner":"d392ccfc07f1:1","concurrency":2,"lease":60000000000,"service.name":"messenger-worker"}
|
||||
@@ -0,0 +1,3 @@
|
||||
PASS: whitespace-only password rejected with 400 (3 variants)
|
||||
PASS: Redis ACL -incr: message visible on every next read (3 in history)
|
||||
PASS: Redis ACL -incr -del -unlink -scan: message visible on every next read (4 in history)
|
||||
@@ -0,0 +1,92 @@
|
||||
-- до:
|
||||
mtusi2026lab6-api-1 84c136cad07b Up 2 minutes (healthy)
|
||||
mtusi2026lab6-api-2 1354d20e1722 Up 2 minutes (healthy)
|
||||
#1 [internal] load local bake definitions
|
||||
#1 reading from stdin 380B done
|
||||
#1 DONE 0.0s
|
||||
|
||||
#2 [internal] load build definition from Dockerfile
|
||||
#2 transferring dockerfile: 2.91kB done
|
||||
#2 DONE 0.0s
|
||||
|
||||
#3 [internal] load metadata for docker.io/library/alpine:3.20@sha256:d9e853e87e55526f6b2917df91a2115c36dd7c696a35be12163d44e6e2a4b6bc
|
||||
#3 DONE 0.0s
|
||||
|
||||
#4 [internal] load metadata for docker.io/library/golang:1.26-alpine@sha256:27f2d3b83cfc5803c8078f8e730af0f7c431bf067afaec8f60a36fb763187cba
|
||||
#4 DONE 0.0s
|
||||
|
||||
#5 [internal] load .dockerignore
|
||||
#5 transferring context: 212B 0.0s done
|
||||
#5 DONE 0.0s
|
||||
|
||||
#6 [builder 1/6] FROM docker.io/library/golang:1.26-alpine@sha256:27f2d3b83cfc5803c8078f8e730af0f7c431bf067afaec8f60a36fb763187cba
|
||||
#6 DONE 0.0s
|
||||
|
||||
#7 [stage-1 1/7] FROM docker.io/library/alpine:3.20@sha256:d9e853e87e55526f6b2917df91a2115c36dd7c696a35be12163d44e6e2a4b6bc
|
||||
#7 DONE 0.0s
|
||||
|
||||
#8 [internal] load build context
|
||||
#8 transferring context: 9.04kB 0.0s done
|
||||
#8 DONE 0.0s
|
||||
|
||||
#9 [builder 6/6] RUN CGO_ENABLED=0 GOOS=linux go build -ldflags="-s -w -X github.com/charawein/mtusi2026lab6/internal/telemetry.Version=lab6" -trimpath -o /out/ ./cmd/api ./cmd/reconcile ./cmd/pipeline
|
||||
#9 CACHED
|
||||
|
||||
#10 [builder 3/6] COPY go.mod go.sum ./
|
||||
#10 CACHED
|
||||
|
||||
#11 [builder 2/6] WORKDIR /src
|
||||
#11 CACHED
|
||||
|
||||
#12 [stage-1 5/7] COPY --from=builder --chown=appuser:appgroup /out/api /app/api
|
||||
#12 CACHED
|
||||
|
||||
#13 [builder 5/6] COPY . .
|
||||
#13 CACHED
|
||||
|
||||
#14 [stage-1 6/7] COPY --from=builder --chown=appuser:appgroup /out/reconcile /app/reconcile
|
||||
#14 CACHED
|
||||
|
||||
#15 [stage-1 3/7] RUN addgroup -g 1001 -S appgroup && adduser -u 1001 -S appuser -G appgroup
|
||||
#15 CACHED
|
||||
|
||||
#16 [stage-1 4/7] WORKDIR /app
|
||||
#16 CACHED
|
||||
|
||||
#17 [builder 4/6] RUN go mod download
|
||||
#17 CACHED
|
||||
|
||||
#18 [stage-1 2/7] RUN apk --no-cache add ca-certificates tzdata libwebp-tools
|
||||
#18 CACHED
|
||||
|
||||
#19 [stage-1 7/7] COPY --from=builder --chown=appuser:appgroup /out/pipeline /app/pipeline
|
||||
#19 CACHED
|
||||
|
||||
#20 exporting to image
|
||||
#20 exporting layers done
|
||||
#20 writing image sha256:11e3e3f4cb401048102ebf29aa2365118d1cad928fbc0f682535f043ff700ec5 done
|
||||
#20 naming to docker.io/library/mtusi2026lab6-api:local done
|
||||
#20 DONE 0.0s
|
||||
|
||||
#21 resolving provenance for metadata file
|
||||
#21 DONE 0.0s
|
||||
api Built
|
||||
Container mtusi2026lab6-postgres-1 Running
|
||||
no change
|
||||
Container mtusi2026lab6-api-1 Running
|
||||
Container mtusi2026lab6-api-2 Running
|
||||
Container mtusi2026lab6-api-3 Creating
|
||||
Container mtusi2026lab6-api-3 Created
|
||||
Container mtusi2026lab6-api-3 Starting
|
||||
Container mtusi2026lab6-api-3 Started
|
||||
replaced /mtusi2026lab6-api-1
|
||||
Container mtusi2026lab6-api-2 Running
|
||||
Container mtusi2026lab6-api-3 Running
|
||||
Container mtusi2026lab6-api-4 Creating
|
||||
Container mtusi2026lab6-api-4 Created
|
||||
Container mtusi2026lab6-api-4 Starting
|
||||
Container mtusi2026lab6-api-4 Started
|
||||
replaced /mtusi2026lab6-api-2
|
||||
mtusi2026lab6-api-3 0cc85552e20a mtusi2026lab6-api:local Up 26 seconds (healthy)
|
||||
mtusi2026lab6-api-4 2b9b764811a6 mtusi2026lab6-api:local Up 13 seconds (healthy)
|
||||
-- замена заняла 61 s (внутри 90 s нагрузки)
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 6,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 90.0,
|
||||
"elapsed_seconds": 90.053,
|
||||
"concurrency": 4,
|
||||
"successful_requests": 29983,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 332.947,
|
||||
"successful_latency_ms": {
|
||||
"p50": 9.107,
|
||||
"p95": 24.061,
|
||||
"p99": 65.137
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
2
|
||||
total used free shared buff/cache available
|
||||
Mem: 7932 1968 1612 102 4769 5963
|
||||
Swap: 3071 0 3071
|
||||
28.0.4
|
||||
Docker Compose version v2.38.2
|
||||
@@ -0,0 +1,6 @@
|
||||
2
|
||||
total used free shared buff/cache available
|
||||
Mem: 7932 2242 1097 83 4991 5690
|
||||
Swap: 3071 0 3071
|
||||
28.0.4
|
||||
Docker Compose version v2.38.2
|
||||
@@ -0,0 +1,6 @@
|
||||
2
|
||||
total used free shared buff/cache available
|
||||
Mem: 7932 2079 917 147 5397 5853
|
||||
Swap: 3071 21 3050
|
||||
28.0.4
|
||||
Docker Compose version v2.38.2
|
||||
@@ -0,0 +1,6 @@
|
||||
2
|
||||
total used free shared buff/cache available
|
||||
Mem: 7932 2735 675 83 4920 5196
|
||||
Swap: 3071 0 3071
|
||||
28.0.4
|
||||
Docker Compose version v2.38.2
|
||||
@@ -0,0 +1,6 @@
|
||||
2
|
||||
total used free shared buff/cache available
|
||||
Mem: 7932 2073 1076 173 5271 5859
|
||||
Swap: 3071 0 3071
|
||||
28.0.4
|
||||
Docker Compose version v2.38.2
|
||||
@@ -0,0 +1 @@
|
||||
PASS: lab6 public API scenario; infrastructure and grading evidence remain separate.
|
||||
@@ -0,0 +1,18 @@
|
||||
=== RUN TestTracePropagationMapAndAMQP
|
||||
--- PASS: TestTracePropagationMapAndAMQP (0.00s)
|
||||
=== RUN TestTraceUnsampledSuppressesBackgroundSpans
|
||||
--- PASS: TestTraceUnsampledSuppressesBackgroundSpans (0.00s)
|
||||
=== RUN TestTraceLogCorrelation
|
||||
--- PASS: TestTraceLogCorrelation (0.00s)
|
||||
=== RUN TestTelemetrySetupBuildsProviders
|
||||
--- PASS: TestTelemetrySetupBuildsProviders (0.01s)
|
||||
PASS
|
||||
ok github.com/charawein/mtusi2026lab6/internal/telemetry 0.013s
|
||||
=== RUN TestTelemetryRouteTemplateAndStatus
|
||||
--- PASS: TestTelemetryRouteTemplateAndStatus (0.00s)
|
||||
PASS
|
||||
ok github.com/charawein/mtusi2026lab6/internal/transport/http/middleware 0.007s
|
||||
=== RUN TestPipelineTracePropagationThroughOutbox
|
||||
--- PASS: TestPipelineTracePropagationThroughOutbox (0.75s)
|
||||
PASS
|
||||
ok github.com/charawein/mtusi2026lab6/internal/worker 0.754s
|
||||
@@ -0,0 +1,999 @@
|
||||
{
|
||||
"method": {
|
||||
"duration_s": 45,
|
||||
"warmup_s": 10,
|
||||
"concurrency": 4,
|
||||
"locks_at_s": [
|
||||
12,
|
||||
28
|
||||
],
|
||||
"lock_seconds": 6.5,
|
||||
"slow_threshold_s": 0.25,
|
||||
"rounds": 3,
|
||||
"handler_timeout_s": 5
|
||||
},
|
||||
"summary": {
|
||||
"off": {
|
||||
"runs": 3,
|
||||
"ok_rps_median": 247.6,
|
||||
"p50_ms_median": 10.06,
|
||||
"p95_ms_median": 20.22,
|
||||
"p99_ms_median": 31.5,
|
||||
"errors_total": 24,
|
||||
"slow_total": 24,
|
||||
"api_cpu_cores_median": 0.316,
|
||||
"api_cpu_ms_per_request_median": 1.282,
|
||||
"api_mem_peak_mib_median": 13.1,
|
||||
"collector_cpu_cores_median": 0.001,
|
||||
"collector_mem_peak_mib_median": 30.1,
|
||||
"spans_received_per_1000_req": 0.0,
|
||||
"spans_exported_per_1000_req": 0.0,
|
||||
"tempo_kib_per_1000_req": 0.0,
|
||||
"loki_kib_per_1000_req": 0.0,
|
||||
"log_records_per_1000_req": 0.0,
|
||||
"metric_points_per_1000_req": 0.0,
|
||||
"traces_kept_errors": {
|
||||
"kept": 0,
|
||||
"total": 24,
|
||||
"ratio": 0.0
|
||||
},
|
||||
"traces_kept_slow": {
|
||||
"kept": 0,
|
||||
"total": 24,
|
||||
"ratio": 0.0
|
||||
},
|
||||
"traces_kept_normal_sample": {
|
||||
"kept": 0,
|
||||
"total": 120,
|
||||
"ratio": 0.0
|
||||
}
|
||||
},
|
||||
"full": {
|
||||
"runs": 3,
|
||||
"ok_rps_median": 197.8,
|
||||
"p50_ms_median": 12.26,
|
||||
"p95_ms_median": 27.91,
|
||||
"p99_ms_median": 46.15,
|
||||
"errors_total": 24,
|
||||
"slow_total": 24,
|
||||
"api_cpu_cores_median": 0.371,
|
||||
"api_cpu_ms_per_request_median": 1.875,
|
||||
"api_mem_peak_mib_median": 22.4,
|
||||
"collector_cpu_cores_median": 0.051,
|
||||
"collector_mem_peak_mib_median": 55.1,
|
||||
"spans_received_per_1000_req": 14018.6,
|
||||
"spans_exported_per_1000_req": 14018.6,
|
||||
"tempo_kib_per_1000_req": 4547.652,
|
||||
"loki_kib_per_1000_req": 339.364,
|
||||
"log_records_per_1000_req": 1018.3,
|
||||
"metric_points_per_1000_req": 243.9,
|
||||
"traces_kept_errors": {
|
||||
"kept": 24,
|
||||
"total": 24,
|
||||
"ratio": 1.0
|
||||
},
|
||||
"traces_kept_slow": {
|
||||
"kept": 24,
|
||||
"total": 24,
|
||||
"ratio": 1.0
|
||||
},
|
||||
"traces_kept_normal_sample": {
|
||||
"kept": 120,
|
||||
"total": 120,
|
||||
"ratio": 1.0
|
||||
}
|
||||
},
|
||||
"head10": {
|
||||
"runs": 3,
|
||||
"ok_rps_median": 234.3,
|
||||
"p50_ms_median": 10.38,
|
||||
"p95_ms_median": 21.88,
|
||||
"p99_ms_median": 34.23,
|
||||
"errors_total": 24,
|
||||
"slow_total": 24,
|
||||
"api_cpu_cores_median": 0.332,
|
||||
"api_cpu_ms_per_request_median": 1.414,
|
||||
"api_mem_peak_mib_median": 21.3,
|
||||
"collector_cpu_cores_median": 0.014,
|
||||
"collector_mem_peak_mib_median": 55.9,
|
||||
"spans_received_per_1000_req": 1397.7,
|
||||
"spans_exported_per_1000_req": 1397.7,
|
||||
"tempo_kib_per_1000_req": 454.021,
|
||||
"loki_kib_per_1000_req": 334.58,
|
||||
"log_records_per_1000_req": 1015.5,
|
||||
"metric_points_per_1000_req": 203.1,
|
||||
"traces_kept_errors": {
|
||||
"kept": 2,
|
||||
"total": 24,
|
||||
"ratio": 0.083
|
||||
},
|
||||
"traces_kept_slow": {
|
||||
"kept": 2,
|
||||
"total": 24,
|
||||
"ratio": 0.083
|
||||
},
|
||||
"traces_kept_normal_sample": {
|
||||
"kept": 14,
|
||||
"total": 120,
|
||||
"ratio": 0.117
|
||||
}
|
||||
},
|
||||
"tail": {
|
||||
"runs": 3,
|
||||
"ok_rps_median": 195.4,
|
||||
"p50_ms_median": 12.45,
|
||||
"p95_ms_median": 27.68,
|
||||
"p99_ms_median": 41.48,
|
||||
"errors_total": 24,
|
||||
"slow_total": 24,
|
||||
"api_cpu_cores_median": 0.372,
|
||||
"api_cpu_ms_per_request_median": 1.905,
|
||||
"api_mem_peak_mib_median": 22.3,
|
||||
"collector_cpu_cores_median": 0.044,
|
||||
"collector_mem_peak_mib_median": 106.5,
|
||||
"spans_received_per_1000_req": 14016.5,
|
||||
"spans_exported_per_1000_req": 1784.9,
|
||||
"tempo_kib_per_1000_req": 587.593,
|
||||
"loki_kib_per_1000_req": 339.584,
|
||||
"log_records_per_1000_req": 1018.6,
|
||||
"metric_points_per_1000_req": 246.3,
|
||||
"traces_kept_errors": {
|
||||
"kept": 24,
|
||||
"total": 24,
|
||||
"ratio": 1.0
|
||||
},
|
||||
"traces_kept_slow": {
|
||||
"kept": 24,
|
||||
"total": 24,
|
||||
"ratio": 1.0
|
||||
},
|
||||
"traces_kept_normal_sample": {
|
||||
"kept": 13,
|
||||
"total": 120,
|
||||
"ratio": 0.108
|
||||
}
|
||||
}
|
||||
},
|
||||
"runs": [
|
||||
{
|
||||
"config": "off",
|
||||
"round": 0,
|
||||
"load": {
|
||||
"requests": 11155,
|
||||
"ok": 11147,
|
||||
"errors": 8,
|
||||
"error_ratio": 0.00072,
|
||||
"ok_rps": 247.7,
|
||||
"seconds": 45.0,
|
||||
"latency_ms_all_ok": {
|
||||
"p50": 10.12,
|
||||
"p95": 20.92,
|
||||
"p99": 32.92
|
||||
},
|
||||
"latency_ms_outside_faults": {
|
||||
"p50": 10.0,
|
||||
"p95": 20.32,
|
||||
"p99": 31.5
|
||||
}
|
||||
},
|
||||
"slow_requests": 8,
|
||||
"error_requests": 8,
|
||||
"traces_kept": {
|
||||
"errors": [
|
||||
0,
|
||||
8
|
||||
],
|
||||
"slow": [
|
||||
0,
|
||||
8
|
||||
],
|
||||
"normal_sample": [
|
||||
0,
|
||||
40
|
||||
]
|
||||
},
|
||||
"api_cpu_cores": 0.315,
|
||||
"api_cpu_ms_per_request": 1.271,
|
||||
"api_mem_peak_mib": [
|
||||
13.1,
|
||||
12.7
|
||||
],
|
||||
"api_mem_now_mib": [
|
||||
8.7,
|
||||
8.3
|
||||
],
|
||||
"collector_cpu_cores": 0.002,
|
||||
"collector_mem_peak_mib": [
|
||||
42.3
|
||||
],
|
||||
"ingest_delta": {
|
||||
"spans_received": 0,
|
||||
"spans_exported": 0,
|
||||
"log_records_received": 0,
|
||||
"metric_points_received": 0,
|
||||
"tempo_bytes": 0,
|
||||
"loki_bytes": 0,
|
||||
"prometheus_samples": 52263
|
||||
},
|
||||
"ingest_per_1000_requests": {
|
||||
"spans_received": 0.0,
|
||||
"spans_exported": 0.0,
|
||||
"log_records_received": 0.0,
|
||||
"metric_points_received": 0.0,
|
||||
"tempo_bytes": 0.0,
|
||||
"loki_bytes": 0.0,
|
||||
"prometheus_samples": 4688.5
|
||||
}
|
||||
},
|
||||
{
|
||||
"config": "full",
|
||||
"round": 0,
|
||||
"load": {
|
||||
"requests": 9033,
|
||||
"ok": 9025,
|
||||
"errors": 8,
|
||||
"error_ratio": 0.00089,
|
||||
"ok_rps": 200.6,
|
||||
"seconds": 45.0,
|
||||
"latency_ms_all_ok": {
|
||||
"p50": 11.94,
|
||||
"p95": 28.43,
|
||||
"p99": 48.57
|
||||
},
|
||||
"latency_ms_outside_faults": {
|
||||
"p50": 11.68,
|
||||
"p95": 26.75,
|
||||
"p99": 46.8
|
||||
}
|
||||
},
|
||||
"slow_requests": 8,
|
||||
"error_requests": 8,
|
||||
"traces_kept": {
|
||||
"errors": [
|
||||
8,
|
||||
8
|
||||
],
|
||||
"slow": [
|
||||
8,
|
||||
8
|
||||
],
|
||||
"normal_sample": [
|
||||
40,
|
||||
40
|
||||
]
|
||||
},
|
||||
"api_cpu_cores": 0.372,
|
||||
"api_cpu_ms_per_request": 1.857,
|
||||
"api_mem_peak_mib": [
|
||||
21.9,
|
||||
21.0
|
||||
],
|
||||
"api_mem_now_mib": [
|
||||
14.5,
|
||||
14.8
|
||||
],
|
||||
"collector_cpu_cores": 0.052,
|
||||
"collector_mem_peak_mib": [
|
||||
54.5
|
||||
],
|
||||
"ingest_delta": {
|
||||
"spans_received": 126498,
|
||||
"spans_exported": 126498,
|
||||
"log_records_received": 9189,
|
||||
"metric_points_received": 2170,
|
||||
"tempo_bytes": 42025732,
|
||||
"loki_bytes": 3136265,
|
||||
"prometheus_samples": 53356
|
||||
},
|
||||
"ingest_per_1000_requests": {
|
||||
"spans_received": 14016.4,
|
||||
"spans_exported": 14016.4,
|
||||
"log_records_received": 1018.2,
|
||||
"metric_points_received": 240.4,
|
||||
"tempo_bytes": 4656590.8,
|
||||
"loki_bytes": 347508.6,
|
||||
"prometheus_samples": 5912.0
|
||||
}
|
||||
},
|
||||
{
|
||||
"config": "head10",
|
||||
"round": 0,
|
||||
"load": {
|
||||
"requests": 10825,
|
||||
"ok": 10817,
|
||||
"errors": 8,
|
||||
"error_ratio": 0.00074,
|
||||
"ok_rps": 240.3,
|
||||
"seconds": 45.0,
|
||||
"latency_ms_all_ok": {
|
||||
"p50": 10.23,
|
||||
"p95": 22.32,
|
||||
"p99": 37.12
|
||||
},
|
||||
"latency_ms_outside_faults": {
|
||||
"p50": 10.14,
|
||||
"p95": 21.88,
|
||||
"p99": 36.09
|
||||
}
|
||||
},
|
||||
"slow_requests": 8,
|
||||
"error_requests": 8,
|
||||
"traces_kept": {
|
||||
"errors": [
|
||||
0,
|
||||
8
|
||||
],
|
||||
"slow": [
|
||||
0,
|
||||
8
|
||||
],
|
||||
"normal_sample": [
|
||||
7,
|
||||
40
|
||||
]
|
||||
},
|
||||
"api_cpu_cores": 0.335,
|
||||
"api_cpu_ms_per_request": 1.394,
|
||||
"api_mem_peak_mib": [
|
||||
21.3,
|
||||
20.4
|
||||
],
|
||||
"api_mem_now_mib": [
|
||||
15.8,
|
||||
15.9
|
||||
],
|
||||
"collector_cpu_cores": 0.014,
|
||||
"collector_mem_peak_mib": [
|
||||
56.5
|
||||
],
|
||||
"ingest_delta": {
|
||||
"spans_received": 15119,
|
||||
"spans_exported": 15119,
|
||||
"log_records_received": 10979,
|
||||
"metric_points_received": 2166,
|
||||
"tempo_bytes": 5029007,
|
||||
"loki_bytes": 3703341,
|
||||
"prometheus_samples": 58820
|
||||
},
|
||||
"ingest_per_1000_requests": {
|
||||
"spans_received": 1397.7,
|
||||
"spans_exported": 1397.7,
|
||||
"log_records_received": 1015.0,
|
||||
"metric_points_received": 200.2,
|
||||
"tempo_bytes": 464917.0,
|
||||
"loki_bytes": 342363.0,
|
||||
"prometheus_samples": 5437.7
|
||||
}
|
||||
},
|
||||
{
|
||||
"config": "tail",
|
||||
"round": 0,
|
||||
"load": {
|
||||
"requests": 9661,
|
||||
"ok": 9653,
|
||||
"errors": 8,
|
||||
"error_ratio": 0.00083,
|
||||
"ok_rps": 214.5,
|
||||
"seconds": 45.0,
|
||||
"latency_ms_all_ok": {
|
||||
"p50": 11.51,
|
||||
"p95": 25.35,
|
||||
"p99": 42.72
|
||||
},
|
||||
"latency_ms_outside_faults": {
|
||||
"p50": 11.51,
|
||||
"p95": 24.51,
|
||||
"p99": 38.18
|
||||
}
|
||||
},
|
||||
"slow_requests": 8,
|
||||
"error_requests": 8,
|
||||
"traces_kept": {
|
||||
"errors": [
|
||||
8,
|
||||
8
|
||||
],
|
||||
"slow": [
|
||||
8,
|
||||
8
|
||||
],
|
||||
"normal_sample": [
|
||||
4,
|
||||
40
|
||||
]
|
||||
},
|
||||
"api_cpu_cores": 0.398,
|
||||
"api_cpu_ms_per_request": 1.854,
|
||||
"api_mem_peak_mib": [
|
||||
22.3,
|
||||
20.8
|
||||
],
|
||||
"api_mem_now_mib": [
|
||||
15.9,
|
||||
14.5
|
||||
],
|
||||
"collector_cpu_cores": 0.044,
|
||||
"collector_mem_peak_mib": [
|
||||
115.5
|
||||
],
|
||||
"ingest_delta": {
|
||||
"spans_received": 135292,
|
||||
"spans_exported": 16105,
|
||||
"log_records_received": 9816,
|
||||
"metric_points_received": 2157,
|
||||
"tempo_bytes": 6135814,
|
||||
"loki_bytes": 3346785,
|
||||
"prometheus_samples": 56512
|
||||
},
|
||||
"ingest_per_1000_requests": {
|
||||
"spans_received": 14015.5,
|
||||
"spans_exported": 1668.4,
|
||||
"log_records_received": 1016.9,
|
||||
"metric_points_received": 223.5,
|
||||
"tempo_bytes": 635638.0,
|
||||
"loki_bytes": 346709.3,
|
||||
"prometheus_samples": 5854.3
|
||||
}
|
||||
},
|
||||
{
|
||||
"config": "full",
|
||||
"round": 1,
|
||||
"load": {
|
||||
"requests": 8909,
|
||||
"ok": 8901,
|
||||
"errors": 8,
|
||||
"error_ratio": 0.0009,
|
||||
"ok_rps": 197.8,
|
||||
"seconds": 45.0,
|
||||
"latency_ms_all_ok": {
|
||||
"p50": 12.28,
|
||||
"p95": 28.01,
|
||||
"p99": 46.37
|
||||
},
|
||||
"latency_ms_outside_faults": {
|
||||
"p50": 12.26,
|
||||
"p95": 27.91,
|
||||
"p99": 46.15
|
||||
}
|
||||
},
|
||||
"slow_requests": 8,
|
||||
"error_requests": 8,
|
||||
"traces_kept": {
|
||||
"errors": [
|
||||
8,
|
||||
8
|
||||
],
|
||||
"slow": [
|
||||
8,
|
||||
8
|
||||
],
|
||||
"normal_sample": [
|
||||
40,
|
||||
40
|
||||
]
|
||||
},
|
||||
"api_cpu_cores": 0.371,
|
||||
"api_cpu_ms_per_request": 1.875,
|
||||
"api_mem_peak_mib": [
|
||||
22.5,
|
||||
21.5
|
||||
],
|
||||
"api_mem_now_mib": [
|
||||
14.9,
|
||||
16.7
|
||||
],
|
||||
"collector_cpu_cores": 0.051,
|
||||
"collector_mem_peak_mib": [
|
||||
55.9
|
||||
],
|
||||
"ingest_delta": {
|
||||
"spans_received": 124841,
|
||||
"spans_exported": 124841,
|
||||
"log_records_received": 9064,
|
||||
"metric_points_received": 2171,
|
||||
"tempo_bytes": 41450142,
|
||||
"loki_bytes": 3091868,
|
||||
"prometheus_samples": 55341
|
||||
},
|
||||
"ingest_per_1000_requests": {
|
||||
"spans_received": 14025.5,
|
||||
"spans_exported": 14025.5,
|
||||
"log_records_received": 1018.3,
|
||||
"metric_points_received": 243.9,
|
||||
"tempo_bytes": 4656796.1,
|
||||
"loki_bytes": 347361.9,
|
||||
"prometheus_samples": 6217.4
|
||||
}
|
||||
},
|
||||
{
|
||||
"config": "head10",
|
||||
"round": 1,
|
||||
"load": {
|
||||
"requests": 10539,
|
||||
"ok": 10531,
|
||||
"errors": 8,
|
||||
"error_ratio": 0.00076,
|
||||
"ok_rps": 234.0,
|
||||
"seconds": 45.0,
|
||||
"latency_ms_all_ok": {
|
||||
"p50": 10.66,
|
||||
"p95": 22.47,
|
||||
"p99": 34.71
|
||||
},
|
||||
"latency_ms_outside_faults": {
|
||||
"p50": 10.68,
|
||||
"p95": 22.17,
|
||||
"p99": 34.23
|
||||
}
|
||||
},
|
||||
"slow_requests": 8,
|
||||
"error_requests": 8,
|
||||
"traces_kept": {
|
||||
"errors": [
|
||||
2,
|
||||
8
|
||||
],
|
||||
"slow": [
|
||||
1,
|
||||
8
|
||||
],
|
||||
"normal_sample": [
|
||||
1,
|
||||
40
|
||||
]
|
||||
},
|
||||
"api_cpu_cores": 0.331,
|
||||
"api_cpu_ms_per_request": 1.414,
|
||||
"api_mem_peak_mib": [
|
||||
20.6,
|
||||
20.4
|
||||
],
|
||||
"api_mem_now_mib": [
|
||||
14.5,
|
||||
15.8
|
||||
],
|
||||
"collector_cpu_cores": 0.015,
|
||||
"collector_mem_peak_mib": [
|
||||
55.9
|
||||
],
|
||||
"ingest_delta": {
|
||||
"spans_received": 14486,
|
||||
"spans_exported": 14486,
|
||||
"log_records_received": 10695,
|
||||
"metric_points_received": 2157,
|
||||
"tempo_bytes": 4820676,
|
||||
"loki_bytes": 3608439,
|
||||
"prometheus_samples": 59603
|
||||
},
|
||||
"ingest_per_1000_requests": {
|
||||
"spans_received": 1375.6,
|
||||
"spans_exported": 1375.6,
|
||||
"log_records_received": 1015.6,
|
||||
"metric_points_received": 204.8,
|
||||
"tempo_bytes": 457760.5,
|
||||
"loki_bytes": 342649.2,
|
||||
"prometheus_samples": 5659.8
|
||||
}
|
||||
},
|
||||
{
|
||||
"config": "tail",
|
||||
"round": 1,
|
||||
"load": {
|
||||
"requests": 8802,
|
||||
"ok": 8794,
|
||||
"errors": 8,
|
||||
"error_ratio": 0.00091,
|
||||
"ok_rps": 195.4,
|
||||
"seconds": 45.0,
|
||||
"latency_ms_all_ok": {
|
||||
"p50": 12.42,
|
||||
"p95": 28.49,
|
||||
"p99": 48.03
|
||||
},
|
||||
"latency_ms_outside_faults": {
|
||||
"p50": 12.45,
|
||||
"p95": 27.68,
|
||||
"p99": 41.48
|
||||
}
|
||||
},
|
||||
"slow_requests": 8,
|
||||
"error_requests": 8,
|
||||
"traces_kept": {
|
||||
"errors": [
|
||||
8,
|
||||
8
|
||||
],
|
||||
"slow": [
|
||||
8,
|
||||
8
|
||||
],
|
||||
"normal_sample": [
|
||||
6,
|
||||
40
|
||||
]
|
||||
},
|
||||
"api_cpu_cores": 0.372,
|
||||
"api_cpu_ms_per_request": 1.905,
|
||||
"api_mem_peak_mib": [
|
||||
22.3,
|
||||
22.3
|
||||
],
|
||||
"api_mem_now_mib": [
|
||||
16.4,
|
||||
15.9
|
||||
],
|
||||
"collector_cpu_cores": 0.044,
|
||||
"collector_mem_peak_mib": [
|
||||
106.5
|
||||
],
|
||||
"ingest_delta": {
|
||||
"spans_received": 123261,
|
||||
"spans_exported": 16559,
|
||||
"log_records_received": 8958,
|
||||
"metric_points_received": 2166,
|
||||
"tempo_bytes": 5291306,
|
||||
"loki_bytes": 3058512,
|
||||
"prometheus_samples": 57176
|
||||
},
|
||||
"ingest_per_1000_requests": {
|
||||
"spans_received": 14016.5,
|
||||
"spans_exported": 1883.0,
|
||||
"log_records_received": 1018.6,
|
||||
"metric_points_received": 246.3,
|
||||
"tempo_bytes": 601695.0,
|
||||
"loki_bytes": 347795.3,
|
||||
"prometheus_samples": 6501.7
|
||||
}
|
||||
},
|
||||
{
|
||||
"config": "off",
|
||||
"round": 1,
|
||||
"load": {
|
||||
"requests": 11150,
|
||||
"ok": 11142,
|
||||
"errors": 8,
|
||||
"error_ratio": 0.00072,
|
||||
"ok_rps": 247.6,
|
||||
"seconds": 45.0,
|
||||
"latency_ms_all_ok": {
|
||||
"p50": 10.08,
|
||||
"p95": 20.47,
|
||||
"p99": 33.29
|
||||
},
|
||||
"latency_ms_outside_faults": {
|
||||
"p50": 10.06,
|
||||
"p95": 19.79,
|
||||
"p99": 31.2
|
||||
}
|
||||
},
|
||||
"slow_requests": 8,
|
||||
"error_requests": 8,
|
||||
"traces_kept": {
|
||||
"errors": [
|
||||
0,
|
||||
8
|
||||
],
|
||||
"slow": [
|
||||
0,
|
||||
8
|
||||
],
|
||||
"normal_sample": [
|
||||
0,
|
||||
40
|
||||
]
|
||||
},
|
||||
"api_cpu_cores": 0.317,
|
||||
"api_cpu_ms_per_request": 1.282,
|
||||
"api_mem_peak_mib": [
|
||||
13.1,
|
||||
13.1
|
||||
],
|
||||
"api_mem_now_mib": [
|
||||
8.6,
|
||||
8.5
|
||||
],
|
||||
"collector_cpu_cores": 0.001,
|
||||
"collector_mem_peak_mib": [
|
||||
30.1
|
||||
],
|
||||
"ingest_delta": {
|
||||
"spans_received": 0,
|
||||
"spans_exported": 0,
|
||||
"log_records_received": 0,
|
||||
"metric_points_received": 0,
|
||||
"tempo_bytes": 0,
|
||||
"loki_bytes": 0,
|
||||
"prometheus_samples": 47494
|
||||
},
|
||||
"ingest_per_1000_requests": {
|
||||
"spans_received": 0.0,
|
||||
"spans_exported": 0.0,
|
||||
"log_records_received": 0.0,
|
||||
"metric_points_received": 0.0,
|
||||
"tempo_bytes": 0.0,
|
||||
"loki_bytes": 0.0,
|
||||
"prometheus_samples": 4262.6
|
||||
}
|
||||
},
|
||||
{
|
||||
"config": "head10",
|
||||
"round": 2,
|
||||
"load": {
|
||||
"requests": 10553,
|
||||
"ok": 10545,
|
||||
"errors": 8,
|
||||
"error_ratio": 0.00076,
|
||||
"ok_rps": 234.3,
|
||||
"seconds": 45.0,
|
||||
"latency_ms_all_ok": {
|
||||
"p50": 10.59,
|
||||
"p95": 22.86,
|
||||
"p99": 36.32
|
||||
},
|
||||
"latency_ms_outside_faults": {
|
||||
"p50": 10.38,
|
||||
"p95": 21.55,
|
||||
"p99": 33.46
|
||||
}
|
||||
},
|
||||
"slow_requests": 8,
|
||||
"error_requests": 8,
|
||||
"traces_kept": {
|
||||
"errors": [
|
||||
0,
|
||||
8
|
||||
],
|
||||
"slow": [
|
||||
1,
|
||||
8
|
||||
],
|
||||
"normal_sample": [
|
||||
6,
|
||||
40
|
||||
]
|
||||
},
|
||||
"api_cpu_cores": 0.332,
|
||||
"api_cpu_ms_per_request": 1.416,
|
||||
"api_mem_peak_mib": [
|
||||
20.6,
|
||||
21.9
|
||||
],
|
||||
"api_mem_now_mib": [
|
||||
14.4,
|
||||
15.9
|
||||
],
|
||||
"collector_cpu_cores": 0.013,
|
||||
"collector_mem_peak_mib": [
|
||||
54.4
|
||||
],
|
||||
"ingest_delta": {
|
||||
"spans_received": 14771,
|
||||
"spans_exported": 14771,
|
||||
"log_records_received": 10708,
|
||||
"metric_points_received": 2142,
|
||||
"tempo_bytes": 5161389,
|
||||
"loki_bytes": 3612825,
|
||||
"prometheus_samples": 55781
|
||||
},
|
||||
"ingest_per_1000_requests": {
|
||||
"spans_received": 1400.8,
|
||||
"spans_exported": 1400.8,
|
||||
"log_records_received": 1015.5,
|
||||
"metric_points_received": 203.1,
|
||||
"tempo_bytes": 489463.2,
|
||||
"loki_bytes": 342610.2,
|
||||
"prometheus_samples": 5289.8
|
||||
}
|
||||
},
|
||||
{
|
||||
"config": "tail",
|
||||
"round": 2,
|
||||
"load": {
|
||||
"requests": 8738,
|
||||
"ok": 8730,
|
||||
"errors": 8,
|
||||
"error_ratio": 0.00092,
|
||||
"ok_rps": 194.0,
|
||||
"seconds": 45.0,
|
||||
"latency_ms_all_ok": {
|
||||
"p50": 12.45,
|
||||
"p95": 29.33,
|
||||
"p99": 44.71
|
||||
},
|
||||
"latency_ms_outside_faults": {
|
||||
"p50": 12.75,
|
||||
"p95": 28.68,
|
||||
"p99": 42.09
|
||||
}
|
||||
},
|
||||
"slow_requests": 8,
|
||||
"error_requests": 8,
|
||||
"traces_kept": {
|
||||
"errors": [
|
||||
8,
|
||||
8
|
||||
],
|
||||
"slow": [
|
||||
8,
|
||||
8
|
||||
],
|
||||
"normal_sample": [
|
||||
3,
|
||||
40
|
||||
]
|
||||
},
|
||||
"api_cpu_cores": 0.371,
|
||||
"api_cpu_ms_per_request": 1.913,
|
||||
"api_mem_peak_mib": [
|
||||
21.3,
|
||||
21.7
|
||||
],
|
||||
"api_mem_now_mib": [
|
||||
16.5,
|
||||
16.0
|
||||
],
|
||||
"collector_cpu_cores": 0.043,
|
||||
"collector_mem_peak_mib": [
|
||||
104.6
|
||||
],
|
||||
"ingest_delta": {
|
||||
"spans_received": 122371,
|
||||
"spans_exported": 15582,
|
||||
"log_records_received": 8892,
|
||||
"metric_points_received": 2164,
|
||||
"tempo_bytes": 4960426,
|
||||
"loki_bytes": 3035717,
|
||||
"prometheus_samples": 57370
|
||||
},
|
||||
"ingest_per_1000_requests": {
|
||||
"spans_received": 14017.3,
|
||||
"spans_exported": 1784.9,
|
||||
"log_records_received": 1018.6,
|
||||
"metric_points_received": 247.9,
|
||||
"tempo_bytes": 568204.6,
|
||||
"loki_bytes": 347733.9,
|
||||
"prometheus_samples": 6571.6
|
||||
}
|
||||
},
|
||||
{
|
||||
"config": "off",
|
||||
"round": 2,
|
||||
"load": {
|
||||
"requests": 11068,
|
||||
"ok": 11060,
|
||||
"errors": 8,
|
||||
"error_ratio": 0.00072,
|
||||
"ok_rps": 245.7,
|
||||
"seconds": 45.0,
|
||||
"latency_ms_all_ok": {
|
||||
"p50": 10.2,
|
||||
"p95": 20.91,
|
||||
"p99": 34.15
|
||||
},
|
||||
"latency_ms_outside_faults": {
|
||||
"p50": 10.1,
|
||||
"p95": 20.22,
|
||||
"p99": 32.38
|
||||
}
|
||||
},
|
||||
"slow_requests": 8,
|
||||
"error_requests": 8,
|
||||
"traces_kept": {
|
||||
"errors": [
|
||||
0,
|
||||
8
|
||||
],
|
||||
"slow": [
|
||||
0,
|
||||
8
|
||||
],
|
||||
"normal_sample": [
|
||||
0,
|
||||
40
|
||||
]
|
||||
},
|
||||
"api_cpu_cores": 0.316,
|
||||
"api_cpu_ms_per_request": 1.284,
|
||||
"api_mem_peak_mib": [
|
||||
13.0,
|
||||
13.0
|
||||
],
|
||||
"api_mem_now_mib": [
|
||||
8.2,
|
||||
8.3
|
||||
],
|
||||
"collector_cpu_cores": 0.001,
|
||||
"collector_mem_peak_mib": [
|
||||
28.6
|
||||
],
|
||||
"ingest_delta": {
|
||||
"spans_received": 0,
|
||||
"spans_exported": 0,
|
||||
"log_records_received": 0,
|
||||
"metric_points_received": 0,
|
||||
"tempo_bytes": 0,
|
||||
"loki_bytes": 0,
|
||||
"prometheus_samples": 47826
|
||||
},
|
||||
"ingest_per_1000_requests": {
|
||||
"spans_received": 0.0,
|
||||
"spans_exported": 0.0,
|
||||
"log_records_received": 0.0,
|
||||
"metric_points_received": 0.0,
|
||||
"tempo_bytes": 0.0,
|
||||
"loki_bytes": 0.0,
|
||||
"prometheus_samples": 4324.2
|
||||
}
|
||||
},
|
||||
{
|
||||
"config": "full",
|
||||
"round": 2,
|
||||
"load": {
|
||||
"requests": 8607,
|
||||
"ok": 8599,
|
||||
"errors": 8,
|
||||
"error_ratio": 0.00093,
|
||||
"ok_rps": 191.1,
|
||||
"seconds": 45.0,
|
||||
"latency_ms_all_ok": {
|
||||
"p50": 12.57,
|
||||
"p95": 29.92,
|
||||
"p99": 46.42
|
||||
},
|
||||
"latency_ms_outside_faults": {
|
||||
"p50": 12.37,
|
||||
"p95": 28.49,
|
||||
"p99": 42.89
|
||||
}
|
||||
},
|
||||
"slow_requests": 8,
|
||||
"error_requests": 8,
|
||||
"traces_kept": {
|
||||
"errors": [
|
||||
8,
|
||||
8
|
||||
],
|
||||
"slow": [
|
||||
8,
|
||||
8
|
||||
],
|
||||
"normal_sample": [
|
||||
40,
|
||||
40
|
||||
]
|
||||
},
|
||||
"api_cpu_cores": 0.364,
|
||||
"api_cpu_ms_per_request": 1.907,
|
||||
"api_mem_peak_mib": [
|
||||
21.3,
|
||||
22.4
|
||||
],
|
||||
"api_mem_now_mib": [
|
||||
16.4,
|
||||
16.2
|
||||
],
|
||||
"collector_cpu_cores": 0.05,
|
||||
"collector_mem_peak_mib": [
|
||||
55.1
|
||||
],
|
||||
"ingest_delta": {
|
||||
"spans_received": 120546,
|
||||
"spans_exported": 120546,
|
||||
"log_records_received": 8762,
|
||||
"metric_points_received": 2166,
|
||||
"tempo_bytes": 40248112,
|
||||
"loki_bytes": 2992045,
|
||||
"prometheus_samples": 55820
|
||||
},
|
||||
"ingest_per_1000_requests": {
|
||||
"spans_received": 14018.6,
|
||||
"spans_exported": 14018.6,
|
||||
"log_records_received": 1019.0,
|
||||
"metric_points_received": 251.9,
|
||||
"tempo_bytes": 4680557.3,
|
||||
"loki_bytes": 347952.7,
|
||||
"prometheus_samples": 6491.5
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
=== RUN TestOptimizedWebP
|
||||
--- PASS: TestOptimizedWebP (0.32s)
|
||||
PASS
|
||||
ok github.com/charawein/mtusi2026lab6/internal/imaging 0.328s
|
||||
@@ -0,0 +1,74 @@
|
||||
module github.com/charawein/mtusi2026lab6
|
||||
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
github.com/exaring/otelpgx v0.13.0
|
||||
github.com/google/uuid v1.6.0
|
||||
github.com/jackc/pgx/v5 v5.11.0
|
||||
github.com/minio/minio-go/v7 v7.4.1
|
||||
github.com/rabbitmq/amqp091-go v1.16.0
|
||||
github.com/redis/go-redis/extra/redisotel/v9 v9.23.0
|
||||
github.com/redis/go-redis/v9 v9.23.0
|
||||
go.opentelemetry.io/contrib/bridges/otelslog v0.21.0
|
||||
go.opentelemetry.io/contrib/instrumentation/host v0.72.0
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.72.0
|
||||
go.opentelemetry.io/contrib/instrumentation/runtime v0.72.0
|
||||
go.opentelemetry.io/otel v1.47.0
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.23.0
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.47.0
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.47.0
|
||||
go.opentelemetry.io/otel/log v1.47.0
|
||||
go.opentelemetry.io/otel/metric v1.47.0
|
||||
go.opentelemetry.io/otel/sdk v1.47.0
|
||||
go.opentelemetry.io/otel/sdk/log v1.47.0
|
||||
go.opentelemetry.io/otel/sdk/metric v1.47.0
|
||||
go.opentelemetry.io/otel/trace v1.47.0
|
||||
golang.org/x/crypto v0.58.0
|
||||
golang.org/x/image v0.47.0
|
||||
golang.org/x/sync v0.25.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/cenkalti/backoff/v5 v5.0.3 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/ebitengine/purego v0.11.1 // indirect
|
||||
github.com/felixge/httpsnoop v1.1.0 // indirect
|
||||
github.com/go-logr/logr v1.4.4 // indirect
|
||||
github.com/go-logr/stdr v1.2.2 // indirect
|
||||
github.com/go-ole/go-ole v1.3.0 // indirect
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 // indirect
|
||||
github.com/jackc/pgpassfile v1.0.0 // indirect
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
|
||||
github.com/jackc/puddle/v2 v2.2.2 // indirect
|
||||
github.com/klauspost/compress v1.19.2 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.4.0 // indirect
|
||||
github.com/klauspost/crc32 v1.3.0 // indirect
|
||||
github.com/lufia/plan9stats v0.0.0-20260802145828-341c2f0c90b5 // indirect
|
||||
github.com/minio/crc64nvme v1.1.1 // indirect
|
||||
github.com/minio/md5-simd v1.1.2 // indirect
|
||||
github.com/philhofer/fwd v1.2.0 // indirect
|
||||
github.com/power-devops/perfstat v0.0.0-20260916203055-22a1a467d9f0 // indirect
|
||||
github.com/redis/go-redis/extra/rediscmd/v9 v9.23.0 // indirect
|
||||
github.com/rs/xid v1.6.0 // indirect
|
||||
github.com/shirou/gopsutil/v4 v4.26.9 // indirect
|
||||
github.com/tinylib/msgp v1.6.4 // indirect
|
||||
github.com/tklauser/go-sysconf v0.4.0 // indirect
|
||||
github.com/tklauser/numcpus v0.12.0 // indirect
|
||||
github.com/yusufpapurcu/wmi v1.2.4 // indirect
|
||||
github.com/zeebo/xxh3 v1.1.0 // indirect
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.47.0 // indirect
|
||||
go.opentelemetry.io/proto/otlp v1.11.0 // indirect
|
||||
go.uber.org/atomic v1.12.0 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.5 // indirect
|
||||
golang.org/x/net v0.60.0 // indirect
|
||||
golang.org/x/sys v0.49.0 // indirect
|
||||
golang.org/x/text v0.43.0 // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260928230214-8a89bd6388cc // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260928230214-8a89bd6388cc // indirect
|
||||
google.golang.org/grpc v1.83.2 // indirect
|
||||
google.golang.org/protobuf v1.36.12 // indirect
|
||||
gopkg.in/ini.v1 v1.67.3 // indirect
|
||||
)
|
||||
@@ -0,0 +1,172 @@
|
||||
github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
|
||||
github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
|
||||
github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
|
||||
github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
|
||||
github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM=
|
||||
github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw=
|
||||
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||
github.com/ebitengine/purego v0.11.1 h1:2zpWRSQNVKN4eKsKO9eM1ILDgWfYMY9GwqRmK6XeQ/0=
|
||||
github.com/ebitengine/purego v0.11.1/go.mod h1:DCHPP08djqhNSoTfImcnHYQRZmd0qhakvrozqaEYhGQ=
|
||||
github.com/exaring/otelpgx v0.13.0 h1:Vi0I2GWRS3nnVzgr73L/XbSLqthR0h6A1rm4Wt3W4aA=
|
||||
github.com/exaring/otelpgx v0.13.0/go.mod h1:3OojrUKhhy3lTbYIMBijP3YjMey/jo14eHAW5cXcUdk=
|
||||
github.com/felixge/httpsnoop v1.1.0 h1:3YtUj32ZZkqZtt3sZZsClsymw/QDuVfpNhoA31zeORc=
|
||||
github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9vDSi5UZCE=
|
||||
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
|
||||
github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8=
|
||||
github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
|
||||
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
|
||||
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
|
||||
github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
|
||||
github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE=
|
||||
github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78=
|
||||
github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
|
||||
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
|
||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 h1:/Tnpcb2E0Pz/tN9s3bfEY2Q8ePCEX9iuS+cneUwncnw=
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0/go.mod h1:zOBXOsUaBSjKgmH4OGzV1esUpR3oUSCPYVd2cUBjKYY=
|
||||
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
|
||||
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
|
||||
github.com/jackc/pgx/v5 v5.11.0 h1:IzBBtyK9AHqf98cctWFifYSci2hgQR/cd56wB4p+ogg=
|
||||
github.com/jackc/pgx/v5 v5.11.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4=
|
||||
github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
|
||||
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
|
||||
github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8=
|
||||
github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
|
||||
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw=
|
||||
github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU=
|
||||
github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM=
|
||||
github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw=
|
||||
github.com/lufia/plan9stats v0.0.0-20260802145828-341c2f0c90b5 h1:eveIIGn4BGM3qknO74omf6HYr30/exH+eVUTuAgwjZ0=
|
||||
github.com/lufia/plan9stats v0.0.0-20260802145828-341c2f0c90b5/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg=
|
||||
github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI=
|
||||
github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg=
|
||||
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
|
||||
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
|
||||
github.com/minio/minio-go/v7 v7.4.1 h1:PtU/zIOfQRWZ+QydXAZysc+xZYHdG+BBR4Z6TUvy73w=
|
||||
github.com/minio/minio-go/v7 v7.4.1/go.mod h1:nQnQi7zTfXR4lREtYXZQWqo5aw0Rqv+ZqCGrTORygRk=
|
||||
github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM=
|
||||
github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/power-devops/perfstat v0.0.0-20260916203055-22a1a467d9f0 h1:XA01Vk/wv9YikCi1V51yRzIHPMT5of9+cMpZoZvDn/M=
|
||||
github.com/power-devops/perfstat v0.0.0-20260916203055-22a1a467d9f0/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
|
||||
github.com/rabbitmq/amqp091-go v1.16.0 h1:1O8oQqnj9hLACoo6CVMgn57z2pLWXvgemhh4GsquV+Y=
|
||||
github.com/rabbitmq/amqp091-go v1.16.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o=
|
||||
github.com/redis/go-redis/extra/rediscmd/v9 v9.23.0 h1:YOQCtePRR/cBrxtZB5u2ueYei3zGJm1n0KB2tvth0k0=
|
||||
github.com/redis/go-redis/extra/rediscmd/v9 v9.23.0/go.mod h1:BzBb5fEjIwijUn8VVkXfQUwFTGyk2vGIVU19zv2GhsM=
|
||||
github.com/redis/go-redis/extra/redisotel/v9 v9.23.0 h1:EBQ01zUjwarZZ4IoSTQNU9EEc223eB9tpSun6n5EaUU=
|
||||
github.com/redis/go-redis/extra/redisotel/v9 v9.23.0/go.mod h1:4xBZ7QF+cd90pc58g+cXH1aLJBgOGd4Gp5BBQvNtPn0=
|
||||
github.com/redis/go-redis/v9 v9.23.0 h1:/TTVdJa8BRNsybwwGGnmiYktHNlUXtbOQxlWpHBPL3s=
|
||||
github.com/redis/go-redis/v9 v9.23.0/go.mod h1:EWP2UUk+XDaI1s5nQB5mablZN6vccXlmBQTyfnF87D0=
|
||||
github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
|
||||
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
|
||||
github.com/shirou/gopsutil/v4 v4.26.9 h1:CaBo/hBFqvlJoLvAQEKdHdf5GD97/MN2ACfyv63+2Ig=
|
||||
github.com/shirou/gopsutil/v4 v4.26.9/go.mod h1:nKH+8wX2zxr/mDbsR+AAy6Qd2z4ZOQ7wfQaEp+yrYmg=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
|
||||
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
|
||||
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
|
||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||
github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE=
|
||||
github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg=
|
||||
github.com/tinylib/msgp v1.6.4 h1:mOwYbyYDLPj35mkA2BjjYejgJk9BuHxDdvRnb6v2ZcQ=
|
||||
github.com/tinylib/msgp v1.6.4/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA=
|
||||
github.com/tklauser/go-sysconf v0.4.0 h1:7H0uAN+7RkwWRaxhYXDLqa5V3LPrJeV8wmD9dRUgPQU=
|
||||
github.com/tklauser/go-sysconf v0.4.0/go.mod h1:8mTNWyog7H+MpKijp4VmKJAd2bbYQ2zuUwkYRbUArPI=
|
||||
github.com/tklauser/numcpus v0.12.0 h1:NR85qdvHA9pFse3x3weVZ0r0ST8R6l5RHbZrlRaqob4=
|
||||
github.com/tklauser/numcpus v0.12.0/go.mod h1:ABHeXzJnr/qqwguhClkZKT1/8VABcYrsyUiUGobwWJg=
|
||||
github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=
|
||||
github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
|
||||
github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
|
||||
github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
|
||||
github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs=
|
||||
github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s=
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
|
||||
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
|
||||
go.opentelemetry.io/contrib/bridges/otelslog v0.21.0 h1:/CdgE52i1OEPHhsGUqrggYNRaZy4v4YgVenGd52dRBI=
|
||||
go.opentelemetry.io/contrib/bridges/otelslog v0.21.0/go.mod h1:9pBdbVbuehIDzxN4accnzzxEUPT8Nq7grbZMo+J1VN4=
|
||||
go.opentelemetry.io/contrib/instrumentation/host v0.72.0 h1:DqxFkQ1eCzAQNPJNog2+tL9k3OaULuiYHiZaz2Waq54=
|
||||
go.opentelemetry.io/contrib/instrumentation/host v0.72.0/go.mod h1:Jn5AUVzW8ZWjm4S1uRMlio5gZxVpmXRr/2PmdESRE7Y=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.72.0 h1:LxwW/9ctSCv+QkE/cLR7M91ZIkXNMqJtEMi1vCw9U8s=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.72.0/go.mod h1:tOsftB4SslBwwErVEPaenU2RpThXWPIU8DoJHEC4dyw=
|
||||
go.opentelemetry.io/contrib/instrumentation/runtime v0.72.0 h1:rv8t3kKjyC2/9D99L1MZt4tJSZO4aYUmHus7L//iKKg=
|
||||
go.opentelemetry.io/contrib/instrumentation/runtime v0.72.0/go.mod h1:ZzG+JmzLLvKzx6tMeLrw2l7F8ik2GeRZJALoPBqnM0c=
|
||||
go.opentelemetry.io/otel v1.47.0 h1:j7ALJ/zgkS7Z6aeJW09p8VC9804bC+PpeTfCD4XPnOM=
|
||||
go.opentelemetry.io/otel v1.47.0/go.mod h1:8wS9O2qfXrYrzp6hIF/HOYJJf/wIhFPhR2xLuP+iXQU=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.23.0 h1:5WVp40k2fJHuc5BtNTgKk+EugKFT8Eg3lywFfCPI/nE=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.23.0/go.mod h1:59nH0RYCGHRlwt15WyioeSYINBbuObeBpTowPhEKWHY=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.47.0 h1:7kszRN4Ze5GeAkUnBjBSUIunVBMA+mZIG04auueqguM=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.47.0/go.mod h1:CLzakYy8jOdkBfcveCWYkeKJxujcF4xPhh2Z7IfE98U=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.47.0 h1:julhjPeUH/q/7hinbSdDdqt5h7Zw9YWmRlWRhI0jd54=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.47.0/go.mod h1:Ao2mz688LH/tFf0yMAenidq6k2YNSx6SIY2q6jDACck=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.47.0 h1:aXZXsZ012wOgVkqaiQv+Z/d8V0xKjmg70F6m4CE94lc=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.47.0/go.mod h1:ziu1gUIJhAaxM3EV1e3Ralk5AAyL1UOYz3lJJ4VSI38=
|
||||
go.opentelemetry.io/otel/log v1.47.0 h1:cOTS1CcLbSQeZKanGJ+0JpF/+t4PELi3O3bbl2lqCcI=
|
||||
go.opentelemetry.io/otel/log v1.47.0/go.mod h1:9byitSQ5pLC6PpqwGXjqdMKya6ZTswHRZh2vvXT33nw=
|
||||
go.opentelemetry.io/otel/metric v1.47.0 h1:4PptaldXx3Eat1XjMZ68pPJEs5wrhlemctZE9a3UdWY=
|
||||
go.opentelemetry.io/otel/metric v1.47.0/go.mod h1:ADGSXxRrXM6bjbvLo535EstVFlPpPYZm4LBKixjDHwU=
|
||||
go.opentelemetry.io/otel/metric/x v0.69.0 h1:DjRLr15H83v+hCW7JA9NoJvOkYTtmq5YoDRbe9deYpM=
|
||||
go.opentelemetry.io/otel/metric/x v0.69.0/go.mod h1:uVvsMPMFFyj/HUQfrUnH3JjnOQ1dwFDorgFLRBasM0k=
|
||||
go.opentelemetry.io/otel/sdk v1.47.0 h1:zWXEr4j2lFefG87TU6Yg8a7ngfohIKFZHKp0Hf5hC6I=
|
||||
go.opentelemetry.io/otel/sdk v1.47.0/go.mod h1:VUc24kiOeoGsxG8G9ULx3fWKvB7jMhnGE8Oi607lgR0=
|
||||
go.opentelemetry.io/otel/sdk/log v1.47.0 h1:W4N/ZfgpkRRnmARnqYoQ6BrWA0mrj8IzVDNLlpmUJZY=
|
||||
go.opentelemetry.io/otel/sdk/log v1.47.0/go.mod h1:Wu74Z2EmOKXi2FleSNAb0Hyau2G+r5kubSEMJw5aAMs=
|
||||
go.opentelemetry.io/otel/sdk/log/logtest v0.23.0 h1:j3pm42QCrKYYLLkI6OrV1QHFkUfDA6/4EP+Y1PzggLg=
|
||||
go.opentelemetry.io/otel/sdk/log/logtest v0.23.0/go.mod h1:zt0gEnuTZgDvqvczHwYswz93eqMaIr6upTTsdV/HUlU=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.47.0 h1:lfISg2j93VT6yqdk9OfUaZmw/GfcZqCCV3jdXtsPnKw=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.47.0/go.mod h1:ypLp+mW1Nt2x+Szt3b5/i1syodyts49lMOwxpDI3VGw=
|
||||
go.opentelemetry.io/otel/trace v1.47.0 h1:JOjX/Oci8K94QHddo+bbfya/Ai/nf6/dt9ZfrFNWSrM=
|
||||
go.opentelemetry.io/otel/trace v1.47.0/go.mod h1:jNaSLa2PZEYFG6fRjJABAu+bw4FS08uDmPg28lTghu0=
|
||||
go.opentelemetry.io/proto/otlp v1.11.0 h1:5rrYs0Ykyj50sdU/JU0x8etU+LubXWb+gED6TbEdMIk=
|
||||
go.opentelemetry.io/proto/otlp v1.11.0/go.mod h1:SmVizdCOAm3XBtG1g1NnOdhW6jtddT72hLMhv8VwA8E=
|
||||
go.uber.org/atomic v1.12.0 h1:BvcXdFKuviU4fTL/f+SxdQ5qJX/Jix8pAkgdUcb3XOE=
|
||||
go.uber.org/atomic v1.12.0/go.mod h1:I6c4cg+6HCxRjfjSsYtApoFILnpc0CGUdGkXVqbYVNk=
|
||||
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
|
||||
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
|
||||
go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
|
||||
go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
|
||||
golang.org/x/crypto v0.58.0 h1:COkYLr4k7nDI4r6QtMML2AfzeIswWHAJthLS6K/M/54=
|
||||
golang.org/x/crypto v0.58.0/go.mod h1:Xh+kl5A+M0gMiWDytH2Hhr5Z7WbNCpv/eB/EFgfSk+w=
|
||||
golang.org/x/image v0.47.0 h1:za/8gVz++PPZqmZJBNSISICN5Rrj5zixP/8K++fgmqo=
|
||||
golang.org/x/image v0.47.0/go.mod h1:IkN5o4A7vq55vXN8FWjjnhg9Rs2GOtffqsSi9mWAKYE=
|
||||
golang.org/x/net v0.60.0 h1:79p50tfZlm0J9YfoDsSi639qSXNGVwEzOPLCxM2FsYU=
|
||||
golang.org/x/net v0.60.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg=
|
||||
golang.org/x/sync v0.25.0 h1:dJnifr0nMV8/SBnBvdo4eSGGnrc3O4FFx6IqvtLcmRs=
|
||||
golang.org/x/sync v0.25.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
|
||||
golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA=
|
||||
golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||
golang.org/x/text v0.43.0 h1:1QivrlhwAsnMOcqgOdl9my7s9OyZl8O5/UGUTJVXoWc=
|
||||
golang.org/x/text v0.43.0/go.mod h1:hbSIYA/amXcRXTFSZqh8tWHEoqRvvbKPJ2uz/KU+a+s=
|
||||
gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=
|
||||
gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260928230214-8a89bd6388cc h1:rZHRz0ogq4Pid1IKtA/ivHi8XONVF5tqUQXA2TtgpkE=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260928230214-8a89bd6388cc/go.mod h1:K/n0XLazCJHjD73YuMyNJsN38BPPy1kfFnKxUXaThFU=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260928230214-8a89bd6388cc h1:4bNTbnb44EqGVy9HaQxSY2jnafSUdgV3qHtedvNpDKg=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260928230214-8a89bd6388cc/go.mod h1:OaIUM3+LpYcK2GXM4FTmhWoIq371Owdr+Cc7/BsYHHc=
|
||||
google.golang.org/grpc v1.83.2 h1:EManeRomTObA0BU7I8vXgg/78uE5MJ9M8B39EX2WscU=
|
||||
google.golang.org/grpc v1.83.2/go.mod h1:YPI1hK3kDked6iHvgX3tR0y+nX/qpMFKhPgFsokw1S8=
|
||||
google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc=
|
||||
google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/ini.v1 v1.67.3 h1:iM9Lhz5MRSGhHVGGwCuzG9KO8PoirCXj/m/qTmOJJQw=
|
||||
gopkg.in/ini.v1 v1.67.3/go.mod h1:x/cyOwCgZqOkJoDIJ3c1KNHMo10+nLGAhh+kn3Zizss=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
@@ -0,0 +1,35 @@
|
||||
# Runbooks alert-ов мессенджера
|
||||
|
||||
Все шаги выполняются только по сохранённым наблюдениям: Grafana (dashboard «Messenger — API, worker, очередь», Explore → Prometheus / Loki / Tempo). Доступ к контейнерам нужен только для устранения причины, не для поиска.
|
||||
|
||||
## MessengerHighErrorRate
|
||||
|
||||
**Условие:** доля 5xx среди бизнес-запросов (все маршруты, кроме `/health/*`) за 1 минуту > 5% при ≥ 0.5 запроса/с, 30 s подряд. 4xx — ошибки клиента — не учитываются.
|
||||
|
||||
1. Dashboard → «Доля ошибок 5xx и 4xx» и «p95 по маршруту»: один маршрут или все? Если все — общая зависимость (БД, Redis, S3); если один — его обработчик.
|
||||
2. «Пул БД: занято / ожидания»: `pgxpool_acquired_connections` упёрся в `pgxpool_max_connections` и растут ожидания (`pgxpool_empty_acquire_total`) — БД не отдаёт соединения или запросы висят.
|
||||
3. Explore → Tempo: `{resource.service.name="messenger-api" && span.http.response.status_code >= 500}`. Открыть трассу и посмотреть, какой дочерний span занимает время:
|
||||
* длинный `query …` / `acquire` (PostgreSQL) — проблема БД (блокировка, недоступность); подтвердить в БД `pg_stat_activity` (wait_event = Lock);
|
||||
* длинный `S3 …` — хранилище файлов;
|
||||
* дочерние spans короткие, а server span длинный — медленный сам обработчик/тело запроса.
|
||||
4. Explore → Loki: `{service_name="messenger-api"} | detected_level=~"error|ERROR"` за то же окно; по `trace_id` из лога — переход в трассу.
|
||||
5. Устранить причину; alert перейдёт в resolved, когда окно 1m очистится от ошибок.
|
||||
|
||||
## MessengerQueueLagGrowing
|
||||
|
||||
**Условие:** старейшая незавершённая job (queued/processing) старше 60 s, 30 s подряд. Возраст считается по PostgreSQL (источник истины), не по брокеру.
|
||||
|
||||
1. Dashboard → «Очередь: jobs по состоянию»: растёт `queued` при нулевом `processing` — задания никто не берёт; растёт `processing` — worker берёт, но не завершает.
|
||||
2. «Jobs по исходу (rate)»: нет `done` — worker стоит; много `retry`/`requeue` — worker работает, но падает на зависимости (S3/БД).
|
||||
3. `messenger_worker_capacity` есть? Нет серии ≥ 1 мин → worker не запущен (сработает и `MessengerWorkerAbsent`).
|
||||
4. «RabbitMQ: сообщения в очередях»: ready растёт — сообщения есть, потребителя нет; ready = 0 при queued в БД — relay не публикует outbox (см. «outbox не опубликовано» и возраст outbox).
|
||||
5. Tempo: последние `attachment.process process` (worker) и `attachment.process publish` (relay) — что выполнялось последним; Loki `{service_name="messenger-worker"}` — последние события (`worker.start`, `job.retry`, `shutdown`).
|
||||
6. После запуска/починки worker backlog уменьшается (панель возраста падает до 0), alert — resolved.
|
||||
|
||||
## MessengerWorkerAbsent
|
||||
|
||||
Нет метрик ни одного worker 1 минуту (серии исчезают через 1 мин после остановки процесса: `metric_expiration` Collector). Проверить `docker compose ps worker`, логи запуска (`worker.start`) и доступность Collector (если Collector недоступен, исчезают метрики всех сервисов — смотреть `MessengerTelemetryExportFailing` и метрики самого Collector).
|
||||
|
||||
## MessengerTelemetryExportFailing
|
||||
|
||||
Collector не может доставить spans/логи в Tempo/Loki (после ограниченной очереди и 60 s повторов данные отбрасываются, счётчики `otelcol_exporter_send_failed_*`). Бизнес-API не затронут. Проверить доступность Tempo/Loki, `otelcol_exporter_queue_size` относительно `otelcol_exporter_queue_capacity`. Потерянные за время отказа трассы/логи не восстанавливаются; метрики приложения при этом продолжают собираться (Prometheus забирает их напрямую из Collector).
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,9 @@
|
||||
apiVersion: 1
|
||||
providers:
|
||||
- name: messenger
|
||||
folder: Messenger
|
||||
type: file
|
||||
disableDeletion: true
|
||||
allowUiUpdates: false
|
||||
options:
|
||||
path: /var/lib/grafana/dashboards
|
||||
@@ -0,0 +1,42 @@
|
||||
# Источники Grafana: Grafana сама данные не хранит — только запрашивает.
|
||||
# Связи: метрика (exemplar) → трасса; лог (trace_id) → трасса; трасса → логи.
|
||||
apiVersion: 1
|
||||
datasources:
|
||||
- name: Prometheus
|
||||
uid: prometheus
|
||||
type: prometheus
|
||||
access: proxy
|
||||
url: http://prometheus:9090
|
||||
isDefault: true
|
||||
jsonData:
|
||||
timeInterval: 5s
|
||||
exemplarTraceIdDestinations:
|
||||
- name: trace_id
|
||||
datasourceUid: tempo
|
||||
- name: Loki
|
||||
uid: loki
|
||||
type: loki
|
||||
access: proxy
|
||||
url: http://loki:3100
|
||||
jsonData:
|
||||
derivedFields:
|
||||
- name: trace
|
||||
matcherType: label
|
||||
matcherRegex: trace_id
|
||||
datasourceUid: tempo
|
||||
url: "$${__value.raw}"
|
||||
- name: Tempo
|
||||
uid: tempo
|
||||
type: tempo
|
||||
access: proxy
|
||||
url: http://tempo:3200
|
||||
jsonData:
|
||||
tracesToLogsV2:
|
||||
datasourceUid: loki
|
||||
filterByTraceID: true
|
||||
spanStartTimeShift: "-5m"
|
||||
spanEndTimeShift: "5m"
|
||||
serviceMap:
|
||||
datasourceUid: prometheus
|
||||
nodeGraph:
|
||||
enabled: true
|
||||
@@ -0,0 +1,59 @@
|
||||
# HAProxy — единственная публичная точка входа (lab3).
|
||||
# Backend-адреса берутся из встроенного DNS Docker (service name "api"),
|
||||
# поэтому масштабирование/пересоздание реплик не требует перезапуска LB.
|
||||
|
||||
global
|
||||
log stdout format raw local0 info
|
||||
maxconn 2048
|
||||
|
||||
defaults
|
||||
mode http
|
||||
log global
|
||||
option httplog
|
||||
option dontlognull
|
||||
timeout connect 2s
|
||||
timeout client 30s
|
||||
timeout server 15s
|
||||
timeout http-request 10s
|
||||
timeout queue 5s
|
||||
# Повтор только при ошибке установки соединения (запрос ещё не отправлен),
|
||||
# поэтому неидемпотентные POST /users, /chats не дублируются.
|
||||
retries 2
|
||||
retry-on conn-failure
|
||||
option redispatch 1
|
||||
|
||||
resolvers docker
|
||||
nameserver dns 127.0.0.11:53
|
||||
resolve_retries 3
|
||||
timeout resolve 1s
|
||||
timeout retry 1s
|
||||
# Как долго держать последний ответ DNS при ошибке/пропаже записи.
|
||||
hold valid 5s
|
||||
hold other 5s
|
||||
hold refused 5s
|
||||
hold nx 2s
|
||||
hold timeout 5s
|
||||
hold obsolete 2s
|
||||
accepted_payload_size 8192
|
||||
|
||||
frontend public
|
||||
bind :8080
|
||||
default_backend api
|
||||
|
||||
backend api
|
||||
# roundrobin по умолчанию; leastconn — вариант для эксперимента трека 1 (env LB_BALANCE).
|
||||
balance "${LB_BALANCE}"
|
||||
# Активная проверка готовности: 503 во время drain или без БД/Redis.
|
||||
option httpchk
|
||||
http-check send meth GET uri /health/ready
|
||||
http-check expect status 200
|
||||
# Период проверки 1s, исключение после 2 неудач (~2s), возврат после 2 успехов.
|
||||
default-server check inter 1s fastinter 500ms downinter 1s fall 2 rise 2 resolvers docker resolve-prefer ipv4 init-addr none
|
||||
# До 8 слотов под реплики сервиса api; лишние слоты остаются в MAINT.
|
||||
server-template api 8 api:8080
|
||||
|
||||
frontend stats
|
||||
bind :8404
|
||||
stats enable
|
||||
stats uri /stats
|
||||
stats refresh 2s
|
||||
@@ -0,0 +1,39 @@
|
||||
# Loki 3 — хранилище логов. Приём OTLP (/otlp) от Collector: service.name и др.
|
||||
# resource attributes → индексные метки (service_name, …), атрибуты записи
|
||||
# (event, trace_id, span_id, severity) → structured metadata, без индексации.
|
||||
auth_enabled: false
|
||||
server:
|
||||
http_listen_port: 3100
|
||||
grpc_listen_port: 9096
|
||||
log_level: warn
|
||||
common:
|
||||
instance_addr: 127.0.0.1
|
||||
path_prefix: /loki
|
||||
storage:
|
||||
filesystem:
|
||||
chunks_directory: /loki/chunks
|
||||
rules_directory: /loki/rules
|
||||
replication_factor: 1
|
||||
ring:
|
||||
kvstore:
|
||||
store: inmemory
|
||||
schema_config:
|
||||
configs:
|
||||
- from: 2024-01-01
|
||||
store: tsdb
|
||||
object_store: filesystem
|
||||
schema: v13
|
||||
index:
|
||||
prefix: index_
|
||||
period: 24h
|
||||
limits_config:
|
||||
allow_structured_metadata: true
|
||||
retention_period: 168h
|
||||
ingestion_rate_mb: 8
|
||||
ingestion_burst_size_mb: 16
|
||||
compactor:
|
||||
working_directory: /loki/compactor
|
||||
retention_enabled: true
|
||||
delete_request_store: filesystem
|
||||
analytics:
|
||||
reporting_enabled: false
|
||||
@@ -0,0 +1,125 @@
|
||||
# ВАРИАНТ ТРЕКА 1 (tail sampling): SDK отправляет 100% spans, Collector хранит
|
||||
# только трассы с ошибкой, медленные (≥ 250 ms) и 10% остальных.
|
||||
#
|
||||
# OpenTelemetry Collector (contrib) — приём OTLP от API/relay/worker и раздача
|
||||
# по хранилищам: traces → Tempo, logs → Loki (OTLP/HTTP), metrics → Prometheus (pull).
|
||||
#
|
||||
# Ограничение памяти и буферов:
|
||||
# memory_limiter — первым в каждом pipeline: при превышении soft limit Collector
|
||||
# отказывает receiver-у (SDK получает ошибку и отбрасывает данные по своей
|
||||
# ограниченной очереди), вместо бесконечного роста RSS;
|
||||
# batch — пакеты до 1024 элементов / 2 s;
|
||||
# sending_queue (in-memory, queue_size пакетов) + retry_on_failure (≤ 60 s) —
|
||||
# размер подобран так, чтобы очередь целиком (20 пакетов × 1024 spans) помещалась
|
||||
# ниже soft limit memory_limiter: при queue_size 200 и пакетах до 2048 очередь при
|
||||
# отказе Tempo упиралась в hard limit, и Collector отвергал НОВЫЕ данные (прогон 4);
|
||||
# при недоступном Tempo/Loki данные ждут ограниченное время в ограниченной
|
||||
# очереди, затем отбрасываются и считаются в otelcol_exporter_send_failed_*.
|
||||
# Метрики не буферизуются вовсе: exporter prometheus хранит только текущее
|
||||
# cumulative-состояние серий, Prometheus забирает его сам (pull).
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
grpc:
|
||||
endpoint: 0.0.0.0:4317
|
||||
http:
|
||||
endpoint: 0.0.0.0:4318
|
||||
|
||||
processors:
|
||||
# Решение по трассе целиком через decision_wait после её первого span;
|
||||
# num_traces ограничивает память буфера решений. decision_wait должен быть больше
|
||||
# самой длинной трассы: при 5 s терялась часть ошибок по таймауту обработчика (5 s) —
|
||||
# решение принималось раньше, чем приходил корневой span со статусом ERROR (прогон 3).
|
||||
tail_sampling:
|
||||
decision_wait: 10s
|
||||
num_traces: 20000
|
||||
expected_new_traces_per_sec: 200
|
||||
policies:
|
||||
- name: errors
|
||||
type: status_code
|
||||
status_code: { status_codes: [ERROR] }
|
||||
- name: slow
|
||||
type: latency
|
||||
latency: { threshold_ms: 250 }
|
||||
- name: baseline
|
||||
type: probabilistic
|
||||
probabilistic: { sampling_percentage: 10 }
|
||||
memory_limiter:
|
||||
check_interval: 1s
|
||||
limit_mib: 200
|
||||
spike_limit_mib: 50
|
||||
batch:
|
||||
send_batch_size: 1024
|
||||
send_batch_max_size: 1024
|
||||
timeout: 2s
|
||||
# Защита от утечек на случай ошибки инструментирования: такие ключи не должны
|
||||
# появляться, но если появятся — удаляются до экспорта.
|
||||
attributes/scrub:
|
||||
actions:
|
||||
- { key: http.request.header.authorization, action: delete }
|
||||
- { key: http.request.header.cookie, action: delete }
|
||||
- { key: authorization, action: delete }
|
||||
- { key: password, action: delete }
|
||||
- { key: token, action: delete }
|
||||
- { key: db.query.parameters, action: delete }
|
||||
|
||||
exporters:
|
||||
otlp/tempo:
|
||||
endpoint: tempo:4317
|
||||
tls:
|
||||
insecure: true # внутренняя сеть telemetry; TLS — в lab7
|
||||
sending_queue:
|
||||
enabled: true
|
||||
num_consumers: 2
|
||||
queue_size: 20 # ≤ 20 пакетов × ≤ 1024 элемента: очередь помещается в memory_limiter
|
||||
retry_on_failure:
|
||||
enabled: true
|
||||
initial_interval: 1s
|
||||
max_interval: 10s
|
||||
max_elapsed_time: 60s
|
||||
otlphttp/loki:
|
||||
endpoint: http://loki:3100/otlp
|
||||
sending_queue:
|
||||
enabled: true
|
||||
num_consumers: 2
|
||||
queue_size: 20 # ≤ 20 пакетов × ≤ 1024 элемента: очередь помещается в memory_limiter
|
||||
retry_on_failure:
|
||||
enabled: true
|
||||
initial_interval: 1s
|
||||
max_interval: 10s
|
||||
max_elapsed_time: 60s
|
||||
prometheus:
|
||||
endpoint: 0.0.0.0:8889
|
||||
enable_open_metrics: true # exemplars (trace_id) в histogram buckets
|
||||
metric_expiration: 1m # серии остановленного процесса исчезают через 1 мин (absent() в alerts)
|
||||
|
||||
extensions:
|
||||
health_check:
|
||||
endpoint: 0.0.0.0:13133
|
||||
|
||||
service:
|
||||
extensions: [health_check]
|
||||
telemetry:
|
||||
logs:
|
||||
level: warn
|
||||
metrics:
|
||||
level: detailed
|
||||
readers:
|
||||
- pull:
|
||||
exporter:
|
||||
prometheus:
|
||||
host: 0.0.0.0
|
||||
port: 8888
|
||||
pipelines:
|
||||
traces:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, tail_sampling, attributes/scrub, batch]
|
||||
exporters: [otlp/tempo]
|
||||
metrics:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, batch]
|
||||
exporters: [prometheus]
|
||||
logs:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, attributes/scrub, batch]
|
||||
exporters: [otlphttp/loki]
|
||||
@@ -0,0 +1,104 @@
|
||||
# OpenTelemetry Collector (contrib) — приём OTLP от API/relay/worker и раздача
|
||||
# по хранилищам: traces → Tempo, logs → Loki (OTLP/HTTP), metrics → Prometheus (pull).
|
||||
#
|
||||
# Ограничение памяти и буферов:
|
||||
# memory_limiter — первым в каждом pipeline: при превышении soft limit Collector
|
||||
# отказывает receiver-у (SDK получает ошибку и отбрасывает данные по своей
|
||||
# ограниченной очереди), вместо бесконечного роста RSS;
|
||||
# batch — пакеты до 1024 элементов / 2 s;
|
||||
# sending_queue (in-memory, queue_size пакетов) + retry_on_failure (≤ 60 s) —
|
||||
# размер подобран так, чтобы очередь целиком (20 пакетов × 1024 spans) помещалась
|
||||
# ниже soft limit memory_limiter: при queue_size 200 и пакетах до 2048 очередь при
|
||||
# отказе Tempo упиралась в hard limit, и Collector отвергал НОВЫЕ данные (прогон 4);
|
||||
# при недоступном Tempo/Loki данные ждут ограниченное время в ограниченной
|
||||
# очереди, затем отбрасываются и считаются в otelcol_exporter_send_failed_*.
|
||||
# Метрики не буферизуются вовсе: exporter prometheus хранит только текущее
|
||||
# cumulative-состояние серий, Prometheus забирает его сам (pull).
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
grpc:
|
||||
endpoint: 0.0.0.0:4317
|
||||
http:
|
||||
endpoint: 0.0.0.0:4318
|
||||
|
||||
processors:
|
||||
memory_limiter:
|
||||
check_interval: 1s
|
||||
limit_mib: 200
|
||||
spike_limit_mib: 50
|
||||
batch:
|
||||
send_batch_size: 1024
|
||||
send_batch_max_size: 1024
|
||||
timeout: 2s
|
||||
# Защита от утечек на случай ошибки инструментирования: такие ключи не должны
|
||||
# появляться, но если появятся — удаляются до экспорта.
|
||||
attributes/scrub:
|
||||
actions:
|
||||
- { key: http.request.header.authorization, action: delete }
|
||||
- { key: http.request.header.cookie, action: delete }
|
||||
- { key: authorization, action: delete }
|
||||
- { key: password, action: delete }
|
||||
- { key: token, action: delete }
|
||||
- { key: db.query.parameters, action: delete }
|
||||
|
||||
exporters:
|
||||
otlp/tempo:
|
||||
endpoint: tempo:4317
|
||||
tls:
|
||||
insecure: true # внутренняя сеть telemetry; TLS — в lab7
|
||||
sending_queue:
|
||||
enabled: true
|
||||
num_consumers: 2
|
||||
queue_size: 20 # ≤ 20 пакетов × ≤ 1024 элемента: очередь помещается в memory_limiter
|
||||
retry_on_failure:
|
||||
enabled: true
|
||||
initial_interval: 1s
|
||||
max_interval: 10s
|
||||
max_elapsed_time: 60s
|
||||
otlphttp/loki:
|
||||
endpoint: http://loki:3100/otlp
|
||||
sending_queue:
|
||||
enabled: true
|
||||
num_consumers: 2
|
||||
queue_size: 20 # ≤ 20 пакетов × ≤ 1024 элемента: очередь помещается в memory_limiter
|
||||
retry_on_failure:
|
||||
enabled: true
|
||||
initial_interval: 1s
|
||||
max_interval: 10s
|
||||
max_elapsed_time: 60s
|
||||
prometheus:
|
||||
endpoint: 0.0.0.0:8889
|
||||
enable_open_metrics: true # exemplars (trace_id) в histogram buckets
|
||||
metric_expiration: 1m # серии остановленного процесса исчезают через 1 мин (absent() в alerts)
|
||||
|
||||
extensions:
|
||||
health_check:
|
||||
endpoint: 0.0.0.0:13133
|
||||
|
||||
service:
|
||||
extensions: [health_check]
|
||||
telemetry:
|
||||
logs:
|
||||
level: warn
|
||||
metrics:
|
||||
level: detailed
|
||||
readers:
|
||||
- pull:
|
||||
exporter:
|
||||
prometheus:
|
||||
host: 0.0.0.0
|
||||
port: 8888
|
||||
pipelines:
|
||||
traces:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, attributes/scrub, batch]
|
||||
exporters: [otlp/tempo]
|
||||
metrics:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, batch]
|
||||
exporters: [prometheus]
|
||||
logs:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, attributes/scrub, batch]
|
||||
exporters: [otlphttp/loki]
|
||||
@@ -0,0 +1,33 @@
|
||||
# Prometheus — хранилище метрик (TSDB на volume, retention 7d) и вычисление alerts.
|
||||
# Метрики приложения приходят OTLP → Collector → exporter prometheus (:8889):
|
||||
# honor_labels сохраняет job=service.name и instance=service.instance.id реплики.
|
||||
global:
|
||||
scrape_interval: 5s
|
||||
evaluation_interval: 5s
|
||||
external_labels:
|
||||
deployment_environment: lab
|
||||
|
||||
rule_files:
|
||||
- /etc/prometheus/rules/*.yml
|
||||
|
||||
scrape_configs:
|
||||
- job_name: otel-collector-app
|
||||
honor_labels: true
|
||||
static_configs:
|
||||
- targets: ["otel-collector:8889"]
|
||||
- job_name: otel-collector
|
||||
static_configs:
|
||||
- targets: ["otel-collector:8888"]
|
||||
- job_name: rabbitmq
|
||||
metrics_path: /metrics/per-object # глубина очередей attachment.process / attachment.dead
|
||||
static_configs:
|
||||
- targets: ["rabbitmq:15692"]
|
||||
- job_name: loki
|
||||
static_configs:
|
||||
- targets: ["loki:3100"]
|
||||
- job_name: tempo
|
||||
static_configs:
|
||||
- targets: ["tempo:3200"]
|
||||
- job_name: prometheus
|
||||
static_configs:
|
||||
- targets: ["localhost:9090"]
|
||||
@@ -0,0 +1,61 @@
|
||||
# Recording rules (SLI) и alert rules. Состояние alerts — локально в Prometheus
|
||||
# (/api/v1/alerts, Grafana → Alerting); доставка наружу не настроена (не требуется).
|
||||
# Runbooks: infra/RUNBOOKS.md.
|
||||
#
|
||||
# SLI/SLO (окно SLO — 28 дней; на стенде проверяются короткие окна):
|
||||
# доступность API: доля бизнес-запросов (кроме /health/*) без 5xx ≥ 99.5%;
|
||||
# задержка API: доля бизнес-запросов быстрее 250 ms ≥ 95%;
|
||||
# время до ready: доля вложений, ставших ready быстрее 30 s после 202, ≥ 95%.
|
||||
groups:
|
||||
- name: messenger-sli
|
||||
interval: 5s
|
||||
rules:
|
||||
- record: messenger:http_requests:rate1m
|
||||
expr: sum(rate(http_server_request_duration_seconds_count{job="messenger-api", http_route!~"/health/.*"}[1m]))
|
||||
- record: messenger:http_errors:rate1m
|
||||
expr: sum(rate(http_server_request_duration_seconds_count{job="messenger-api", http_route!~"/health/.*", http_response_status_class="5xx"}[1m])) or vector(0)
|
||||
- record: messenger:http_error_ratio:rate1m
|
||||
expr: messenger:http_errors:rate1m / messenger:http_requests:rate1m
|
||||
- record: messenger:http_fast_ratio:rate5m
|
||||
expr: |
|
||||
sum(rate(http_server_request_duration_seconds_bucket{job="messenger-api", http_route!~"/health/.*", le="0.25"}[5m]))
|
||||
/ sum(rate(http_server_request_duration_seconds_count{job="messenger-api", http_route!~"/health/.*"}[5m]))
|
||||
- record: messenger:time_to_ready_fast_ratio:rate5m
|
||||
expr: |
|
||||
sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="30"}[5m]))
|
||||
/ sum(rate(messenger_attachment_time_to_ready_seconds_count[5m]))
|
||||
|
||||
- name: messenger-alerts
|
||||
interval: 5s
|
||||
rules:
|
||||
- alert: MessengerHighErrorRate
|
||||
expr: messenger:http_error_ratio:rate1m > 0.05 and messenger:http_requests:rate1m > 0.5
|
||||
for: 30s
|
||||
labels: { severity: page }
|
||||
annotations:
|
||||
summary: "API 5xx > 5% бизнес-запросов (окно 1m, 30s подряд)"
|
||||
description: "Доля 5xx {{ $value | humanizePercentage }}. Числитель — 5xx, знаменатель — все запросы кроме /health/*; 4xx (ошибки клиента) не считаются."
|
||||
runbook: "infra/RUNBOOKS.md#messengerhigherrorrate"
|
||||
- alert: MessengerQueueLagGrowing
|
||||
expr: max(messenger_jobs_oldest_age_seconds) > 60
|
||||
for: 30s
|
||||
labels: { severity: page }
|
||||
annotations:
|
||||
summary: "Старейшая незавершённая job старше 60 s"
|
||||
description: "Возраст {{ $value | humanizeDuration }}: задания принимаются (202), но не завершаются — worker остановлен/перегружен или broker недоступен."
|
||||
runbook: "infra/RUNBOOKS.md#messengerqueuelaggrowing"
|
||||
- alert: MessengerWorkerAbsent
|
||||
expr: absent(messenger_worker_capacity)
|
||||
for: 1m
|
||||
labels: { severity: warning }
|
||||
annotations:
|
||||
summary: "Нет метрик ни одного worker 1 минуту"
|
||||
runbook: "infra/RUNBOOKS.md#messengerworkerabsent"
|
||||
- alert: MessengerTelemetryExportFailing
|
||||
expr: (sum(rate({__name__=~"otelcol_exporter_send_failed_spans(_total)?"}[1m])) or vector(0)) + (sum(rate({__name__=~"otelcol_exporter_send_failed_log_records(_total)?"}[1m])) or vector(0)) > 0
|
||||
for: 30s
|
||||
labels: { severity: warning }
|
||||
annotations:
|
||||
summary: "Collector не может доставить spans/logs в Tempo/Loki"
|
||||
description: "Данные отбрасываются после ограниченной очереди и 60 s повторов; API не затронут."
|
||||
runbook: "infra/RUNBOOKS.md#messengertelemetryexportfailing"
|
||||
@@ -0,0 +1,35 @@
|
||||
#!/bin/sh
|
||||
# Идемпотентный bootstrap MinIO: повторный запуск ничего не ломает и не стирает данные.
|
||||
# 1. приватный bucket (anonymous access: none);
|
||||
# 2. политика messenger-api: только Get/Put/Delete объектов и List этого bucket;
|
||||
# 3. отдельный пользователь API с этой политикой (root-ключ API не получает).
|
||||
# Секреты читаются из /run/secrets и не печатаются.
|
||||
set -eu
|
||||
: "${S3_BUCKET:?}"
|
||||
ROOT_USER=$(cat /run/secrets/s3_root_user)
|
||||
ROOT_PASS=$(cat /run/secrets/s3_root_password)
|
||||
API_KEY=$(cat /run/secrets/s3_access_key_id)
|
||||
API_SECRET=$(cat /run/secrets/s3_secret_access_key)
|
||||
|
||||
mc alias set local http://s3:9000 "$ROOT_USER" "$ROOT_PASS" >/dev/null
|
||||
mc mb --ignore-existing "local/$S3_BUCKET"
|
||||
mc anonymous set none "local/$S3_BUCKET" >/dev/null
|
||||
|
||||
cat > /tmp/messenger-api.json <<EOF
|
||||
{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{"Effect": "Allow", "Action": ["s3:GetBucketLocation", "s3:ListBucket"],
|
||||
"Resource": ["arn:aws:s3:::$S3_BUCKET"]},
|
||||
{"Effect": "Allow", "Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"],
|
||||
"Resource": ["arn:aws:s3:::$S3_BUCKET/*"]}
|
||||
]
|
||||
}
|
||||
EOF
|
||||
mc admin policy create local messenger-api /tmp/messenger-api.json >/dev/null
|
||||
# user add для существующего пользователя обновляет секрет — повтор безопасен.
|
||||
mc admin user add local "$API_KEY" "$API_SECRET" >/dev/null
|
||||
if ! mc admin policy entities local --user "$API_KEY" 2>/dev/null | grep -q messenger-api; then
|
||||
mc admin policy attach local messenger-api --user "$API_KEY" >/dev/null
|
||||
fi
|
||||
echo "s3 bootstrap ok: bucket=$S3_BUCKET private, api user with messenger-api policy"
|
||||
@@ -0,0 +1,31 @@
|
||||
# Tempo 2 — хранилище трасс (локальный backend на volume). Приём OTLP gRPC от
|
||||
# Collector; поиск TraceQL и выдача трассы по ID для Grafana.
|
||||
stream_over_http_enabled: true
|
||||
server:
|
||||
http_listen_port: 3200
|
||||
log_level: warn
|
||||
distributor:
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
grpc:
|
||||
endpoint: 0.0.0.0:4317
|
||||
# Явный адрес frontend для querier в single-binary: автоматическая настройка после
|
||||
# рестарта Tempo оставляла запросы без ответа (прогон 4, отказ backend).
|
||||
querier:
|
||||
frontend_worker:
|
||||
frontend_address: 127.0.0.1:9095
|
||||
ingester:
|
||||
max_block_duration: 5m
|
||||
compactor:
|
||||
compaction:
|
||||
block_retention: 168h
|
||||
storage:
|
||||
trace:
|
||||
backend: local
|
||||
wal:
|
||||
path: /var/tempo/wal
|
||||
local:
|
||||
path: /var/tempo/blocks
|
||||
usage_report:
|
||||
reporting_enabled: false
|
||||
@@ -0,0 +1,189 @@
|
||||
// Package broker — RabbitMQ для заданий обработки вложений.
|
||||
//
|
||||
// Топология (объявляется идемпотентно при каждом подключении):
|
||||
//
|
||||
// exchange attachments (direct, durable) ──process──► queue attachment.process
|
||||
// quorum-очередь (реплицируемый журнал на диске), x-delivery-limit=10:
|
||||
// сообщение, которое 10 раз вернулось без ack (poison), уходит в DLX
|
||||
// exchange attachments.dlx (fanout, durable) ──► queue attachment.dead (реестр «мёртвых»)
|
||||
//
|
||||
// Publisher confirm подтверждает, что broker устойчиво ПРИНЯЛ сообщение (записал
|
||||
// в quorum-журнал); consumer ack подтверждает, что worker устойчиво ОБРАБОТАЛ его.
|
||||
// Это разные границы: между ними сообщение может быть доставлено несколько раз.
|
||||
package broker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
amqp "github.com/rabbitmq/amqp091-go"
|
||||
"go.opentelemetry.io/otel"
|
||||
)
|
||||
|
||||
// HeaderCarrier — заголовки AMQP как TextMapCarrier (traceparent, tracestate).
|
||||
type HeaderCarrier amqp.Table
|
||||
|
||||
func (c HeaderCarrier) Get(key string) string {
|
||||
if v, ok := c[key].(string); ok {
|
||||
return v
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (c HeaderCarrier) Set(key, value string) { c[key] = value }
|
||||
|
||||
func (c HeaderCarrier) Keys() []string {
|
||||
keys := make([]string, 0, len(c))
|
||||
for k := range c {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
return keys
|
||||
}
|
||||
|
||||
const (
|
||||
Exchange = "attachments"
|
||||
RoutingKey = "process"
|
||||
Queue = "attachment.process"
|
||||
DeadExchange = "attachments.dlx"
|
||||
DeadQueue = "attachment.dead"
|
||||
DeliveryLimit = 10
|
||||
)
|
||||
|
||||
// Declare объявляет топологию; повторный вызов с теми же аргументами — no-op.
|
||||
func Declare(ch *amqp.Channel) error {
|
||||
if err := ch.ExchangeDeclare(Exchange, "direct", true, false, false, false, nil); err != nil {
|
||||
return fmt.Errorf("declare exchange: %w", err)
|
||||
}
|
||||
if err := ch.ExchangeDeclare(DeadExchange, "fanout", true, false, false, false, nil); err != nil {
|
||||
return fmt.Errorf("declare dlx: %w", err)
|
||||
}
|
||||
if _, err := ch.QueueDeclare(DeadQueue, true, false, false, false, amqp.Table{"x-queue-type": "quorum"}); err != nil {
|
||||
return fmt.Errorf("declare dead queue: %w", err)
|
||||
}
|
||||
if err := ch.QueueBind(DeadQueue, "", DeadExchange, false, nil); err != nil {
|
||||
return fmt.Errorf("bind dead queue: %w", err)
|
||||
}
|
||||
args := amqp.Table{
|
||||
"x-queue-type": "quorum",
|
||||
"x-dead-letter-exchange": DeadExchange,
|
||||
"x-delivery-limit": int32(DeliveryLimit),
|
||||
}
|
||||
if _, err := ch.QueueDeclare(Queue, true, false, false, false, args); err != nil {
|
||||
return fmt.Errorf("declare queue: %w", err)
|
||||
}
|
||||
if err := ch.QueueBind(Queue, RoutingKey, Exchange, false, nil); err != nil {
|
||||
return fmt.Errorf("bind queue: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Publisher публикует с publisher confirms.
|
||||
type Publisher struct {
|
||||
conn *amqp.Connection
|
||||
ch *amqp.Channel
|
||||
}
|
||||
|
||||
func NewPublisher(url string) (*Publisher, error) {
|
||||
conn, err := amqp.DialConfig(url, amqp.Config{Heartbeat: 10 * time.Second, Properties: amqp.Table{"connection_name": "relay"}})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("dial broker: %w", err)
|
||||
}
|
||||
ch, err := conn.Channel()
|
||||
if err != nil {
|
||||
conn.Close()
|
||||
return nil, err
|
||||
}
|
||||
if err := Declare(ch); err != nil {
|
||||
conn.Close()
|
||||
return nil, err
|
||||
}
|
||||
if err := ch.Confirm(false); err != nil {
|
||||
conn.Close()
|
||||
return nil, fmt.Errorf("enable confirms: %w", err)
|
||||
}
|
||||
return &Publisher{conn: conn, ch: ch}, nil
|
||||
}
|
||||
|
||||
func (p *Publisher) Closed() bool { return p.conn.IsClosed() || p.ch.IsClosed() }
|
||||
|
||||
func (p *Publisher) Close() { _ = p.conn.Close() }
|
||||
|
||||
// Publish отправляет persistent-сообщение и ждёт confirm (ack от broker).
|
||||
// nack или таймаут — ошибка: outbox не помечается опубликованным.
|
||||
// W3C trace context из ctx (producer span) уходит в заголовки AMQP traceparent/tracestate.
|
||||
func (p *Publisher) Publish(ctx context.Context, messageID string, body []byte) error {
|
||||
headers := amqp.Table{}
|
||||
otel.GetTextMapPropagator().Inject(ctx, HeaderCarrier(headers))
|
||||
ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
dc, err := p.ch.PublishWithDeferredConfirmWithContext(ctx, Exchange, RoutingKey, true, false, amqp.Publishing{
|
||||
ContentType: "application/json",
|
||||
DeliveryMode: amqp.Persistent,
|
||||
MessageId: messageID,
|
||||
Timestamp: time.Now(),
|
||||
Type: "attachment.process.v1",
|
||||
Headers: headers,
|
||||
Body: body,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("publish: %w", err)
|
||||
}
|
||||
ok, err := dc.WaitContext(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("publish confirm: %w", err)
|
||||
}
|
||||
if !ok {
|
||||
return errors.New("publish nacked by broker")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Consumer — подписка с ручным ack и prefetch.
|
||||
type Consumer struct {
|
||||
conn *amqp.Connection
|
||||
ch *amqp.Channel
|
||||
}
|
||||
|
||||
func NewConsumer(url string, prefetch int, name string) (*Consumer, <-chan amqp.Delivery, error) {
|
||||
conn, err := amqp.DialConfig(url, amqp.Config{Heartbeat: 10 * time.Second, Properties: amqp.Table{"connection_name": name}})
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("dial broker: %w", err)
|
||||
}
|
||||
ch, err := conn.Channel()
|
||||
if err != nil {
|
||||
conn.Close()
|
||||
return nil, nil, err
|
||||
}
|
||||
if err := Declare(ch); err != nil {
|
||||
conn.Close()
|
||||
return nil, nil, err
|
||||
}
|
||||
// prefetch = максимум неподтверждённых сообщений у этого worker:
|
||||
// ограничивает конкуренцию и не даёт одному worker забрать всю очередь.
|
||||
if err := ch.Qos(prefetch, 0, false); err != nil {
|
||||
conn.Close()
|
||||
return nil, nil, fmt.Errorf("qos: %w", err)
|
||||
}
|
||||
deliveries, err := ch.Consume(Queue, name, false, false, false, false, nil)
|
||||
if err != nil {
|
||||
conn.Close()
|
||||
return nil, nil, fmt.Errorf("consume: %w", err)
|
||||
}
|
||||
return &Consumer{conn: conn, ch: ch}, deliveries, nil
|
||||
}
|
||||
|
||||
// Cancel прекращает получение новых сообщений; неподтверждённые вернутся в очередь.
|
||||
func (c *Consumer) Cancel(name string) { _ = c.ch.Cancel(name, false) }
|
||||
|
||||
func (c *Consumer) Close() { _ = c.conn.Close() }
|
||||
|
||||
// Ping проверяет соединение (для readiness/heartbeat-проверок).
|
||||
func Ping(url string) error {
|
||||
conn, err := amqp.DialConfig(url, amqp.Config{Heartbeat: 10 * time.Second})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return conn.Close()
|
||||
}
|
||||
@@ -0,0 +1,148 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"log/slog"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
Host string
|
||||
Port string
|
||||
Env string
|
||||
SessionTTLSeconds int
|
||||
DatabaseURL string
|
||||
CacheURL string
|
||||
SessionStore string
|
||||
LogLevel slog.Level
|
||||
ShutdownTimeout time.Duration
|
||||
CacheEnabled bool
|
||||
InstanceID string
|
||||
IdempotencyTTL time.Duration
|
||||
DrainDelay time.Duration
|
||||
DBPoolMax int32
|
||||
CacheInvalidation string
|
||||
InternalAddr string
|
||||
S3Endpoint string
|
||||
S3Region string
|
||||
S3Bucket string
|
||||
S3AccessKey string
|
||||
S3SecretKey string
|
||||
MaxUploadBytes int64
|
||||
MaxImagePixels int64
|
||||
// Pipeline (lab5)
|
||||
BrokerURL string
|
||||
PipelineVersion int
|
||||
JobMaxAttempts int
|
||||
JobTimeout time.Duration
|
||||
WorkerConcurrency int
|
||||
StagingRetention time.Duration
|
||||
WebPQuality int
|
||||
PixelBudget int64
|
||||
RetryBaseDelay time.Duration
|
||||
RetryMaxDelay time.Duration
|
||||
}
|
||||
|
||||
func Load() Config {
|
||||
return Config{
|
||||
Host: getEnv("APP_HOST", "0.0.0.0"),
|
||||
Port: getEnv("APP_PORT", "8080"),
|
||||
Env: getEnv("APP_ENV", "lab"),
|
||||
SessionTTLSeconds: getInt("SESSION_TTL_SECONDS", 3600),
|
||||
DatabaseURL: readSecretFile("DATABASE_URL_FILE"),
|
||||
CacheURL: readSecretFile("CACHE_URL_FILE"),
|
||||
SessionStore: getEnv("SESSION_STORE", "postgres"),
|
||||
LogLevel: parseLevel(getEnv("LOG_LEVEL", "info")),
|
||||
ShutdownTimeout: parseDuration(getEnv("SHUTDOWN_TIMEOUT", "8s")),
|
||||
CacheEnabled: getEnv("CACHE_ENABLED", "true") == "true",
|
||||
InstanceID: instanceID(),
|
||||
IdempotencyTTL: time.Duration(getInt("IDEMPOTENCY_TTL_SECONDS", 86400)) * time.Second,
|
||||
DrainDelay: parseDuration(getEnv("DRAIN_DELAY", "3s")),
|
||||
DBPoolMax: int32(max(1, getInt("DB_POOL_MAX", 20))),
|
||||
CacheInvalidation: getEnv("CACHE_INVALIDATION", "version"),
|
||||
InternalAddr: getEnv("INTERNAL_ADDR", ":9090"),
|
||||
S3Endpoint: getEnv("S3_ENDPOINT", "http://s3:9000"),
|
||||
S3Region: getEnv("S3_REGION", "us-east-1"),
|
||||
S3Bucket: getEnv("S3_BUCKET", "messenger-attachments"),
|
||||
S3AccessKey: readSecretFile("S3_ACCESS_KEY_ID_FILE"),
|
||||
S3SecretKey: readSecretFile("S3_SECRET_ACCESS_KEY_FILE"),
|
||||
MaxUploadBytes: int64(getInt("MAX_UPLOAD_BYTES", 10*1024*1024)),
|
||||
MaxImagePixels: int64(getInt("MAX_IMAGE_PIXELS", 40_000_000)),
|
||||
BrokerURL: readSecretFile("BROKER_URL_FILE"),
|
||||
PipelineVersion: max(1, getInt("PIPELINE_VERSION", 1)),
|
||||
JobMaxAttempts: max(1, getInt("JOB_MAX_ATTEMPTS", 5)),
|
||||
JobTimeout: time.Duration(max(5, getInt("JOB_TIMEOUT_SECONDS", 60))) * time.Second,
|
||||
WorkerConcurrency: max(1, getInt("WORKER_CONCURRENCY", 2)),
|
||||
StagingRetention: time.Duration(max(60, getInt("STAGING_RETENTION_SECONDS", 86400))) * time.Second,
|
||||
WebPQuality: min(100, max(1, getInt("WEBP_QUALITY", 80))),
|
||||
PixelBudget: int64(max(1_000_000, getInt("WORKER_PIXEL_BUDGET", 60_000_000))),
|
||||
RetryBaseDelay: parseDuration(getEnv("RETRY_BASE_DELAY", "2s")),
|
||||
RetryMaxDelay: parseDuration(getEnv("RETRY_MAX_DELAY", "60s")),
|
||||
}
|
||||
}
|
||||
|
||||
// instanceID — идентификатор реплики для X-Instance-Id. По умолчанию hostname,
|
||||
// который в Docker равен короткому ID контейнера: так stop-one.sh однозначно
|
||||
// находит контейнер. В lab7 его заменит непрозрачный alias.
|
||||
func instanceID() string {
|
||||
if v := os.Getenv("INSTANCE_ID"); v != "" {
|
||||
return v
|
||||
}
|
||||
if h, err := os.Hostname(); err == nil && h != "" {
|
||||
return h
|
||||
}
|
||||
return "unknown"
|
||||
}
|
||||
|
||||
func (c Config) SessionTTL() time.Duration {
|
||||
return time.Duration(c.SessionTTLSeconds) * time.Second
|
||||
}
|
||||
|
||||
func (c Config) Addr() string { return c.Host + ":" + c.Port }
|
||||
|
||||
func getEnv(key, def string) string {
|
||||
if v := os.Getenv(key); v != "" {
|
||||
return v
|
||||
}
|
||||
return def
|
||||
}
|
||||
|
||||
func getInt(key string, def int) int {
|
||||
if v := os.Getenv(key); v != "" {
|
||||
if n, err := strconv.Atoi(v); err == nil {
|
||||
return n
|
||||
}
|
||||
}
|
||||
return def
|
||||
}
|
||||
|
||||
func readSecretFile(envKey string) string {
|
||||
path := os.Getenv(envKey)
|
||||
if path == "" {
|
||||
return ""
|
||||
}
|
||||
b, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
slog.Warn("secret file not readable", "path", path, "err", err)
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(string(b))
|
||||
}
|
||||
|
||||
func parseLevel(s string) slog.Level {
|
||||
var l slog.Level
|
||||
if err := l.UnmarshalText([]byte(s)); err != nil {
|
||||
return slog.LevelInfo
|
||||
}
|
||||
return l
|
||||
}
|
||||
|
||||
func parseDuration(s string) time.Duration {
|
||||
d, err := time.ParseDuration(s)
|
||||
if err != nil {
|
||||
return 8 * time.Second
|
||||
}
|
||||
return d
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
package domain
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
AttachmentUploading = "uploading" // внутреннее намерение до 202, клиенту не видно
|
||||
AttachmentQueued = "queued"
|
||||
AttachmentProcessing = "processing"
|
||||
AttachmentReady = "ready"
|
||||
AttachmentFailed = "failed"
|
||||
AttachmentMissing = "missing" // карантин reconciler
|
||||
)
|
||||
|
||||
// Variant — результат обработки одной версии pipeline.
|
||||
type Variant struct {
|
||||
Name string `json:"name"`
|
||||
ContentType string `json:"content_type"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
Width int `json:"width"`
|
||||
Height int `json:"height"`
|
||||
SHA256 string `json:"sha256"`
|
||||
ObjectKey string `json:"-"`
|
||||
PipelineVersion int `json:"-"`
|
||||
}
|
||||
|
||||
// Attachment — метаданные вложения. Ключи S3 никогда не уходят клиенту.
|
||||
type Attachment struct {
|
||||
ID string
|
||||
ChatID string
|
||||
OwnerID string
|
||||
Filename string
|
||||
ObjectKey string // текущее место оригинала: staging до ready, конечный prefix после
|
||||
StagingKey string // staging-объект, пока он не удалён после публикации
|
||||
Status string
|
||||
ContentType string
|
||||
SizeBytes int64
|
||||
SHA256 string
|
||||
ErrorCode string
|
||||
PipelineVersion int // версия, чей результат опубликован (0 — ещё нет)
|
||||
Variants []Variant
|
||||
CreatedAt time.Time
|
||||
UpdatedAt time.Time
|
||||
}
|
||||
|
||||
// MarshalJSON — представление по контракту: queued/processing/ready/failed,
|
||||
// варианты текущей опубликованной версии, error_code только у failed.
|
||||
func (a Attachment) MarshalJSON() ([]byte, error) {
|
||||
variants := a.Variants
|
||||
if variants == nil {
|
||||
variants = []Variant{}
|
||||
}
|
||||
var errCode *string
|
||||
if a.Status == AttachmentFailed && a.ErrorCode != "" {
|
||||
ec := a.ErrorCode
|
||||
errCode = &ec
|
||||
}
|
||||
return json.Marshal(struct {
|
||||
ID string `json:"id"`
|
||||
ChatID string `json:"chat_id"`
|
||||
OwnerID string `json:"owner_id"`
|
||||
Filename string `json:"filename"`
|
||||
ContentType string `json:"content_type"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
SHA256 string `json:"sha256"`
|
||||
Status string `json:"status"`
|
||||
Variants []Variant `json:"variants"`
|
||||
ErrorCode *string `json:"error_code"`
|
||||
CreatedAt string `json:"created_at"`
|
||||
UpdatedAt string `json:"updated_at"`
|
||||
}{
|
||||
ID: a.ID, ChatID: a.ChatID, OwnerID: a.OwnerID, Filename: a.Filename,
|
||||
ContentType: a.ContentType, SizeBytes: a.SizeBytes, SHA256: a.SHA256,
|
||||
Status: a.Status, Variants: variants, ErrorCode: errCode,
|
||||
CreatedAt: a.CreatedAt.UTC().Truncate(time.Second).Format(time.RFC3339),
|
||||
UpdatedAt: a.UpdatedAt.UTC().Truncate(time.Second).Format(time.RFC3339),
|
||||
})
|
||||
}
|
||||
|
||||
// Job — сообщение broker по contracts/jobs.schema.json. Только ID и служебные
|
||||
// поля: ни байтов, ни URL, ни credentials.
|
||||
type Job struct {
|
||||
SchemaVersion int `json:"schema_version"`
|
||||
JobID string `json:"job_id"`
|
||||
Type string `json:"type"`
|
||||
AttachmentID string `json:"attachment_id"`
|
||||
PipelineVersion int `json:"pipeline_version"`
|
||||
Attempt int `json:"attempt"`
|
||||
CreatedAt string `json:"created_at"`
|
||||
Traceparent string `json:"traceparent,omitempty"`
|
||||
}
|
||||
|
||||
const JobType = "attachment.process.v1"
|
||||
@@ -0,0 +1,30 @@
|
||||
package domain
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Chat struct {
|
||||
ID string `json:"id"`
|
||||
Title string `json:"title"`
|
||||
MemberIDs []string `json:"member_ids"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
}
|
||||
|
||||
func (c Chat) MarshalJSON() ([]byte, error) {
|
||||
type Alias Chat
|
||||
members := c.MemberIDs
|
||||
if members == nil {
|
||||
members = []string{}
|
||||
}
|
||||
return json.Marshal(&struct {
|
||||
Alias
|
||||
MemberIDs []string `json:"member_ids"`
|
||||
CreatedAt string `json:"created_at"`
|
||||
}{
|
||||
Alias: Alias(c),
|
||||
MemberIDs: members,
|
||||
CreatedAt: c.CreatedAt.UTC().Truncate(time.Second).Format(time.RFC3339),
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
package domain
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Message struct {
|
||||
ID string `json:"id"`
|
||||
ChatID string `json:"chat_id"`
|
||||
SenderID string `json:"sender_id"`
|
||||
Text string `json:"text"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
// AttachmentIDs — в порядке отправки; всегда массив (пустой, если вложений нет).
|
||||
AttachmentIDs []string `json:"attachment_ids"`
|
||||
}
|
||||
|
||||
func (m Message) MarshalJSON() ([]byte, error) {
|
||||
type Alias Message
|
||||
if m.AttachmentIDs == nil {
|
||||
m.AttachmentIDs = []string{}
|
||||
}
|
||||
return json.Marshal(&struct {
|
||||
Alias
|
||||
CreatedAt string `json:"created_at"`
|
||||
}{
|
||||
Alias: Alias(m),
|
||||
CreatedAt: m.CreatedAt.UTC().Truncate(time.Second).Format(time.RFC3339),
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
package domain
|
||||
|
||||
import "time"
|
||||
|
||||
type Session struct {
|
||||
Token string `json:"token"`
|
||||
UserID string `json:"-"`
|
||||
ExpiresAt time.Time `json:"expires_at"`
|
||||
CreatedAt time.Time `json:"-"`
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
package domain
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"time"
|
||||
)
|
||||
|
||||
type User struct {
|
||||
ID string `json:"id"`
|
||||
Username string `json:"username"`
|
||||
DisplayName string `json:"display_name"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
}
|
||||
|
||||
// MarshalJSON форматирует CreatedAt в RFC3339 без микросекунд.
|
||||
func (u User) MarshalJSON() ([]byte, error) {
|
||||
type Alias User
|
||||
return json.Marshal(&struct {
|
||||
Alias
|
||||
CreatedAt string `json:"created_at"`
|
||||
}{
|
||||
Alias: Alias(u),
|
||||
CreatedAt: u.CreatedAt.UTC().Truncate(time.Second).Format(time.RFC3339),
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,592 @@
|
||||
// Package imagecheck потоково проверяет PNG/JPEG без декодирования пикселей.
|
||||
//
|
||||
// Проверка идёт по мере чтения (используется параллельно с PUT в S3), память
|
||||
// O(1) от размера изображения: размеры берутся из заголовка (IHDR/SOF) и
|
||||
// сравниваются с лимитом до чтения тела; PNG — CRC каждого chunk, длина
|
||||
// распакованного IDAT-потока (zlib → io.Discard) и байт фильтра каждой строки,
|
||||
// JPEG — структура сегментов и маркер EOI. «Заведомо повреждённые» файлы
|
||||
// (обрыв, битый CRC, мусор вместо данных, чужой формат) дают ErrInvalid,
|
||||
// слишком большие по пикселям — ErrTooLarge.
|
||||
package imagecheck
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"compress/zlib"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"fmt"
|
||||
"hash/crc32"
|
||||
"io"
|
||||
)
|
||||
|
||||
const (
|
||||
PNG = "image/png"
|
||||
JPEG = "image/jpeg"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrInvalid = errors.New("invalid or unsupported image")
|
||||
ErrTooLarge = errors.New("image dimensions exceed limit")
|
||||
)
|
||||
|
||||
var pngSignature = []byte("\x89PNG\r\n\x1a\n")
|
||||
|
||||
// Sniff определяет формат по первым байтам (не по заголовку Content-Type клиента).
|
||||
func Sniff(head []byte) (string, bool) {
|
||||
switch {
|
||||
case bytes.HasPrefix(head, pngSignature):
|
||||
return PNG, true
|
||||
case len(head) >= 3 && head[0] == 0xFF && head[1] == 0xD8 && head[2] == 0xFF:
|
||||
return JPEG, true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
// Result — что удалось узнать при проверке.
|
||||
type Result struct {
|
||||
Format string
|
||||
Width int
|
||||
Height int
|
||||
}
|
||||
|
||||
// Validate читает r до конца и проверяет изображение формата format.
|
||||
// maxPixels ограничивает width*height. Хвост после конца изображения
|
||||
// дочитывается и игнорируется, чтобы не блокировать писателя.
|
||||
func Validate(r io.Reader, format string, maxPixels int64) (Result, error) {
|
||||
br := bufio.NewReaderSize(r, 32*1024)
|
||||
var res Result
|
||||
var err error
|
||||
switch format {
|
||||
case PNG:
|
||||
res, err = validatePNG(br, maxPixels)
|
||||
case JPEG:
|
||||
res, err = validateJPEG(br, maxPixels)
|
||||
default:
|
||||
return Result{}, ErrInvalid
|
||||
}
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
_, _ = io.Copy(io.Discard, br)
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func invalid(format string, args ...any) error {
|
||||
return fmt.Errorf("%w: %s", ErrInvalid, fmt.Sprintf(format, args...))
|
||||
}
|
||||
|
||||
// --- PNG ---
|
||||
|
||||
func channels(colorType byte) (int, bool) {
|
||||
switch colorType {
|
||||
case 0:
|
||||
return 1, true // grayscale
|
||||
case 2:
|
||||
return 3, true // RGB
|
||||
case 3:
|
||||
return 1, true // palette
|
||||
case 4:
|
||||
return 2, true // gray + alpha
|
||||
case 6:
|
||||
return 4, true // RGBA
|
||||
}
|
||||
return 0, false
|
||||
}
|
||||
|
||||
func validBitDepth(colorType, depth byte) bool {
|
||||
switch colorType {
|
||||
case 0:
|
||||
return depth == 1 || depth == 2 || depth == 4 || depth == 8 || depth == 16
|
||||
case 3:
|
||||
return depth == 1 || depth == 2 || depth == 4 || depth == 8
|
||||
case 2, 4, 6:
|
||||
return depth == 8 || depth == 16
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// pngRowLen — длина одной отсканированной строки прохода: 1 байт фильтра +
|
||||
// данные пикселей.
|
||||
func pngRowLen(pw, bitsPerPixel int64) int64 {
|
||||
return 1 + (pw*bitsPerPixel+7)/8
|
||||
}
|
||||
|
||||
// expectedRaw — длина распакованного потока IDAT: строки с байтом фильтра.
|
||||
func expectedRaw(w, h int64, bitsPerPixel int64, interlaced bool) int64 {
|
||||
if !interlaced {
|
||||
return h * pngRowLen(w, bitsPerPixel)
|
||||
}
|
||||
passes := [7][4]int64{{0, 0, 8, 8}, {4, 0, 8, 8}, {0, 4, 4, 8}, {2, 0, 4, 4}, {0, 2, 2, 4}, {1, 0, 2, 2}, {0, 1, 1, 2}}
|
||||
var total int64
|
||||
for _, p := range passes {
|
||||
if w <= p[0] || h <= p[1] {
|
||||
continue
|
||||
}
|
||||
pw := (w - p[0] + p[2] - 1) / p[2]
|
||||
ph := (h - p[1] + p[3] - 1) / p[3]
|
||||
total += ph * pngRowLen(pw, bitsPerPixel)
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
// pngRows — итератор по (pw, ph) проходов Adam7 или единственного прохода.
|
||||
// Используется, чтобы валидировать байт фильтра каждой строки, не зная
|
||||
// заранее, как разбивается распакованный поток на проходы.
|
||||
type pngPass struct {
|
||||
pw, ph int64
|
||||
}
|
||||
|
||||
func pngPasses(w, h int64, interlaced bool) []pngPass {
|
||||
if !interlaced {
|
||||
return []pngPass{{w, h}}
|
||||
}
|
||||
coords := [7][4]int64{{0, 0, 8, 8}, {4, 0, 8, 8}, {0, 4, 4, 8}, {2, 0, 4, 4}, {0, 2, 2, 4}, {1, 0, 2, 2}, {0, 1, 1, 2}}
|
||||
out := make([]pngPass, 0, len(coords))
|
||||
for _, p := range coords {
|
||||
if w <= p[0] || h <= p[1] {
|
||||
continue
|
||||
}
|
||||
pw := (w - p[0] + p[2] - 1) / p[2]
|
||||
ph := (h - p[1] + p[3] - 1) / p[3]
|
||||
out = append(out, pngPass{pw, ph})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
type pngChunk struct {
|
||||
length uint32
|
||||
typ [4]byte
|
||||
}
|
||||
|
||||
func readChunkHeader(br *bufio.Reader) (pngChunk, error) {
|
||||
var hdr [8]byte
|
||||
if _, err := io.ReadFull(br, hdr[:]); err != nil {
|
||||
return pngChunk{}, invalid("truncated chunk header")
|
||||
}
|
||||
c := pngChunk{length: binary.BigEndian.Uint32(hdr[:4])}
|
||||
copy(c.typ[:], hdr[4:])
|
||||
if c.length > 1<<31-1 {
|
||||
return c, invalid("chunk too long")
|
||||
}
|
||||
for _, b := range c.typ {
|
||||
if !((b >= 'A' && b <= 'Z') || (b >= 'a' && b <= 'z')) {
|
||||
return c, invalid("bad chunk type")
|
||||
}
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
|
||||
// readChunkData читает тело chunk в dst (если не nil) и проверяет CRC.
|
||||
func readChunkData(br *bufio.Reader, c pngChunk, dst io.Writer) error {
|
||||
h := crc32.NewIEEE()
|
||||
h.Write(c.typ[:])
|
||||
w := io.Writer(h)
|
||||
if dst != nil {
|
||||
w = io.MultiWriter(h, dst)
|
||||
}
|
||||
if _, err := io.CopyN(w, br, int64(c.length)); err != nil {
|
||||
return invalid("truncated %s chunk", c.typ[:])
|
||||
}
|
||||
var crc [4]byte
|
||||
if _, err := io.ReadFull(br, crc[:]); err != nil {
|
||||
return invalid("truncated CRC")
|
||||
}
|
||||
if binary.BigEndian.Uint32(crc[:]) != h.Sum32() {
|
||||
return invalid("CRC mismatch in %s", c.typ[:])
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// idatStream отдаёт данные подряд идущих IDAT-chunk как один поток.
|
||||
type idatStream struct {
|
||||
br *bufio.Reader
|
||||
remaining uint32
|
||||
crc uint32
|
||||
next *pngChunk // первый chunk после последнего IDAT
|
||||
err error
|
||||
}
|
||||
|
||||
func (s *idatStream) Read(p []byte) (int, error) {
|
||||
for s.remaining == 0 {
|
||||
if s.err != nil || s.next != nil {
|
||||
if s.err != nil {
|
||||
return 0, s.err
|
||||
}
|
||||
return 0, io.EOF
|
||||
}
|
||||
if err := s.finishChunk(); err != nil {
|
||||
s.err = err
|
||||
return 0, err
|
||||
}
|
||||
c, err := readChunkHeader(s.br)
|
||||
if err != nil {
|
||||
s.err = err
|
||||
return 0, err
|
||||
}
|
||||
if string(c.typ[:]) != "IDAT" {
|
||||
s.next = &c
|
||||
return 0, io.EOF
|
||||
}
|
||||
s.startChunk(c)
|
||||
}
|
||||
if uint32(len(p)) > s.remaining {
|
||||
p = p[:s.remaining]
|
||||
}
|
||||
n, err := s.br.Read(p)
|
||||
s.crc = crc32.Update(s.crc, crc32.IEEETable, p[:n])
|
||||
s.remaining -= uint32(n)
|
||||
if err != nil && err != io.EOF {
|
||||
s.err = err
|
||||
return n, err
|
||||
}
|
||||
if n == 0 && err == io.EOF {
|
||||
s.err = invalid("truncated IDAT")
|
||||
return 0, s.err
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
|
||||
func (s *idatStream) startChunk(c pngChunk) {
|
||||
s.remaining = c.length
|
||||
s.crc = crc32.Update(0, crc32.IEEETable, c.typ[:])
|
||||
}
|
||||
|
||||
func (s *idatStream) finishChunk() error {
|
||||
var crc [4]byte
|
||||
if _, err := io.ReadFull(s.br, crc[:]); err != nil {
|
||||
return invalid("truncated CRC")
|
||||
}
|
||||
if binary.BigEndian.Uint32(crc[:]) != s.crc {
|
||||
return invalid("CRC mismatch in IDAT")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// validatePNGStream проверяет распакованный поток: длину, кратность строкам
|
||||
// и байты фильтра (0..4) в начале каждой строки каждого прохода.
|
||||
func validatePNGStream(zr io.Reader, passes []pngPass, bitsPerPixel, want int64) error {
|
||||
total := int64(0)
|
||||
buf := make([]byte, 32*1024)
|
||||
for _, pass := range passes {
|
||||
rowLen := pngRowLen(pass.pw, bitsPerPixel)
|
||||
for y := int64(0); y < pass.ph; y++ {
|
||||
// читаем ровно одну строку, но чанками, чтобы не аллоцировать
|
||||
// rowLen (может быть большой для широких изображений)
|
||||
left := rowLen
|
||||
first := true
|
||||
for left > 0 {
|
||||
chunk := int64(len(buf))
|
||||
if left < chunk {
|
||||
chunk = left
|
||||
}
|
||||
n, err := io.ReadFull(zr, buf[:chunk])
|
||||
if err != nil {
|
||||
if errors.Is(err, io.ErrUnexpectedEOF) || errors.Is(err, io.EOF) {
|
||||
return invalid("truncated IDAT (got %d of %d)", total, want)
|
||||
}
|
||||
return invalid("corrupt image data")
|
||||
}
|
||||
if first {
|
||||
if buf[0] > 4 {
|
||||
return invalid("bad row filter %d", buf[0])
|
||||
}
|
||||
first = false
|
||||
}
|
||||
total += int64(n)
|
||||
left -= int64(n)
|
||||
}
|
||||
}
|
||||
}
|
||||
// Проверяем, что распакованный поток ровно want байт (ни больше, ни меньше).
|
||||
var extra [1]byte
|
||||
n, err := zr.Read(extra[:])
|
||||
if n != 0 {
|
||||
return invalid("image data longer than expected")
|
||||
}
|
||||
if err != nil && !errors.Is(err, io.EOF) {
|
||||
return invalid("corrupt image data")
|
||||
}
|
||||
if total != want {
|
||||
return invalid("image data length %d, want %d", total, want)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validatePNG(br *bufio.Reader, maxPixels int64) (Result, error) {
|
||||
res := Result{Format: PNG}
|
||||
sig := make([]byte, 8)
|
||||
if _, err := io.ReadFull(br, sig); err != nil || !bytes.Equal(sig, pngSignature) {
|
||||
return res, invalid("not a PNG")
|
||||
}
|
||||
c, err := readChunkHeader(br)
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
if string(c.typ[:]) != "IHDR" || c.length != 13 {
|
||||
return res, invalid("IHDR must be first")
|
||||
}
|
||||
var ihdr bytes.Buffer
|
||||
if err := readChunkData(br, c, &ihdr); err != nil {
|
||||
return res, err
|
||||
}
|
||||
b := ihdr.Bytes()
|
||||
w := int64(binary.BigEndian.Uint32(b[0:4]))
|
||||
h := int64(binary.BigEndian.Uint32(b[4:8]))
|
||||
depth, colorType, compression, filter, interlace := b[8], b[9], b[10], b[11], b[12]
|
||||
res.Width, res.Height = int(w), int(h)
|
||||
if w < 1 || h < 1 || w > 1<<31-1 || h > 1<<31-1 {
|
||||
return res, invalid("bad dimensions")
|
||||
}
|
||||
// Лимит пикселей — до чтения и распаковки данных изображения.
|
||||
if w*h > maxPixels {
|
||||
return res, ErrTooLarge
|
||||
}
|
||||
ch, ok := channels(colorType)
|
||||
if !ok || !validBitDepth(colorType, depth) || compression != 0 || filter != 0 || interlace > 1 {
|
||||
return res, invalid("unsupported IHDR")
|
||||
}
|
||||
bpp := int64(ch) * int64(depth)
|
||||
want := expectedRaw(w, h, bpp, interlace == 1)
|
||||
passes := pngPasses(w, h, interlace == 1)
|
||||
|
||||
sawIDAT := false
|
||||
for {
|
||||
c, err := readChunkHeader(br)
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
switch string(c.typ[:]) {
|
||||
case "IDAT":
|
||||
if sawIDAT {
|
||||
return res, invalid("non-consecutive IDAT")
|
||||
}
|
||||
sawIDAT = true
|
||||
s := &idatStream{br: br}
|
||||
s.startChunk(c)
|
||||
zr, err := zlib.NewReader(s)
|
||||
if err != nil {
|
||||
return res, invalid("bad zlib header")
|
||||
}
|
||||
// Распаковка и проверка фильтров строк — без хранения пикселей.
|
||||
if err := validatePNGStream(zr, passes, bpp, want); err != nil {
|
||||
// Закрываем zlib, чтобы дочитать хвост CRC в idatStream, но
|
||||
// игнорируем ошибку закрытия: она не важна для результата.
|
||||
_ = zr.Close()
|
||||
return res, err
|
||||
}
|
||||
if err := zr.Close(); err != nil && !errors.Is(err, io.EOF) {
|
||||
return res, invalid("bad zlib trailer")
|
||||
}
|
||||
// Остаток IDAT (например, пустые chunk) дочитываем с проверкой CRC.
|
||||
if _, err := io.Copy(io.Discard, s); err != nil {
|
||||
return res, err
|
||||
}
|
||||
if s.next == nil {
|
||||
return res, invalid("missing IEND")
|
||||
}
|
||||
c = *s.next
|
||||
if string(c.typ[:]) == "IEND" {
|
||||
if err := readChunkData(br, c, nil); err != nil {
|
||||
return res, err
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
if err := readChunkData(br, c, nil); err != nil {
|
||||
return res, err
|
||||
}
|
||||
case "IEND":
|
||||
if !sawIDAT {
|
||||
return res, invalid("no image data")
|
||||
}
|
||||
if err := readChunkData(br, c, nil); err != nil {
|
||||
return res, err
|
||||
}
|
||||
return res, nil
|
||||
case "IHDR":
|
||||
return res, invalid("duplicate IHDR")
|
||||
default:
|
||||
if err := readChunkData(br, c, nil); err != nil {
|
||||
return res, err
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --- JPEG ---
|
||||
|
||||
func isSOF(m byte) bool {
|
||||
switch m {
|
||||
case 0xC0, 0xC1, 0xC2, 0xC3, 0xC5, 0xC6, 0xC7, 0xC9, 0xCA, 0xCB, 0xCD, 0xCE, 0xCF:
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// readMarker читает 0xFF (с заполнителями) и код маркера.
|
||||
func readMarker(br *bufio.Reader) (byte, error) {
|
||||
b, err := br.ReadByte()
|
||||
if err != nil {
|
||||
return 0, invalid("truncated JPEG")
|
||||
}
|
||||
if b != 0xFF {
|
||||
return 0, invalid("expected JPEG marker")
|
||||
}
|
||||
for b == 0xFF {
|
||||
if b, err = br.ReadByte(); err != nil {
|
||||
return 0, invalid("truncated JPEG")
|
||||
}
|
||||
}
|
||||
return b, nil
|
||||
}
|
||||
|
||||
func segmentLength(br *bufio.Reader) (int, error) {
|
||||
var l [2]byte
|
||||
if _, err := io.ReadFull(br, l[:]); err != nil {
|
||||
return 0, invalid("truncated JPEG segment")
|
||||
}
|
||||
n := int(binary.BigEndian.Uint16(l[:]))
|
||||
if n < 2 {
|
||||
return 0, invalid("bad JPEG segment length")
|
||||
}
|
||||
return n - 2, nil
|
||||
}
|
||||
|
||||
// skipEntropy пропускает сжатые данные скана и возвращает следующий маркер.
|
||||
func skipEntropy(br *bufio.Reader) (byte, error) {
|
||||
for {
|
||||
b, err := br.ReadByte()
|
||||
if err != nil {
|
||||
return 0, invalid("truncated JPEG scan")
|
||||
}
|
||||
if b != 0xFF {
|
||||
continue
|
||||
}
|
||||
for b == 0xFF {
|
||||
if b, err = br.ReadByte(); err != nil {
|
||||
return 0, invalid("truncated JPEG scan")
|
||||
}
|
||||
}
|
||||
if b == 0x00 || (b >= 0xD0 && b <= 0xD7) {
|
||||
continue // байт-заполнитель или RSTn внутри скана
|
||||
}
|
||||
return b, nil
|
||||
}
|
||||
}
|
||||
|
||||
// readSOF читает сегмент SOF и проверяет его структуру (ITU T.81, B.2.2):
|
||||
// P(1) Y(2) X(2) Nf(1), затем Nf компонент по 3 байта. n — длина сегмента
|
||||
// без двух байт поля длины. Возвращает (width, height).
|
||||
func readSOF(br *bufio.Reader, n int) (int64, int64, error) {
|
||||
if n < 6 {
|
||||
return 0, 0, invalid("short SOF")
|
||||
}
|
||||
hdr := make([]byte, 6)
|
||||
if _, err := io.ReadFull(br, hdr); err != nil {
|
||||
return 0, 0, invalid("truncated SOF")
|
||||
}
|
||||
// hdr[0] — точность выборки (precision), не число компонент.
|
||||
h := int64(binary.BigEndian.Uint16(hdr[1:3]))
|
||||
w := int64(binary.BigEndian.Uint16(hdr[3:5]))
|
||||
nf := int(hdr[5])
|
||||
if nf == 0 {
|
||||
return 0, 0, invalid("SOF without components")
|
||||
}
|
||||
if n != 6+3*nf {
|
||||
return 0, 0, invalid("SOF length %d, want %d", n, 6+3*nf)
|
||||
}
|
||||
if _, err := br.Discard(3 * nf); err != nil {
|
||||
return 0, 0, invalid("truncated SOF")
|
||||
}
|
||||
return w, h, nil
|
||||
}
|
||||
|
||||
func validateJPEG(br *bufio.Reader, maxPixels int64) (Result, error) {
|
||||
res := Result{Format: JPEG}
|
||||
var soi [2]byte
|
||||
if _, err := io.ReadFull(br, soi[:]); err != nil || soi != [2]byte{0xFF, 0xD8} {
|
||||
return res, invalid("not a JPEG")
|
||||
}
|
||||
sawSOF, sawSOS := false, false
|
||||
marker, err := readMarker(br)
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
for {
|
||||
switch {
|
||||
case marker == 0xD9: // EOI
|
||||
if !sawSOF || !sawSOS {
|
||||
return res, invalid("JPEG without frame or scan")
|
||||
}
|
||||
return res, nil
|
||||
case marker == 0xD8:
|
||||
return res, invalid("unexpected SOI")
|
||||
case marker == 0x01 || (marker >= 0xD0 && marker <= 0xD7):
|
||||
// маркеры без длины
|
||||
case isSOF(marker):
|
||||
n, err := segmentLength(br)
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
if sawSOF {
|
||||
// читаем сегмент, но так и так вернём ошибку
|
||||
_, _, _ = readSOF(br, n)
|
||||
return res, invalid("multiple frames")
|
||||
}
|
||||
w, h, err := readSOF(br, n)
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
res.Width, res.Height = int(w), int(h)
|
||||
if w < 1 || h < 1 {
|
||||
return res, invalid("bad JPEG dimensions")
|
||||
}
|
||||
if w*h > maxPixels {
|
||||
return res, ErrTooLarge
|
||||
}
|
||||
sawSOF = true
|
||||
case marker == 0xDA: // SOS
|
||||
if !sawSOF {
|
||||
return res, invalid("scan before frame")
|
||||
}
|
||||
n, err := segmentLength(br)
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
// SOS: Ns + Ss + Se + Ah/Al + 2*Ns = 1 + 3 + 2*Ns = 4 + 2*Ns
|
||||
if n < 4 {
|
||||
return res, invalid("short SOS")
|
||||
}
|
||||
nsBuf := make([]byte, 1)
|
||||
if _, err := io.ReadFull(br, nsBuf); err != nil {
|
||||
return res, invalid("truncated SOS")
|
||||
}
|
||||
ns := int(nsBuf[0])
|
||||
if ns < 1 {
|
||||
return res, invalid("SOS without components")
|
||||
}
|
||||
if n != 4+2*ns {
|
||||
return res, invalid("SOS length %d, want %d", n, 4+2*ns)
|
||||
}
|
||||
if _, err := br.Discard(n - 1); err != nil {
|
||||
return res, invalid("truncated SOS")
|
||||
}
|
||||
sawSOS = true
|
||||
if marker, err = skipEntropy(br); err != nil {
|
||||
return res, err
|
||||
}
|
||||
continue
|
||||
default:
|
||||
n, err := segmentLength(br)
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
if _, err := br.Discard(n); err != nil {
|
||||
return res, invalid("truncated JPEG segment")
|
||||
}
|
||||
}
|
||||
if marker, err = readMarker(br); err != nil {
|
||||
return res, err
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,187 @@
|
||||
package imagecheck
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/zlib"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"hash/crc32"
|
||||
"image"
|
||||
"image/color"
|
||||
"image/jpeg"
|
||||
"image/png"
|
||||
"os"
|
||||
"testing"
|
||||
)
|
||||
|
||||
const limit = 40_000_000
|
||||
|
||||
func testImage(w, h int) *image.RGBA {
|
||||
img := image.NewRGBA(image.Rect(0, 0, w, h))
|
||||
for y := 0; y < h; y++ {
|
||||
for x := 0; x < w; x++ {
|
||||
img.Set(x, y, color.RGBA{uint8(x * 7), uint8(y * 3), uint8(x ^ y), 255})
|
||||
}
|
||||
}
|
||||
return img
|
||||
}
|
||||
|
||||
func encodePNG(t *testing.T, w, h int) []byte {
|
||||
var b bytes.Buffer
|
||||
if err := png.Encode(&b, testImage(w, h)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return b.Bytes()
|
||||
}
|
||||
|
||||
func encodeJPEG(t *testing.T, w, h int, progressive bool) []byte {
|
||||
var b bytes.Buffer
|
||||
if err := jpeg.Encode(&b, testImage(w, h), &jpeg.Options{Quality: 80}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return b.Bytes()
|
||||
}
|
||||
|
||||
func chunk(typ string, data []byte) []byte {
|
||||
var b bytes.Buffer
|
||||
binary.Write(&b, binary.BigEndian, uint32(len(data)))
|
||||
b.WriteString(typ)
|
||||
b.Write(data)
|
||||
crc := crc32.NewIEEE()
|
||||
crc.Write([]byte(typ))
|
||||
crc.Write(data)
|
||||
binary.Write(&b, binary.BigEndian, crc.Sum32())
|
||||
return b.Bytes()
|
||||
}
|
||||
|
||||
// craftPNG собирает PNG вручную: так можно задать любые IHDR и поток данных.
|
||||
func craftPNG(w, h uint32, colorType, depth, interlace byte, raw []byte) []byte {
|
||||
var b bytes.Buffer
|
||||
b.Write(pngSignature)
|
||||
ihdr := make([]byte, 13)
|
||||
binary.BigEndian.PutUint32(ihdr[0:], w)
|
||||
binary.BigEndian.PutUint32(ihdr[4:], h)
|
||||
ihdr[8], ihdr[9], ihdr[12] = depth, colorType, interlace
|
||||
b.Write(chunk("IHDR", ihdr))
|
||||
var z bytes.Buffer
|
||||
zw := zlib.NewWriter(&z)
|
||||
zw.Write(raw)
|
||||
zw.Close()
|
||||
b.Write(chunk("IDAT", z.Bytes()))
|
||||
b.Write(chunk("IEND", nil))
|
||||
return b.Bytes()
|
||||
}
|
||||
|
||||
func check(t *testing.T, name string, data []byte, format string, want error) {
|
||||
t.Helper()
|
||||
_, err := Validate(bytes.NewReader(data), format, limit)
|
||||
if want == nil && err != nil {
|
||||
t.Errorf("%s: unexpected error %v", name, err)
|
||||
}
|
||||
if want != nil && !errors.Is(err, want) {
|
||||
t.Errorf("%s: got %v, want %v", name, err, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidPNGAndJPEG(t *testing.T) {
|
||||
p := encodePNG(t, 320, 200)
|
||||
res, err := Validate(bytes.NewReader(p), PNG, limit)
|
||||
if err != nil || res.Width != 320 || res.Height != 200 {
|
||||
t.Fatalf("png: %+v %v", res, err)
|
||||
}
|
||||
j := encodeJPEG(t, 321, 199, false)
|
||||
res, err = Validate(bytes.NewReader(j), JPEG, limit)
|
||||
if err != nil || res.Width != 321 || res.Height != 199 {
|
||||
t.Fatalf("jpeg: %+v %v", res, err)
|
||||
}
|
||||
check(t, "png with trailing bytes", append(p, []byte("trailer")...), PNG, nil)
|
||||
}
|
||||
|
||||
func TestCourseFixture(t *testing.T) {
|
||||
data, err := os.ReadFile("../../tests/fixtures/sample.png")
|
||||
if err != nil {
|
||||
t.Skip("fixture not found")
|
||||
}
|
||||
res, err := Validate(bytes.NewReader(data), PNG, limit)
|
||||
if err != nil || res.Width != 320 || res.Height != 200 {
|
||||
t.Fatalf("fixture: %+v %v", res, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInterlacedPNG(t *testing.T) {
|
||||
// 10×7 RGB, Adam7: длина сырых данных по семи проходам.
|
||||
w, h := int64(10), int64(7)
|
||||
raw := make([]byte, expectedRaw(w, h, 24, true))
|
||||
check(t, "interlaced", craftPNG(uint32(w), uint32(h), 2, 8, 1, raw), PNG, nil)
|
||||
check(t, "interlaced short data", craftPNG(uint32(w), uint32(h), 2, 8, 1, raw[:len(raw)-1]), PNG, ErrInvalid)
|
||||
}
|
||||
|
||||
func TestBrokenImages(t *testing.T) {
|
||||
p := encodePNG(t, 64, 64)
|
||||
j := encodeJPEG(t, 64, 64, false)
|
||||
|
||||
check(t, "png truncated", p[:len(p)/2], PNG, ErrInvalid)
|
||||
check(t, "png without IEND", p[:len(p)-12], PNG, ErrInvalid)
|
||||
bad := bytes.Clone(p)
|
||||
bad[len(bad)/2] ^= 0xFF
|
||||
check(t, "png corrupted byte", bad, PNG, ErrInvalid)
|
||||
check(t, "jpeg truncated", j[:len(j)/2], JPEG, ErrInvalid)
|
||||
check(t, "jpeg without EOI", j[:len(j)-2], JPEG, ErrInvalid)
|
||||
check(t, "png declared as jpeg", p, JPEG, ErrInvalid)
|
||||
check(t, "text as png", []byte("hello, not an image at all"), PNG, ErrInvalid)
|
||||
check(t, "empty", nil, PNG, ErrInvalid)
|
||||
check(t, "png short raw data", craftPNG(8, 8, 2, 8, 0, make([]byte, 8*(1+24)-1)), PNG, ErrInvalid)
|
||||
check(t, "png bad bit depth", craftPNG(8, 8, 2, 4, 0, make([]byte, 8*(1+12))), PNG, ErrInvalid)
|
||||
}
|
||||
|
||||
func TestPixelLimitBeforeData(t *testing.T) {
|
||||
// «Бомба»: IHDR на 100000×100000, данных почти нет. Должно быть ErrTooLarge,
|
||||
// а не попытка распаковать 30 ГБ.
|
||||
bomb := craftPNG(100000, 100000, 2, 8, 0, []byte{0})
|
||||
check(t, "png bomb", bomb, PNG, ErrTooLarge)
|
||||
// Ровно на границе лимита — допустимо по пикселям (данных не хватает → invalid).
|
||||
check(t, "png at limit", craftPNG(8000, 5000, 0, 1, 0, nil), PNG, ErrInvalid)
|
||||
check(t, "png over limit", craftPNG(8000, 5001, 0, 1, 0, nil), PNG, ErrTooLarge)
|
||||
|
||||
// JPEG: подменяем размеры в SOF0 на 60000×60000.
|
||||
j := encodeJPEG(t, 16, 16, false)
|
||||
i := bytes.Index(j, []byte{0xFF, 0xC0})
|
||||
if i < 0 {
|
||||
t.Fatal("no SOF0")
|
||||
}
|
||||
big := bytes.Clone(j)
|
||||
binary.BigEndian.PutUint16(big[i+5:], 60000)
|
||||
binary.BigEndian.PutUint16(big[i+7:], 60000)
|
||||
check(t, "jpeg bomb", big, JPEG, ErrTooLarge)
|
||||
}
|
||||
|
||||
func TestJPEGSOFStructure(t *testing.T) {
|
||||
// Серый JPEG: одна компонента (Nf=1), длина SOF = 6 + 3*1.
|
||||
var g bytes.Buffer
|
||||
gray := image.NewGray(image.Rect(0, 0, 33, 17))
|
||||
if err := jpeg.Encode(&g, gray, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
res, err := Validate(bytes.NewReader(g.Bytes()), JPEG, limit)
|
||||
if err != nil || res.Width != 33 || res.Height != 17 {
|
||||
t.Fatalf("grayscale jpeg: %+v %v", res, err)
|
||||
}
|
||||
// Цветной JPEG с испорченным Nf (длина сегмента больше не сходится).
|
||||
j := encodeJPEG(t, 16, 16, false)
|
||||
i := bytes.Index(j, []byte{0xFF, 0xC0})
|
||||
bad := bytes.Clone(j)
|
||||
bad[i+9] = 7 // Nf: было 3
|
||||
check(t, "jpeg SOF length mismatch", bad, JPEG, ErrInvalid)
|
||||
}
|
||||
|
||||
func TestSniff(t *testing.T) {
|
||||
if f, ok := Sniff(encodePNG(t, 2, 2)); !ok || f != PNG {
|
||||
t.Error("png sniff")
|
||||
}
|
||||
if f, ok := Sniff(encodeJPEG(t, 2, 2, false)); !ok || f != JPEG {
|
||||
t.Error("jpeg sniff")
|
||||
}
|
||||
if _, ok := Sniff([]byte("GIF89a")); ok {
|
||||
t.Error("gif must not be accepted")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,291 @@
|
||||
// Package imaging строит варианты вложения по контракту lab5.
|
||||
//
|
||||
// thumbnail JPEG, вписан в 256×256 с сохранением пропорций, без увеличения,
|
||||
// EXIF orientation применён, alpha компонуется на белый,
|
||||
// размеры округляются вниз, минимум 1 px (320×200 → 256×160)
|
||||
// crop JPEG, центрированный квадрат 128×128 (маленькие увеличиваются)
|
||||
// optimized WebP, вписан в 1280×1280 без увеличения; lossy, качество WEBP_QUALITY,
|
||||
// метаданные удалены (cwebp -metadata none; пиксели и так
|
||||
// перекодированы из декодированного изображения, EXIF не переносится)
|
||||
//
|
||||
// Все кодировщики детерминированы: одинаковый вход даёт одинаковые байты, поэтому
|
||||
// повтор job перезаписывает объект тем же содержимым.
|
||||
package imaging
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/binary"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"image"
|
||||
"image/color"
|
||||
"image/draw"
|
||||
"image/jpeg"
|
||||
"image/png"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
|
||||
xdraw "golang.org/x/image/draw"
|
||||
)
|
||||
|
||||
// ErrDecode — изображение не декодируется: постоянная ошибка (повтор не поможет).
|
||||
var ErrDecode = errors.New("cannot decode image")
|
||||
|
||||
// Params — параметры версии pipeline. Новая версия может менять их, не трогая
|
||||
// уже опубликованные результаты старой (ключ эффекта включает версию).
|
||||
type Params struct {
|
||||
ThumbQuality int
|
||||
CropQuality int
|
||||
WebPQuality int
|
||||
}
|
||||
|
||||
// ParamsFor — параметры по номеру версии pipeline.
|
||||
func ParamsFor(version, webpQuality int) Params {
|
||||
p := Params{ThumbQuality: 85, CropQuality: 85, WebPQuality: webpQuality}
|
||||
if version >= 2 {
|
||||
// v2 (трек 5): чуть выше качество превью — другой, но детерминированный результат.
|
||||
p.ThumbQuality, p.CropQuality = 92, 92
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
// Output — закодированный вариант.
|
||||
type Output struct {
|
||||
Name string
|
||||
ContentType string
|
||||
Data []byte
|
||||
Width int
|
||||
Height int
|
||||
SHA256 string
|
||||
}
|
||||
|
||||
func newOutput(name, ctype string, data []byte, w, h int) Output {
|
||||
sum := sha256.Sum256(data)
|
||||
return Output{Name: name, ContentType: ctype, Data: data, Width: w, Height: h, SHA256: hex.EncodeToString(sum[:])}
|
||||
}
|
||||
|
||||
// Decode декодирует оригинал, применяет EXIF orientation (JPEG) и компонует
|
||||
// прозрачность на белый фон.
|
||||
func Decode(data []byte) (*image.RGBA, error) {
|
||||
src, _, err := image.Decode(bytes.NewReader(data))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: %v", ErrDecode, err)
|
||||
}
|
||||
b := src.Bounds()
|
||||
// Непрозрачный RGBA (типичный PNG RGB) уже готов: вторая полноразмерная копия
|
||||
// (4 байта × пиксель) не нужна — для 36 Мпикс это 137 MiB памяти worker.
|
||||
if rgba, ok := src.(*image.RGBA); ok && b.Min == (image.Point{}) && rgba.Opaque() {
|
||||
return Orient(rgba, ExifOrientation(data)), nil
|
||||
}
|
||||
out := image.NewRGBA(image.Rect(0, 0, b.Dx(), b.Dy()))
|
||||
draw.Draw(out, out.Bounds(), image.NewUniform(color.White), image.Point{}, draw.Src)
|
||||
draw.Draw(out, out.Bounds(), src, b.Min, draw.Over) // alpha → на белый
|
||||
return Orient(out, ExifOrientation(data)), nil
|
||||
}
|
||||
|
||||
// FitSize — вписать w×h в box×box с сохранением пропорций; без увеличения,
|
||||
// если allowUpscale=false; округление вниз, минимум 1.
|
||||
func FitSize(w, h, box int, allowUpscale bool) (int, int) {
|
||||
if !allowUpscale && w <= box && h <= box {
|
||||
return w, h
|
||||
}
|
||||
if w >= h {
|
||||
nh := h * box / w
|
||||
return box, max(1, nh)
|
||||
}
|
||||
nw := w * box / h
|
||||
return max(1, nw), box
|
||||
}
|
||||
|
||||
func resize(src image.Image, w, h int) *image.RGBA {
|
||||
dst := image.NewRGBA(image.Rect(0, 0, w, h))
|
||||
xdraw.CatmullRom.Scale(dst, dst.Bounds(), src, src.Bounds(), xdraw.Src, nil)
|
||||
return dst
|
||||
}
|
||||
|
||||
func encodeJPEG(img image.Image, quality int) ([]byte, error) {
|
||||
var b bytes.Buffer
|
||||
if err := jpeg.Encode(&b, img, &jpeg.Options{Quality: quality}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return b.Bytes(), nil
|
||||
}
|
||||
|
||||
// Thumbnail — JPEG, вписан в 256×256 без увеличения.
|
||||
func Thumbnail(img *image.RGBA, p Params) (Output, error) {
|
||||
w, h := FitSize(img.Bounds().Dx(), img.Bounds().Dy(), 256, false)
|
||||
var src image.Image = img
|
||||
if w != img.Bounds().Dx() || h != img.Bounds().Dy() {
|
||||
src = resize(img, w, h)
|
||||
}
|
||||
data, err := encodeJPEG(src, p.ThumbQuality)
|
||||
if err != nil {
|
||||
return Output{}, err
|
||||
}
|
||||
return newOutput("thumbnail", "image/jpeg", data, w, h), nil
|
||||
}
|
||||
|
||||
// Crop — JPEG, центрированный квадрат 128×128 (маленькие исходники увеличиваются).
|
||||
func Crop(img *image.RGBA, p Params) (Output, error) {
|
||||
b := img.Bounds()
|
||||
side := min(b.Dx(), b.Dy())
|
||||
x0 := b.Min.X + (b.Dx()-side)/2
|
||||
y0 := b.Min.Y + (b.Dy()-side)/2
|
||||
sq := img.SubImage(image.Rect(x0, y0, x0+side, y0+side))
|
||||
data, err := encodeJPEG(resize(sq, 128, 128), p.CropQuality)
|
||||
if err != nil {
|
||||
return Output{}, err
|
||||
}
|
||||
return newOutput("crop", "image/jpeg", data, 128, 128), nil
|
||||
}
|
||||
|
||||
// WebPEncoder кодирует RGBA в WebP.
|
||||
type WebPEncoder func(ctx context.Context, img image.Image, quality int) ([]byte, error)
|
||||
|
||||
// CWebP — кодировщик через cwebp (libwebp-tools): lossy, -metadata none.
|
||||
// Вход — PNG во временном каталоге процесса (эфемерный, удаляется сразу).
|
||||
func CWebP(ctx context.Context, img image.Image, quality int) ([]byte, error) {
|
||||
dir, err := os.MkdirTemp("", "webp-")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer os.RemoveAll(dir)
|
||||
in, out := filepath.Join(dir, "in.png"), filepath.Join(dir, "out.webp")
|
||||
f, err := os.Create(in)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
enc := png.Encoder{CompressionLevel: png.BestSpeed}
|
||||
if err := enc.Encode(f, img); err != nil {
|
||||
f.Close()
|
||||
return nil, err
|
||||
}
|
||||
f.Close()
|
||||
cmd := exec.CommandContext(ctx, "cwebp", "-quiet", "-q", fmt.Sprint(quality), "-m", "4", "-metadata", "none", in, "-o", out)
|
||||
if msg, err := cmd.CombinedOutput(); err != nil {
|
||||
return nil, fmt.Errorf("cwebp: %v: %s", err, bytes.TrimSpace(msg))
|
||||
}
|
||||
return os.ReadFile(out)
|
||||
}
|
||||
|
||||
// Optimized — WebP, вписан в 1280×1280 без увеличения.
|
||||
func Optimized(ctx context.Context, img *image.RGBA, p Params, enc WebPEncoder) (Output, error) {
|
||||
w, h := FitSize(img.Bounds().Dx(), img.Bounds().Dy(), 1280, false)
|
||||
var src image.Image = img
|
||||
if w != img.Bounds().Dx() || h != img.Bounds().Dy() {
|
||||
src = resize(img, w, h)
|
||||
}
|
||||
data, err := enc(ctx, src, p.WebPQuality)
|
||||
if err != nil {
|
||||
return Output{}, err
|
||||
}
|
||||
if len(data) < 12 || string(data[0:4]) != "RIFF" || string(data[8:12]) != "WEBP" {
|
||||
return Output{}, errors.New("encoder produced non-WebP data")
|
||||
}
|
||||
return newOutput("optimized", "image/webp", data, w, h), nil
|
||||
}
|
||||
|
||||
// --- EXIF orientation ---
|
||||
|
||||
// ExifOrientation возвращает тег 0x0112 из APP1 Exif JPEG (1 — нет поворота).
|
||||
func ExifOrientation(data []byte) int {
|
||||
if len(data) < 4 || data[0] != 0xFF || data[1] != 0xD8 {
|
||||
return 1
|
||||
}
|
||||
i := 2
|
||||
for i+4 <= len(data) && data[i] == 0xFF {
|
||||
marker := data[i+1]
|
||||
if marker == 0xDA || marker == 0xD9 {
|
||||
break
|
||||
}
|
||||
segLen := int(binary.BigEndian.Uint16(data[i+2 : i+4]))
|
||||
if segLen < 2 || i+2+segLen > len(data) {
|
||||
break
|
||||
}
|
||||
seg := data[i+4 : i+2+segLen]
|
||||
if marker == 0xE1 && len(seg) > 14 && string(seg[:6]) == "Exif\x00\x00" {
|
||||
if o := tiffOrientation(seg[6:]); o >= 1 && o <= 8 {
|
||||
return o
|
||||
}
|
||||
}
|
||||
i += 2 + segLen
|
||||
}
|
||||
return 1
|
||||
}
|
||||
|
||||
func tiffOrientation(t []byte) int {
|
||||
if len(t) < 8 {
|
||||
return 1
|
||||
}
|
||||
var bo binary.ByteOrder
|
||||
switch string(t[:2]) {
|
||||
case "II":
|
||||
bo = binary.LittleEndian
|
||||
case "MM":
|
||||
bo = binary.BigEndian
|
||||
default:
|
||||
return 1
|
||||
}
|
||||
off := int(bo.Uint32(t[4:8]))
|
||||
if off+2 > len(t) {
|
||||
return 1
|
||||
}
|
||||
n := int(bo.Uint16(t[off : off+2]))
|
||||
for k := 0; k < n; k++ {
|
||||
e := off + 2 + 12*k
|
||||
if e+12 > len(t) {
|
||||
return 1
|
||||
}
|
||||
if bo.Uint16(t[e:e+2]) == 0x0112 {
|
||||
return int(bo.Uint16(t[e+8 : e+10]))
|
||||
}
|
||||
}
|
||||
return 1
|
||||
}
|
||||
|
||||
// Orient применяет EXIF orientation 1..8 (поворот/отражение).
|
||||
func Orient(src *image.RGBA, o int) *image.RGBA {
|
||||
if o <= 1 || o > 8 {
|
||||
return src
|
||||
}
|
||||
b := src.Bounds()
|
||||
w, h := b.Dx(), b.Dy()
|
||||
dw, dh := w, h
|
||||
if o >= 5 {
|
||||
dw, dh = h, w
|
||||
}
|
||||
dst := image.NewRGBA(image.Rect(0, 0, dw, dh))
|
||||
for y := 0; y < h; y++ {
|
||||
for x := 0; x < w; x++ {
|
||||
var dx, dy int
|
||||
switch o {
|
||||
case 2:
|
||||
dx, dy = w-1-x, y
|
||||
case 3:
|
||||
dx, dy = w-1-x, h-1-y
|
||||
case 4:
|
||||
dx, dy = x, h-1-y
|
||||
case 5:
|
||||
dx, dy = y, x
|
||||
case 6:
|
||||
dx, dy = h-1-y, x
|
||||
case 7:
|
||||
dx, dy = h-1-y, w-1-x
|
||||
case 8:
|
||||
dx, dy = y, w-1-x
|
||||
}
|
||||
dst.SetRGBA(dx, dy, src.RGBAAt(b.Min.X+x, b.Min.Y+y))
|
||||
}
|
||||
}
|
||||
return dst
|
||||
}
|
||||
|
||||
// SHA256Hex — lowercase hex SHA-256.
|
||||
func SHA256Hex(data []byte) string {
|
||||
sum := sha256.Sum256(data)
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
@@ -0,0 +1,208 @@
|
||||
package imaging
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"image"
|
||||
"image/color"
|
||||
"image/jpeg"
|
||||
"image/png"
|
||||
"os"
|
||||
"os/exec"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func solid(w, h int, c color.Color) *image.NRGBA {
|
||||
m := image.NewNRGBA(image.Rect(0, 0, w, h))
|
||||
for y := 0; y < h; y++ {
|
||||
for x := 0; x < w; x++ {
|
||||
m.Set(x, y, c)
|
||||
}
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
func encPNG(t *testing.T, img image.Image) []byte {
|
||||
var b bytes.Buffer
|
||||
if err := png.Encode(&b, img); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return b.Bytes()
|
||||
}
|
||||
|
||||
func TestFitSize(t *testing.T) {
|
||||
cases := []struct {
|
||||
w, h, box int
|
||||
up bool
|
||||
ww, wh int
|
||||
}{
|
||||
{320, 200, 256, false, 256, 160}, // fixture курса
|
||||
{200, 320, 256, false, 160, 256},
|
||||
{100, 50, 256, false, 100, 50}, // без увеличения
|
||||
{5000, 1, 256, false, 256, 1}, // минимум 1 px
|
||||
{1, 5000, 256, false, 1, 256},
|
||||
{257, 255, 256, false, 256, 254}, // округление вниз
|
||||
{40, 30, 128, true, 128, 96},
|
||||
{3000, 2000, 1280, false, 1280, 853},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if w, h := FitSize(c.w, c.h, c.box, c.up); w != c.ww || h != c.wh {
|
||||
t.Errorf("FitSize(%d,%d,%d,%v)=%d×%d want %d×%d", c.w, c.h, c.box, c.up, w, h, c.ww, c.wh)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func jpegSize(t *testing.T, data []byte) (int, int) {
|
||||
cfg, err := jpeg.DecodeConfig(bytes.NewReader(data))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return cfg.Width, cfg.Height
|
||||
}
|
||||
|
||||
func TestCourseFixtureThumbnail(t *testing.T) {
|
||||
data, err := os.ReadFile("../../tests/fixtures/sample.png")
|
||||
if err != nil {
|
||||
t.Skip("fixture not found")
|
||||
}
|
||||
img, err := Decode(data)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
th, err := Thumbnail(img, ParamsFor(1, 80))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if w, h := jpegSize(t, th.Data); w != 256 || h != 160 || th.Width != 256 || th.Height != 160 {
|
||||
t.Fatalf("thumbnail %d×%d (meta %d×%d), want 256×160", w, h, th.Width, th.Height)
|
||||
}
|
||||
again, _ := Thumbnail(img, ParamsFor(1, 80))
|
||||
if !bytes.Equal(th.Data, again.Data) {
|
||||
t.Error("thumbnail encoding is not deterministic")
|
||||
}
|
||||
cr, err := Crop(img, ParamsFor(1, 80))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if w, h := jpegSize(t, cr.Data); w != 128 || h != 128 {
|
||||
t.Fatalf("crop %d×%d", w, h)
|
||||
}
|
||||
v2, _ := Thumbnail(img, ParamsFor(2, 80))
|
||||
if bytes.Equal(v2.Data, th.Data) {
|
||||
t.Error("pipeline v2 must produce a different thumbnail")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSmallImageNotUpscaledButCropIs(t *testing.T) {
|
||||
img, err := Decode(encPNG(t, solid(40, 30, color.RGBA{10, 20, 30, 255})))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
th, _ := Thumbnail(img, ParamsFor(1, 80))
|
||||
if th.Width != 40 || th.Height != 30 {
|
||||
t.Errorf("small thumbnail upscaled to %d×%d", th.Width, th.Height)
|
||||
}
|
||||
cr, _ := Crop(img, ParamsFor(1, 80))
|
||||
if w, h := jpegSize(t, cr.Data); w != 128 || h != 128 {
|
||||
t.Errorf("crop of small image %d×%d", w, h)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAlphaOnWhite(t *testing.T) {
|
||||
img, err := Decode(encPNG(t, solid(10, 10, color.NRGBA{0, 0, 0, 0})))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if c := img.RGBAAt(5, 5); c.R != 255 || c.G != 255 || c.B != 255 {
|
||||
t.Errorf("transparent pixel composited to %v, want white", c)
|
||||
}
|
||||
half, _ := Decode(encPNG(t, solid(4, 4, color.NRGBA{0, 0, 0, 128})))
|
||||
if c := half.RGBAAt(1, 1); c.R < 120 || c.R > 135 {
|
||||
t.Errorf("50%% black over white = %v, want ~127", c)
|
||||
}
|
||||
}
|
||||
|
||||
// withExif вставляет APP1 Exif с orientation сразу после SOI.
|
||||
func withExif(jpg []byte, orientation uint16) []byte {
|
||||
var tiff bytes.Buffer
|
||||
tiff.WriteString("MM\x00\x2a")
|
||||
binary.Write(&tiff, binary.BigEndian, uint32(8))
|
||||
binary.Write(&tiff, binary.BigEndian, uint16(1))
|
||||
binary.Write(&tiff, binary.BigEndian, uint16(0x0112))
|
||||
binary.Write(&tiff, binary.BigEndian, uint16(3))
|
||||
binary.Write(&tiff, binary.BigEndian, uint32(1))
|
||||
binary.Write(&tiff, binary.BigEndian, orientation)
|
||||
tiff.Write([]byte{0, 0})
|
||||
binary.Write(&tiff, binary.BigEndian, uint32(0))
|
||||
seg := append([]byte("Exif\x00\x00"), tiff.Bytes()...)
|
||||
var out bytes.Buffer
|
||||
out.Write(jpg[:2])
|
||||
out.Write([]byte{0xFF, 0xE1})
|
||||
binary.Write(&out, binary.BigEndian, uint16(len(seg)+2))
|
||||
out.Write(seg)
|
||||
out.Write(jpg[2:])
|
||||
return out.Bytes()
|
||||
}
|
||||
|
||||
func TestExifOrientation(t *testing.T) {
|
||||
// 300×100: слева красный, справа синий.
|
||||
src := image.NewRGBA(image.Rect(0, 0, 300, 100))
|
||||
for y := 0; y < 100; y++ {
|
||||
for x := 0; x < 300; x++ {
|
||||
c := color.RGBA{255, 0, 0, 255}
|
||||
if x >= 150 {
|
||||
c = color.RGBA{0, 0, 255, 255}
|
||||
}
|
||||
src.Set(x, y, c)
|
||||
}
|
||||
}
|
||||
var b bytes.Buffer
|
||||
jpeg.Encode(&b, src, &jpeg.Options{Quality: 95})
|
||||
data := withExif(b.Bytes(), 6) // 90° по часовой
|
||||
if o := ExifOrientation(data); o != 6 {
|
||||
t.Fatalf("orientation %d, want 6", o)
|
||||
}
|
||||
img, err := Decode(data)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if img.Bounds().Dx() != 100 || img.Bounds().Dy() != 300 {
|
||||
t.Fatalf("rotated size %v, want 100×300", img.Bounds())
|
||||
}
|
||||
// После поворота на 90° по часовой левая (красная) половина оказывается сверху.
|
||||
if c := img.RGBAAt(50, 20); c.R < 200 || c.B > 60 {
|
||||
t.Errorf("top after rotation = %v, want red", c)
|
||||
}
|
||||
th, _ := Thumbnail(img, ParamsFor(1, 80))
|
||||
if th.Width != 85 || th.Height != 256 {
|
||||
t.Errorf("thumbnail of rotated %d×%d, want 85×256", th.Width, th.Height)
|
||||
}
|
||||
if ExifOrientation(b.Bytes()) != 1 {
|
||||
t.Error("no exif must mean orientation 1")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeGarbage(t *testing.T) {
|
||||
if _, err := Decode([]byte("not an image")); err == nil {
|
||||
t.Fatal("garbage decoded")
|
||||
}
|
||||
}
|
||||
|
||||
func TestOptimizedWebP(t *testing.T) {
|
||||
if _, err := exec.LookPath("cwebp"); err != nil {
|
||||
t.Skip("cwebp not installed (worker image has libwebp-tools)")
|
||||
}
|
||||
img, _ := Decode(encPNG(t, solid(2000, 1000, color.RGBA{200, 100, 50, 255})))
|
||||
out, err := Optimized(context.Background(), img, ParamsFor(1, 80), CWebP)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if out.Width != 1280 || out.Height != 640 || out.ContentType != "image/webp" {
|
||||
t.Fatalf("optimized %+v", out)
|
||||
}
|
||||
again, _ := Optimized(context.Background(), img, ParamsFor(1, 80), CWebP)
|
||||
if !bytes.Equal(out.Data, again.Data) {
|
||||
t.Error("cwebp output not deterministic")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,404 @@
|
||||
// Package reconcile сверяет метаданные вложений (PostgreSQL) и объекты (S3).
|
||||
//
|
||||
// Обход — один merge-проход по двум потокам, отсортированным одинаково
|
||||
// (object_key COLLATE "C" и лексикографический листинг S3), поэтому память
|
||||
// O(1) от числа вложений, а стоимость — O(N) строк + ceil(N/1000) LIST-запросов.
|
||||
//
|
||||
// Расхождения и безопасные действия:
|
||||
//
|
||||
// stale_intent строка uploading старше grace (брошенная загрузка) → удалить объект, затем строку
|
||||
// orphan_object объект без строки, старше grace → удалить объект (после перепроверки строки)
|
||||
// missing_object строка ready без объекта → карантин: status=missing (после перепроверки Stat)
|
||||
// size_mismatch строка ready, размер объекта другой → карантин: status=missing
|
||||
// restored строка missing, объект снова есть и размер совпадает → status=ready
|
||||
// purge_missing строка missing старше grace, ни одно сообщение не ссылается → удалить строку
|
||||
// missing_variant у ready-варианта нет объекта → только отчёт (без автоисправления)
|
||||
//
|
||||
// С lab5 вложения в queued/processing/failed и их staging принадлежат pipeline
|
||||
// (lease, retry, retention): reconciler их не трогает. Объекты без строки
|
||||
// (например, частичные результаты попытки, закончившейся failed) удаляются как
|
||||
// orphan после grace — так же, как в lab4.
|
||||
//
|
||||
// Свежие объекты и намерения (моложе grace) и вложения, на которые ссылаются
|
||||
// сообщения, не удаляются никогда. Режим по умолчанию — dry-run (только отчёт).
|
||||
package reconcile
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"runtime"
|
||||
"time"
|
||||
|
||||
"github.com/charawein/mtusi2026lab6/internal/domain"
|
||||
"github.com/charawein/mtusi2026lab6/internal/storage"
|
||||
)
|
||||
|
||||
// Repo — нужные reconciler операции над метаданными.
|
||||
type Repo interface {
|
||||
StreamByKey(ctx context.Context, prefix string, fn func(*domain.Attachment) error) error
|
||||
HasKey(ctx context.Context, key string) (bool, error)
|
||||
DeleteStaleIntent(ctx context.Context, id string, cutoff time.Time) (bool, error)
|
||||
SetStatus(ctx context.Context, id, from, to string) (bool, error)
|
||||
DeleteUnreferencedMissing(ctx context.Context, id string, cutoff time.Time) (bool, error)
|
||||
IsReferenced(ctx context.Context, id string) (bool, error)
|
||||
}
|
||||
|
||||
type Options struct {
|
||||
Prefix string
|
||||
Grace time.Duration
|
||||
Apply bool
|
||||
Now time.Time // общий «сейчас» для строк и объектов (время БД)
|
||||
// MaxItems ограничивает список примеров в отчёте на каждый вид расхождения.
|
||||
MaxItems int
|
||||
}
|
||||
|
||||
type Item struct {
|
||||
Kind string `json:"kind"`
|
||||
Key string `json:"key"`
|
||||
AttachmentID string `json:"attachment_id,omitempty"`
|
||||
Action string `json:"action"`
|
||||
Detail string `json:"detail,omitempty"`
|
||||
}
|
||||
|
||||
type Report struct {
|
||||
DryRun bool `json:"dry_run"`
|
||||
GraceSeconds float64 `json:"grace_seconds"`
|
||||
Cutoff time.Time `json:"cutoff"`
|
||||
DurationMS int64 `json:"duration_ms"`
|
||||
RowsScanned int `json:"rows_scanned"`
|
||||
ObjectsScanned int `json:"objects_scanned"`
|
||||
Consistent int `json:"consistent"`
|
||||
Found map[string]int `json:"found"`
|
||||
Fixed map[string]int `json:"fixed"`
|
||||
Skipped map[string]int `json:"skipped"`
|
||||
Calls map[string]int `json:"calls"`
|
||||
HeapAllocMB float64 `json:"heap_alloc_mb"`
|
||||
// WaitMS — сколько merge-цикл простоял в ожидании очередного элемента каждого
|
||||
// потока: показывает, кто (S3-листинг или запрос к БД) задаёт время обхода.
|
||||
WaitMS map[string]int64 `json:"wait_ms"`
|
||||
Errors []string `json:"errors"`
|
||||
Items []Item `json:"items"`
|
||||
}
|
||||
|
||||
type reconciler struct {
|
||||
repo Repo
|
||||
store storage.Store
|
||||
opt Options
|
||||
rep *Report
|
||||
items map[string]int
|
||||
}
|
||||
|
||||
func (r *reconciler) note(kind string, it Item) {
|
||||
r.rep.Found[kind]++
|
||||
if r.items[kind] < r.opt.MaxItems {
|
||||
r.items[kind]++
|
||||
r.rep.Items = append(r.rep.Items, it)
|
||||
}
|
||||
}
|
||||
|
||||
func (r *reconciler) fail(format string, args ...any) {
|
||||
if len(r.rep.Errors) < 100 {
|
||||
r.rep.Errors = append(r.rep.Errors, fmt.Sprintf(format, args...))
|
||||
}
|
||||
}
|
||||
|
||||
// Run выполняет один проход. Ошибки отдельных действий попадают в отчёт и не
|
||||
// прерывают обход; ошибка листинга/запроса прерывает проход целиком.
|
||||
func Run(ctx context.Context, repo Repo, store storage.Store, opt Options) (*Report, error) {
|
||||
start := time.Now()
|
||||
if opt.MaxItems == 0 {
|
||||
opt.MaxItems = 50
|
||||
}
|
||||
cutoff := opt.Now.Add(-opt.Grace)
|
||||
rep := &Report{
|
||||
DryRun: !opt.Apply, GraceSeconds: opt.Grace.Seconds(), Cutoff: cutoff,
|
||||
Found: map[string]int{}, Fixed: map[string]int{}, Skipped: map[string]int{}, Calls: map[string]int{},
|
||||
Errors: []string{}, Items: []Item{}, WaitMS: map[string]int64{},
|
||||
}
|
||||
rc := &reconciler{repo: repo, store: store, opt: opt, rep: rep, items: map[string]int{}}
|
||||
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
|
||||
objs := make(chan storage.Object, 512)
|
||||
objErr := make(chan error, 1)
|
||||
go func() {
|
||||
defer close(objs)
|
||||
objErr <- store.List(ctx, opt.Prefix, func(o storage.Object) error {
|
||||
select {
|
||||
case objs <- o:
|
||||
return nil
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
})
|
||||
}()
|
||||
rows := make(chan *domain.Attachment, 512)
|
||||
rowErr := make(chan error, 1)
|
||||
go func() {
|
||||
defer close(rows)
|
||||
rowErr <- repo.StreamByKey(ctx, opt.Prefix, func(a *domain.Attachment) error {
|
||||
select {
|
||||
case rows <- a:
|
||||
return nil
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
})
|
||||
}()
|
||||
|
||||
var waitS3, waitDB time.Duration
|
||||
nextObj := func() (storage.Object, bool) {
|
||||
t := time.Now()
|
||||
o, ok := <-objs
|
||||
waitS3 += time.Since(t)
|
||||
return o, ok
|
||||
}
|
||||
nextRow := func() (*domain.Attachment, bool) {
|
||||
t := time.Now()
|
||||
a, ok := <-rows
|
||||
waitDB += time.Since(t)
|
||||
return a, ok
|
||||
}
|
||||
o, okO := nextObj()
|
||||
a, okA := nextRow()
|
||||
for okO || okA {
|
||||
switch {
|
||||
case okA && (!okO || a.ObjectKey < o.Key):
|
||||
rep.RowsScanned++
|
||||
rc.rowWithoutObject(ctx, a, cutoff)
|
||||
a, okA = nextRow()
|
||||
case okO && (!okA || o.Key < a.ObjectKey):
|
||||
rep.ObjectsScanned++
|
||||
rc.objectWithoutRow(ctx, o, cutoff)
|
||||
o, okO = nextObj()
|
||||
default:
|
||||
rep.RowsScanned++
|
||||
rep.ObjectsScanned++
|
||||
rc.both(ctx, a, o, cutoff)
|
||||
a, okA = nextRow()
|
||||
o, okO = nextObj()
|
||||
}
|
||||
}
|
||||
rep.WaitMS["s3_list"] = waitS3.Milliseconds()
|
||||
rep.WaitMS["db_stream"] = waitDB.Milliseconds()
|
||||
if err := <-objErr; err != nil {
|
||||
return rep, fmt.Errorf("list objects: %w", err)
|
||||
}
|
||||
if err := <-rowErr; err != nil {
|
||||
return rep, fmt.Errorf("stream rows: %w", err)
|
||||
}
|
||||
rep.Calls["s3_list_pages"] = (rep.ObjectsScanned + 999) / 1000
|
||||
var ms runtime.MemStats
|
||||
runtime.ReadMemStats(&ms)
|
||||
rep.HeapAllocMB = float64(ms.HeapAlloc) / (1 << 20)
|
||||
rep.DurationMS = time.Since(start).Milliseconds()
|
||||
return rep, nil
|
||||
}
|
||||
|
||||
func (r *reconciler) rowWithoutObject(ctx context.Context, a *domain.Attachment, cutoff time.Time) {
|
||||
switch a.Status {
|
||||
case "variant":
|
||||
if r.objectExists(ctx, a.ObjectKey) {
|
||||
r.rep.Skipped["recheck_object_present"]++
|
||||
return
|
||||
}
|
||||
r.note("missing_variant", Item{Kind: "missing_variant", Key: a.ObjectKey, AttachmentID: a.ID, Action: "report"})
|
||||
case "staging", domain.AttachmentQueued, domain.AttachmentProcessing, domain.AttachmentFailed:
|
||||
r.rep.Skipped["owned_by_pipeline"]++
|
||||
case domain.AttachmentUploading:
|
||||
if !a.UpdatedAt.Before(cutoff) {
|
||||
r.rep.Skipped["fresh_intent"]++ // загрузка может быть в процессе
|
||||
return
|
||||
}
|
||||
r.staleIntent(ctx, a, false)
|
||||
case domain.AttachmentReady:
|
||||
// Листинг мог начаться раньше, чем объект записан: перепроверяем точечно.
|
||||
if r.objectExists(ctx, a.ObjectKey) {
|
||||
r.rep.Skipped["recheck_object_present"]++
|
||||
return
|
||||
}
|
||||
r.quarantine(ctx, a, "missing_object", "")
|
||||
case domain.AttachmentMissing:
|
||||
r.purgeMissing(ctx, a, cutoff)
|
||||
}
|
||||
}
|
||||
|
||||
func (r *reconciler) objectWithoutRow(ctx context.Context, o storage.Object, cutoff time.Time) {
|
||||
if !o.LastModified.Before(cutoff) {
|
||||
r.rep.Skipped["fresh_object"]++
|
||||
return
|
||||
}
|
||||
// Строка могла появиться после начала обхода — перепроверяем перед удалением.
|
||||
r.rep.Calls["db_haskey"]++
|
||||
has, err := r.repo.HasKey(ctx, o.Key)
|
||||
if err != nil {
|
||||
r.fail("haskey %s: %v", o.Key, err)
|
||||
return
|
||||
}
|
||||
if has {
|
||||
r.rep.Skipped["recheck_row_present"]++
|
||||
return
|
||||
}
|
||||
it := Item{Kind: "orphan_object", Key: o.Key, Action: "delete_object",
|
||||
Detail: fmt.Sprintf("size=%d modified=%s", o.Size, o.LastModified.UTC().Format(time.RFC3339))}
|
||||
if !r.opt.Apply {
|
||||
it.Action = "would_delete_object"
|
||||
r.note("orphan_object", it)
|
||||
return
|
||||
}
|
||||
r.rep.Calls["s3_delete"]++
|
||||
if err := r.store.Delete(ctx, o.Key); err != nil {
|
||||
r.fail("delete orphan %s: %v", o.Key, err)
|
||||
return
|
||||
}
|
||||
r.note("orphan_object", it)
|
||||
r.rep.Fixed["orphan_object"]++
|
||||
}
|
||||
|
||||
func (r *reconciler) both(ctx context.Context, a *domain.Attachment, o storage.Object, cutoff time.Time) {
|
||||
switch a.Status {
|
||||
case "variant":
|
||||
if o.Size != a.SizeBytes {
|
||||
r.note("variant_size_mismatch", Item{Kind: "variant_size_mismatch", Key: a.ObjectKey, AttachmentID: a.ID,
|
||||
Action: "report", Detail: fmt.Sprintf("db=%d s3=%d", a.SizeBytes, o.Size)})
|
||||
return
|
||||
}
|
||||
r.rep.Consistent++
|
||||
case "staging", domain.AttachmentQueued, domain.AttachmentProcessing, domain.AttachmentFailed:
|
||||
r.rep.Consistent++
|
||||
case domain.AttachmentUploading:
|
||||
if !a.UpdatedAt.Before(cutoff) {
|
||||
r.rep.Skipped["fresh_intent"]++
|
||||
return
|
||||
}
|
||||
r.staleIntent(ctx, a, true)
|
||||
case domain.AttachmentReady:
|
||||
if o.Size != a.SizeBytes {
|
||||
r.quarantine(ctx, a, "size_mismatch", fmt.Sprintf("db=%d s3=%d", a.SizeBytes, o.Size))
|
||||
return
|
||||
}
|
||||
r.rep.Consistent++
|
||||
case domain.AttachmentMissing:
|
||||
if o.Size != a.SizeBytes {
|
||||
r.purgeMissing(ctx, a, cutoff)
|
||||
return
|
||||
}
|
||||
it := Item{Kind: "restored", Key: a.ObjectKey, AttachmentID: a.ID, Action: "set_ready"}
|
||||
if !r.opt.Apply {
|
||||
it.Action = "would_set_ready"
|
||||
r.note("restored", it)
|
||||
return
|
||||
}
|
||||
r.rep.Calls["db_update"]++
|
||||
ok, err := r.repo.SetStatus(ctx, a.ID, domain.AttachmentMissing, domain.AttachmentReady)
|
||||
if err != nil {
|
||||
r.fail("restore %s: %v", a.ID, err)
|
||||
return
|
||||
}
|
||||
r.note("restored", it)
|
||||
if ok {
|
||||
r.rep.Fixed["restored"]++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (r *reconciler) objectExists(ctx context.Context, key string) bool {
|
||||
r.rep.Calls["s3_stat"]++
|
||||
_, err := r.store.Stat(ctx, key)
|
||||
if err == nil {
|
||||
return true
|
||||
}
|
||||
if !errors.Is(err, storage.ErrNotFound) {
|
||||
r.fail("stat %s: %v", key, err)
|
||||
return true // при ошибке хранилища ничего не трогаем
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (r *reconciler) staleIntent(ctx context.Context, a *domain.Attachment, hasObject bool) {
|
||||
it := Item{Kind: "stale_intent", Key: a.ObjectKey, AttachmentID: a.ID, Action: "delete_intent",
|
||||
Detail: "updated_at=" + a.UpdatedAt.UTC().Format(time.RFC3339)}
|
||||
if hasObject {
|
||||
it.Action = "delete_object_and_intent"
|
||||
}
|
||||
if !r.opt.Apply {
|
||||
it.Action = "would_" + it.Action
|
||||
r.note("stale_intent", it)
|
||||
return
|
||||
}
|
||||
// Сначала объект: если упадём между шагами, останется намерение без объекта,
|
||||
// и следующий проход его доберёт (обратный порядок оставил бы сироту без учёта).
|
||||
if hasObject {
|
||||
r.rep.Calls["s3_delete"]++
|
||||
if err := r.store.Delete(ctx, a.ObjectKey); err != nil {
|
||||
r.fail("delete intent object %s: %v", a.ObjectKey, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
r.rep.Calls["db_delete"]++
|
||||
ok, err := r.repo.DeleteStaleIntent(ctx, a.ID, r.opt.Now.Add(-r.opt.Grace))
|
||||
if err != nil {
|
||||
r.fail("delete intent %s: %v", a.ID, err)
|
||||
return
|
||||
}
|
||||
r.note("stale_intent", it)
|
||||
if ok {
|
||||
r.rep.Fixed["stale_intent"]++
|
||||
}
|
||||
}
|
||||
|
||||
func (r *reconciler) quarantine(ctx context.Context, a *domain.Attachment, kind, detail string) {
|
||||
it := Item{Kind: kind, Key: a.ObjectKey, AttachmentID: a.ID, Action: "quarantine", Detail: detail}
|
||||
if !r.opt.Apply {
|
||||
it.Action = "would_quarantine"
|
||||
r.note(kind, it)
|
||||
return
|
||||
}
|
||||
r.rep.Calls["db_update"]++
|
||||
ok, err := r.repo.SetStatus(ctx, a.ID, domain.AttachmentReady, domain.AttachmentMissing)
|
||||
if err != nil {
|
||||
r.fail("quarantine %s: %v", a.ID, err)
|
||||
return
|
||||
}
|
||||
r.note(kind, it)
|
||||
if ok {
|
||||
r.rep.Fixed[kind]++
|
||||
}
|
||||
}
|
||||
|
||||
// purgeMissing удаляет запись карантина, только если она старше grace и на неё
|
||||
// не ссылается ни одно сообщение. Используемые вложения остаются (их история цела).
|
||||
func (r *reconciler) purgeMissing(ctx context.Context, a *domain.Attachment, cutoff time.Time) {
|
||||
if !a.UpdatedAt.Before(cutoff) {
|
||||
r.rep.Skipped["fresh_missing"]++
|
||||
return
|
||||
}
|
||||
r.rep.Calls["db_referenced"]++
|
||||
ref, err := r.repo.IsReferenced(ctx, a.ID)
|
||||
if err != nil {
|
||||
r.fail("referenced %s: %v", a.ID, err)
|
||||
return
|
||||
}
|
||||
if ref {
|
||||
r.rep.Skipped["referenced_missing_kept"]++
|
||||
return
|
||||
}
|
||||
it := Item{Kind: "purge_missing", Key: a.ObjectKey, AttachmentID: a.ID, Action: "delete_row"}
|
||||
if !r.opt.Apply {
|
||||
it.Action = "would_delete_row"
|
||||
r.note("purge_missing", it)
|
||||
return
|
||||
}
|
||||
r.rep.Calls["db_delete"]++
|
||||
ok, err := r.repo.DeleteUnreferencedMissing(ctx, a.ID, cutoff)
|
||||
if err != nil {
|
||||
r.fail("purge %s: %v", a.ID, err)
|
||||
return
|
||||
}
|
||||
r.note("purge_missing", it)
|
||||
if ok {
|
||||
r.rep.Fixed["purge_missing"]++
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,241 @@
|
||||
package reconcile
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
"sort"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/charawein/mtusi2026lab6/internal/domain"
|
||||
"github.com/charawein/mtusi2026lab6/internal/storage"
|
||||
)
|
||||
|
||||
type fakeStore struct{ objs map[string]storage.Object }
|
||||
|
||||
func (f *fakeStore) Put(_ context.Context, key string, r io.Reader, size int64, _ string) error {
|
||||
f.objs[key] = storage.Object{Key: key, Size: size, LastModified: time.Now()}
|
||||
return nil
|
||||
}
|
||||
func (f *fakeStore) Get(context.Context, string) (io.ReadCloser, int64, error) {
|
||||
return io.NopCloser(bytes.NewReader(nil)), 0, nil
|
||||
}
|
||||
func (f *fakeStore) Stat(_ context.Context, key string) (storage.Object, error) {
|
||||
o, ok := f.objs[key]
|
||||
if !ok {
|
||||
return o, storage.ErrNotFound
|
||||
}
|
||||
return o, nil
|
||||
}
|
||||
func (f *fakeStore) Delete(_ context.Context, key string) error { delete(f.objs, key); return nil }
|
||||
func (f *fakeStore) Copy(_ context.Context, src, dst string) error {
|
||||
o, ok := f.objs[src]
|
||||
if !ok {
|
||||
return storage.ErrNotFound
|
||||
}
|
||||
o.Key = dst
|
||||
f.objs[dst] = o
|
||||
return nil
|
||||
}
|
||||
func (f *fakeStore) List(_ context.Context, prefix string, fn func(storage.Object) error) error {
|
||||
keys := make([]string, 0, len(f.objs))
|
||||
for k := range f.objs {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, k := range keys {
|
||||
if err := fn(f.objs[k]); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type fakeRepo struct {
|
||||
rows map[string]*domain.Attachment // by id
|
||||
refs map[string]bool
|
||||
}
|
||||
|
||||
func (f *fakeRepo) StreamByKey(_ context.Context, _ string, fn func(*domain.Attachment) error) error {
|
||||
list := make([]*domain.Attachment, 0, len(f.rows))
|
||||
for _, a := range f.rows {
|
||||
c := *a
|
||||
list = append(list, &c)
|
||||
}
|
||||
sort.Slice(list, func(i, j int) bool { return list[i].ObjectKey < list[j].ObjectKey })
|
||||
for _, a := range list {
|
||||
if err := fn(a); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func (f *fakeRepo) HasKey(_ context.Context, key string) (bool, error) {
|
||||
for _, a := range f.rows {
|
||||
if a.ObjectKey == key {
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
func (f *fakeRepo) DeleteStaleIntent(_ context.Context, id string, cutoff time.Time) (bool, error) {
|
||||
a, ok := f.rows[id]
|
||||
if !ok || a.Status != domain.AttachmentUploading || !a.UpdatedAt.Before(cutoff) {
|
||||
return false, nil
|
||||
}
|
||||
delete(f.rows, id)
|
||||
return true, nil
|
||||
}
|
||||
func (f *fakeRepo) SetStatus(_ context.Context, id, from, to string) (bool, error) {
|
||||
a, ok := f.rows[id]
|
||||
if !ok || a.Status != from {
|
||||
return false, nil
|
||||
}
|
||||
a.Status = to
|
||||
a.UpdatedAt = now
|
||||
return true, nil
|
||||
}
|
||||
func (f *fakeRepo) DeleteUnreferencedMissing(_ context.Context, id string, cutoff time.Time) (bool, error) {
|
||||
a, ok := f.rows[id]
|
||||
if !ok || a.Status != domain.AttachmentMissing || !a.UpdatedAt.Before(cutoff) || f.refs[id] {
|
||||
return false, nil
|
||||
}
|
||||
delete(f.rows, id)
|
||||
return true, nil
|
||||
}
|
||||
func (f *fakeRepo) IsReferenced(_ context.Context, id string) (bool, error) { return f.refs[id], nil }
|
||||
|
||||
var now = time.Date(2026, 10, 30, 12, 0, 0, 0, time.UTC)
|
||||
|
||||
const grace = time.Hour
|
||||
|
||||
func old() time.Time { return now.Add(-2 * grace) }
|
||||
func fresh() time.Time { return now.Add(-time.Minute) }
|
||||
|
||||
func setup() (*fakeRepo, *fakeStore) {
|
||||
r := &fakeRepo{rows: map[string]*domain.Attachment{}, refs: map[string]bool{}}
|
||||
s := &fakeStore{objs: map[string]storage.Object{}}
|
||||
row := func(id, status string, size int64, at time.Time) {
|
||||
r.rows[id] = &domain.Attachment{ID: id, ObjectKey: "a/c/" + id, Status: status, SizeBytes: size, UpdatedAt: at}
|
||||
}
|
||||
obj := func(id string, size int64, at time.Time) {
|
||||
s.objs["a/c/"+id] = storage.Object{Key: "a/c/" + id, Size: size, LastModified: at}
|
||||
}
|
||||
// согласованное
|
||||
row("ok", domain.AttachmentReady, 10, old())
|
||||
obj("ok", 10, old())
|
||||
// брошенная загрузка с объектом и без
|
||||
row("stale1", domain.AttachmentUploading, 0, old())
|
||||
obj("stale1", 7, old())
|
||||
row("stale2", domain.AttachmentUploading, 0, old())
|
||||
// загрузка в процессе — не трогать
|
||||
row("inflight", domain.AttachmentUploading, 0, fresh())
|
||||
obj("inflight", 5, fresh())
|
||||
// сирота старая и свежая
|
||||
obj("orphan", 3, old())
|
||||
obj("orphan-fresh", 3, fresh())
|
||||
// ready без объекта: используемое и нет
|
||||
row("lost-used", domain.AttachmentReady, 9, old())
|
||||
r.refs["lost-used"] = true
|
||||
row("lost-free", domain.AttachmentReady, 9, old())
|
||||
// подменённый размер
|
||||
row("resized", domain.AttachmentReady, 10, old())
|
||||
obj("resized", 11, old())
|
||||
return r, s
|
||||
}
|
||||
|
||||
func run(t *testing.T, r *fakeRepo, s *fakeStore, apply bool, at time.Time) *Report {
|
||||
t.Helper()
|
||||
rep, err := Run(context.Background(), r, s, Options{Prefix: "a/", Grace: grace, Apply: apply, Now: at})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(rep.Errors) > 0 {
|
||||
t.Fatalf("errors: %v", rep.Errors)
|
||||
}
|
||||
return rep
|
||||
}
|
||||
|
||||
func TestDryRunChangesNothing(t *testing.T) {
|
||||
r, s := setup()
|
||||
nRows, nObjs := len(r.rows), len(s.objs)
|
||||
rep := run(t, r, s, false, now)
|
||||
if len(r.rows) != nRows || len(s.objs) != nObjs || r.rows["lost-free"].Status != domain.AttachmentReady {
|
||||
t.Fatal("dry-run mutated state")
|
||||
}
|
||||
want := map[string]int{"stale_intent": 2, "orphan_object": 1, "missing_object": 2, "size_mismatch": 1}
|
||||
for k, v := range want {
|
||||
if rep.Found[k] != v {
|
||||
t.Errorf("found[%s]=%d want %d (%v)", k, rep.Found[k], v, rep.Found)
|
||||
}
|
||||
}
|
||||
if len(rep.Fixed) != 0 {
|
||||
t.Errorf("dry-run fixed %v", rep.Fixed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyFixesAndIsIdempotent(t *testing.T) {
|
||||
r, s := setup()
|
||||
run(t, r, s, true, now)
|
||||
|
||||
for _, id := range []string{"stale1", "stale2"} {
|
||||
if _, ok := r.rows[id]; ok {
|
||||
t.Errorf("stale intent %s not removed", id)
|
||||
}
|
||||
}
|
||||
if _, ok := s.objs["a/c/stale1"]; ok {
|
||||
t.Error("stale intent object not removed")
|
||||
}
|
||||
if _, ok := s.objs["a/c/orphan"]; ok {
|
||||
t.Error("old orphan not removed")
|
||||
}
|
||||
// Свежие — никогда.
|
||||
if _, ok := s.objs["a/c/orphan-fresh"]; !ok {
|
||||
t.Error("fresh orphan deleted")
|
||||
}
|
||||
if _, ok := r.rows["inflight"]; !ok {
|
||||
t.Error("in-flight intent deleted")
|
||||
}
|
||||
if _, ok := s.objs["a/c/inflight"]; !ok {
|
||||
t.Error("in-flight object deleted")
|
||||
}
|
||||
for _, id := range []string{"lost-used", "lost-free", "resized"} {
|
||||
if r.rows[id].Status != domain.AttachmentMissing {
|
||||
t.Errorf("%s not quarantined", id)
|
||||
}
|
||||
}
|
||||
if r.rows["ok"].Status != domain.AttachmentReady {
|
||||
t.Error("consistent row touched")
|
||||
}
|
||||
|
||||
// Повтор сразу: исправлять нечего, карантин ещё моложе grace.
|
||||
rep := run(t, r, s, true, now)
|
||||
if len(rep.Fixed) != 0 {
|
||||
t.Errorf("second pass fixed %v", rep.Fixed)
|
||||
}
|
||||
|
||||
// Через grace: неиспользуемая запись карантина удаляется, используемая остаётся.
|
||||
later := now.Add(2 * grace)
|
||||
run(t, r, s, true, later)
|
||||
if _, ok := r.rows["lost-free"]; ok {
|
||||
t.Error("unreferenced missing row not purged")
|
||||
}
|
||||
if _, ok := r.rows["lost-used"]; !ok {
|
||||
t.Error("referenced missing row deleted")
|
||||
}
|
||||
if _, ok := s.objs["a/c/orphan-fresh"]; ok {
|
||||
t.Error("orphan not removed after grace")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRestoredObjectLeavesQuarantine(t *testing.T) {
|
||||
r, s := setup()
|
||||
run(t, r, s, true, now)
|
||||
// Объект вернули (например, из бэкапа) с правильным размером.
|
||||
s.objs["a/c/lost-used"] = storage.Object{Key: "a/c/lost-used", Size: 9, LastModified: now}
|
||||
rep := run(t, r, s, true, now)
|
||||
if rep.Fixed["restored"] != 1 || r.rows["lost-used"].Status != domain.AttachmentReady {
|
||||
t.Errorf("restore failed: %v %s", rep.Fixed, r.rows["lost-used"].Status)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,242 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"github.com/charawein/mtusi2026lab6/internal/domain"
|
||||
"github.com/charawein/mtusi2026lab6/internal/repository"
|
||||
)
|
||||
|
||||
type AttachmentRepository struct {
|
||||
pool *pgxpool.Pool
|
||||
}
|
||||
|
||||
func NewAttachmentRepository(pool *pgxpool.Pool) *AttachmentRepository {
|
||||
return &AttachmentRepository{pool: pool}
|
||||
}
|
||||
|
||||
const attachmentColumns = `id, chat_id, owner_id, filename, object_key, status,
|
||||
COALESCE(content_type, ''), COALESCE(size_bytes, 0), COALESCE(sha256, ''), created_at, updated_at,
|
||||
COALESCE(error_code, ''), pipeline_version, COALESCE(staging_key, '')`
|
||||
|
||||
func scanAttachment(row pgx.Row) (*domain.Attachment, error) {
|
||||
var a domain.Attachment
|
||||
err := row.Scan(&a.ID, &a.ChatID, &a.OwnerID, &a.Filename, &a.ObjectKey, &a.Status,
|
||||
&a.ContentType, &a.SizeBytes, &a.SHA256, &a.CreatedAt, &a.UpdatedAt,
|
||||
&a.ErrorCode, &a.PipelineVersion, &a.StagingKey)
|
||||
if err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, repository.ErrNotFound
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
return &a, nil
|
||||
}
|
||||
|
||||
// CreateIntent записывает намерение загрузки (status=uploading) ДО записи в S3.
|
||||
func (r *AttachmentRepository) CreateIntent(ctx context.Context, a *domain.Attachment) error {
|
||||
const q = `
|
||||
INSERT INTO attachments (id, chat_id, owner_id, filename, object_key, staging_key, status, created_at, updated_at)
|
||||
VALUES ($1::uuid, $2::uuid, $3::uuid, $4, $5, $5, 'uploading', $6, $6)
|
||||
`
|
||||
if _, err := r.pool.Exec(ctx, q, a.ID, a.ChatID, a.OwnerID, a.Filename, a.ObjectKey, a.CreatedAt); err != nil {
|
||||
return fmt.Errorf("insert attachment intent: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// MarkQueued — точка 202. В ОДНОЙ транзакции: метаданные проверенного оригинала
|
||||
// и status=queued, логическое задание (jobs) и сообщение для broker (outbox).
|
||||
// После COMMIT работа не теряется: даже если broker недоступен или relay упал,
|
||||
// outbox будет опубликован позже. Если строки uploading уже нет — ErrNotFound.
|
||||
func (r *AttachmentRepository) MarkQueued(ctx context.Context, a *domain.Attachment, pipelineVersion, maxAttempts int) (string, error) {
|
||||
tx, err := r.pool.Begin(ctx)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("begin tx: %w", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback(ctx) }()
|
||||
const q = `
|
||||
UPDATE attachments
|
||||
SET status = 'queued', content_type = $2, size_bytes = $3, sha256 = $4, updated_at = $5
|
||||
WHERE id = $1::uuid AND status = 'uploading'
|
||||
`
|
||||
tag, err := tx.Exec(ctx, q, a.ID, a.ContentType, a.SizeBytes, a.SHA256, a.UpdatedAt)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("mark attachment queued: %w", err)
|
||||
}
|
||||
if tag.RowsAffected() != 1 {
|
||||
return "", repository.ErrNotFound
|
||||
}
|
||||
jobID, err := EnqueueJobTx(ctx, tx, a.ID, pipelineVersion, maxAttempts, "upload")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return "", fmt.Errorf("commit tx: %w", err)
|
||||
}
|
||||
return jobID, nil
|
||||
}
|
||||
|
||||
// Variants — варианты указанной версии pipeline.
|
||||
func (r *AttachmentRepository) Variants(ctx context.Context, attachmentID string, version int) ([]domain.Variant, error) {
|
||||
const q = `
|
||||
SELECT name, content_type, size_bytes, width, height, sha256, object_key, pipeline_version
|
||||
FROM attachment_variants
|
||||
WHERE attachment_id = $1::uuid AND pipeline_version = $2
|
||||
ORDER BY CASE name WHEN 'thumbnail' THEN 1 WHEN 'crop' THEN 2 ELSE 3 END
|
||||
`
|
||||
rows, err := r.pool.Query(ctx, q, attachmentID, version)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("select variants: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
out := []domain.Variant{}
|
||||
for rows.Next() {
|
||||
var v domain.Variant
|
||||
if err := rows.Scan(&v.Name, &v.ContentType, &v.SizeBytes, &v.Width, &v.Height, &v.SHA256, &v.ObjectKey, &v.PipelineVersion); err != nil {
|
||||
return nil, fmt.Errorf("scan variant: %w", err)
|
||||
}
|
||||
out = append(out, v)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// DeleteIntent убирает намерение, если загрузка не удалась (компенсация).
|
||||
func (r *AttachmentRepository) DeleteIntent(ctx context.Context, id string) error {
|
||||
const q = `DELETE FROM attachments WHERE id = $1::uuid AND status = 'uploading'`
|
||||
if _, err := r.pool.Exec(ctx, q, id); err != nil {
|
||||
return fmt.Errorf("delete attachment intent: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetForMember возвращает вложение, если пользователь — участник его чата.
|
||||
// Чужое и несуществующее вложение неразличимы (ErrNotFound).
|
||||
func (r *AttachmentRepository) GetForMember(ctx context.Context, id, userID string) (*domain.Attachment, error) {
|
||||
q := `SELECT ` + attachmentColumns + `
|
||||
FROM attachments a
|
||||
WHERE a.id = $1::uuid
|
||||
AND EXISTS (SELECT 1 FROM chat_members m WHERE m.chat_id = a.chat_id AND m.user_id = $2::uuid)`
|
||||
a, err := scanAttachment(r.pool.QueryRow(ctx, q, id, userID))
|
||||
if err != nil && !errors.Is(err, repository.ErrNotFound) {
|
||||
return nil, fmt.Errorf("select attachment: %w", err)
|
||||
}
|
||||
return a, err
|
||||
}
|
||||
|
||||
// --- для reconciler ---
|
||||
|
||||
// StreamByKey отдаёт все известные БД ключи объектов в байтовом порядке (как
|
||||
// ListObjectsV2): оригинал вложения (object_key), отдельный staging-объект и
|
||||
// объекты вариантов. Для staging и вариантов Status — «staging»/«variant», чтобы
|
||||
// reconciler не путал их с оригиналом.
|
||||
func (r *AttachmentRepository) StreamByKey(ctx context.Context, prefix string, fn func(*domain.Attachment) error) error {
|
||||
const q = `
|
||||
SELECT key, id::text, status, size_bytes, updated_at FROM (
|
||||
SELECT object_key AS key, id, status, COALESCE(size_bytes, 0) AS size_bytes, updated_at FROM attachments
|
||||
UNION ALL
|
||||
SELECT staging_key, id, 'staging', COALESCE(size_bytes, 0), updated_at FROM attachments
|
||||
WHERE staging_key IS NOT NULL AND staging_key <> object_key
|
||||
UNION ALL
|
||||
SELECT object_key, attachment_id, 'variant', size_bytes, created_at FROM attachment_variants
|
||||
) k
|
||||
WHERE starts_with(key, $1)
|
||||
ORDER BY key COLLATE "C"`
|
||||
rows, err := r.pool.Query(ctx, q, prefix)
|
||||
if err != nil {
|
||||
return fmt.Errorf("stream object keys: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var a domain.Attachment
|
||||
if err := rows.Scan(&a.ObjectKey, &a.ID, &a.Status, &a.SizeBytes, &a.UpdatedAt); err != nil {
|
||||
return fmt.Errorf("scan object key: %w", err)
|
||||
}
|
||||
if err := fn(&a); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return rows.Err()
|
||||
}
|
||||
|
||||
// HasKey — есть ли строка с этим object_key (повторная проверка перед удалением объекта).
|
||||
func (r *AttachmentRepository) HasKey(ctx context.Context, key string) (bool, error) {
|
||||
var ok bool
|
||||
err := r.pool.QueryRow(ctx, `SELECT EXISTS (SELECT 1 FROM attachments WHERE object_key = $1 OR staging_key = $1)
|
||||
OR EXISTS (SELECT 1 FROM attachment_variants WHERE object_key = $1)`, key).Scan(&ok)
|
||||
return ok, err
|
||||
}
|
||||
|
||||
// DeleteStaleIntent удаляет намерение, только если оно всё ещё uploading и старше cutoff.
|
||||
func (r *AttachmentRepository) DeleteStaleIntent(ctx context.Context, id string, cutoff time.Time) (bool, error) {
|
||||
const q = `DELETE FROM attachments WHERE id = $1::uuid AND status = 'uploading' AND updated_at < $2`
|
||||
tag, err := r.pool.Exec(ctx, q, id, cutoff)
|
||||
return tag.RowsAffected() == 1, err
|
||||
}
|
||||
|
||||
// SetStatus меняет статус ready↔missing (карантин и восстановление).
|
||||
func (r *AttachmentRepository) SetStatus(ctx context.Context, id, from, to string) (bool, error) {
|
||||
const q = `UPDATE attachments SET status = $3, updated_at = now() WHERE id = $1::uuid AND status = $2`
|
||||
tag, err := r.pool.Exec(ctx, q, id, from, to)
|
||||
return tag.RowsAffected() == 1, err
|
||||
}
|
||||
|
||||
// DeleteUnreferencedMissing удаляет запись карантина старше cutoff, если на неё
|
||||
// не ссылается ни одно сообщение. Блокировка строки (FOR UPDATE) конфликтует с
|
||||
// FOR SHARE при создании сообщения, поэтому гонки «удалили, пока прикрепляли» нет.
|
||||
func (r *AttachmentRepository) DeleteUnreferencedMissing(ctx context.Context, id string, cutoff time.Time) (bool, error) {
|
||||
tx, err := r.pool.Begin(ctx)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
defer func() { _ = tx.Rollback(ctx) }()
|
||||
var locked string
|
||||
err = tx.QueryRow(ctx, `SELECT id FROM attachments WHERE id = $1::uuid AND status = 'missing' AND updated_at < $2 FOR UPDATE`, id, cutoff).Scan(&locked)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
var referenced bool
|
||||
if err := tx.QueryRow(ctx, `SELECT EXISTS (SELECT 1 FROM message_attachments WHERE attachment_id = $1::uuid)`, id).Scan(&referenced); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if referenced {
|
||||
return false, nil
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `DELETE FROM attachments WHERE id = $1::uuid`, id); err != nil {
|
||||
return false, err
|
||||
}
|
||||
return true, tx.Commit(ctx)
|
||||
}
|
||||
|
||||
// IsReferenced — используется ли вложение каким-либо сообщением.
|
||||
func (r *AttachmentRepository) IsReferenced(ctx context.Context, id string) (bool, error) {
|
||||
var ok bool
|
||||
err := r.pool.QueryRow(ctx, `SELECT EXISTS (SELECT 1 FROM message_attachments WHERE attachment_id = $1::uuid)`, id).Scan(&ok)
|
||||
return ok, err
|
||||
}
|
||||
|
||||
// TryLock — advisory lock, чтобы одновременно работал один reconciler.
|
||||
func (r *AttachmentRepository) TryLock(ctx context.Context) (func(), bool, error) {
|
||||
conn, err := r.pool.Acquire(ctx)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
var ok bool
|
||||
if err := conn.QueryRow(ctx, `SELECT pg_try_advisory_lock(4242004)`).Scan(&ok); err != nil || !ok {
|
||||
conn.Release()
|
||||
return nil, false, err
|
||||
}
|
||||
return func() {
|
||||
_, _ = conn.Exec(context.Background(), `SELECT pg_advisory_unlock(4242004)`)
|
||||
conn.Release()
|
||||
}, true, nil
|
||||
}
|
||||
@@ -0,0 +1,180 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"github.com/charawein/mtusi2026lab6/internal/domain"
|
||||
"github.com/charawein/mtusi2026lab6/internal/repository"
|
||||
)
|
||||
|
||||
type ChatRepository struct {
|
||||
pool *pgxpool.Pool
|
||||
}
|
||||
|
||||
func NewChatRepository(pool *pgxpool.Pool) *ChatRepository {
|
||||
return &ChatRepository{pool: pool}
|
||||
}
|
||||
|
||||
func (r *ChatRepository) Create(ctx context.Context, chat *domain.Chat) error {
|
||||
tx, err := r.pool.Begin(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("begin tx: %w", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback(ctx) }()
|
||||
|
||||
const insertChat = `INSERT INTO chats (id, title, created_at) VALUES ($1::uuid, $2, $3)`
|
||||
if _, err := tx.Exec(ctx, insertChat, chat.ID, chat.Title, chat.CreatedAt); err != nil {
|
||||
return fmt.Errorf("insert chat: %w", err)
|
||||
}
|
||||
|
||||
const insertMember = `INSERT INTO chat_members (chat_id, user_id) VALUES ($1::uuid, $2::uuid)`
|
||||
for _, uid := range chat.MemberIDs {
|
||||
if _, err := tx.Exec(ctx, insertMember, chat.ID, uid); err != nil {
|
||||
return fmt.Errorf("insert member %s: %w", uid, err)
|
||||
}
|
||||
}
|
||||
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return fmt.Errorf("commit tx: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *ChatRepository) GetByID(ctx context.Context, id string) (*domain.Chat, error) {
|
||||
const q = `SELECT id, title, created_at FROM chats WHERE id = $1::uuid`
|
||||
var c domain.Chat
|
||||
err := r.pool.QueryRow(ctx, q, id).Scan(&c.ID, &c.Title, &c.CreatedAt)
|
||||
if err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, repository.ErrNotFound
|
||||
}
|
||||
return nil, fmt.Errorf("select chat: %w", err)
|
||||
}
|
||||
|
||||
const qMembers = `SELECT user_id FROM chat_members WHERE chat_id = $1::uuid ORDER BY user_id`
|
||||
rows, err := r.pool.Query(ctx, qMembers, id)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("select members: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
c.MemberIDs = []string{}
|
||||
for rows.Next() {
|
||||
var uid string
|
||||
if err := rows.Scan(&uid); err != nil {
|
||||
return nil, fmt.Errorf("scan member: %w", err)
|
||||
}
|
||||
c.MemberIDs = append(c.MemberIDs, uid)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("iterate members: %w", err)
|
||||
}
|
||||
return &c, nil
|
||||
}
|
||||
|
||||
func (r *ChatRepository) IsMember(ctx context.Context, chatID, userID string) (bool, error) {
|
||||
const q = `SELECT 1 FROM chat_members WHERE chat_id = $1::uuid AND user_id = $2::uuid`
|
||||
var one int
|
||||
err := r.pool.QueryRow(ctx, q, chatID, userID).Scan(&one)
|
||||
if err == nil {
|
||||
return true, nil
|
||||
}
|
||||
if !errors.Is(err, pgx.ErrNoRows) {
|
||||
return false, fmt.Errorf("check membership: %w", err)
|
||||
}
|
||||
const qChat = `SELECT 1 FROM chats WHERE id = $1::uuid`
|
||||
err = r.pool.QueryRow(ctx, qChat, chatID).Scan(&one)
|
||||
if err == nil {
|
||||
return false, nil
|
||||
}
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return false, repository.ErrNotFound
|
||||
}
|
||||
return false, fmt.Errorf("check chat exists: %w", err)
|
||||
}
|
||||
|
||||
// MemberVersion одним запросом возвращает версию истории чата и признак членства.
|
||||
// Несуществующий чат — ErrNotFound.
|
||||
func (r *ChatRepository) MemberVersion(ctx context.Context, chatID, userID string) (int64, bool, error) {
|
||||
const q = `
|
||||
SELECT c.version,
|
||||
EXISTS (SELECT 1 FROM chat_members m WHERE m.chat_id = c.id AND m.user_id = $2::uuid)
|
||||
FROM chats c
|
||||
WHERE c.id = $1::uuid
|
||||
`
|
||||
var version int64
|
||||
var member bool
|
||||
err := r.pool.QueryRow(ctx, q, chatID, userID).Scan(&version, &member)
|
||||
if err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return 0, false, repository.ErrNotFound
|
||||
}
|
||||
return 0, false, fmt.Errorf("select chat version: %w", err)
|
||||
}
|
||||
return version, member, nil
|
||||
}
|
||||
|
||||
func (r *ChatRepository) ListByMember(ctx context.Context, userID string, cursor time.Time, cursorID string, limit int) ([]*domain.Chat, error) {
|
||||
const qChats = `
|
||||
SELECT c.id, c.title, c.created_at
|
||||
FROM chats c
|
||||
JOIN chat_members m ON m.chat_id = c.id
|
||||
WHERE m.user_id = $1::uuid
|
||||
AND (c.created_at, c.id) > ($2::timestamptz, $3::uuid)
|
||||
ORDER BY c.created_at, c.id
|
||||
LIMIT $4
|
||||
`
|
||||
rows, err := r.pool.Query(ctx, qChats, userID, cursor, cursorID, limit)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list chats: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
chats := []*domain.Chat{}
|
||||
var chatIDs []string
|
||||
for rows.Next() {
|
||||
var c domain.Chat
|
||||
if err := rows.Scan(&c.ID, &c.Title, &c.CreatedAt); err != nil {
|
||||
return nil, fmt.Errorf("scan chat: %w", err)
|
||||
}
|
||||
c.MemberIDs = []string{}
|
||||
chats = append(chats, &c)
|
||||
chatIDs = append(chatIDs, c.ID)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("iterate chats: %w", err)
|
||||
}
|
||||
if len(chats) == 0 {
|
||||
return chats, nil
|
||||
}
|
||||
|
||||
const qMembers = `SELECT chat_id, user_id FROM chat_members WHERE chat_id = ANY($1::uuid[]) ORDER BY chat_id, user_id`
|
||||
memberRows, err := r.pool.Query(ctx, qMembers, chatIDs)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list members: %w", err)
|
||||
}
|
||||
defer memberRows.Close()
|
||||
|
||||
byChat := make(map[string][]string)
|
||||
for memberRows.Next() {
|
||||
var chatID, uid string
|
||||
if err := memberRows.Scan(&chatID, &uid); err != nil {
|
||||
return nil, fmt.Errorf("scan member: %w", err)
|
||||
}
|
||||
byChat[chatID] = append(byChat[chatID], uid)
|
||||
}
|
||||
if err := memberRows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("iterate members: %w", err)
|
||||
}
|
||||
for _, c := range chats {
|
||||
if members, ok := byChat[c.ID]; ok {
|
||||
c.MemberIDs = members
|
||||
}
|
||||
}
|
||||
return chats, nil
|
||||
}
|
||||
@@ -0,0 +1,201 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"github.com/charawein/mtusi2026lab6/internal/domain"
|
||||
"github.com/charawein/mtusi2026lab6/internal/repository"
|
||||
)
|
||||
|
||||
type MessageRepository struct {
|
||||
pool *pgxpool.Pool
|
||||
}
|
||||
|
||||
func NewMessageRepository(pool *pgxpool.Pool) *MessageRepository {
|
||||
return &MessageRepository{pool: pool}
|
||||
}
|
||||
|
||||
func (r *MessageRepository) Create(ctx context.Context, msg *domain.Message) error {
|
||||
const q = `
|
||||
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
|
||||
VALUES ($1::uuid, $2::uuid, $3::uuid, $4, $5)
|
||||
`
|
||||
if _, err := r.pool.Exec(ctx, q, msg.ID, msg.ChatID, msg.SenderID, msg.Text, msg.CreatedAt); err != nil {
|
||||
return fmt.Errorf("insert message: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// CreateIdempotent вставляет сообщение и ключ в одной транзакции.
|
||||
// Конкурентные запросы с одним ключом сериализуются уникальным индексом
|
||||
// idempotency_keys: второй INSERT ждёт завершения первой транзакции, после
|
||||
// COMMIT получает конфликт (DO NOTHING), откатывает своё сообщение и читает
|
||||
// уже сохранённое. Если первая транзакция откатилась (транзиентная ошибка),
|
||||
// вторая просто продолжает — неуспешный результат не фиксируется.
|
||||
func (r *MessageRepository) CreateIdempotent(ctx context.Context, msg *domain.Message, key, requestHash string, ttl time.Duration) (*domain.Message, bool, error) {
|
||||
tx, err := r.pool.Begin(ctx)
|
||||
if err != nil {
|
||||
return nil, false, fmt.Errorf("begin tx: %w", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback(ctx) }()
|
||||
|
||||
// Ключ старше TTL больше не защищает от повтора — освобождаем его.
|
||||
const deleteExpired = `
|
||||
DELETE FROM idempotency_keys
|
||||
WHERE user_id = $1::uuid AND chat_id = $2::uuid AND key = $3
|
||||
AND created_at < now() - make_interval(secs => $4)
|
||||
`
|
||||
if _, err := tx.Exec(ctx, deleteExpired, msg.SenderID, msg.ChatID, key, ttl.Seconds()); err != nil {
|
||||
return nil, false, fmt.Errorf("delete expired key: %w", err)
|
||||
}
|
||||
|
||||
const insertMsg = `
|
||||
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
|
||||
VALUES ($1::uuid, $2::uuid, $3::uuid, $4, $5)
|
||||
`
|
||||
if _, err := tx.Exec(ctx, insertMsg, msg.ID, msg.ChatID, msg.SenderID, msg.Text, msg.CreatedAt); err != nil {
|
||||
return nil, false, fmt.Errorf("insert message: %w", err)
|
||||
}
|
||||
|
||||
if len(msg.AttachmentIDs) > 0 {
|
||||
if err := attachToMessage(ctx, tx, msg); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
}
|
||||
|
||||
const insertKey = `
|
||||
INSERT INTO idempotency_keys (user_id, chat_id, key, request_hash, message_id)
|
||||
VALUES ($1::uuid, $2::uuid, $3, $4, $5::uuid)
|
||||
ON CONFLICT (user_id, chat_id, key) DO NOTHING
|
||||
`
|
||||
tag, err := tx.Exec(ctx, insertKey, msg.SenderID, msg.ChatID, key, requestHash, msg.ID)
|
||||
if err != nil {
|
||||
return nil, false, fmt.Errorf("insert idempotency key: %w", err)
|
||||
}
|
||||
if tag.RowsAffected() == 1 {
|
||||
// Версия истории растёт атомарно с сообщением: после COMMIT любое чтение
|
||||
// увидит новую версию и не попадёт в кэш, записанный до этого сообщения.
|
||||
const bumpVersion = `UPDATE chats SET version = version + 1 WHERE id = $1::uuid`
|
||||
if _, err := tx.Exec(ctx, bumpVersion, msg.ChatID); err != nil {
|
||||
return nil, false, fmt.Errorf("bump chat version: %w", err)
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return nil, false, fmt.Errorf("commit tx: %w", err)
|
||||
}
|
||||
return msg, true, nil
|
||||
}
|
||||
|
||||
// Ключ уже занят подтверждённой записью: наше сообщение не сохраняем.
|
||||
_ = tx.Rollback(ctx)
|
||||
|
||||
const selectStored = `
|
||||
SELECT k.request_hash, m.id, m.chat_id, m.sender_id, m.text, m.created_at,
|
||||
` + attachmentIDsExpr + `
|
||||
FROM idempotency_keys k
|
||||
JOIN messages m ON m.id = k.message_id
|
||||
WHERE k.user_id = $1::uuid AND k.chat_id = $2::uuid AND k.key = $3
|
||||
`
|
||||
var storedHash string
|
||||
var m domain.Message
|
||||
err = r.pool.QueryRow(ctx, selectStored, msg.SenderID, msg.ChatID, key).
|
||||
Scan(&storedHash, &m.ID, &m.ChatID, &m.SenderID, &m.Text, &m.CreatedAt, &m.AttachmentIDs)
|
||||
if err != nil {
|
||||
return nil, false, fmt.Errorf("select stored message: %w", err)
|
||||
}
|
||||
if storedHash != requestHash {
|
||||
return nil, false, repository.ErrConflict
|
||||
}
|
||||
return &m, false, nil
|
||||
}
|
||||
|
||||
// DeleteExpiredKeys удаляет ключи старше TTL. Безопасно выполнять с любой реплики.
|
||||
func (r *MessageRepository) DeleteExpiredKeys(ctx context.Context, ttl time.Duration) (int64, error) {
|
||||
const q = `DELETE FROM idempotency_keys WHERE created_at < now() - make_interval(secs => $1)`
|
||||
tag, err := r.pool.Exec(ctx, q, ttl.Seconds())
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("delete expired keys: %w", err)
|
||||
}
|
||||
return tag.RowsAffected(), nil
|
||||
}
|
||||
|
||||
func (r *MessageRepository) ListByChat(ctx context.Context, chatID string, cursor time.Time, cursorID string, limit int) ([]*domain.Message, error) {
|
||||
q := `
|
||||
SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at, ` + attachmentIDsExpr + `
|
||||
FROM messages m
|
||||
WHERE m.chat_id = $1::uuid
|
||||
AND (m.created_at, m.id) > ($2::timestamptz, $3::uuid)
|
||||
ORDER BY m.created_at, m.id
|
||||
LIMIT $4
|
||||
`
|
||||
rows, err := r.pool.Query(ctx, q, chatID, cursor, cursorID, limit)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list messages: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
msgs := []*domain.Message{}
|
||||
for rows.Next() {
|
||||
var m domain.Message
|
||||
if err := rows.Scan(&m.ID, &m.ChatID, &m.SenderID, &m.Text, &m.CreatedAt, &m.AttachmentIDs); err != nil {
|
||||
return nil, fmt.Errorf("scan message: %w", err)
|
||||
}
|
||||
msgs = append(msgs, &m)
|
||||
}
|
||||
return msgs, rows.Err()
|
||||
}
|
||||
|
||||
// attachmentIDsExpr — id вложений сообщения m в порядке отправки (всегда массив).
|
||||
const attachmentIDsExpr = `ARRAY(SELECT ma.attachment_id::text FROM message_attachments ma
|
||||
WHERE ma.message_id = m.id ORDER BY ma.position)`
|
||||
|
||||
// attachToMessage проверяет и привязывает вложения в транзакции сообщения.
|
||||
// FOR SHARE блокирует строки вложений до COMMIT: reconciler (FOR UPDATE) не
|
||||
// удалит вложение, которое сейчас прикрепляют. Вложение должно быть из того же
|
||||
// чата и в статусе ready; чужое, неизвестное и незавершённое — ErrNotFound.
|
||||
func attachToMessage(ctx context.Context, tx pgx.Tx, msg *domain.Message) error {
|
||||
const q = `
|
||||
SELECT id::text, status FROM attachments
|
||||
WHERE id = ANY($1::uuid[]) AND chat_id = $2::uuid
|
||||
FOR SHARE
|
||||
`
|
||||
rows, err := tx.Query(ctx, q, msg.AttachmentIDs, msg.ChatID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("select attachments: %w", err)
|
||||
}
|
||||
status := map[string]string{}
|
||||
for rows.Next() {
|
||||
var id, st string
|
||||
if err := rows.Scan(&id, &st); err != nil {
|
||||
rows.Close()
|
||||
return fmt.Errorf("scan attachment: %w", err)
|
||||
}
|
||||
status[id] = st
|
||||
}
|
||||
rows.Close()
|
||||
if err := rows.Err(); err != nil {
|
||||
return fmt.Errorf("select attachments: %w", err)
|
||||
}
|
||||
for _, id := range msg.AttachmentIDs {
|
||||
switch status[id] {
|
||||
case domain.AttachmentReady, domain.AttachmentQueued, domain.AttachmentProcessing:
|
||||
// с lab5 можно ссылаться на ещё обрабатываемое вложение
|
||||
case domain.AttachmentFailed, domain.AttachmentMissing:
|
||||
return repository.ErrConflict
|
||||
default:
|
||||
return repository.ErrNotFound // чужое, неизвестное, не принятое (uploading)
|
||||
}
|
||||
}
|
||||
batch := &pgx.Batch{}
|
||||
for i, id := range msg.AttachmentIDs {
|
||||
batch.Queue(`INSERT INTO message_attachments (message_id, attachment_id, position) VALUES ($1::uuid, $2::uuid, $3)`,
|
||||
msg.ID, id, i)
|
||||
}
|
||||
if err := tx.SendBatch(ctx, batch).Close(); err != nil {
|
||||
return fmt.Errorf("insert message attachments: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user