ЛР 6 — БВТ2303 — Вейнмайер — на 5 #63

Open
Weinmaier_Andrew wants to merge 9 commits from Weinmaier_Andrew/mtusi2026lab6:solution/bvt2303-weinmaier into main
180 changed files with 19382 additions and 21 deletions
+2
View File
@@ -18,3 +18,5 @@ __pycache__
node_modules
evidence
.DS_Store
bin
experiments
+26 -2
View File
@@ -7,7 +7,7 @@ SESSION_TTL_SECONDS=3600
# Choose credentials locally. File paths contain no secret values.
DATABASE_URL_FILE=/run/secrets/database_url
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
SESSION_STORE=postgres
SESSION_STORE=redis
CACHE_URL_FILE=/run/secrets/cache_url
API_REPLICAS=2
IDEMPOTENCY_TTL_SECONDS=86400
@@ -20,11 +20,35 @@ MAX_UPLOAD_BYTES=10485760
MAX_IMAGE_PIXELS=40000000
BROKER_URL_FILE=/run/secrets/broker_url
WORKER_CONCURRENCY=2
# Мягкий лимит кучи Go в worker (≈75% WORKER_MEMORY)
WORKER_GOMEMLIMIT=560MiB
JOB_MAX_ATTEMPTS=5
JOB_TIMEOUT_SECONDS=60
STAGING_RETENTION_SECONDS=86400
# Graceful shutdown: after SIGTERM readiness=503 for DRAIN_DELAY, then Shutdown up to SHUTDOWN_TIMEOUT.
DRAIN_DELAY=3s
SHUTDOWN_TIMEOUT=8s
# Reconciler (PostgreSQL <-> S3): background pass interval and safe grace period.
RECONCILE_INTERVAL=10m
RECONCILE_GRACE=1h
# Pipeline (lab5): published version, retry backoff (exponential with jitter), WebP quality, memory budget.
PIPELINE_VERSION=1
RETRY_BASE_DELAY=2s
RETRY_MAX_DELAY=60s
WEBP_QUALITY=80
WORKER_PIXEL_BUDGET=60000000
WORKER_REPLICAS=1
# --- Наблюдаемость (lab6) ---
OTEL_SERVICE_NAME=messenger-api
# Worker must set a distinct service.name.
# Worker must set a distinct service.name (compose: messenger-worker, relay: messenger-relay).
OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318
OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab
# Трек 1: false/true выключает экспорт; sampler — стандартные переменные SDK.
OTEL_SDK_DISABLED=false
OTEL_TRACES_SAMPLER=parentbased_always_on
OTEL_TRACES_SAMPLER_ARG=1.0
# collector.yaml (все трассы) или collector-tail.yaml (tail sampling)
COLLECTOR_CONFIG=collector.yaml
GRAFANA_PORT=3000
PROMETHEUS_PORT=9090
+133
View File
@@ -0,0 +1,133 @@
# Реальный прогон Docker-стенда lab6 на GitHub-hosted runner.
# Нужен потому, что на машине разработки нет аппаратной виртуализации для Docker Desktop.
# Все результаты — настоящие прогоны; публикуются в ветку lab6-results.
name: lab6 stand
on:
push:
branches: [lab6]
workflow_dispatch:
permissions:
contents: read
concurrency:
group: lab6-stand-${{ github.ref }}
cancel-in-progress: true
env:
PYTHONUTF8: "1"
jobs:
# Unit-тесты и интеграционный fault-injection pipeline (трек 5) на настоящем PostgreSQL.
go-tests:
name: Go unit + интеграционный fault-injection
runs-on: ubuntu-24.04
timeout-minutes: 20
services:
postgres:
image: postgres:16.4-alpine
env:
POSTGRES_USER: it
POSTGRES_PASSWORD: it
POSTGRES_DB: postgres
ports: ["5432:5432"]
options: >-
--health-cmd "pg_isready -U it" --health-interval 2s --health-timeout 2s --health-retries 30
steps:
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
with:
go-version-file: go.mod
- name: cwebp для теста варианта optimized
run: sudo apt-get update -qq && sudo apt-get install -y -qq webp > /dev/null && cwebp -version
- name: go test
env:
TEST_DATABASE_URL: postgres://it:it@localhost:5432/postgres?sslmode=disable
run: |
mkdir -p evidence/track2
go test ./... -count=1 2>&1 | tee evidence/go_test.txt
go test ./internal/worker/ -run Pipeline -count=1 -v 2>&1 | grep -E '^(=== RUN|--- |PASS|FAIL|ok)|FAULT INJECTION|duplicate|lease|stale|noop|attempt failed|early delivery' \
| tee evidence/track2/pipeline_fault_injection.txt
go test ./internal/imaging/ -run OptimizedWebP -count=1 -v 2>&1 | tee evidence/webp_test.txt
go test ./internal/telemetry/ ./internal/transport/http/middleware/ ./internal/worker/ -run 'Telemetry|Trace|Route|Propag' -count=1 -v 2>&1 \
| grep -E '^(=== RUN|--- |PASS|FAIL|ok)' | tee evidence/telemetry_tests.txt
- uses: actions/upload-artifact@v4
if: always()
with:
name: evidence-go
path: evidence/
stand:
name: Стенд — ${{ matrix.part }}
runs-on: ubuntu-24.04
timeout-minutes: 80
strategy:
fail-fast: false
matrix:
part: [base, files, pipeline, obs, track1]
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: "3.12"
- name: Секреты и запуск стенда
shell: bash
run: |
set -o pipefail
mkdir -p evidence
chmod +x scripts/*.sh infra/s3/init.sh
sh scripts/init-secrets.sh
docker compose --env-file .env config --quiet
for img in $(docker compose config --images | sort -u | grep -v -- '-api:local'); do
if docker pull -q "$img" > /dev/null 2>> evidence/pull_${{ matrix.part }}.log; then echo "ok $img"; else echo "FAIL $img"; fi
done | tee -a evidence/pull_${{ matrix.part }}.log
docker compose --env-file .env build 2>&1 | tail -30 > evidence/build_${{ matrix.part }}.log
# Снимок dashboard в CI делается анонимным Viewer; по умолчанию анонимный доступ выключен.
if [ "${{ matrix.part }}" = obs ]; then export GRAFANA_ANONYMOUS=true; fi
docker compose --env-file .env up -d --wait 2>&1 | tee evidence/up_${{ matrix.part }}.log
- name: make check + доказательства (${{ matrix.part }})
run: |
make check PYTHON=python
PYTHON=python sh scripts/collect-evidence.sh ${{ matrix.part }}
- name: Окружение runner
if: always()
run: |
mkdir -p evidence
{ nproc; free -m; docker version --format '{{.Server.Version}}'; docker compose version; } > evidence/runner_${{ matrix.part }}.txt
docker compose ps -a > evidence/docker_ps_final_${{ matrix.part }}.txt || true
for c in $(docker compose ps -aq); do docker inspect -f '{{.Name}} restarts={{.RestartCount}} oom_killed={{.State.OOMKilled}} exit={{.State.ExitCode}}' "$c"; done > evidence/containers_state_${{ matrix.part }}.txt || true
docker compose logs --no-color --tail 300 > evidence/compose_logs_${{ matrix.part }}.log 2>&1 || true
- uses: actions/upload-artifact@v4
if: always()
with:
name: evidence-${{ matrix.part }}
path: evidence/
publish:
name: Публикация результатов в ветку lab6-results
needs: [go-tests, stand]
if: always()
runs-on: ubuntu-24.04
permissions:
contents: write
steps:
- uses: actions/checkout@v4
- uses: actions/download-artifact@v4
with:
path: collected
- name: Commit в lab6-results
run: |
git config user.name "github-actions[bot]"
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
git checkout -q --orphan lab6-results
git rm -rf -q .
{ echo "source_commit=${{ github.sha }}"
echo "run=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
echo "go_tests=${{ needs.go-tests.result }}"
echo "stand=${{ needs.stand.result }}"; } > collected/RUN.txt
cp -r collected/. .
rm -rf collected
git add -A
git commit -q -m "results for ${{ github.sha }}"
git push -f origin lab6-results
+59
View File
@@ -0,0 +1,59 @@
# ============================================================
# Stage 1: builder — компилируем статический бинарник
# ============================================================
FROM golang:1.26-alpine@sha256:27f2d3b83cfc5803c8078f8e730af0f7c431bf067afaec8f60a36fb763187cba AS builder
WORKDIR /src
# Сначала манифесты зависимостей — кэшируем слой.
COPY go.mod go.sum ./
RUN go mod download
# Затем исходники.
COPY . .
# Статический бинарник с оптимизациями:
# CGO_ENABLED=0 — статическая линковка (не требует libc)
# -ldflags="-s -w" — убираем отладочную информацию
# -trimpath — убираем пути сборки из бинарника
RUN CGO_ENABLED=0 GOOS=linux go build \
-ldflags="-s -w -X github.com/charawein/mtusi2026lab6/internal/telemetry.Version=lab6" \
-trimpath \
-o /out/ \
./cmd/api ./cmd/reconcile ./cmd/pipeline
# ============================================================
# Stage 2: runtime — минимальный образ только с бинарником
# ============================================================
FROM alpine:3.20@sha256:d9e853e87e55526f6b2917df91a2115c36dd7c696a35be12163d44e6e2a4b6bc
# ca-certificates — для HTTPS в lab2+
# tzdata — для корректной работы с временными зонами
# libwebp-tools — cwebp для варианта optimized (WebP) в worker.
RUN apk --no-cache add ca-certificates tzdata libwebp-tools
# Непривилегированный пользователь (UID/GID 1001).
RUN addgroup -g 1001 -S appgroup && \
adduser -u 1001 -S appuser -G appgroup
WORKDIR /app
# Копируем бинарник и сразу выставляем владельца.
COPY --from=builder --chown=appuser:appgroup /out/api /app/api
# reconcile — сверка PostgreSQL и S3 (сервис reconciler и ручной запуск).
COPY --from=builder --chown=appuser:appgroup /out/reconcile /app/reconcile
# pipeline — worker, relay (outbox) и операторские команды (replay, failed, reprocess).
COPY --from=builder --chown=appuser:appgroup /out/pipeline /app/pipeline
# Переключаемся на непривилегированного пользователя.
USER 1001:1001
# Информативно: какой порт слушает приложение.
EXPOSE 8080
# Healthcheck: readiness (БД + Redis + не в режиме drain).
HEALTHCHECK --interval=5s --timeout=2s --start-period=5s --retries=3 \
CMD wget --spider -q http://localhost:8080/health/ready || exit 1
# Exec-форма CMD — критично для доставки SIGTERM в приложение.
CMD ["/app/api"]
+31 -1
View File
@@ -4,7 +4,7 @@ CA_FILE ?=
ACTION_SCRIPT ?=
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
.PHONY: help check validate test load test-persistence test-failover
.PHONY: help check validate test load test-persistence test-failover up down test-files test-pipeline test-faults failed replay evidence
help:
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
@@ -28,3 +28,33 @@ test-persistence:
test-failover:
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
# --- Студенческие цели (lab5) ---
up:
sh scripts/init-secrets.sh
docker compose --env-file .env up -d --build --wait
down:
docker compose --env-file .env stop
test-files:
$(PYTHON) student-tests/attachments_negative.py --base-url "$(BASE_URL)"
# Сценарии pipeline на стенде: make test-pipeline SCENARIO=demo (список — student-tests/pipeline_ops.py)
SCENARIO ?= demo
test-pipeline:
$(PYTHON) student-tests/pipeline_ops.py $(SCENARIO) --base-url "$(BASE_URL)"
# Интеграционный fault-injection pipeline (трек 5); нужен PostgreSQL: TEST_DATABASE_URL=postgres://...
test-faults:
go test ./internal/worker/ -run Pipeline -count=1 -v
failed:
docker compose run --rm --no-deps -T --entrypoint /app/pipeline relay failed
# make replay ID=<attachment_id>
replay:
docker compose run --rm --no-deps -T --entrypoint /app/pipeline relay replay $(ID) --actor operator
evidence:
sh scripts/collect-evidence.sh all
+176 -18
View File
@@ -1,37 +1,195 @@
# Отчёт — лабораторная 6
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
- Автор, группа: Вейнмайер Андрей Андреевич, БВТ2303
- Целевая оценка: 5 (уровни 3, 4 и трек 1 «Цена наблюдаемости»)
- Commit/PR: ветка `solution/bvt2303-weinmaier` в https://gitea.privcloud.tech/Weinmaier_Andrew/mtusi2026lab6 (PR не открывался)
- Основа: собственная lab5 (PR mtusi2026/mtusi2026lab5#35) — RabbitMQ, outbox, worker, versioned pipeline; через неё lab3–4
- Использованные источники:
- Контракт: contracts/observability.md, contracts/README.md
- Библиотеки: go.opentelemetry.io/otel v1.47 (SDK traces/metrics/logs, OTLP/HTTP exporters), contrib: otelslog, otelhttp, runtime, host; github.com/exaring/otelpgx; go-redis/extra/redisotel
- Документация: OpenTelemetry Collector (receivers/processors/exporters, memory_limiter, exporterhelper sending_queue, tail_sampling), W3C Trace Context, Prometheus histograms, Loki OTLP ingestion, Tempo TraceQL, Grafana provisioning
- Помощь ИИ: ИИ-ассистент использовался при реализации и отладке; все результаты ниже — реальные прогоны, ссылки на них приведены.
## Запуск из чистого clone
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
| Роль | Сервис | Образ | Хранение |
| --- | --- | --- | --- |
| Collector | `otel-collector` | otel/opentelemetry-collector-contrib:0.116.1 | — (не БД: только приём, обработка, экспорт) |
| Метрики + alerts | `prometheus` | prom/prometheus:v3.1.0 | volume `prometheus_data`, 7 дней |
| Логи | `loki` | grafana/loki:3.3.2 | volume `loki_data`, 7 дней |
| Трассы | `tempo` (+ `tempo-init`) | grafana/tempo:2.6.1 | volume `tempo_data`, 7 дней |
| Интерфейс | `grafana` | grafana/grafana:11.4.0 | volume `grafana_data` (только состояние Grafana) |
Остальное — как в lab5 (HAProxy, 2 × API, PostgreSQL, Redis, MinIO, RabbitMQ, relay, worker, reconciler). Digest всех образов — `evidence/images.txt`.
```sh
sh scripts/init-secrets.sh # .env и secrets/ (в т.ч. пароль admin Grafana; значения не печатаются)
docker compose --env-file .env up -d --build --wait # стенд + Collector, Prometheus, Loki, Tempo, Grafana (provisioning из Git)
make test # публичный smoke lab6
python3 student-tests/observability_ops.py signals # сигналы, сквозная трасса, утечки, dashboard (и retry-trace, incident-worker,
# incident-db, backend-down)
python3 student-tests/telemetry_cost.py --rounds 3 # трек 1
sh scripts/collect-evidence.sh all # все доказательства в evidence/
```
Grafana: `http://127.0.0.1:3000` (HTTPS — в lab7), пользователь admin, пароль — `secrets/grafana_admin_password`; dashboard «Messenger — API, worker, очередь» и три источника появляются без ручных действий. Prometheus UI — `127.0.0.1:9090`. Остановка: `docker compose stop`; удаление учебных данных вместе с телеметрией: `docker compose down -v`.
## Где запускался стенд
Docker-проверки — на GitHub-hosted runner (2 vCPU, 8 GB, Docker 28.0.4, Compose v2.38.2) workflow [.github/workflows/lab6-stand.yml](.github/workflows/lab6-stand.yml): Go-тесты и пять независимых стендов (`base`, `files`, `pipeline`, `obs`, `track1`). Итоговый прогон: https://github.com/CharaWein/2026_lab3/actions/runs/43208190207 (commit d90aef7). Все файлы `evidence/` — из него.
## Архитектура и решения
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
```
api ×2 / relay / worker ── OTLP/HTTP :4318 ──► otel-collector ──► otlp/tempo ──► Tempo (traces)
(SDK: BatchSpanProcessor, memory_limiter ──► otlphttp ───► Loki (logs, /otlp)
PeriodicReader 5s, slog→otelslog) batch, scrub ──► prometheus :8889 ◄── Prometheus (pull 5s)
├─ rules (SLI, alerts)
Grafana ◄──── datasources (provisioning) ┘
```
* **Почему три хранилища.** Prometheus — временные ряды (счётчики, histogram, alert rules), Loki — события (структурные логи, индекс только по меткам потока, атрибуты — structured metadata), Tempo — причинность одного запроса/задания (span-дерево по trace_id). Collector хранит только очереди в памяти; Grafana — интерфейс и связи между источниками (exemplar → трасса, лог `trace_id` → трасса, трасса → логи).
* **SDK** ([internal/telemetry](internal/telemetry/telemetry.go)): ресурс `service.name` (messenger-api / messenger-worker / messenger-relay), `service.version`, `service.instance.id` (= X-Instance-Id), `deployment.environment.name=lab`. Логи пишутся в stdout (JSON с trace_id/span_id) и через мост otelslog в Collector. `OTEL_SDK_DISABLED=true` выключает экспорт; sampler — стандартные `OTEL_TRACES_SAMPLER(_ARG)`.
* **HTTP** ([middleware/telemetry.go](internal/transport/http/middleware/telemetry.go)): server span `METHOD /шаблон`, метрика `http.server.request.duration` (секунды, buckets 2 ms…30 s); её `_count` — счётчик запросов (одна метрика, не две). Метки: метод, `http.route` (шаблон ServeMux), код и класс ответа. `/health/*` метрики пишет, трасс нет. Ответ несёт `traceresponse` (W3C).
* **Зависимости**: otelpgx — spans SQL (текст с плейсхолдерами, без значений) и ожидания пула (`pool.acquire`), метрики `pgxpool_*`; otelhttp вокруг minio-go — spans `S3 GET/PUT`; redisotel — spans команд без аргументов.
* **Очередь** (W3C через outbox и AMQP, [0006_telemetry.up.sql](migrations/0006_telemetry.up.sql)): upload пишет `traceparent` активного span в `outbox.trace_context` (и `jobs.trace_context`); relay создаёт producer span `attachment.process publish` дочерним к нему и кладёт его контекст в заголовки AMQP; worker — consumer span `attachment.process process` дочерний к producer. Повтор пишет в outbox контекст неудачной попытки — вся цепочка retry в одной трассе; sweeper (без своего span) берёт контекст исходного upload из jobs.
* **Worker-метрики**: `messenger.jobs.processed{job.type,outcome}`, `messenger.job.duration` (histogram), `messenger.attachment.time_to_ready` (от создания job при 202 до ready), `messenger.worker.busy` / `messenger.worker.capacity` (saturation). **Очередь** (relay, по PostgreSQL — источнику истины): `messenger.jobs.backlog{state}`, `messenger.jobs.oldest_age` (s), `messenger.outbox.pending`, `messenger.outbox.oldest_age`; плюс глубина очередей RabbitMQ (`rabbitmq_queue_messages_ready`).
* **Фоновые циклы** relay (опрос outbox каждые 300 ms, sweeper, gauges) идут под не-сэмплированным контекстом — иначе каждый опрос был бы отдельной трассой.
**Соответствие контракту наблюдаемости** (имена в Prometheus — после трансляции OTel → Prometheus в exporter Collector; полный список — `evidence/obs/prometheus_state.json`):
| Сигнал контракта | Имя | Метки |
| --- | --- | --- |
| HTTP requests | `http_server_request_duration_seconds_count` | `http_request_method`, `http_route`, `http_response_status_code`, `http_response_status_class`, `job`, `instance` |
| HTTP duration | `http_server_request_duration_seconds_bucket` (s) | те же |
| Jobs processed / failed | `messenger_jobs_processed_total` | `job_type`, `outcome` (done, retry, failed, duplicate, not_due, requeue…) |
| Job duration | `messenger_job_duration_seconds_bucket` | `job_type`, `outcome` |
| Логи | Loki: `{service_name=…}`, structured metadata `event`, `trace_id`, `span_id`, `severity_text`, `request_id`, `route`, `status` | — |
| Traces | Tempo: server / producer / consumer / client spans | — |
| Queue depth / oldest age | `messenger_jobs_backlog`, `messenger_jobs_oldest_age_seconds`, `rabbitmq_queue_messages_ready` | `state`, `queue` |
| Pool / saturation | `pgxpool_acquired_connections`, `pgxpool_max_connections`, `pgxpool_empty_acquire_total`; `messenger_worker_busy / messenger_worker_capacity`; `process_cpu_time_seconds_total`, `go_memory_used_bytes` | `job`, `instance` |
| Сквозная связь | `traceparent` в outbox и заголовках AMQP, parent-child | — |
| Time to ready | `messenger_attachment_time_to_ready_seconds_bucket` | `job_type` |
**Error ratio:** числитель — запросы с `http_response_status_class="5xx"`, знаменатель — все запросы, кроме `/health/*`; 4xx (неверный пароль, чужой чат) — ошибки клиента, в SLO не входят. p95/p99 — `histogram_quantile` по `sum by (le)` buckets всех реплик (никакого усреднения p99 реплик).
**SLI/SLO** ([rules/messenger.yml](infra/prometheus/rules/messenger.yml), окно SLO — 28 дней, на стенде видны короткие окна): доступность API ≥ 99.5% бизнес-запросов без 5xx; задержка — ≥ 95% быстрее 250 ms; время до ready — ≥ 95% быстрее 30 s. **Alerts** (состояние firing/resolved — в Prometheus и на dashboard как аннотации; runbooks — [infra/RUNBOOKS.md](infra/RUNBOOKS.md)): `MessengerHighErrorRate` (5xx > 5% за 1m, 30 s), `MessengerQueueLagGrowing` (старейшая job > 60 s, 30 s), `MessengerWorkerAbsent`, `MessengerTelemetryExportFailing`.
**Ограниченная буферизация:** SDK — очередь BatchSpanProcessor 4096 spans, логов 4096 (переполнение — отбрасывание); Collector — `memory_limiter` 200 MiB (spike 50) первым в pipeline, `sending_queue` 20 пакетов × ≤ 1024 элементов и `retry_on_failure` ≤ 60 s на экспортёр (Tempo, Loki), лимит контейнера 256 MiB; метрики не буферизуются (Prometheus pull, `metric_expiration` 1m).
## Исправления, найденные прогонами
| Что показал прогон | Причина | Исправление |
| --- | --- | --- |
| API не стартовал (прогон 1) | конфликт schema URL ресурса: semconv 1.40 vs 1.43 в SDK | semconv 1.43 + unit-тест `Setup` с настоящими экспортёрами |
| alert `MessengerTelemetryExportFailing` и панели Collector без данных | внутренние метрики Collector 0.116 без суффикса `_total` | запросы `{__name__=~"otelcol_…(_total)?"}` |
| tail sampling терял 1 из 24 ошибок (прогон 3) | `decision_wait: 5s` ≤ длительности трассы ошибки (таймаут обработчика 5 s): решение до прихода корневого span со статусом ERROR | `decision_wait: 10s`; в итоговом прогоне потерь ошибок нет |
| после отказа Tempo новые трассы не появлялись (прогоны 4–5) | очередь экспорта 200 пакетов × до 2048 spans упёрлась в hard limit `memory_limiter` — Collector отвергал новые данные; в Tempo single-binary querier без явного адреса frontend | очередь 20 × 1024 (помещается в soft limit), `querier.frontend_worker.frontend_address`; поиск трассы — по ID из `traceresponse` |
| Tempo/Loki перезапускались под нагрузкой трека 1 (прогон 5) | лимит 512 MiB при ~14 тыс. spans на 1000 запросов | 1 GiB; в доказательства добавлено состояние контейнеров (`containers_state_*.txt`: restarts, OOMKilled) |
## Доказательства критериев
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
### Уровень 3
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
| Критерий | Сценарий | Результат | Файл |
| --- | --- | --- | --- |
| Минимум lab5 работает | smoke, регрессии, failover ×2, persistence (перезапуск включает стек наблюдаемости), сценарии pipeline | все PASS; failover — восстановление записи 3.535 / 3.478 s; demo: ready через 2.6 s | evidence/smoke.txt, failover_*.txt, persistence.txt, pipeline/*.json |
| Collector, 3 хранилища, Grafana в стенде; конфиги и provisioning в Git; volumes | compose.yaml, infra/ | datasources из provisioning: loki:loki, prometheus:prometheus, tempo:tempo; dashboard «Messenger — API, worker, очередь» (30 панелей) | evidence/obs/signals.json |
| API и worker → OTLP; RPS/errors/duration API, jobs и duration worker, логи, трассы | `observability_ops.py signals` | все сигналы контракта найдены в Prometheus (число серий в signals.json); трассы трёх сервисов; логи с trace_id | evidence/obs/signals.json, prometheus_state.json |
| Импортируемый dashboard, не скриншот | provisioning + проверка запросов | 39 запросов панелей выполнены через Grafana без ошибок; без данных в момент проверки: 2 (окно 5m без загрузок, нет 4xx) | infra/grafana/dashboards/messenger.json, evidence/obs/dashboard.png |
| Низкая кардинальность, trace/span ID в логах, без секретов | проверка меток и поиск утечек | запрещённых меток (user_id, chat_id, message_id, attachment_id, request_id, url, filename…) нет; `http_route` — только шаблоны (8 значений); пароль, токен, текст сообщения, заголовок Bearer: 0 вхождений в Loki, Tempo и stdout | evidence/obs/signals.json |
| Контролируемый запрос → trace → лог; job → worker trace и лог | `signals` | запрос `obs-msg-249e829f8c1e4049` → trace `c275fd38f588c08d29aea802ac7c9c83` → лог API `event=http.request`, `route=/api/v1/chats/{chat_id}/messages`, тот же `trace_id`/`span_id`; upload → trace `cf8628e62914dd27bef25fbfca80ba11` с логами worker `job.claimed`, `job.done` | evidence/obs/signals.json |
| make test при включённой телеметрии | smoke | PASS | evidence/smoke.txt |
### Уровень 4
| Критерий | Сценарий | Результат | Файл |
| --- | --- | --- | --- |
| W3C context API → broker → worker; producer/consumer; цепочка до S3/DB, включая retry | `signals`, `retry-trace` | одна трасса upload: messenger-api, messenger-relay, messenger-worker, 65 spans; связи parent-child: API server → relay producer → worker consumer → `image.decode`, `S3 PUT`, SQL (BEGIN, COMMIT, INSERT, SELECT, UPDATE…). Retry при остановленном S3: попытки 1=retry, 2=retry, 3=retry, 4=done в той же трассе, 4 producer spans | evidence/obs/signals.json, retry-trace.json |
| p50/p95/p99, доля ошибок, queue depth/возраст, DB pool, saturation; запросы и единицы | dashboard | панели с `histogram_quantile` по buckets, единицы (s, %, req/s, bytes) в JSON | infra/grafana/dashboards/messenger.json |
| SLI/SLO и окно; alert rules high error rate и queue lag, проверены отказом | инциденты 1 и 2 | `MessengerQueueLagGrowing` firing через 96.7 s после остановки worker, resolved через 10.6 s после запуска; `MessengerHighErrorRate` firing через 98.4 s, resolved через 18.3 s после снятия блокировки | evidence/obs/incident-*.json |
| Collector: memory limiter/batch, ограниченный буфер; остановка backend не ломает сообщения и RAM | `backend-down` | Tempo и Loki остановлены на 155 s под нагрузкой: **0 ошибок из 100015** запросов; Collector ≤ 108.4 MiB (лимит 256); очередь экспортёров ≤ 40 из 40; потери посчитаны: 4204 spans и 2843 записей логов не доставлены, 728158 spans не приняты в полную очередь; alert `MessengerTelemetryExportFailing` через 81.0 s; трассы снова в Tempo через 20.9 s после возврата; затем Collector остановлен на 40 s — API без ошибок (SDK отбрасывает по ограниченной очереди), пик памяти API ≤ 224.5 MiB (лимит 256) | evidence/obs/backend-down.json |
| Разбор двух инцидентов | ниже | — | evidence/obs/incident-*.json |
## Инциденты: поиск причины по сохранённым наблюдениям
**Инцидент 1 — остановлен worker.** Загружены 3 файла (202). Через 96.7 s — `MessengerQueueLagGrowing` firing. Диагностика без доступа к контейнерам (снимок в момент firing):
1. Prometheus: `messenger_jobs_backlog{state="queued"}` = 3, возраст старейшей job 91 s, скорость обработки `rate(messenger_jobs_processed_total[1m])` = 0 — задания принимаются, но не обрабатываются; ошибок API нет.
2. `messenger_worker_capacity` — 0 серий: ни одного живого worker (а не «worker медленный»).
3. Tempo: после остановки 3 producer spans relay и 0 consumer spans — сообщения публикуются, но никто не потребляет (broker и relay исправны).
4. Loki: последние события worker — job.claimed, job.done, job.retry, worker.start; после остановки событий нет.
Вывод — worker не запущен. После запуска backlog обработан за 2.6 s, alert resolved через 10.6 s.
**Инцидент 2 — БД «зависла»** (`LOCK TABLE messages IN ACCESS EXCLUSIVE MODE` на 110 s при нагрузке 4 клиентов). `MessengerHighErrorRate` firing через 98.4 s (окно 1m сначала содержит успешные запросы до блокировки — поэтому обнаружение ≈ окно + `for`). В момент firing: доля 5xx 100%, p99 9.95 s, и только у маршрута `/api/v1/chats/{chat_id}/messages`; занято соединений пула API 4; `/health/ready` без 5xx. Трассы ошибок (Tempo, `status_code >= 500`): server span ≈ 5000 ms, самый длинный дочерний span — SQL `INSERT` 4978 ms (доля 100%) — время ушло в ожидание БД, а не в обработчик. Для сравнения медленный по другой причине запрос (загрузка 4 MiB): server span 77.38 ms, самый длинный SQL 2.2 ms, S3 68.0 ms — БД не причина. Логи ошибок (88) — только маршрут сообщений. Alert resolved через 18.3 s после снятия блокировки. За инцидент клиенты получили 88 ошибок из 15036 запросов.
## Трек 1: цена наблюдаемости
### Гипотеза
Полная трассировка (100%) заметно дороже по CPU API и объёму ingest, чем логи и метрики; head sampling 10% снижает стоимость почти до «телеметрии нет», но теряет ~90% ошибок и редких медленных запросов; tail sampling в Collector сохраняет все ошибки и медленные запросы при объёме хранения, близком к head 10%, — ценой CPU/памяти Collector, а не API.
### Методика
[student-tests/telemetry_cost.py](student-tests/telemetry_cost.py). Одинаковая нагрузка: 4 клиента closed loop, POST сообщения, 45 s после прогрева 10 s; лимиты контейнеров неизменны (API 0.5 CPU / 256 MiB × 2, Collector 0.5 CPU / 256 MiB). В каждом прогоне два внедрённых инцидента `LOCK TABLE messages` по 6.5 s (на 12-й и 28-й секунде): запросы, попавшие в блокировку, частично завершаются 5xx по таймауту 5 s (**ошибки**), частично — успешно, но дольше 250 ms (**редкие медленные**). У каждого запроса свой `X-Request-Id`; после прогона по Tempo проверяется, сохранилась ли трасса **каждой** ошибки, **каждого** медленного запроса и случайной выборки 40 обычных. Ресурсы — cgroup контейнеров (`cpu.stat`, `memory.peak`), ingest — счётчики Collector/Tempo/Loki. 3 повтора, порядок конфигураций чередуется, приводятся медианы.
| Конфигурация | Что меняется |
| --- | --- |
| off | `OTEL_SDK_DISABLED=true` (stdout-логи остаются) |
| full | 100% трасс, Collector пропускает всё |
| head10 | `parentbased_traceidratio` 0.1 в SDK |
| tail | SDK 100%, Collector `tail_sampling`: все ERROR + ≥ 250 ms + 10% остальных ([collector-tail.yaml](infra/otel/collector-tail.yaml)) |
### Результаты (медианы 3 прогонов; latency — вне окон инцидентов)
| | off | full | head10 | tail |
| --- | --- | --- | --- | --- |
| Успешных RPS | 247.6 | 197.8 (-20%) | 234.3 (-5%) | 195.4 (-21%) |
| p50 / p95 / p99, ms | 10.06 / 20.22 / 31.5 | 12.26 / 27.91 / 46.15 | 10.38 / 21.88 / 34.23 | 12.45 / 27.68 / 41.48 |
| CPU API на запрос, ms | 1.282 | 1.875 (+46%) | 1.414 (+10%) | 1.905 (+49%) |
| Пик памяти реплики API, MiB | 13.1 | 22.4 | 21.3 | 22.3 |
| CPU / пик памяти Collector | 0.001 ядра / 30.1 MiB | 0.051 / 55.1 MiB | 0.014 / 55.9 MiB | 0.044 / 106.5 MiB |
| Spans принято / сохранено на 1000 запросов | 0 / 0 | 14019 / 14019 | 1398 / 1398 | 14016 / 1785 |
| Ingest Tempo / Loki, KiB на 1000 запросов | 0 / 0 | 4548 / 339 | 454 / 335 | 588 / 340 |
| Трассы ошибок сохранены | 0/24 | 24/24 | **2/24** | **24/24** |
| Трассы медленных сохранены | 0/24 | 24/24 | **2/24** | **24/24** |
| Выборка обычных сохранена | 0/120 | 120/120 | 14/120 | 13/120 |
Сырые данные каждого прогона — evidence/track1/telemetry_cost.json.
### Выводы
* **Цена полной трассировки** на этом сервисе — в первую очередь CPU API: +46% CPU на запрос относительно «off» и -20% успешных RPS (closed loop: те же 4 клиента ждут более дорогие ответы — запросов за то же время меньше). Одна запись сообщения даёт ≈ 14 spans (server, pool.acquire, SQL, Redis) и ≈ 4.5 KiB в Tempo — трассы в 13 раз объёмнее логов.
* **Head 10%** возвращает почти всю цену (CPU +10%, ingest ≈ 10%), но решение принимается в начале запроса, когда исход неизвестен: сохранились лишь 2/24 трасс ошибок и 2/24 медленных — ровно то, ради чего трассы нужны при разборе инцидента.
* **Tail** (сбалансированная конфигурация) сохранил 24/24 ошибок и 24/24 медленных при объёме Tempo 588 KiB на 1000 запросов против 4548 у full (в 7.7 раза меньше). Цена: CPU API как у full (SDK по-прежнему создаёт и отправляет все spans), Collector держит буфер решений — пик памяти 106.5 MiB против 55.1 у full.
* **Метрики от sampling не зависят**: доля ошибок, p95/p99 и alerts считаются в SDK до sampling — во всех конфигурациях, кроме off, они точны; sampling влияет только на то, по каким запросам можно открыть трассу.
* **Найденная ошибка конфигурации**: при `decision_wait: 5s` tail терял трассы ошибок длительностью 5 s (решение раньше прихода корневого span, прогон https://github.com/CharaWein/2026_lab3/actions/runs/43191617128: 23/24); итоговая конфигурация — 10 s.
**Рекомендация:** tail sampling в Collector (ошибки + медленные + небольшая доля остальных) при `decision_wait` больше максимальной длительности запроса; если CPU API критичнее — head-sampling малой доли **плюс** метрики и логи для ошибок (логи сохраняют trace_id даже для не-сэмплированных запросов).
### Ограничения трека
* Один runner 2 vCPU, closed loop с координированным пропуском; абсолютные RPS зависят от машины, сравнение — в одинаковых условиях одного прогона.
* «Медленные» здесь — следствие блокировки БД; естественно редкие медленные запросы распределены иначе. Ошибок и медленных по 24 на конфигурацию — доли сохранения грубые (±1 трасса).
* Tail sampling на одном Collector; при нескольких Collector нужен балансировщик по trace_id (loadbalancing exporter) — не проверялось.
* Объём Prometheus от sampling не зависит и в таблицу не вынесен (метрики считаются в SDK).
## Эксперименты и отказы
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| Воздействие | Ожидание | Наблюдение | Восстановление / потери |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
| Остановка worker | alert queue lag | firing через 96.7 s | resolved через 10.6 s после запуска; задачи не потеряны |
| Блокировка таблицы messages 110 s | alert error rate; трасса указывает на БД | firing через 98.4 s; самый длинный span — SQL | resolved через 18.3 s |
| S3 недоступен при обработке | retry видны в трассе | попытки 1–3 retry, 4 done в одной трассе | ready |
| Tempo + Loki остановлены 155 s | API работает, память ограничена | 0 ошибок API, Collector ≤ 108.4 MiB | трассы через 20.9 s; трассы/логи периода отказа потеряны (счётчики) |
| Collector остановлен 40 s | API работает | 0 ошибок, API ≤ 224.5 MiB | телеметрия периода потеряна (SDK) |
| Остановка API-реплики под нагрузкой | 0 ошибок | 0 из 16801 | — |
| Rolling update | 0 ошибок | 0 из 29983, p99 65.137 ms | — |
## Ограничения и следующие шаги
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
* Память контейнеров в `backend-down` — `memory.current` cgroup (включает page cache), поэтому у API она близка к лимиту при полной нагрузке; OOM-kill ни у одного контейнера не было (`evidence/containers_state_*.txt`).
* Телеметрия внутри доверенной сети по HTTP без аутентификации (TLS/mTLS — lab7). Grafana на 127.0.0.1 с паролем администратора.
* Один экземпляр Collector и каждого хранилища: их отказ — потеря телеметрии за период (посчитана, но не восстанавливается); persistent queue Collector не включалась.
* Алерты без доставки наружу (по заданию); burn-rate по нескольким окнам — следующий шаг (трек 2 не выбирался).
* `messenger.jobs.oldest_age` и backlog считаются запросом к PostgreSQL каждые 5 s — при больших таблицах нужен индекс (добавлен `jobs_open_idx`) или учёт в самом relay.
+178
View File
@@ -0,0 +1,178 @@
package main
import (
"context"
"errors"
"log/slog"
"net/http"
"os"
"os/signal"
"sync/atomic"
"syscall"
"time"
"github.com/charawein/mtusi2026lab6/internal/config"
"github.com/charawein/mtusi2026lab6/internal/repository/postgres"
redisrepo "github.com/charawein/mtusi2026lab6/internal/repository/redis"
"github.com/charawein/mtusi2026lab6/internal/service"
"github.com/charawein/mtusi2026lab6/internal/storage"
"github.com/charawein/mtusi2026lab6/internal/telemetry"
transporthttp "github.com/charawein/mtusi2026lab6/internal/transport/http"
)
func main() {
cfg := config.Load()
slog.SetDefault(slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: cfg.LogLevel})))
// OTLP → Collector: трассы, метрики, логи (service.name=messenger-api,
// service.instance.id = X-Instance-Id реплики). Логи дублируются JSON в stdout.
shutdownTelemetry, err := telemetry.Setup(context.Background(), "messenger-api", cfg.InstanceID, cfg.LogLevel)
if err != nil {
slog.Error("telemetry setup", "err", err)
os.Exit(1)
}
defer func() {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
_ = shutdownTelemetry(ctx)
}()
if cfg.DatabaseURL == "" {
slog.Error("DATABASE_URL_FILE is not set or empty")
os.Exit(1)
}
if cfg.CacheURL == "" {
slog.Error("CACHE_URL_FILE is not set or empty")
os.Exit(1)
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
pool, err := postgres.NewPool(ctx, cfg.DatabaseURL, cfg.DBPoolMax)
cancel()
if err != nil {
slog.Error("connect to postgres", "err", err)
os.Exit(1)
}
defer pool.Close()
rctx, rcancel := context.WithTimeout(context.Background(), 10*time.Second)
redisClient, err := redisrepo.NewClient(rctx, cfg.CacheURL)
rcancel()
if err != nil {
slog.Error("connect to redis", "err", err)
os.Exit(1)
}
defer redisClient.Close()
if cfg.S3AccessKey == "" || cfg.S3SecretKey == "" {
slog.Error("S3_ACCESS_KEY_ID_FILE / S3_SECRET_ACCESS_KEY_FILE are not set or empty")
os.Exit(1)
}
store, err := storage.NewS3(cfg.S3Endpoint, cfg.S3Region, cfg.S3Bucket, cfg.S3AccessKey, cfg.S3SecretKey)
if err != nil {
slog.Error("s3 client", "err", err)
os.Exit(1)
}
userRepo := postgres.NewUserRepository(pool)
chatRepo := postgres.NewChatRepository(pool)
msgRepo := postgres.NewMessageRepository(pool)
sessRepo := redisrepo.NewSessionRepository(redisClient)
userSvc := service.NewUserService(userRepo)
chatSvc := service.NewChatService(chatRepo, userRepo)
var msgCache *redisrepo.MessageCache
if cfg.CacheEnabled {
msgCache = redisrepo.NewMessageCache(redisClient, cfg.CacheInvalidation)
}
msgSvc := service.NewMessageService(msgRepo, chatRepo, msgCache, cfg.IdempotencyTTL)
attSvc := service.NewAttachmentService(postgres.NewAttachmentRepository(pool), chatRepo, store,
cfg.MaxUploadBytes, cfg.MaxImagePixels, cfg.PipelineVersion, cfg.JobMaxAttempts)
authSvc := service.NewAuthService(userRepo, sessRepo, cfg.SessionTTL())
// draining выставляется по SIGTERM: /health/ready начинает отвечать 503,
// LB исключает реплику, и только затем закрывается listener.
var draining atomic.Bool
router := transporthttp.NewRouter(userSvc, chatSvc, msgSvc, attSvc, authSvc, pool, redisClient, cfg.InstanceID, &draining)
// Очистка просроченных ключей идемпотентности. Запрос идемпотентен,
// поэтому параллельный запуск на всех репликах безопасен.
cleanupCtx, cleanupStop := context.WithCancel(context.Background())
defer cleanupStop()
go func() {
t := time.NewTicker(time.Hour)
defer t.Stop()
for {
select {
case <-cleanupCtx.Done():
return
case <-t.C:
ctx, cancel := context.WithTimeout(cleanupCtx, 30*time.Second)
if n, err := msgRepo.DeleteExpiredKeys(ctx, cfg.IdempotencyTTL); err != nil {
slog.Warn("idempotency cleanup failed", "err", err)
} else if n > 0 {
slog.Info("idempotency keys expired", "deleted", n)
}
cancel()
}
}
}()
srv := &http.Server{
Addr: cfg.Addr(),
Handler: router,
ReadHeaderTimeout: 5 * time.Second,
// Загрузка до 10 MiB должна уложиться в ReadTimeout; медленный клиент
// дольше не держит соединение и память.
ReadTimeout: 30 * time.Second,
WriteTimeout: 35 * time.Second,
IdleTimeout: 60 * time.Second,
}
// Внутренний endpoint метрик для экспериментов (трек 1). Порт не публикуется
// и не проксируется LB: читается только через docker exec внутри контейнера.
internalSrv := &http.Server{
Addr: cfg.InternalAddr,
Handler: transporthttp.NewInternalStatsHandler(pool, cfg),
ReadHeaderTimeout: 2 * time.Second,
}
go func() {
if err := internalSrv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
slog.Warn("internal stats server failed", "err", err)
}
}()
defer internalSrv.Close()
errCh := make(chan error, 1)
go func() {
slog.Info("server starting", "addr", cfg.Addr(), "instance_id", cfg.InstanceID)
if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
errCh <- err
}
}()
sigCtx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT)
defer stop()
select {
case err := <-errCh:
slog.Error("server failed", "err", err)
os.Exit(1)
case <-sigCtx.Done():
slog.Info("shutdown signal received, draining", "delay", cfg.DrainDelay)
}
// Перестаём быть ready и даём LB время заметить это (inter*fall у HAProxy),
// продолжая обслуживать уже направленные к нам запросы.
draining.Store(true)
time.Sleep(cfg.DrainDelay)
shutdownCtx, cancel := context.WithTimeout(context.Background(), cfg.ShutdownTimeout)
defer cancel()
if err := srv.Shutdown(shutdownCtx); err != nil {
slog.Error("graceful shutdown failed", "err", err)
os.Exit(1)
}
slog.Info("server stopped")
}
+371
View File
@@ -0,0 +1,371 @@
// Command pipeline — асинхронная обработка вложений (lab5) с телеметрией (lab6).
//
// pipeline worker потребитель broker (сервис worker)
// pipeline relay outbox → broker, возврат истёкших lease, очистка staging
// pipeline failed реестр failed-заданий (JSON)
// pipeline replay <attachment_id> [--version N] [--actor NAME] явный повтор failed
// pipeline reprocess <attachment_id> --version N работа новой версии (трек 5)
package main
import (
"context"
"encoding/json"
"errors"
"flag"
"fmt"
"log/slog"
"os"
"os/signal"
"strings"
"sync"
"syscall"
"time"
"github.com/charawein/mtusi2026lab6/internal/broker"
"github.com/charawein/mtusi2026lab6/internal/config"
"github.com/charawein/mtusi2026lab6/internal/imaging"
"github.com/charawein/mtusi2026lab6/internal/repository/postgres"
"github.com/charawein/mtusi2026lab6/internal/storage"
"github.com/charawein/mtusi2026lab6/internal/telemetry"
"github.com/charawein/mtusi2026lab6/internal/worker"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/metric"
"go.opentelemetry.io/otel/trace"
)
func main() {
slog.SetDefault(slog.New(slog.NewJSONHandler(os.Stdout, nil)))
if len(os.Args) < 2 {
fmt.Fprintln(os.Stderr, "usage: pipeline worker|relay|failed|replay|reprocess")
os.Exit(2)
}
cfg := config.Load()
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT)
defer stop()
// Телеметрия только у долгоживущих процессов; служебные команды (failed,
// replay) пишут JSON в stdout. service.name: messenger-worker / messenger-relay
// (OTEL_SERVICE_NAME из compose имеет приоритет).
if os.Args[1] == "worker" || os.Args[1] == "relay" {
host, _ := os.Hostname()
shutdown, err := telemetry.Setup(ctx, "messenger-"+os.Args[1], host, slog.LevelInfo)
if err != nil {
slog.Error("telemetry setup", "err", err)
os.Exit(1)
}
defer func() {
sctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
_ = shutdown(sctx)
}()
}
pool, err := postgres.NewPool(ctx, cfg.DatabaseURL, int32(cfg.WorkerConcurrency+4))
if err != nil {
slog.Error("connect to postgres", "err", err)
os.Exit(1)
}
defer pool.Close()
repo := postgres.NewPipelineRepository(pool)
switch os.Args[1] {
case "worker":
code := runWorker(ctx, cfg, repo)
if code != 0 {
os.Exit(code)
}
case "relay":
code := runRelay(ctx, cfg, repo)
if code != 0 {
os.Exit(code)
}
case "failed":
jobs, err := repo.FailedJobs(ctx, 200)
if err != nil {
slog.Error("failed jobs", "err", err)
os.Exit(1)
}
out, _ := json.MarshalIndent(jobs, "", " ")
fmt.Println(string(out))
case "replay", "reprocess":
fs := flag.NewFlagSet(os.Args[1], flag.ExitOnError)
version := fs.Int("version", cfg.PipelineVersion, "pipeline version")
actor := fs.String("actor", "operator", "who requested the replay (audit)")
if len(os.Args) < 3 {
fmt.Fprintln(os.Stderr, "attachment_id required")
os.Exit(2)
}
_ = fs.Parse(os.Args[3:])
var jobID string
if os.Args[1] == "replay" {
jobID, err = repo.Replay(ctx, os.Args[2], *version, *actor)
} else {
jobID, err = repo.Reprocess(ctx, os.Args[2], *version, cfg.JobMaxAttempts)
}
out, _ := json.Marshal(map[string]any{"command": os.Args[1], "attachment_id": os.Args[2], "version": *version,
"job_id": jobID, "ok": err == nil, "error": errString(err)})
fmt.Println(string(out))
if err != nil {
os.Exit(3)
}
default:
fmt.Fprintln(os.Stderr, "unknown command", os.Args[1])
os.Exit(2)
}
}
func errString(err error) string {
if err == nil {
return ""
}
return err.Error()
}
func faultPoints() map[string]bool {
m := map[string]bool{}
for _, p := range strings.Split(os.Getenv("FAULT_POINTS"), ",") {
if p = strings.TrimSpace(p); p != "" {
m[p] = true
}
}
return m
}
func newStore(cfg config.Config) storage.Store {
store, err := storage.NewS3(cfg.S3Endpoint, cfg.S3Region, cfg.S3Bucket, cfg.S3AccessKey, cfg.S3SecretKey)
if err != nil {
slog.Error("s3 client", "err", err)
os.Exit(1)
}
return store
}
func runWorker(ctx context.Context, cfg config.Config, repo *postgres.PipelineRepository) int {
host, _ := os.Hostname()
owner := fmt.Sprintf("%s:%d", host, os.Getpid())
w := worker.New(repo, newStore(cfg), imaging.CWebP, worker.Config{
Owner: owner, Lease: cfg.JobTimeout, JobTimeout: cfg.JobTimeout,
MaxBytes: cfg.MaxUploadBytes, MaxPixels: cfg.MaxImagePixels, PixelBudget: cfg.PixelBudget,
WebPQuality: cfg.WebPQuality, RetryBase: cfg.RetryBaseDelay, RetryMax: cfg.RetryMaxDelay,
FaultPoints: faultPoints(), FaultDir: os.TempDir(), DeleteStaging: true,
})
slog.Info("worker starting", "event", "worker.start", "owner", owner, "concurrency", cfg.WorkerConcurrency, "lease", cfg.JobTimeout)
// Ёмкость worker (saturation = messenger.worker.busy / capacity).
capacity, _ := telemetry.Meter().Int64ObservableGauge("messenger.worker.capacity", metric.WithUnit("{job}"),
metric.WithDescription("Concurrent job slots of this worker (prefetch)"))
_, _ = telemetry.Meter().RegisterCallback(func(_ context.Context, o metric.Observer) error {
o.ObserveInt64(capacity, int64(cfg.WorkerConcurrency))
return nil
}, capacity)
tracer := telemetry.Tracer()
backoff := time.Second
for ctx.Err() == nil {
cons, deliveries, err := broker.NewConsumer(cfg.BrokerURL, cfg.WorkerConcurrency, "worker-"+owner)
if err != nil {
slog.Warn("broker unavailable, retrying", "err", err, "in", backoff)
select {
case <-ctx.Done():
case <-time.After(backoff):
}
backoff = min(backoff*2, 30*time.Second)
continue
}
backoff = time.Second
var wg sync.WaitGroup
for i := 0; i < cfg.WorkerConcurrency; i++ {
wg.Add(1)
go func() {
defer wg.Done()
for d := range deliveries {
// Consumer span — дочерний для producer span relay (W3C traceparent
// из заголовков AMQP), тот — для upload: одна трасса от HTTP до S3/DB.
mctx := otel.GetTextMapPropagator().Extract(context.WithoutCancel(ctx), broker.HeaderCarrier(d.Headers))
mctx, span := tracer.Start(mctx, broker.Queue+" process", trace.WithSpanKind(trace.SpanKindConsumer),
trace.WithAttributes(
attribute.String("messaging.system", "rabbitmq"),
attribute.String("messaging.operation.type", "process"),
attribute.String("messaging.destination.name", broker.Queue),
attribute.String("messaging.message.id", d.MessageId),
attribute.Bool("messaging.rabbitmq.redelivered", d.Redelivered),
))
decision := w.Handle(mctx, d.Body)
span.SetAttributes(attribute.String("messenger.delivery.decision", decision.String()))
span.End()
switch decision {
case worker.Ack:
_ = d.Ack(false)
case worker.Requeue:
time.Sleep(time.Second)
_ = d.Nack(false, true)
case worker.Reject:
_ = d.Nack(false, false) // → DLX attachment.dead
}
}
}()
}
select {
case <-ctx.Done():
// Graceful: прекращаем получать новые сообщения, дорабатываем текущие.
slog.Info("shutdown: cancel consumer, finishing in-flight jobs")
cons.Cancel("worker-" + owner)
done := make(chan struct{})
go func() { wg.Wait(); close(done) }()
select {
case <-done:
case <-time.After(cfg.JobTimeout):
slog.Warn("shutdown timeout: unacked jobs will be redelivered")
}
cons.Close()
return 0
case <-waitAll(&wg):
slog.Warn("broker connection lost, reconnecting")
cons.Close()
}
}
return 0
}
func waitAll(wg *sync.WaitGroup) <-chan struct{} {
ch := make(chan struct{})
go func() { wg.Wait(); close(ch) }()
return ch
}
func runRelay(ctx context.Context, cfg config.Config, repo *postgres.PipelineRepository) int {
store := newStore(cfg)
points := faultPoints()
afterPublish := func() { worker.Fault(points, os.TempDir(), "relay.after_publish", os.Exit) }
var pub *broker.Publisher
defer func() {
if pub != nil {
pub.Close()
}
}()
tPublish := time.NewTicker(300 * time.Millisecond)
tSweep := time.NewTicker(5 * time.Second)
tClean := time.NewTicker(time.Minute)
defer tPublish.Stop()
defer tSweep.Stop()
defer tClean.Stop()
slog.Info("relay starting", "event", "relay.start")
registerQueueGauges(repo)
tracer := telemetry.Tracer()
published, _ := telemetry.Meter().Int64Counter("messenger.outbox.published", metric.WithUnit("{message}"),
metric.WithDescription("Outbox messages confirmed by broker"))
publishDur, _ := telemetry.Meter().Float64Histogram("messenger.outbox.publish.duration", metric.WithUnit("s"),
metric.WithDescription("Publish + publisher confirm latency"))
for {
select {
case <-ctx.Done():
return 0
case <-tPublish.C:
if pub == nil || pub.Closed() {
p, err := broker.NewPublisher(cfg.BrokerURL)
if err != nil {
slog.Warn("broker unavailable; outbox keeps messages", "err", err)
continue
}
pub = p
}
for {
// Цикл опроса outbox не трассируется (Unsampled); producer span каждого
// сообщения — дочерний для span, записавшего его в outbox (upload/попытка).
n, err := repo.PublishDue(telemetry.Unsampled(ctx), 100, func(c context.Context, m postgres.OutboxMessage) error {
pctx := telemetry.ExtractMap(context.WithoutCancel(ctx), m.Trace)
pctx, span := tracer.Start(pctx, broker.Queue+" publish", trace.WithSpanKind(trace.SpanKindProducer),
trace.WithAttributes(
attribute.String("messaging.system", "rabbitmq"),
attribute.String("messaging.operation.type", "publish"),
attribute.String("messaging.destination.name", broker.Exchange),
attribute.String("messaging.message.id", m.JobID),
attribute.Float64("messenger.outbox.wait_seconds", time.Since(m.Created).Seconds()),
))
start := time.Now()
err := pub.Publish(pctx, m.JobID, m.Payload)
res := "ok"
if err != nil {
res = "error"
span.RecordError(err)
span.SetStatus(codes.Error, "publish failed")
}
publishDur.Record(pctx, time.Since(start).Seconds(), metric.WithAttributes(attribute.String("outcome", res)))
if err == nil {
published.Add(pctx, 1)
}
span.End()
return err
}, afterPublish)
if err != nil {
slog.Warn("outbox publish failed; will retry", "err", err)
break
}
if n > 0 {
slog.Info("outbox published", "messages", n)
}
if n < 100 {
break
}
}
case <-tSweep.C:
if n, err := repo.SweepExpiredLeases(telemetry.Unsampled(ctx)); err != nil {
slog.Warn("lease sweep failed", "err", err)
} else if n > 0 {
slog.Warn("expired leases requeued", "jobs", n)
}
case <-tClean.C:
cleanStaging(telemetry.Unsampled(ctx), cfg, repo, store)
}
}
}
// cleanStaging удаляет staging после безопасной публикации (ready, ≥10 мин назад —
// fallback, если worker не удалил сразу) и у failed старше STAGING_RETENTION.
func cleanStaging(ctx context.Context, cfg config.Config, repo *postgres.PipelineRepository, store storage.Store) {
cands, err := repo.StagingToClean(ctx, 10*time.Minute, cfg.StagingRetention, 200)
if err != nil {
slog.Warn("staging cleanup query failed", "err", err)
return
}
for _, c := range cands {
if err := store.Delete(ctx, c.Key); err != nil && !errors.Is(err, storage.ErrNotFound) {
slog.Warn("staging delete failed", "key", c.Key, "err", err)
continue
}
_ = repo.ClearStaging(ctx, c.AttachmentID, c.Key)
}
if len(cands) > 0 {
slog.Info("staging cleaned", "objects", len(cands))
}
}
// registerQueueGauges — глубина очереди и возраст старейшей работы по PostgreSQL
// (источник истины; broker-метрики RabbitMQ собираются Prometheus отдельно).
func registerQueueGauges(repo *postgres.PipelineRepository) {
m := telemetry.Meter()
jobs, _ := m.Int64ObservableGauge("messenger.jobs.backlog", metric.WithUnit("{job}"),
metric.WithDescription("Jobs by state (queued, processing, failed)"))
oldest, _ := m.Float64ObservableGauge("messenger.jobs.oldest_age", metric.WithUnit("s"),
metric.WithDescription("Age of the oldest unfinished job (queued or processing)"))
outbox, _ := m.Int64ObservableGauge("messenger.outbox.pending", metric.WithUnit("{message}"),
metric.WithDescription("Due outbox messages not yet confirmed by broker"))
outboxAge, _ := m.Float64ObservableGauge("messenger.outbox.oldest_age", metric.WithUnit("s"),
metric.WithDescription("Age of the oldest due unpublished outbox message"))
q := attribute.String("queue", broker.Queue)
_, _ = m.RegisterCallback(func(ctx context.Context, o metric.Observer) error {
qctx, cancel := context.WithTimeout(telemetry.Unsampled(ctx), 2*time.Second)
defer cancel()
st, err := repo.QueueStats(qctx)
if err != nil {
return nil // нет данных — пропуск точки, а не ошибка экспорта
}
o.ObserveInt64(jobs, st.Queued, metric.WithAttributes(q, attribute.String("state", "queued")))
o.ObserveInt64(jobs, st.Processing, metric.WithAttributes(q, attribute.String("state", "processing")))
o.ObserveInt64(jobs, st.Failed, metric.WithAttributes(q, attribute.String("state", "failed")))
o.ObserveFloat64(oldest, st.OldestQueuedAge, metric.WithAttributes(q))
o.ObserveInt64(outbox, st.OutboxPending)
o.ObserveFloat64(outboxAge, st.OldestOutboxAge)
return nil
}, jobs, oldest, outbox, outboxAge)
}
+105
View File
@@ -0,0 +1,105 @@
// Command reconcile сверяет PostgreSQL и S3 и безопасно убирает расхождения.
//
// reconcile # dry-run, grace 1h, отчёт JSON в stdout
// reconcile --apply # исправить
// reconcile --apply --interval 10m # фоновый режим (сервис reconciler в compose)
//
// Grace period должен быть больше максимальной длительности загрузки
// (ReadTimeout 30 s + PUT) с запасом на расхождение часов; значения < 5m
// разрешены только с --unsafe-short-grace (для тестов).
package main
import (
"context"
"encoding/json"
"flag"
"fmt"
"log/slog"
"os"
"os/signal"
"syscall"
"time"
"github.com/charawein/mtusi2026lab6/internal/config"
"github.com/charawein/mtusi2026lab6/internal/reconcile"
"github.com/charawein/mtusi2026lab6/internal/repository/postgres"
"github.com/charawein/mtusi2026lab6/internal/storage"
)
func main() {
apply := flag.Bool("apply", false, "apply fixes (default: dry-run, report only)")
grace := flag.Duration("grace", time.Hour, "do not touch objects/intents/quarantine younger than this")
unsafe := flag.Bool("unsafe-short-grace", false, "allow grace < 5m (tests only)")
interval := flag.Duration("interval", 0, "repeat every interval (0 = run once)")
flag.Parse()
slog.SetDefault(slog.New(slog.NewJSONHandler(os.Stderr, nil)))
if *grace < 5*time.Minute && !*unsafe {
fmt.Fprintln(os.Stderr, "grace < 5m is unsafe: in-flight uploads could be deleted; pass --unsafe-short-grace for tests")
os.Exit(2)
}
cfg := config.Load()
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT)
defer stop()
pool, err := postgres.NewPool(ctx, cfg.DatabaseURL, 4)
if err != nil {
slog.Error("connect to postgres", "err", err)
os.Exit(1)
}
defer pool.Close()
store, err := storage.NewS3(cfg.S3Endpoint, cfg.S3Region, cfg.S3Bucket, cfg.S3AccessKey, cfg.S3SecretKey)
if err != nil {
slog.Error("s3 client", "err", err)
os.Exit(1)
}
repo := postgres.NewAttachmentRepository(pool)
runOnce := func() int {
unlock, ok, err := repo.TryLock(ctx)
if err != nil {
slog.Error("advisory lock", "err", err)
return 1
}
if !ok {
slog.Warn("another reconciler is running, skipping this pass")
return 3
}
defer unlock()
var now time.Time
if err := pool.QueryRow(ctx, `SELECT now()`).Scan(&now); err != nil {
slog.Error("db time", "err", err)
return 1
}
rep, err := reconcile.Run(ctx, repo, store, reconcile.Options{
Prefix: "", Grace: *grace, Apply: *apply, Now: now, // весь bucket: a/, s/, o/, v/
})
out, _ := json.Marshal(rep)
fmt.Println(string(out))
if err != nil {
slog.Error("reconcile pass failed", "err", err)
return 1
}
slog.Info("reconcile pass done", "dry_run", rep.DryRun, "found", rep.Found, "fixed", rep.Fixed,
"rows", rep.RowsScanned, "objects", rep.ObjectsScanned, "ms", rep.DurationMS)
if len(rep.Errors) > 0 {
return 4
}
return 0
}
if *interval <= 0 {
os.Exit(runOnce())
}
t := time.NewTicker(*interval)
defer t.Stop()
for {
runOnce()
select {
case <-ctx.Done():
return
case <-t.C:
}
}
}
+476
View File
@@ -0,0 +1,476 @@
# Lab5: клиент → HAProxy → API × N → PostgreSQL (данные, метаданные, jobs, outbox), Redis (сессии, кэш),
# приватный S3 (MinIO: staging s/, оригиналы o/, варианты v/).
# relay: outbox → RabbitMQ (publisher confirms); worker: RabbitMQ → варианты → S3 + PostgreSQL.
# Lab6: API / relay / worker —OTLP/HTTP→ otel-collector → Tempo (traces), Loki (logs), Prometheus (metrics,
# pull :8889, alert rules) → Grafana (provisioning из infra/grafana). Сеть telemetry отделена от back.
# Запуск: scripts/init-secrets.sh && docker compose --env-file .env up -d --build --wait
name: mtusi2026lab6
x-api-env: &api-env
APP_HOST: ${APP_HOST:-0.0.0.0}
APP_PORT: ${APP_PORT:-8080}
APP_ENV: ${APP_ENV:-lab}
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
SESSION_STORE: ${SESSION_STORE:-redis}
DATABASE_URL_FILE: /run/secrets/database_url
CACHE_URL_FILE: /run/secrets/cache_url
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
DRAIN_DELAY: ${DRAIN_DELAY:-3s}
SHUTDOWN_TIMEOUT: ${SHUTDOWN_TIMEOUT:-8s}
DB_POOL_MAX: ${DB_POOL_MAX:-20}
CACHE_INVALIDATION: ${CACHE_INVALIDATION:-version}
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000}
S3_REGION: ${S3_REGION:-us-east-1}
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
PIPELINE_VERSION: ${PIPELINE_VERSION:-1}
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
WEBP_QUALITY: ${WEBP_QUALITY:-80}
WORKER_PIXEL_BUDGET: ${WORKER_PIXEL_BUDGET:-60000000}
RETRY_BASE_DELAY: ${RETRY_BASE_DELAY:-2s}
RETRY_MAX_DELAY: ${RETRY_MAX_DELAY:-60s}
# OpenTelemetry SDK (стандартные переменные; service.name задаёт каждый сервис)
OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT:-http://otel-collector:4318}
OTEL_EXPORTER_OTLP_PROTOCOL: ${OTEL_EXPORTER_OTLP_PROTOCOL:-http/protobuf}
OTEL_RESOURCE_ATTRIBUTES: ${OTEL_RESOURCE_ATTRIBUTES:-deployment.environment.name=lab}
OTEL_SDK_DISABLED: ${OTEL_SDK_DISABLED:-false}
OTEL_TRACES_SAMPLER: ${OTEL_TRACES_SAMPLER:-parentbased_always_on}
OTEL_TRACES_SAMPLER_ARG: ${OTEL_TRACES_SAMPLER_ARG:-1.0}
services:
lb:
image: haproxy:3.0.5-alpine
ports:
- "127.0.0.1:${LB_PORT:-8080}:8080"
- "127.0.0.1:${LB_STATS_PORT:-8404}:8404"
environment:
LB_BALANCE: ${LB_BALANCE:-roundrobin}
volumes:
- ./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
depends_on:
api:
condition: service_healthy
restart: unless-stopped
networks: [front]
deploy:
resources:
limits: { cpus: "${LB_CPUS:-0.50}", memory: 128M }
api:
build: .
image: mtusi2026lab6-api:local
# Без container_name и без ports: реплики доступны только через lb.
environment:
<<: *api-env
OTEL_SERVICE_NAME: messenger-api
secrets: [database_url, cache_url, s3_access_key_id, s3_secret_access_key]
depends_on:
migrate:
condition: service_completed_successfully
s3-init:
condition: service_completed_successfully
postgres:
condition: service_healthy
redis:
condition: service_healthy
healthcheck:
test: ["CMD", "wget", "--spider", "-q", "http://localhost:8080/health/ready"]
interval: 5s
timeout: 2s
retries: 3
start_period: 5s
# DRAIN_DELAY + SHUTDOWN_TIMEOUT < stop_grace_period, иначе SIGKILL оборвёт запросы.
stop_grace_period: 15s
restart: unless-stopped
networks: [front, back, telemetry]
deploy:
replicas: ${API_REPLICAS:-2}
resources:
limits: { cpus: "${API_CPUS:-0.50}", memory: "${API_MEMORY:-256M}" }
# Однократные миграции до старта API: реплики не гоняются за схему.
migrate:
image: migrate/migrate:v4.17.0
volumes:
- ./migrations:/migrations:ro
secrets: [database_url]
entrypoint: ["/bin/sh", "-c", "exec migrate -path /migrations -database \"$$(cat /run/secrets/database_url)\" up"]
depends_on:
postgres:
condition: service_healthy
restart: "no"
networks: [back]
postgres:
image: postgres:16.4-alpine
command: ["postgres", "-c", "max_connections=${PG_MAX_CONNECTIONS:-100}"]
environment:
POSTGRES_USER: messenger
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
POSTGRES_DB: messenger
secrets: [postgres_password]
volumes:
- pg_data:/var/lib/postgresql/data
healthcheck:
test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"]
interval: 5s
timeout: 3s
retries: 5
restart: unless-stopped
networks: [back]
deploy:
resources:
limits: { cpus: "${PG_CPUS:-1.00}", memory: "${PG_MEMORY:-512M}" }
redis:
image: redis:7.4.1-alpine
command: ["redis-server", "--appendonly", "yes", "--appendfsync", "everysec"]
volumes:
- redis_data:/data
healthcheck:
test: ["CMD", "redis-cli", "ping"]
interval: 5s
timeout: 3s
retries: 5
restart: unless-stopped
networks: [back]
deploy:
resources:
limits: { cpus: "${REDIS_CPUS:-0.50}", memory: 256M }
# S3-совместимое хранилище байтов вложений: MinIO RELEASE.2025-04-22T22-12-26Z.
# Официальные образы minio/minio убраны из Docker Hub и quay.io (проверено в CI),
# поэтому используется сборка Bitnami того же релиза, закреплённая по digest.
# Порты не публикуются, консоль выключена (MINIO_BROWSER=off): доступ только из сети back.
s3:
image: bitnamilegacy/minio:2025.4.22@sha256:50cec18ac4184af4671a78aedd5554942c8ae105d51a465fa82037949046da01
entrypoint: ["/bin/bash", "-c"]
command:
- export MINIO_ROOT_USER="$$(cat /run/secrets/s3_root_user)" MINIO_ROOT_PASSWORD="$$(cat /run/secrets/s3_root_password)"; exec /opt/bitnami/minio/bin/minio server /bitnami/minio/data --address :9000
environment:
MINIO_BROWSER: "off"
secrets: [s3_root_user, s3_root_password]
volumes:
- s3_data:/bitnami/minio/data
healthcheck:
# Без curl в образе: HTTP/1.1-запрос к /minio/health/live через /dev/tcp и read (только bash).
test: ["CMD", "bash", "-c", "exec 3<>/dev/tcp/127.0.0.1/9000 && printf 'GET /minio/health/live HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n' >&3 && read -r line <&3 && [[ $$line == *' 200 '* ]]"]
interval: 5s
timeout: 3s
retries: 20
restart: unless-stopped
networks: [back]
deploy:
resources:
limits: { cpus: "${S3_CPUS:-0.50}", memory: 512M }
# Идемпотентный bootstrap: приватный bucket, политика и отдельный пользователь
# API с правами только на этот bucket. Root-учётка есть только у s3 и s3-init.
s3-init:
image: bitnamilegacy/minio-client:2025.4.16@sha256:8a86e441decf053093c5977d49290298295f850c7843652f93e970ab15871dd1
entrypoint: ["/bin/sh", "/init/init.sh"]
environment:
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
PATH: /opt/bitnami/minio-client/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
MC_CONFIG_DIR: /tmp/.mc
secrets: [s3_root_user, s3_root_password, s3_access_key_id, s3_secret_access_key]
volumes:
- ./infra/s3:/init:ro
depends_on:
s3:
condition: service_healthy
restart: "no"
networks: [back]
# Сверка PostgreSQL и S3: брошенные загрузки, сироты, потерянные объекты.
# Безопасный grace period 1h; одновременно работает один экземпляр (advisory lock).
reconciler:
image: mtusi2026lab6-api:local
entrypoint: ["/app/reconcile", "--apply", "--interval", "${RECONCILE_INTERVAL:-10m}", "--grace", "${RECONCILE_GRACE:-1h}"]
environment:
<<: *api-env
CACHE_URL_FILE: "" # Redis reconciler не нужен
secrets: [database_url, s3_access_key_id, s3_secret_access_key]
# HTTP не обслуживает, поэтому вместо HEALTHCHECK образа API — проверка, что
# процесс reconcile жив (нужна и для docker compose up --wait).
healthcheck:
test: ["CMD-SHELL", "pgrep -x reconcile >/dev/null || pidof reconcile >/dev/null"]
interval: 10s
timeout: 3s
retries: 3
start_period: 5s
depends_on:
migrate:
condition: service_completed_successfully
s3-init:
condition: service_completed_successfully
restart: unless-stopped
networks: [back]
deploy:
resources:
limits: { cpus: "0.25", memory: 128M }
# Broker: quorum-очереди на диске (volume), без management UI, порты не публикуются.
# Пользователь и пароль — из secret rabbitmq.conf (guest отключён вне localhost).
rabbitmq:
image: rabbitmq:4.0.9-alpine
hostname: rabbitmq # имя узла фиксировано: данные в volume привязаны к нему
secrets:
- source: rabbitmq_conf
target: /etc/rabbitmq/conf.d/20-credentials.conf
volumes:
- rabbitmq_data:/var/lib/rabbitmq
healthcheck:
test: ["CMD", "rabbitmq-diagnostics", "-q", "check_port_connectivity"]
interval: 5s
timeout: 10s
retries: 20
start_period: 10s
restart: unless-stopped
# telemetry — только чтобы Prometheus забирал :15692 (метрики очередей, без AMQP-учётки).
networks: [back, telemetry]
deploy:
resources:
limits: { cpus: "0.50", memory: 512M }
# Relay: публикует transactional outbox с publisher confirms, возвращает в очередь
# работу с истёкшим lease, чистит staging. Не зависит от доступности API.
relay:
image: mtusi2026lab6-api:local
entrypoint: ["/app/pipeline", "relay"]
environment:
<<: *api-env
CACHE_URL_FILE: ""
BROKER_URL_FILE: /run/secrets/broker_url
FAULT_POINTS: ${RELAY_FAULT_POINTS:-}
OTEL_SERVICE_NAME: messenger-relay
secrets: [database_url, s3_access_key_id, s3_secret_access_key, broker_url]
healthcheck:
test: ["CMD-SHELL", "pgrep -x pipeline >/dev/null || pidof pipeline >/dev/null"]
interval: 10s
timeout: 3s
retries: 3
start_period: 5s
depends_on:
migrate:
condition: service_completed_successfully
s3-init:
condition: service_completed_successfully
rabbitmq:
condition: service_healthy
restart: unless-stopped
networks: [back, telemetry]
deploy:
resources:
limits: { cpus: "0.25", memory: 128M }
# Worker: отдельный процесс; prefetch = WORKER_CONCURRENCY, lease = JOB_TIMEOUT_SECONDS,
# память ограничена контейнером и пиксельным бюджетом. SIGTERM → дорабатывает текущие задания.
worker:
image: mtusi2026lab6-api:local
entrypoint: ["/app/pipeline", "worker"]
environment:
<<: *api-env
CACHE_URL_FILE: ""
BROKER_URL_FILE: /run/secrets/broker_url
FAULT_POINTS: ${WORKER_FAULT_POINTS:-}
OTEL_SERVICE_NAME: messenger-worker
# Мягкий лимит кучи Go ≈ 75% лимита контейнера: без него GC (GOGC=100) даёт куче
# расти до ~2× живого объёма, и пик подходил вплотную к 768M (замер прогона 4).
GOMEMLIMIT: ${WORKER_GOMEMLIMIT:-560MiB}
secrets: [database_url, s3_access_key_id, s3_secret_access_key, broker_url]
healthcheck:
test: ["CMD-SHELL", "pgrep -x pipeline >/dev/null || pidof pipeline >/dev/null"]
interval: 10s
timeout: 3s
retries: 3
start_period: 5s
stop_grace_period: 70s
depends_on:
migrate:
condition: service_completed_successfully
s3-init:
condition: service_completed_successfully
rabbitmq:
condition: service_healthy
restart: unless-stopped
networks: [back, telemetry]
deploy:
replicas: ${WORKER_REPLICAS:-1}
resources:
limits: { cpus: "${WORKER_CPUS:-1.00}", memory: "${WORKER_MEMORY:-768M}" }
# ---------------- Наблюдаемость (lab6) ----------------
# Collector: приём OTLP (4317 gRPC / 4318 HTTP) только из сети telemetry, порты
# наружу не публикуются. memory_limiter + ограниченные очереди экспортёров:
# отказ Tempo/Loki не приводит к росту памяти (infra/otel/collector.yaml).
otel-collector:
image: otel/opentelemetry-collector-contrib:0.116.1
command: ["--config=/etc/otelcol/${COLLECTOR_CONFIG:-collector.yaml}"]
volumes:
- ./infra/otel:/etc/otelcol:ro
restart: unless-stopped
networks: [telemetry]
deploy:
resources:
limits: { cpus: "${COLLECTOR_CPUS:-0.50}", memory: 256M }
# Метрики: TSDB на volume (7 дней), alert rules, exemplars → Tempo.
# UI/API только на localhost хоста (администрирование, проверки).
prometheus:
image: prom/prometheus:v3.1.0
command:
- --config.file=/etc/prometheus/prometheus.yml
- --storage.tsdb.path=/prometheus
- --storage.tsdb.retention.time=7d
- --enable-feature=exemplar-storage
- --web.enable-lifecycle
volumes:
- ./infra/prometheus:/etc/prometheus:ro
- prometheus_data:/prometheus
ports:
- "127.0.0.1:${PROMETHEUS_PORT:-9090}:9090"
healthcheck:
test: ["CMD", "wget", "-q", "--spider", "http://localhost:9090/-/ready"]
interval: 5s
timeout: 3s
retries: 20
restart: unless-stopped
networks: [telemetry]
deploy:
resources:
limits: { cpus: "0.50", memory: 512M }
# Логи: Loki (OTLP /otlp), файловое хранилище на volume, retention 7 дней.
loki:
image: grafana/loki:3.3.2
command: ["-config.file=/etc/loki/loki.yaml"]
volumes:
- ./infra/loki/loki.yaml:/etc/loki/loki.yaml:ro
- loki_data:/loki
healthcheck:
test: ["CMD", "wget", "-q", "--spider", "http://localhost:3100/ready"]
interval: 5s
timeout: 3s
retries: 30
restart: unless-stopped
networks: [telemetry]
deploy:
resources:
limits: { cpus: "0.50", memory: 1G } # 512M: OOM под нагрузкой трека 1 (~14 тыс. spans на 1000 запросов, прогон 5)
# Трассы: Tempo, локальный backend на volume, retention 7 дней.
# tempo-init один раз выдаёт volume пользователю tempo (uid 10001).
tempo-init:
image: &tempo-image grafana/tempo:2.6.1
user: root
entrypoint: ["chown", "10001:10001", "/var/tempo"]
volumes:
- tempo_data:/var/tempo
restart: "no"
networks: [telemetry]
tempo:
image: *tempo-image
command: ["-config.file=/etc/tempo/tempo.yaml"]
volumes:
- ./infra/tempo/tempo.yaml:/etc/tempo/tempo.yaml:ro
- tempo_data:/var/tempo
depends_on:
tempo-init:
condition: service_completed_successfully
healthcheck:
test: ["CMD", "wget", "-q", "--spider", "http://localhost:3200/ready"]
interval: 5s
timeout: 3s
retries: 30
start_period: 10s
restart: unless-stopped
networks: [telemetry]
deploy:
resources:
limits: { cpus: "0.50", memory: 1G } # 512M: OOM под нагрузкой трека 1 (~14 тыс. spans на 1000 запросов, прогон 5)
# Grafana: только интерфейс и запросы; datasources и dashboard — provisioning из Git.
# Пароль admin — secret; порт только на localhost.
grafana:
image: grafana/grafana:11.4.0
environment:
GF_SECURITY_ADMIN_USER: admin
GF_SECURITY_ADMIN_PASSWORD__FILE: /run/secrets/grafana_admin_password
GF_USERS_ALLOW_SIGN_UP: "false"
GF_ANALYTICS_REPORTING_ENABLED: "false"
GF_ANALYTICS_CHECK_FOR_UPDATES: "false"
GF_FEATURE_TOGGLES_ENABLE: traceqlEditor
# Только для снимка dashboard в CI (GRAFANA_ANONYMOUS=true): анонимный Viewer.
GF_AUTH_ANONYMOUS_ENABLED: ${GRAFANA_ANONYMOUS:-false}
GF_AUTH_ANONYMOUS_ORG_ROLE: Viewer
secrets: [grafana_admin_password]
volumes:
- ./infra/grafana/provisioning:/etc/grafana/provisioning:ro
- ./infra/grafana/dashboards:/var/lib/grafana/dashboards:ro
- grafana_data:/var/lib/grafana
ports:
- "127.0.0.1:${GRAFANA_PORT:-3000}:3000"
depends_on:
prometheus:
condition: service_healthy
loki:
condition: service_healthy
tempo:
condition: service_healthy
healthcheck:
test: ["CMD", "wget", "-q", "--spider", "http://localhost:3000/api/health"]
interval: 5s
timeout: 3s
retries: 30
restart: unless-stopped
networks: [telemetry]
deploy:
resources:
limits: { cpus: "0.50", memory: 256M }
networks:
# front: lb ↔ api; back: api ↔ хранилища и broker. LB не видит БД, S3 и broker.
# telemetry: процессы приложения → Collector → хранилища сигналов → Grafana;
# backend-ы телеметрии не видят БД, S3 и broker (кроме порта метрик RabbitMQ).
front:
back:
telemetry:
secrets:
postgres_password:
file: ./secrets/postgres_password
database_url:
file: ./secrets/database_url
cache_url:
file: ./secrets/cache_url
s3_root_user:
file: ./secrets/s3_root_user
s3_root_password:
file: ./secrets/s3_root_password
s3_access_key_id:
file: ./secrets/s3_access_key_id
s3_secret_access_key:
file: ./secrets/s3_secret_access_key
rabbitmq_conf:
file: ./secrets/rabbitmq.conf
broker_url:
file: ./secrets/broker_url
grafana_admin_password:
file: ./secrets/grafana_admin_password
volumes:
prometheus_data:
loki_data:
tempo_data:
grafana_data:
rabbitmq_data:
s3_data:
pg_data:
redis_data:
+4
View File
@@ -0,0 +1,4 @@
source_commit=d90aef716adeab32daa81ea5fce789ea6b627aaf
run=https://github.com/CharaWein/2026_lab3/actions/runs/43208190207
go_tests=success
stand=success
+18
View File
@@ -0,0 +1,18 @@
/mtusi2026lab6-api-3 restarts=0 oom_killed=false exit=0
/mtusi2026lab6-api-4 restarts=0 oom_killed=false exit=0
/mtusi2026lab6-grafana-1 restarts=0 oom_killed=false exit=0
/mtusi2026lab6-lb-1 restarts=0 oom_killed=false exit=0
/mtusi2026lab6-loki-1 restarts=0 oom_killed=false exit=0
/mtusi2026lab6-migrate-1 restarts=0 oom_killed=false exit=0
/mtusi2026lab6-otel-collector-1 restarts=0 oom_killed=false exit=0
/mtusi2026lab6-postgres-1 restarts=0 oom_killed=false exit=0
/mtusi2026lab6-prometheus-1 restarts=0 oom_killed=false exit=0
/mtusi2026lab6-rabbitmq-1 restarts=0 oom_killed=false exit=0
/mtusi2026lab6-reconciler-1 restarts=0 oom_killed=false exit=0
/mtusi2026lab6-redis-1 restarts=0 oom_killed=false exit=0
/mtusi2026lab6-relay-1 restarts=0 oom_killed=false exit=0
/mtusi2026lab6-s3-1 restarts=0 oom_killed=false exit=0
/mtusi2026lab6-s3-init-1 restarts=0 oom_killed=false exit=0
/mtusi2026lab6-tempo-1 restarts=0 oom_killed=false exit=0
/mtusi2026lab6-tempo-init-1 restarts=0 oom_killed=false exit=0
/mtusi2026lab6-worker-1 restarts=0 oom_killed=false exit=0
+18
View File
@@ -0,0 +1,18 @@
/mtusi2026lab6-api-1 restarts=0 oom_killed=false exit=0
/mtusi2026lab6-api-2 restarts=0 oom_killed=false exit=0
/mtusi2026lab6-grafana-1 restarts=0 oom_killed=false exit=0
/mtusi2026lab6-lb-1 restarts=0 oom_killed=false exit=0
/mtusi2026lab6-loki-1 restarts=0 oom_killed=false exit=0
/mtusi2026lab6-migrate-1 restarts=0 oom_killed=false exit=0
/mtusi2026lab6-otel-collector-1 restarts=0 oom_killed=false exit=0
/mtusi2026lab6-postgres-1 restarts=0 oom_killed=false exit=0
/mtusi2026lab6-prometheus-1 restarts=0 oom_killed=false exit=0
/mtusi2026lab6-rabbitmq-1 restarts=0 oom_killed=false exit=0
/mtusi2026lab6-reconciler-1 restarts=0 oom_killed=false exit=0
/mtusi2026lab6-redis-1 restarts=0 oom_killed=false exit=0
/mtusi2026lab6-relay-1 restarts=0 oom_killed=false exit=0
/mtusi2026lab6-s3-1 restarts=0 oom_killed=false exit=0
/mtusi2026lab6-s3-init-1 restarts=0 oom_killed=false exit=0
/mtusi2026lab6-tempo-1 restarts=0 oom_killed=false exit=0
/mtusi2026lab6-tempo-init-1 restarts=0 oom_killed=false exit=0
/mtusi2026lab6-worker-1 restarts=0 oom_killed=false exit=0
+18
View File
@@ -0,0 +1,18 @@
/mtusi2026lab6-api-1 restarts=0 oom_killed=false exit=0
/mtusi2026lab6-api-2 restarts=0 oom_killed=false exit=0
/mtusi2026lab6-grafana-1 restarts=0 oom_killed=false exit=0
/mtusi2026lab6-lb-1 restarts=0 oom_killed=false exit=0
/mtusi2026lab6-loki-1 restarts=0 oom_killed=false exit=0
/mtusi2026lab6-migrate-1 restarts=0 oom_killed=false exit=0
/mtusi2026lab6-otel-collector-1 restarts=0 oom_killed=false exit=0
/mtusi2026lab6-postgres-1 restarts=0 oom_killed=false exit=0
/mtusi2026lab6-prometheus-1 restarts=0 oom_killed=false exit=0
/mtusi2026lab6-rabbitmq-1 restarts=0 oom_killed=false exit=0
/mtusi2026lab6-reconciler-1 restarts=0 oom_killed=false exit=0
/mtusi2026lab6-redis-1 restarts=0 oom_killed=false exit=0
/mtusi2026lab6-relay-1 restarts=0 oom_killed=false exit=0
/mtusi2026lab6-s3-1 restarts=0 oom_killed=false exit=0
/mtusi2026lab6-s3-init-1 restarts=0 oom_killed=false exit=0
/mtusi2026lab6-tempo-1 restarts=0 oom_killed=false exit=0
/mtusi2026lab6-tempo-init-1 restarts=0 oom_killed=false exit=0
/mtusi2026lab6-worker-1 restarts=0 oom_killed=false exit=0
+18
View File
@@ -0,0 +1,18 @@
/mtusi2026lab6-api-1 restarts=0 oom_killed=false exit=0
/mtusi2026lab6-api-2 restarts=0 oom_killed=false exit=0
/mtusi2026lab6-grafana-1 restarts=0 oom_killed=false exit=0
/mtusi2026lab6-lb-1 restarts=0 oom_killed=false exit=0
/mtusi2026lab6-loki-1 restarts=0 oom_killed=false exit=0
/mtusi2026lab6-migrate-1 restarts=0 oom_killed=false exit=0
/mtusi2026lab6-otel-collector-1 restarts=0 oom_killed=false exit=0
/mtusi2026lab6-postgres-1 restarts=0 oom_killed=false exit=0
/mtusi2026lab6-prometheus-1 restarts=0 oom_killed=false exit=0
/mtusi2026lab6-rabbitmq-1 restarts=0 oom_killed=false exit=0
/mtusi2026lab6-reconciler-1 restarts=0 oom_killed=false exit=0
/mtusi2026lab6-redis-1 restarts=0 oom_killed=false exit=0
/mtusi2026lab6-relay-1 restarts=0 oom_killed=false exit=0
/mtusi2026lab6-s3-1 restarts=0 oom_killed=false exit=0
/mtusi2026lab6-s3-init-1 restarts=0 oom_killed=false exit=0
/mtusi2026lab6-tempo-1 restarts=0 oom_killed=false exit=0
/mtusi2026lab6-tempo-init-1 restarts=0 oom_killed=false exit=0
/mtusi2026lab6-worker-1 restarts=0 oom_killed=false exit=0
+18
View File
@@ -0,0 +1,18 @@
/mtusi2026lab6-api-1 restarts=0 oom_killed=false exit=0
/mtusi2026lab6-api-2 restarts=0 oom_killed=false exit=0
/mtusi2026lab6-grafana-1 restarts=0 oom_killed=false exit=0
/mtusi2026lab6-lb-1 restarts=0 oom_killed=false exit=0
/mtusi2026lab6-loki-1 restarts=0 oom_killed=false exit=0
/mtusi2026lab6-migrate-1 restarts=0 oom_killed=false exit=0
/mtusi2026lab6-otel-collector-1 restarts=0 oom_killed=false exit=0
/mtusi2026lab6-postgres-1 restarts=0 oom_killed=false exit=0
/mtusi2026lab6-prometheus-1 restarts=0 oom_killed=false exit=0
/mtusi2026lab6-rabbitmq-1 restarts=0 oom_killed=false exit=0
/mtusi2026lab6-reconciler-1 restarts=0 oom_killed=false exit=0
/mtusi2026lab6-redis-1 restarts=0 oom_killed=false exit=0
/mtusi2026lab6-relay-1 restarts=0 oom_killed=false exit=0
/mtusi2026lab6-s3-1 restarts=0 oom_killed=false exit=0
/mtusi2026lab6-s3-init-1 restarts=0 oom_killed=false exit=0
/mtusi2026lab6-tempo-1 restarts=0 oom_killed=false exit=0
/mtusi2026lab6-tempo-init-1 restarts=0 oom_killed=false exit=0
/mtusi2026lab6-worker-1 restarts=0 oom_killed=false exit=0
+18
View File
@@ -0,0 +1,18 @@
2026-11-12T11:23:35Z
Docker Compose version v2.38.2
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
mtusi2026lab6-api-1 mtusi2026lab6-api:local "/app/api" api 28 seconds ago Up 19 seconds (healthy) 8080/tcp
mtusi2026lab6-api-2 mtusi2026lab6-api:local "/app/api" api 28 seconds ago Up 19 seconds (healthy) 8080/tcp
mtusi2026lab6-grafana-1 grafana/grafana:11.4.0 "/run.sh" grafana 28 seconds ago Up 6 seconds (healthy) 127.0.0.1:3000->3000/tcp
mtusi2026lab6-lb-1 haproxy:3.0.5-alpine "docker-entrypoint.s…" lb 28 seconds ago Up 13 seconds 127.0.0.1:8080->8080/tcp, 127.0.0.1:8404->8404/tcp
mtusi2026lab6-loki-1 grafana/loki:3.3.2 "/usr/bin/loki -conf…" loki 28 seconds ago Up 27 seconds (healthy) 3100/tcp
mtusi2026lab6-otel-collector-1 otel/opentelemetry-collector-contrib:0.116.1 "/otelcol-contrib --…" otel-collector 28 seconds ago Up 27 seconds 4317-4318/tcp, 55678-55679/tcp
mtusi2026lab6-postgres-1 postgres:16.4-alpine "docker-entrypoint.s…" postgres 28 seconds ago Up 27 seconds (healthy) 5432/tcp
mtusi2026lab6-prometheus-1 prom/prometheus:v3.1.0 "/bin/prometheus --c…" prometheus 28 seconds ago Up 27 seconds (healthy) 127.0.0.1:9090->9090/tcp
mtusi2026lab6-rabbitmq-1 rabbitmq:4.0.9-alpine "docker-entrypoint.s…" rabbitmq 28 seconds ago Up 27 seconds (healthy) 4369/tcp, 5671-5672/tcp, 15691-15692/tcp, 25672/tcp
mtusi2026lab6-reconciler-1 mtusi2026lab6-api:local "/app/reconcile --ap…" reconciler 28 seconds ago Up 19 seconds (healthy) 8080/tcp
mtusi2026lab6-redis-1 redis:7.4.1-alpine "docker-entrypoint.s…" redis 28 seconds ago Up 27 seconds (healthy) 6379/tcp
mtusi2026lab6-relay-1 mtusi2026lab6-api:local "/app/pipeline relay" relay 28 seconds ago Up 13 seconds (healthy) 8080/tcp
mtusi2026lab6-s3-1 bitnamilegacy/minio:2025.4.22@sha256:50cec18ac4184af4671a78aedd5554942c8ae105d51a465fa82037949046da01 "/bin/bash -c 'expor…" s3 28 seconds ago Up 27 seconds (healthy) 9000-9001/tcp
mtusi2026lab6-tempo-1 grafana/tempo:2.6.1 "/tempo -config.file…" tempo 28 seconds ago Up 26 seconds (healthy)
mtusi2026lab6-worker-1 mtusi2026lab6-api:local "/app/pipeline worker" worker 28 seconds ago Up 13 seconds (healthy) 8080/tcp
+19
View File
@@ -0,0 +1,19 @@
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
mtusi2026lab6-api-3 mtusi2026lab6-api:local "/app/api" api 55 seconds ago Up 55 seconds (healthy) 8080/tcp
mtusi2026lab6-api-4 mtusi2026lab6-api:local "/app/api" api 41 seconds ago Up 41 seconds (healthy) 8080/tcp
mtusi2026lab6-grafana-1 grafana/grafana:11.4.0 "/run.sh" grafana 6 minutes ago Up 4 minutes (healthy) 127.0.0.1:3000->3000/tcp
mtusi2026lab6-lb-1 haproxy:3.0.5-alpine "docker-entrypoint.s…" lb 6 minutes ago Up 4 minutes 127.0.0.1:8080->8080/tcp, 127.0.0.1:8404->8404/tcp
mtusi2026lab6-loki-1 grafana/loki:3.3.2 "/usr/bin/loki -conf…" loki 6 minutes ago Up 4 minutes (healthy) 3100/tcp
mtusi2026lab6-migrate-1 migrate/migrate:v4.17.0 "/bin/sh -c 'exec mi…" migrate 6 minutes ago Exited (0) 4 minutes ago
mtusi2026lab6-otel-collector-1 otel/opentelemetry-collector-contrib:0.116.1 "/otelcol-contrib --…" otel-collector 6 minutes ago Up 4 minutes 4317-4318/tcp, 55678-55679/tcp
mtusi2026lab6-postgres-1 postgres:16.4-alpine "docker-entrypoint.s…" postgres 6 minutes ago Up 4 minutes (healthy) 5432/tcp
mtusi2026lab6-prometheus-1 prom/prometheus:v3.1.0 "/bin/prometheus --c…" prometheus 6 minutes ago Up 4 minutes (healthy) 127.0.0.1:9090->9090/tcp
mtusi2026lab6-rabbitmq-1 rabbitmq:4.0.9-alpine "docker-entrypoint.s…" rabbitmq 6 minutes ago Up 4 minutes (healthy) 4369/tcp, 5671-5672/tcp, 15691-15692/tcp, 25672/tcp
mtusi2026lab6-reconciler-1 mtusi2026lab6-api:local "/app/reconcile --ap…" reconciler 6 minutes ago Up 4 minutes (healthy) 8080/tcp
mtusi2026lab6-redis-1 redis:7.4.1-alpine "docker-entrypoint.s…" redis 6 minutes ago Up 4 minutes (healthy) 6379/tcp
mtusi2026lab6-relay-1 mtusi2026lab6-api:local "/app/pipeline relay" relay 6 minutes ago Up 4 minutes (healthy) 8080/tcp
mtusi2026lab6-s3-1 bitnamilegacy/minio:2025.4.22@sha256:50cec18ac4184af4671a78aedd5554942c8ae105d51a465fa82037949046da01 "/bin/bash -c 'expor…" s3 6 minutes ago Up 4 minutes (healthy) 9000-9001/tcp
mtusi2026lab6-s3-init-1 bitnamilegacy/minio-client:2025.4.16@sha256:8a86e441decf053093c5977d49290298295f850c7843652f93e970ab15871dd1 "/bin/sh /init/init.…" s3-init 6 minutes ago Exited (0) 4 minutes ago
mtusi2026lab6-tempo-1 grafana/tempo:2.6.1 "/tempo -config.file…" tempo 6 minutes ago Up 4 minutes (healthy)
mtusi2026lab6-tempo-init-1 grafana/tempo:2.6.1 "chown 10001:10001 /…" tempo-init 6 minutes ago Exited (0) 6 minutes ago
mtusi2026lab6-worker-1 mtusi2026lab6-api:local "/app/pipeline worker" worker 6 minutes ago Up 4 minutes (healthy) 8080/tcp
+19
View File
@@ -0,0 +1,19 @@
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
mtusi2026lab6-api-1 mtusi2026lab6-api:local "/app/api" api 3 minutes ago Up 12 seconds (healthy) 8080/tcp
mtusi2026lab6-api-2 mtusi2026lab6-api:local "/app/api" api 3 minutes ago Up 12 seconds (healthy) 8080/tcp
mtusi2026lab6-grafana-1 grafana/grafana:11.4.0 "/run.sh" grafana 3 minutes ago Up 2 minutes (healthy) 127.0.0.1:3000->3000/tcp
mtusi2026lab6-lb-1 haproxy:3.0.5-alpine "docker-entrypoint.s…" lb 3 minutes ago Up 2 minutes 127.0.0.1:8080->8080/tcp, 127.0.0.1:8404->8404/tcp
mtusi2026lab6-loki-1 grafana/loki:3.3.2 "/usr/bin/loki -conf…" loki 3 minutes ago Up 3 minutes (healthy) 3100/tcp
mtusi2026lab6-migrate-1 migrate/migrate:v4.17.0 "/bin/sh -c 'exec mi…" migrate 3 minutes ago Exited (0) 10 seconds ago
mtusi2026lab6-otel-collector-1 otel/opentelemetry-collector-contrib:0.116.1 "/otelcol-contrib --…" otel-collector 3 minutes ago Up 3 minutes 4317-4318/tcp, 55678-55679/tcp
mtusi2026lab6-postgres-1 postgres:16.4-alpine "docker-entrypoint.s…" postgres 3 minutes ago Up 3 minutes (healthy) 5432/tcp
mtusi2026lab6-prometheus-1 prom/prometheus:v3.1.0 "/bin/prometheus --c…" prometheus 3 minutes ago Up 3 minutes (healthy) 127.0.0.1:9090->9090/tcp
mtusi2026lab6-rabbitmq-1 rabbitmq:4.0.9-alpine "docker-entrypoint.s…" rabbitmq 3 minutes ago Up 3 minutes (healthy) 4369/tcp, 5671-5672/tcp, 15691-15692/tcp, 25672/tcp
mtusi2026lab6-reconciler-1 mtusi2026lab6-api:local "/app/reconcile --ap…" reconciler 3 minutes ago Up 2 minutes (healthy) 8080/tcp
mtusi2026lab6-redis-1 redis:7.4.1-alpine "docker-entrypoint.s…" redis 3 minutes ago Up 3 minutes (healthy) 6379/tcp
mtusi2026lab6-relay-1 mtusi2026lab6-api:local "/app/pipeline relay" relay 3 minutes ago Up 2 minutes (healthy) 8080/tcp
mtusi2026lab6-s3-1 bitnamilegacy/minio:2025.4.22@sha256:50cec18ac4184af4671a78aedd5554942c8ae105d51a465fa82037949046da01 "/bin/bash -c 'expor…" s3 3 minutes ago Up About a minute (healthy) 9000-9001/tcp
mtusi2026lab6-s3-init-1 bitnamilegacy/minio-client:2025.4.16@sha256:8a86e441decf053093c5977d49290298295f850c7843652f93e970ab15871dd1 "/bin/sh /init/init.…" s3-init 3 minutes ago Exited (0) 9 seconds ago
mtusi2026lab6-tempo-1 grafana/tempo:2.6.1 "/tempo -config.file…" tempo 3 minutes ago Up 3 minutes (healthy)
mtusi2026lab6-tempo-init-1 grafana/tempo:2.6.1 "chown 10001:10001 /…" tempo-init 3 minutes ago Exited (0) 3 minutes ago
mtusi2026lab6-worker-1 mtusi2026lab6-api:local "/app/pipeline worker" worker 3 minutes ago Up 2 minutes (healthy) 8080/tcp
+19
View File
@@ -0,0 +1,19 @@
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
mtusi2026lab6-api-1 mtusi2026lab6-api:local "/app/api" api 10 minutes ago Up 10 minutes (healthy) 8080/tcp
mtusi2026lab6-api-2 mtusi2026lab6-api:local "/app/api" api 10 minutes ago Up 10 minutes (healthy) 8080/tcp
mtusi2026lab6-grafana-1 grafana/grafana:11.4.0 "/run.sh" grafana 10 minutes ago Up 9 minutes (healthy) 127.0.0.1:3000->3000/tcp
mtusi2026lab6-lb-1 haproxy:3.0.5-alpine "docker-entrypoint.s…" lb 10 minutes ago Up 10 minutes 127.0.0.1:8080->8080/tcp, 127.0.0.1:8404->8404/tcp
mtusi2026lab6-loki-1 grafana/loki:3.3.2 "/usr/bin/loki -conf…" loki 10 minutes ago Up 2 minutes (healthy) 3100/tcp
mtusi2026lab6-migrate-1 migrate/migrate:v4.17.0 "/bin/sh -c 'exec mi…" migrate 10 minutes ago Exited (0) 7 minutes ago
mtusi2026lab6-otel-collector-1 otel/opentelemetry-collector-contrib:0.116.1 "/otelcol-contrib --…" otel-collector 10 minutes ago Up 39 seconds 4317-4318/tcp, 55678-55679/tcp
mtusi2026lab6-postgres-1 postgres:16.4-alpine "docker-entrypoint.s…" postgres 10 minutes ago Up 10 minutes (healthy) 5432/tcp
mtusi2026lab6-prometheus-1 prom/prometheus:v3.1.0 "/bin/prometheus --c…" prometheus 10 minutes ago Up 10 minutes (healthy) 127.0.0.1:9090->9090/tcp
mtusi2026lab6-rabbitmq-1 rabbitmq:4.0.9-alpine "docker-entrypoint.s…" rabbitmq 10 minutes ago Up 10 minutes (healthy) 4369/tcp, 5671-5672/tcp, 15691-15692/tcp, 25672/tcp
mtusi2026lab6-reconciler-1 mtusi2026lab6-api:local "/app/reconcile --ap…" reconciler 10 minutes ago Up 10 minutes (healthy) 8080/tcp
mtusi2026lab6-redis-1 redis:7.4.1-alpine "docker-entrypoint.s…" redis 10 minutes ago Up 10 minutes (healthy) 6379/tcp
mtusi2026lab6-relay-1 mtusi2026lab6-api:local "/app/pipeline relay" relay 10 minutes ago Up 10 minutes (healthy) 8080/tcp
mtusi2026lab6-s3-1 bitnamilegacy/minio:2025.4.22@sha256:50cec18ac4184af4671a78aedd5554942c8ae105d51a465fa82037949046da01 "/bin/bash -c 'expor…" s3 10 minutes ago Up 9 minutes (healthy) 9000-9001/tcp
mtusi2026lab6-s3-init-1 bitnamilegacy/minio-client:2025.4.16@sha256:8a86e441decf053093c5977d49290298295f850c7843652f93e970ab15871dd1 "/bin/sh /init/init.…" s3-init 10 minutes ago Exited (0) 7 minutes ago
mtusi2026lab6-tempo-1 grafana/tempo:2.6.1 "/tempo -config.file…" tempo 10 minutes ago Up 2 minutes (healthy)
mtusi2026lab6-tempo-init-1 grafana/tempo:2.6.1 "chown 10001:10001 /…" tempo-init 10 minutes ago Exited (0) 2 minutes ago
mtusi2026lab6-worker-1 mtusi2026lab6-api:local "/app/pipeline worker" worker 10 minutes ago Up 7 minutes (healthy) 8080/tcp
+19
View File
@@ -0,0 +1,19 @@
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
mtusi2026lab6-api-1 mtusi2026lab6-api:local "/app/api" api 4 minutes ago Up 2 minutes (healthy) 8080/tcp
mtusi2026lab6-api-2 mtusi2026lab6-api:local "/app/api" api 4 minutes ago Up 2 minutes (healthy) 8080/tcp
mtusi2026lab6-grafana-1 grafana/grafana:11.4.0 "/run.sh" grafana 4 minutes ago Up 3 minutes (healthy) 127.0.0.1:3000->3000/tcp
mtusi2026lab6-lb-1 haproxy:3.0.5-alpine "docker-entrypoint.s…" lb 4 minutes ago Up 3 minutes 127.0.0.1:8080->8080/tcp, 127.0.0.1:8404->8404/tcp
mtusi2026lab6-loki-1 grafana/loki:3.3.2 "/usr/bin/loki -conf…" loki 4 minutes ago Up 4 minutes (healthy) 3100/tcp
mtusi2026lab6-migrate-1 migrate/migrate:v4.17.0 "/bin/sh -c 'exec mi…" migrate 4 minutes ago Exited (0) 36 seconds ago
mtusi2026lab6-otel-collector-1 otel/opentelemetry-collector-contrib:0.116.1 "/otelcol-contrib --…" otel-collector 4 minutes ago Up 4 minutes 4317-4318/tcp, 55678-55679/tcp
mtusi2026lab6-postgres-1 postgres:16.4-alpine "docker-entrypoint.s…" postgres 4 minutes ago Up 4 minutes (healthy) 5432/tcp
mtusi2026lab6-prometheus-1 prom/prometheus:v3.1.0 "/bin/prometheus --c…" prometheus 4 minutes ago Up 4 minutes (healthy) 127.0.0.1:9090->9090/tcp
mtusi2026lab6-rabbitmq-1 rabbitmq:4.0.9-alpine "docker-entrypoint.s…" rabbitmq 4 minutes ago Up About a minute (healthy) 4369/tcp, 5671-5672/tcp, 15691-15692/tcp, 25672/tcp
mtusi2026lab6-reconciler-1 mtusi2026lab6-api:local "/app/reconcile --ap…" reconciler 4 minutes ago Up 3 minutes (healthy) 8080/tcp
mtusi2026lab6-redis-1 redis:7.4.1-alpine "docker-entrypoint.s…" redis 4 minutes ago Up 4 minutes (healthy) 6379/tcp
mtusi2026lab6-relay-1 mtusi2026lab6-api:local "/app/pipeline relay" relay 2 minutes ago Up 2 minutes (healthy) 8080/tcp
mtusi2026lab6-s3-1 bitnamilegacy/minio:2025.4.22@sha256:50cec18ac4184af4671a78aedd5554942c8ae105d51a465fa82037949046da01 "/bin/bash -c 'expor…" s3 4 minutes ago Up 46 seconds (healthy) 9000-9001/tcp
mtusi2026lab6-s3-init-1 bitnamilegacy/minio-client:2025.4.16@sha256:8a86e441decf053093c5977d49290298295f850c7843652f93e970ab15871dd1 "/bin/sh /init/init.…" s3-init 4 minutes ago Exited (0) 35 seconds ago
mtusi2026lab6-tempo-1 grafana/tempo:2.6.1 "/tempo -config.file…" tempo 4 minutes ago Up 3 minutes (healthy)
mtusi2026lab6-tempo-init-1 grafana/tempo:2.6.1 "chown 10001:10001 /…" tempo-init 4 minutes ago Exited (0) 4 minutes ago
mtusi2026lab6-worker-1 mtusi2026lab6-api:local "/app/pipeline worker" worker About a minute ago Up 20 seconds (healthy) 8080/tcp
+19
View File
@@ -0,0 +1,19 @@
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
mtusi2026lab6-api-1 mtusi2026lab6-api:local "/app/api" api About a minute ago Up About a minute (healthy) 8080/tcp
mtusi2026lab6-api-2 mtusi2026lab6-api:local "/app/api" api About a minute ago Up About a minute (healthy) 8080/tcp
mtusi2026lab6-grafana-1 grafana/grafana:11.4.0 "/run.sh" grafana 19 minutes ago Up 19 minutes (healthy) 127.0.0.1:3000->3000/tcp
mtusi2026lab6-lb-1 haproxy:3.0.5-alpine "docker-entrypoint.s…" lb 19 minutes ago Up 19 minutes 127.0.0.1:8080->8080/tcp, 127.0.0.1:8404->8404/tcp
mtusi2026lab6-loki-1 grafana/loki:3.3.2 "/usr/bin/loki -conf…" loki 19 minutes ago Up 19 minutes (healthy) 3100/tcp
mtusi2026lab6-migrate-1 migrate/migrate:v4.17.0 "/bin/sh -c 'exec mi…" migrate 19 minutes ago Exited (0) 19 minutes ago
mtusi2026lab6-otel-collector-1 otel/opentelemetry-collector-contrib:0.116.1 "/otelcol-contrib --…" otel-collector 3 minutes ago Up 3 minutes 4317-4318/tcp, 55678-55679/tcp
mtusi2026lab6-postgres-1 postgres:16.4-alpine "docker-entrypoint.s…" postgres 19 minutes ago Up 19 minutes (healthy) 5432/tcp
mtusi2026lab6-prometheus-1 prom/prometheus:v3.1.0 "/bin/prometheus --c…" prometheus 19 minutes ago Up 19 minutes (healthy) 127.0.0.1:9090->9090/tcp
mtusi2026lab6-rabbitmq-1 rabbitmq:4.0.9-alpine "docker-entrypoint.s…" rabbitmq 19 minutes ago Up 19 minutes (healthy) 4369/tcp, 5671-5672/tcp, 15691-15692/tcp, 25672/tcp
mtusi2026lab6-reconciler-1 mtusi2026lab6-api:local "/app/reconcile --ap…" reconciler 19 minutes ago Up 19 minutes (healthy) 8080/tcp
mtusi2026lab6-redis-1 redis:7.4.1-alpine "docker-entrypoint.s…" redis 19 minutes ago Up 19 minutes (healthy) 6379/tcp
mtusi2026lab6-relay-1 mtusi2026lab6-api:local "/app/pipeline relay" relay About a minute ago Up About a minute (healthy) 8080/tcp
mtusi2026lab6-s3-1 bitnamilegacy/minio:2025.4.22@sha256:50cec18ac4184af4671a78aedd5554942c8ae105d51a465fa82037949046da01 "/bin/bash -c 'expor…" s3 19 minutes ago Up 19 minutes (healthy) 9000-9001/tcp
mtusi2026lab6-s3-init-1 bitnamilegacy/minio-client:2025.4.16@sha256:8a86e441decf053093c5977d49290298295f850c7843652f93e970ab15871dd1 "/bin/sh /init/init.…" s3-init 19 minutes ago Exited (0) 19 minutes ago
mtusi2026lab6-tempo-1 grafana/tempo:2.6.1 "/tempo -config.file…" tempo 19 minutes ago Up 19 minutes (healthy)
mtusi2026lab6-tempo-init-1 grafana/tempo:2.6.1 "chown 10001:10001 /…" tempo-init 19 minutes ago Exited (0) 19 minutes ago
mtusi2026lab6-worker-1 mtusi2026lab6-api:local "/app/pipeline worker" worker About a minute ago Up About a minute (healthy) 8080/tcp
+15
View File
@@ -0,0 +1,15 @@
mtusi2026lab6-api-4 0.33% 15.89MiB / 256MiB
mtusi2026lab6-api-3 0.19% 14.84MiB / 256MiB
mtusi2026lab6-lb-1 0.08% 8.844MiB / 128MiB
mtusi2026lab6-grafana-1 0.14% 51.9MiB / 256MiB
mtusi2026lab6-relay-1 0.14% 11.45MiB / 128MiB
mtusi2026lab6-worker-1 0.02% 11.4MiB / 768MiB
mtusi2026lab6-reconciler-1 0.00% 4.32MiB / 128MiB
mtusi2026lab6-tempo-1 2.87% 156.6MiB / 1GiB
mtusi2026lab6-postgres-1 21.75% 79.28MiB / 512MiB
mtusi2026lab6-s3-1 0.00% 224MiB / 512MiB
mtusi2026lab6-otel-collector-1 1.22% 50.91MiB / 256MiB
mtusi2026lab6-rabbitmq-1 0.11% 69.58MiB / 512MiB
mtusi2026lab6-prometheus-1 0.67% 39.98MiB / 512MiB
mtusi2026lab6-redis-1 0.34% 3.977MiB / 256MiB
mtusi2026lab6-loki-1 3.34% 122.5MiB / 1GiB
+15
View File
@@ -0,0 +1,15 @@
mtusi2026lab6-lb-1 0.09% 8.566MiB / 128MiB
mtusi2026lab6-reconciler-1 0.00% 4.137MiB / 128MiB
mtusi2026lab6-relay-1 0.28% 12.17MiB / 128MiB
mtusi2026lab6-api-2 3.71% 18.97MiB / 256MiB
mtusi2026lab6-api-1 3.16% 18.38MiB / 256MiB
mtusi2026lab6-worker-1 99.82% 153.3MiB / 768MiB
mtusi2026lab6-grafana-1 3.18% 57.65MiB / 256MiB
mtusi2026lab6-tempo-1 1.94% 55.03MiB / 1GiB
mtusi2026lab6-loki-1 0.45% 62.09MiB / 1GiB
mtusi2026lab6-s3-1 7.44% 447.9MiB / 512MiB
mtusi2026lab6-rabbitmq-1 22.57% 85.34MiB / 512MiB
mtusi2026lab6-prometheus-1 0.98% 39.7MiB / 512MiB
mtusi2026lab6-otel-collector-1 0.99% 48.46MiB / 256MiB
mtusi2026lab6-postgres-1 4.18% 84.61MiB / 512MiB
mtusi2026lab6-redis-1 0.47% 3.414MiB / 256MiB
+15
View File
@@ -0,0 +1,15 @@
mtusi2026lab6-lb-1 0.08% 8.562MiB / 128MiB
mtusi2026lab6-grafana-1 0.03% 74.72MiB / 256MiB
mtusi2026lab6-api-1 0.12% 21.26MiB / 256MiB
mtusi2026lab6-reconciler-1 2.26% 4.445MiB / 128MiB
mtusi2026lab6-relay-1 0.16% 12.93MiB / 128MiB
mtusi2026lab6-api-2 0.12% 18.84MiB / 256MiB
mtusi2026lab6-worker-1 0.01% 12.86MiB / 768MiB
mtusi2026lab6-tempo-1 0.03% 145.2MiB / 1GiB
mtusi2026lab6-rabbitmq-1 0.13% 86.12MiB / 512MiB
mtusi2026lab6-otel-collector-1 0.32% 54.26MiB / 256MiB
mtusi2026lab6-prometheus-1 0.18% 57.23MiB / 512MiB
mtusi2026lab6-loki-1 0.37% 130.3MiB / 1GiB
mtusi2026lab6-postgres-1 0.19% 132MiB / 512MiB
mtusi2026lab6-s3-1 2.28% 231.6MiB / 512MiB
mtusi2026lab6-redis-1 0.34% 3.406MiB / 256MiB
+15
View File
@@ -0,0 +1,15 @@
mtusi2026lab6-worker-1 0.01% 528.9MiB / 768MiB
mtusi2026lab6-relay-1 0.20% 11.69MiB / 128MiB
mtusi2026lab6-lb-1 0.10% 8.027MiB / 128MiB
mtusi2026lab6-reconciler-1 0.00% 4.328MiB / 128MiB
mtusi2026lab6-api-1 0.30% 203.4MiB / 256MiB
mtusi2026lab6-api-2 0.24% 142.4MiB / 256MiB
mtusi2026lab6-grafana-1 2.77% 66.38MiB / 256MiB
mtusi2026lab6-tempo-1 2.86% 43.02MiB / 1GiB
mtusi2026lab6-rabbitmq-1 51.00% 123.6MiB / 512MiB
mtusi2026lab6-postgres-1 2.13% 57.27MiB / 512MiB
mtusi2026lab6-redis-1 0.52% 3.598MiB / 256MiB
mtusi2026lab6-s3-1 2.85% 342MiB / 512MiB
mtusi2026lab6-prometheus-1 0.95% 42.73MiB / 512MiB
mtusi2026lab6-otel-collector-1 0.21% 47.2MiB / 256MiB
mtusi2026lab6-loki-1 0.46% 63.41MiB / 1GiB
+15
View File
@@ -0,0 +1,15 @@
mtusi2026lab6-api-1 0.13% 16.12MiB / 256MiB
mtusi2026lab6-relay-1 0.30% 10.56MiB / 128MiB
mtusi2026lab6-api-2 2.94% 16.2MiB / 256MiB
mtusi2026lab6-worker-1 0.01% 9.699MiB / 768MiB
mtusi2026lab6-otel-collector-1 0.11% 50.07MiB / 256MiB
mtusi2026lab6-lb-1 0.10% 8.715MiB / 128MiB
mtusi2026lab6-grafana-1 0.04% 65.7MiB / 256MiB
mtusi2026lab6-reconciler-1 0.00% 4.344MiB / 128MiB
mtusi2026lab6-tempo-1 0.47% 91.59MiB / 1GiB
mtusi2026lab6-postgres-1 3.74% 152.8MiB / 512MiB
mtusi2026lab6-rabbitmq-1 22.76% 119.6MiB / 512MiB
mtusi2026lab6-prometheus-1 0.15% 57.73MiB / 512MiB
mtusi2026lab6-loki-1 0.34% 170.3MiB / 1GiB
mtusi2026lab6-s3-1 0.07% 224.1MiB / 512MiB
mtusi2026lab6-redis-1 0.45% 3.66MiB / 256MiB
+8
View File
@@ -0,0 +1,8 @@
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
{"observed_instances_before": 2, "successful_writes": 143, "failed_attempts": 0, "first_recovery_seconds": 3.535}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
-- контейнеры api после stop-one:
mtusi2026lab6-api-1 84c136cad07b Up 43 seconds (healthy)
mtusi2026lab6-api-2 1354d20e1722 Exited (0) 12 seconds ago
mtusi2026lab6-api-1 Up 51 seconds (healthy)
mtusi2026lab6-api-2 Up 6 seconds (healthy)
+8
View File
@@ -0,0 +1,8 @@
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
{"observed_instances_before": 2, "successful_writes": 144, "failed_attempts": 0, "first_recovery_seconds": 3.478}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
-- контейнеры api после stop-one:
mtusi2026lab6-api-1 84c136cad07b Up About a minute (healthy)
mtusi2026lab6-api-2 1354d20e1722 Exited (0) 12 seconds ago
mtusi2026lab6-api-1 Up About a minute (healthy)
mtusi2026lab6-api-2 Up 6 seconds (healthy)
+64
View File
@@ -0,0 +1,64 @@
{
"concurrency": 4,
"replicas": 2,
"total": {
"seconds": 53.16,
"ok": 16801,
"errors": 0,
"ok_rps": 316.1
},
"confirmed_writes": 8401,
"stored_messages": 8404,
"duplicates": 0,
"lost_confirmed": 0,
"per_replica": [
{
"target_instance": "84c136cad07b",
"docker_stop_seconds": 4.03,
"before": {
"seconds": 5.0,
"ok": 1759,
"errors": 0,
"ok_rps": 351.8
},
"during_stop_and_observe": {
"seconds": 15.0,
"ok": 5183,
"errors": 0,
"ok_rps": 345.5
},
"errors_after_stop": 0,
"recovery_seconds": 0.0,
"target_last_served_after_stop_seconds": 1.15,
"other_instances_after": [
"1354d20e1722"
],
"restore_healthy_seconds": 5.57,
"restore_serving_via_lb_seconds": 5.57
},
{
"target_instance": "1354d20e1722",
"docker_stop_seconds": 4.85,
"before": {
"seconds": 5.0,
"ok": 1717,
"errors": 0,
"ok_rps": 343.4
},
"during_stop_and_observe": {
"seconds": 15.0,
"ok": 4302,
"errors": 0,
"ok_rps": 286.8
},
"errors_after_stop": 0,
"recovery_seconds": 0.0,
"target_last_served_after_stop_seconds": 1.5,
"other_instances_after": [
"84c136cad07b"
],
"restore_healthy_seconds": 6.61,
"restore_serving_via_lb_seconds": 6.61
}
]
}
+29
View File
@@ -0,0 +1,29 @@
ok.png image/png 106 -> 202
ok.jpg image/jpeg 900 -> 202
%D1%84%D0%BE%D1%82%D0%BE%20%F0%9F%98%80. image/png 106 -> 202
params.png image/png; charset=binary 106 -> 202
near-limit.png image/png 10288524 -> 202
too-big.png (Content-Length) image/png 10485761 -> 413
too-big-chunked.png (no Content-Length) image/png 11534336 -> 413
bomb.png image/png 66 -> 413
over-pixels.png image/png 66 -> 413
truncated.png image/png 53 -> 415
corrupt.png image/png 106 -> 415
truncated.jpg image/jpeg 898 -> 415
mime-swap.png image/jpeg 106 -> 415
mime-swap.jpg image/png 900 -> 415
text.png image/png 23 -> 415
gif.gif image/gif 14 -> 415
ctype-missing.png application/octet-stream 106 -> 415
empty.png image/png 0 -> 415
short-data.png image/png 68 -> 415
..%2Fetc%2Fpasswd image/png 106 -> 400
a%5Cb.png image/png 106 -> 400
nul%00.png image/png 106 -> 400
bell%07.png image/png 106 -> 400
%20%20%20 image/png 106 -> 400
xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx image/png 106 -> 400
image/png 106 -> 400
eve.png image/png 106 -> 404
foreign.png image/png 106 -> 202
PASS: attachment negative suite (28 upload cases, no 5xx)
+14
View File
@@ -0,0 +1,14 @@
{
"scenario": "db-gap",
"result": {
"client_status": 504,
"seconds": 54.2,
"after_gap": {
"row_status": "uploading",
"staging_objects": 1
},
"final": {
"row_status": "none"
}
}
}
+11
View File
@@ -0,0 +1,11 @@
{
"scenario": "failover",
"result": {
"uploaded_by": "e8856700a5dd",
"downloaded_by": [
"23f43d87eef5"
],
"sha256": "9cd8ae086706327908419e4e9782e8ed003310802f9381e4f8776a81079460e5",
"stopped_container": "e8856700a5dd"
}
}
+9
View File
@@ -0,0 +1,9 @@
{
"scenario": "interrupt",
"result": {
"sent_bytes": 1048576,
"declared_bytes": 4192910,
"rows_after": 0,
"objects_after": 0
}
}
+40
View File
@@ -0,0 +1,40 @@
{
"scenario": "memory",
"result": {
"file_bytes": 10187085,
"runs": [
{
"parallel": 10,
"ok": 10,
"failed": [],
"max_seconds": 1.21,
"baseline_peak_mib": {
"e8856700a5dd": 12.7,
"23f43d87eef5": 13.4
},
"peak_mib": {
"e8856700a5dd": 20.7,
"23f43d87eef5": 29.5
},
"payload_total_mib": 97.2,
"max_peak_growth_mib": 16.1
},
{
"parallel": 20,
"ok": 20,
"failed": [],
"max_seconds": 2.12,
"baseline_peak_mib": {
"e8856700a5dd": 12.6,
"23f43d87eef5": 13.0
},
"peak_mib": {
"e8856700a5dd": 41.0,
"23f43d87eef5": 38.4
},
"payload_total_mib": 194.3,
"max_peak_growth_mib": 28.5
}
]
}
}
+13
View File
@@ -0,0 +1,13 @@
{
"scenario": "s3-outage",
"result": {
"upload_status_during_outage": 503,
"upload_error_seconds": 0.32,
"download_status_during_outage": 503,
"metadata_during_outage": "ready",
"text_message_during_outage": 201,
"readiness_during_outage": 200,
"intents_left": 0,
"upload_after_recovery": "queued"
}
}
+22
View File
@@ -0,0 +1,22 @@
{
"scenario": "security",
"result": {
"anonymous_get": "403",
"anonymous_list": "403",
"api_key_own_bucket": "allowed",
"api_key_make_bucket": "denied",
"api_key_admin": "denied",
"api_key_visible_buckets": [
"messenger-attachments"
],
"bootstrap_runs": [
0,
0
],
"objects_before_after": [
40,
40
],
"s3_published_ports": "9000-9001/tcp"
}
}
+20
View File
@@ -0,0 +1,20 @@
? github.com/charawein/mtusi2026lab6/cmd/api [no test files]
? github.com/charawein/mtusi2026lab6/cmd/pipeline [no test files]
? github.com/charawein/mtusi2026lab6/cmd/reconcile [no test files]
? github.com/charawein/mtusi2026lab6/internal/broker [no test files]
? github.com/charawein/mtusi2026lab6/internal/config [no test files]
? github.com/charawein/mtusi2026lab6/internal/domain [no test files]
ok github.com/charawein/mtusi2026lab6/internal/imagecheck 0.045s
ok github.com/charawein/mtusi2026lab6/internal/imaging 0.690s
ok github.com/charawein/mtusi2026lab6/internal/reconcile 0.005s
? github.com/charawein/mtusi2026lab6/internal/repository [no test files]
? github.com/charawein/mtusi2026lab6/internal/repository/postgres [no test files]
? github.com/charawein/mtusi2026lab6/internal/repository/redis [no test files]
ok github.com/charawein/mtusi2026lab6/internal/service 0.008s
? github.com/charawein/mtusi2026lab6/internal/storage [no test files]
ok github.com/charawein/mtusi2026lab6/internal/telemetry 0.017s
? github.com/charawein/mtusi2026lab6/internal/transport/http [no test files]
ok github.com/charawein/mtusi2026lab6/internal/transport/http/middleware 0.009s
? github.com/charawein/mtusi2026lab6/internal/transport/httputil [no test files]
ok github.com/charawein/mtusi2026lab6/internal/worker 7.709s
? github.com/charawein/mtusi2026lab6/student-tests/genfixtures [no test files]
+88
View File
@@ -0,0 +1,88 @@
{
"rounds": 10,
"parallel_per_round": 16,
"distinct_instances_seen": 2,
"duplicates_found": 0,
"per_round": [
{
"round": 0,
"requests": 16,
"status_201": 16,
"distinct_messages": 1,
"stored": 1,
"instances": 2
},
{
"round": 1,
"requests": 16,
"status_201": 16,
"distinct_messages": 1,
"stored": 1,
"instances": 2
},
{
"round": 2,
"requests": 16,
"status_201": 16,
"distinct_messages": 1,
"stored": 1,
"instances": 2
},
{
"round": 3,
"requests": 16,
"status_201": 16,
"distinct_messages": 1,
"stored": 1,
"instances": 2
},
{
"round": 4,
"requests": 16,
"status_201": 16,
"distinct_messages": 1,
"stored": 1,
"instances": 2
},
{
"round": 5,
"requests": 16,
"status_201": 16,
"distinct_messages": 1,
"stored": 1,
"instances": 2
},
{
"round": 6,
"requests": 16,
"status_201": 16,
"distinct_messages": 1,
"stored": 1,
"instances": 2
},
{
"round": 7,
"requests": 16,
"status_201": 16,
"distinct_messages": 1,
"stored": 1,
"instances": 2
},
{
"round": 8,
"requests": 16,
"status_201": 16,
"distinct_messages": 1,
"stored": 1,
"instances": 2
},
{
"round": 9,
"requests": 16,
"status_201": 16,
"distinct_messages": 1,
"stored": 1,
"instances": 2
}
]
}
+14
View File
@@ -0,0 +1,14 @@
bitnamilegacy/minio-client:2025.4.16@sha256:8a86e441decf053093c5977d49290298295f850c7843652f93e970ab15871dd1 bitnamilegacy/minio-client@sha256:8a86e441decf053093c5977d49290298295f850c7843652f93e970ab15871dd1
bitnamilegacy/minio:2025.4.22@sha256:50cec18ac4184af4671a78aedd5554942c8ae105d51a465fa82037949046da01 bitnamilegacy/minio@sha256:50cec18ac4184af4671a78aedd5554942c8ae105d51a465fa82037949046da01
grafana/grafana:11.4.0 grafana/grafana@sha256:d8ea37798ccc41061a62ab080f2676dda6bf7815558499f901bdb0f533a456fb
grafana/loki:3.3.2 grafana/loki@sha256:8af2de1abbdd7aa92b27c9bcc96f0f4140c9096b507c77921ffddf1c6ad6c48f
grafana/tempo:2.6.1 grafana/tempo@sha256:ef4384fce6e8ad22b95b243d8fc165628cda655376fd50e7850536ad89d71d50
haproxy:3.0.5-alpine haproxy@sha256:43a2591a6a62a41dd22edb4452880e9c70de489fbd7c54ffb52d035cccbfe949
migrate/migrate:v4.17.0 migrate/migrate@sha256:4d017c6fb5997127093648cab09e63d377997125c3d3dcca18e5d1c847da49fa
mtusi2026lab6-api:local
local-build
otel/opentelemetry-collector-contrib:0.116.1 otel/opentelemetry-collector-contrib@sha256:d0ebf65280da2e1b1491d1b93648281afd353d4b9ea19160090303cec9a233bd
postgres:16.4-alpine postgres@sha256:5660c2cbfea50c7a9127d17dc4e48543eedd3d7a41a595a2dfa572471e37e64c
prom/prometheus:v3.1.0 prom/prometheus@sha256:6559acbd5d770b15bb3c954629ce190ac3cbbdb2b7f1c30f0385c4e05104e218
rabbitmq:4.0.9-alpine rabbitmq@sha256:0a09ddfd00a1ef2e58f497d731e9b2814ef3bebb3668069185c54d085352d2b9
redis:7.4.1-alpine redis@sha256:c1e88455c85225310bbea54816e9c3f4b5295815e6dbf80c34d40afc6df28275
+2
View File
@@ -0,0 +1,2 @@
1354d20e1722 10
84c136cad07b 10
+420
View File
@@ -0,0 +1,420 @@
{
"scenario": "backend-down",
"result": {
"load": {
"requests": 100015,
"ok": 100015,
"errors": 0,
"error_ratio": 0.0,
"ok_rps": 406.0,
"seconds": 246.3,
"latency_ms_all_ok": {
"p50": 8.07,
"p95": 19.0,
"p99": 33.38
},
"latency_ms_outside_faults": {
"p50": 8.07,
"p95": 19.0,
"p99": 33.38
}
},
"collector_limit_mib": 256,
"collector_max_mib": 108.4,
"api_max_mib": 224.5,
"api_limit_mib": 256,
"phases": {
"export_failing_alert_seconds": 81.0,
"during": {
"exporter_queue_size_max": 40.0,
"exporter_queue_capacity": 40.0,
"send_failed_spans": 4203.596068571429,
"send_failed_log_records": 2843.3654285714283,
"enqueue_failed_spans": 728158.464,
"refused_spans_memory_limiter": 0.0,
"metrics_still_scraped": 430.16363636363633
},
"tempo_spans_exported_after_up": 37888.0,
"refused_spans_total": 0.0,
"new_trace_after_recovery": true,
"traces_visible_seconds_after_tempo_up": 20.9,
"probe_requests_after_up": 5
},
"timeline": {
"tempo_loki_down_s": 18,
"tempo_loki_up_s": 173,
"collector_down_s": 195,
"collector_down_for_s": 40
},
"memory_samples": [
{
"t": 0.0,
"collector_mib": 52.1,
"api_mib": [
224.5,
224.3
]
},
{
"t": 5.6,
"collector_mib": 52.4,
"api_mib": [
21.0,
19.4
]
},
{
"t": 10.9,
"collector_mib": 54.3,
"api_mib": [
20.8,
17.3
]
},
{
"t": 16.1,
"collector_mib": 52.3,
"api_mib": [
20.6,
18.2
]
},
{
"t": 21.8,
"collector_mib": 73.3,
"api_mib": [
20.9,
19.9
]
},
{
"t": 27.1,
"collector_mib": 82.3,
"api_mib": [
20.3,
17.4
]
},
{
"t": 32.4,
"collector_mib": 84.0,
"api_mib": [
19.8,
20.6
]
},
{
"t": 37.8,
"collector_mib": 87.8,
"api_mib": [
17.9,
18.9
]
},
{
"t": 43.2,
"collector_mib": 90.8,
"api_mib": [
18.0,
20.1
]
},
{
"t": 49.0,
"collector_mib": 95.5,
"api_mib": [
20.3,
17.6
]
},
{
"t": 54.3,
"collector_mib": 96.0,
"api_mib": [
21.0,
19.7
]
},
{
"t": 59.6,
"collector_mib": 96.4,
"api_mib": [
21.4,
20.0
]
},
{
"t": 64.9,
"collector_mib": 103.6,
"api_mib": [
19.6,
19.1
]
},
{
"t": 70.3,
"collector_mib": 105.3,
"api_mib": [
21.0,
19.0
]
},
{
"t": 75.7,
"collector_mib": 106.6,
"api_mib": [
20.3,
19.0
]
},
{
"t": 81.2,
"collector_mib": 108.4,
"api_mib": [
20.5,
17.2
]
},
{
"t": 86.6,
"collector_mib": 108.3,
"api_mib": [
18.5,
20.0
]
},
{
"t": 91.9,
"collector_mib": 108.4,
"api_mib": [
19.7,
18.6
]
},
{
"t": 97.2,
"collector_mib": 108.4,
"api_mib": [
20.2,
18.8
]
},
{
"t": 102.7,
"collector_mib": 105.1,
"api_mib": [
18.3,
21.1
]
},
{
"t": 108.2,
"collector_mib": 104.8,
"api_mib": [
19.2,
17.5
]
},
{
"t": 113.4,
"collector_mib": 104.8,
"api_mib": [
21.7,
18.2
]
},
{
"t": 118.7,
"collector_mib": 104.8,
"api_mib": [
20.2,
17.8
]
},
{
"t": 124.1,
"collector_mib": 107.8,
"api_mib": [
19.9,
19.0
]
},
{
"t": 129.4,
"collector_mib": 107.8,
"api_mib": [
19.4,
19.5
]
},
{
"t": 135.0,
"collector_mib": 108.1,
"api_mib": [
20.1,
18.2
]
},
{
"t": 140.4,
"collector_mib": 106.0,
"api_mib": [
20.1,
19.0
]
},
{
"t": 145.8,
"collector_mib": 106.4,
"api_mib": [
19.2,
20.4
]
},
{
"t": 151.2,
"collector_mib": 106.8,
"api_mib": [
20.3,
17.9
]
},
{
"t": 156.8,
"collector_mib": 103.8,
"api_mib": [
21.5,
19.1
]
},
{
"t": 163.0,
"collector_mib": 85.3,
"api_mib": [
17.7,
17.0
]
},
{
"t": 168.4,
"collector_mib": 82.5,
"api_mib": [
20.1,
18.7
]
},
{
"t": 173.7,
"collector_mib": 83.8,
"api_mib": [
19.8,
19.8
]
},
{
"t": 179.2,
"collector_mib": 83.8,
"api_mib": [
20.6,
19.2
]
},
{
"t": 184.6,
"collector_mib": 83.9,
"api_mib": [
20.8,
17.2
]
},
{
"t": 189.9,
"collector_mib": 54.8,
"api_mib": [
19.7,
19.1
]
},
{
"t": 195.4,
"collector_mib": 0.0,
"api_mib": [
24.2,
22.9
]
},
{
"t": 200.7,
"collector_mib": 0.0,
"api_mib": [
29.1,
31.2
]
},
{
"t": 206.1,
"collector_mib": 0.0,
"api_mib": [
33.3,
34.8
]
},
{
"t": 211.6,
"collector_mib": 0.0,
"api_mib": [
33.3,
32.8
]
},
{
"t": 217.2,
"collector_mib": 0.0,
"api_mib": [
32.5,
32.9
]
},
{
"t": 222.8,
"collector_mib": 0.0,
"api_mib": [
34.3,
33.7
]
},
{
"t": 228.2,
"collector_mib": 0.0,
"api_mib": [
35.5,
32.1
]
},
{
"t": 233.6,
"collector_mib": 0.0,
"api_mib": [
34.0,
32.4
]
},
{
"t": 239.0,
"collector_mib": 58.3,
"api_mib": [
30.3,
23.8
]
},
{
"t": 244.4,
"collector_mib": 54.7,
"api_mib": [
22.0,
18.7
]
}
]
}
}
Binary file not shown.

After

Width:  |  Height:  |  Size: 485 KiB

+155
View File
@@ -0,0 +1,155 @@
{
"scenario": "incident-db",
"result": {
"fault": "LOCK TABLE messages IN ACCESS EXCLUSIVE MODE for 110 s under load (4 clients)",
"detection_seconds": 98.4,
"resolved_seconds_after_unlock": 18.3,
"metrics_at_firing": {
"error_ratio_1m": 1.0,
"p99_s": 9.95,
"p99_by_route": {
"/api/v1/chats/{chat_id}/attachments": NaN,
"/api/v1/auth/sessions": NaN,
"/api/v1/users": NaN,
"/api/v1/attachments/{attachment_id}": NaN,
"/api/v1/chats/{chat_id}/messages": 9.95,
"/api/v1/chats": NaN
},
"db_pool_acquired": 4.0,
"health_ready_5xx_rate": 0.0
},
"error_trace_breakdown": [
{
"server_span": "POST /api/v1/chats/{chat_id}/messages",
"server_ms": 5000.48,
"status": "500",
"longest_db_span_ms": 4977.9,
"longest_s3_span_ms": 0.0,
"db_share": 0.995,
"db_spans_longest": [
[
"INSERT",
4977.87
],
[
"INSERT",
4977.85
],
[
"pool.acquire",
12.77
],
[
"connect",
12.72
]
]
},
{
"server_span": "POST /api/v1/chats/{chat_id}/messages",
"server_ms": 5000.53,
"status": "500",
"longest_db_span_ms": 4973.6,
"longest_s3_span_ms": 0.0,
"db_share": 0.995,
"db_spans_longest": [
[
"INSERT",
4973.55
],
[
"INSERT",
4973.51
],
[
"SELECT",
10.33
],
[
"SELECT",
6.0
]
]
},
{
"server_span": "POST /api/v1/chats/{chat_id}/messages",
"server_ms": 5001.06,
"status": "500",
"longest_db_span_ms": 4982.3,
"longest_s3_span_ms": 0.0,
"db_share": 0.996,
"db_spans_longest": [
[
"INSERT",
4982.27
],
[
"INSERT",
4982.25
],
[
"DELETE",
6.95
],
[
"DELETE",
5.49
]
]
}
],
"slow_handler_comparison": {
"server_span": "POST /api/v1/chats/{chat_id}/attachments",
"server_ms": 77.38,
"status": "202",
"longest_db_span_ms": 2.2,
"longest_s3_span_ms": 68.0,
"db_share": 0.028,
"db_spans_longest": [
[
"INSERT",
2.18
],
[
"INSERT",
1.33
],
[
"INSERT",
1.31
],
[
"INSERT",
1.22
]
]
},
"error_logs": {
"count": 88,
"events": [
"http.request"
],
"routes": [
"/api/v1/chats/{chat_id}/messages"
]
},
"load": {
"requests": 15036,
"ok": 14948,
"errors": 88,
"error_ratio": 0.00585,
"ok_rps": 100.5,
"seconds": 148.8,
"latency_ms_all_ok": {
"p50": 8.55,
"p95": 20.86,
"p99": 35.95
},
"latency_ms_outside_faults": {
"p50": 8.53,
"p95": 20.73,
"p99": 35.35
}
}
}
}
+86
View File
@@ -0,0 +1,86 @@
{
"scenario": "incident-worker",
"result": {
"detection_seconds_from_worker_stop": 96.7,
"diagnosis": {
"alerts_firing": [
"MessengerQueueLagGrowing"
],
"backlog_queued": 3.0,
"oldest_age_s": 91.144665,
"jobs_processed_rate_1m": 0.0,
"worker_capacity_series": 0,
"rabbitmq_ready": null,
"api_5xx_ratio": NaN,
"producer_spans_since_stop": 3,
"consumer_spans_since_stop": 0,
"last_worker_log_events": [
"",
"job.claimed",
"job.done",
"job.retry",
"worker.start"
]
},
"backlog_ready_seconds_after_start": 2.6,
"resolved_seconds_after_worker_start": 10.6,
"oldest_age_series": [
[
-30,
0.0
],
[
-20,
0.0
],
[
-10,
7.4
],
[
0,
0.0
],
[
10,
6.1
],
[
20,
16.1
],
[
30,
26.1
],
[
40,
36.1
],
[
50,
46.1
],
[
60,
56.1
],
[
70,
66.1
],
[
80,
76.1
],
[
90,
86.1
],
[
100,
96.1
]
]
}
}
+310
View File
@@ -0,0 +1,310 @@
{
"metric_names": [
"go_cgo_go_to_c_calls_calls_total",
"go_config_gogc_percent",
"go_cpu_classes_gc_mark_assist_cpu_seconds_total",
"go_cpu_classes_gc_mark_dedicated_cpu_seconds_total",
"go_cpu_classes_gc_mark_idle_cpu_seconds_total",
"go_cpu_classes_gc_pause_cpu_seconds_total",
"go_cpu_classes_gc_total_cpu_seconds_total",
"go_cpu_classes_idle_cpu_seconds_total",
"go_cpu_classes_scavenge_assist_cpu_seconds_total",
"go_cpu_classes_scavenge_background_cpu_seconds_total",
"go_cpu_classes_scavenge_total_cpu_seconds_total",
"go_cpu_classes_total_cpu_seconds_total",
"go_cpu_classes_user_cpu_seconds_total",
"go_gc_cycles_automatic_gc_cycles_total",
"go_gc_cycles_forced_gc_cycles_total",
"go_gc_cycles_total_gc_cycles_total",
"go_gc_duration_seconds",
"go_gc_duration_seconds_count",
"go_gc_duration_seconds_sum",
"go_gc_gogc_percent",
"go_gc_gomemlimit_bytes",
"go_gc_heap_allocs_by_size_bytes_bucket",
"go_gc_heap_allocs_by_size_bytes_count",
"go_gc_heap_allocs_by_size_bytes_sum",
"go_gc_heap_allocs_bytes_total",
"go_gc_heap_allocs_objects_total",
"go_gc_heap_frees_by_size_bytes_bucket",
"go_gc_heap_frees_by_size_bytes_count",
"go_gc_heap_frees_by_size_bytes_sum",
"go_gc_heap_frees_bytes_total",
"go_gc_heap_frees_objects_total",
"go_gc_heap_goal_bytes",
"go_gc_heap_live_bytes",
"go_gc_heap_objects_objects",
"go_gc_heap_tiny_allocs_objects_total",
"go_gc_limiter_last_enabled_gc_cycle",
"go_gc_pauses_seconds_bucket",
"go_gc_pauses_seconds_count",
"go_gc_pauses_seconds_sum",
"go_gc_scan_globals_bytes",
"go_gc_scan_heap_bytes",
"go_gc_scan_stack_bytes",
"go_gc_scan_total_bytes",
"go_gc_stack_starting_size_bytes",
"go_godebug_non_default_behavior_asynctimerchan_events_total",
"go_godebug_non_default_behavior_execerrdot_events_total",
"go_godebug_non_default_behavior_gocachehash_events_total",
"go_godebug_non_default_behavior_gocachetest_events_total",
"go_godebug_non_default_behavior_gocacheverify_events_total",
"go_godebug_non_default_behavior_gotypesalias_events_total",
"go_godebug_non_default_behavior_http2client_events_total",
"go_godebug_non_default_behavior_http2server_events_total",
"go_godebug_non_default_behavior_httplaxcontentlength_events_total",
"go_godebug_non_default_behavior_httpmuxgo121_events_total",
"go_godebug_non_default_behavior_httpservecontentkeepheaders_events_total",
"go_godebug_non_default_behavior_installgoroot_events_total",
"go_godebug_non_default_behavior_multipartmaxheaders_events_total",
"go_godebug_non_default_behavior_multipartmaxparts_events_total",
"go_godebug_non_default_behavior_multipathtcp_events_total",
"go_godebug_non_default_behavior_netedns0_events_total",
"go_godebug_non_default_behavior_panicnil_events_total",
"go_godebug_non_default_behavior_randautoseed_events_total",
"go_godebug_non_default_behavior_tarinsecurepath_events_total",
"go_godebug_non_default_behavior_tls10server_events_total",
"go_godebug_non_default_behavior_tls3des_events_total",
"go_godebug_non_default_behavior_tlsmaxrsasize_events_total",
"go_godebug_non_default_behavior_tlsrsakex_events_total",
"go_godebug_non_default_behavior_tlsunsafeekm_events_total",
"go_godebug_non_default_behavior_winreadlinkvolume_events_total",
"go_godebug_non_default_behavior_winsymlink_events_total",
"go_godebug_non_default_behavior_x509keypairleaf_events_total",
"go_godebug_non_default_behavior_x509negativeserial_events_total",
"go_godebug_non_default_behavior_x509sha1_events_total",
"go_godebug_non_default_behavior_x509usefallbackroots_events_total",
"go_godebug_non_default_behavior_x509usepolicies_events_total",
"go_godebug_non_default_behavior_zipinsecurepath_events_total",
"go_goroutine_count",
"go_goroutines",
"go_info",
"go_memory_allocated_bytes_total",
"go_memory_allocations_total",
"go_memory_classes_heap_free_bytes",
"go_memory_classes_heap_objects_bytes",
"go_memory_classes_heap_released_bytes",
"go_memory_classes_heap_stacks_bytes",
"go_memory_classes_heap_unused_bytes",
"go_memory_classes_metadata_mcache_free_bytes",
"go_memory_classes_metadata_mcache_inuse_bytes",
"go_memory_classes_metadata_mspan_free_bytes",
"go_memory_classes_metadata_mspan_inuse_bytes",
"go_memory_classes_metadata_other_bytes",
"go_memory_classes_os_stacks_bytes",
"go_memory_classes_other_bytes",
"go_memory_classes_profiling_buckets_bytes",
"go_memory_classes_total_bytes",
"go_memory_gc_goal_bytes",
"go_memory_limit_bytes",
"go_memory_used_bytes",
"go_memstats_alloc_bytes",
"go_memstats_alloc_bytes_total",
"go_memstats_buck_hash_sys_bytes",
"go_memstats_frees_total",
"go_memstats_gc_sys_bytes",
"go_memstats_heap_alloc_bytes",
"go_memstats_heap_idle_bytes",
"go_memstats_heap_inuse_bytes",
"go_memstats_heap_objects",
"go_memstats_heap_released_bytes",
"go_memstats_heap_sys_bytes",
"go_memstats_last_gc_time_seconds",
"go_memstats_lookups_total",
"go_memstats_mallocs_total",
"go_memstats_mcache_inuse_bytes",
"go_memstats_mcache_sys_bytes",
"go_memstats_mspan_inuse_bytes",
"go_memstats_mspan_sys_bytes",
"go_memstats_next_gc_bytes",
"go_memstats_other_sys_bytes",
"go_memstats_stack_inuse_bytes",
"go_memstats_stack_sys_bytes",
"go_memstats_sys_bytes",
"go_processor_limit",
"go_sched_gomaxprocs_threads",
"go_sched_goroutines_goroutines",
"go_sched_latencies_seconds_bucket",
"go_sched_latencies_seconds_count",
"go_sched_latencies_seconds_sum",
"go_sched_pauses_stopping_gc_seconds_bucket",
"go_sched_pauses_stopping_gc_seconds_count",
"go_sched_pauses_stopping_gc_seconds_sum",
"go_sched_pauses_stopping_other_seconds_bucket",
"go_sched_pauses_stopping_other_seconds_count",
"go_sched_pauses_stopping_other_seconds_sum",
"go_sched_pauses_total_gc_seconds_bucket",
"go_sched_pauses_total_gc_seconds_count",
"go_sched_pauses_total_gc_seconds_sum",
"go_sched_pauses_total_other_seconds_bucket",
"go_sched_pauses_total_other_seconds_count",
"go_sched_pauses_total_other_seconds_sum",
"go_sync_mutex_wait_total_seconds_total",
"go_threads",
"http_client_duration_bucket",
"http_client_duration_count",
"http_client_duration_sum",
"http_client_request_body_size_bytes_bucket",
"http_client_request_body_size_bytes_count",
"http_client_request_body_size_bytes_sum",
"http_client_request_duration_seconds_bucket",
"http_client_request_duration_seconds_count",
"http_client_request_duration_seconds_sum",
"http_client_request_size",
"http_client_response_size",
"http_server_active_requests",
"http_server_duration_bucket",
"http_server_duration_count",
"http_server_duration_sum",
"http_server_request_duration_seconds_bucket",
"http_server_request_duration_seconds_count",
"http_server_request_duration_seconds_sum",
"http_server_request_size",
"http_server_response_size",
"messenger:http_error_ratio:rate1m",
"messenger:http_errors:rate1m",
"messenger:http_fast_ratio:rate5m",
"messenger:http_requests:rate1m",
"messenger_attachment_time_to_ready_seconds_bucket",
"messenger_attachment_time_to_ready_seconds_count",
"messenger_attachment_time_to_ready_seconds_sum",
"messenger_job_duration_seconds_bucket",
"messenger_job_duration_seconds_count",
"messenger_job_duration_seconds_sum",
"messenger_jobs_backlog",
"messenger_jobs_oldest_age_seconds",
"messenger_jobs_processed_total",
"messenger_outbox_oldest_age_seconds",
"messenger_outbox_pending",
"messenger_outbox_publish_duration_seconds_bucket",
"messenger_outbox_publish_duration_seconds_count",
"messenger_outbox_publish_duration_seconds_sum",
"messenger_outbox_published_total",
"messenger_worker_busy",
"messenger_worker_capacity",
"otelcol_exporter_enqueue_failed_log_records",
"otelcol_exporter_enqueue_failed_spans",
"otelcol_exporter_queue_capacity",
"otelcol_exporter_queue_size",
"otelcol_exporter_send_failed_log_records",
"otelcol_exporter_send_failed_metric_points",
"otelcol_exporter_send_failed_spans",
"otelcol_exporter_sent_log_records",
"otelcol_exporter_sent_metric_points",
"otelcol_exporter_sent_spans",
"otelcol_receiver_accepted_log_records",
"otelcol_receiver_accepted_metric_points",
"otelcol_receiver_accepted_spans",
"otelcol_receiver_refused_log_records",
"otelcol_receiver_refused_metric_points",
"otelcol_receiver_refused_spans",
"pgxpool_acquire_duration_nanoseconds_total",
"pgxpool_acquired_connections",
"pgxpool_acquires_total",
"pgxpool_canceled_acquires_total",
"pgxpool_constructing_connections",
"pgxpool_empty_acquire_total",
"pgxpool_empty_acquire_wait_time_nanoseconds_total",
"pgxpool_idle_connections",
"pgxpool_max_connections",
"pgxpool_max_idle_destroys_total",
"pgxpool_max_lifetime_destroys_total",
"pgxpool_new_connections_total",
"pgxpool_total_connections",
"process_cpu_seconds_total",
"process_cpu_time_seconds_total",
"process_max_fds",
"process_network_receive_bytes_total",
"process_network_transmit_bytes_total",
"process_open_fds",
"process_resident_memory_bytes",
"process_start_time_seconds",
"process_virtual_memory_bytes",
"process_virtual_memory_max_bytes",
"target_info"
],
"rules": [
{
"group": "messenger-alerts",
"name": "MessengerHighErrorRate",
"type": "alerting",
"health": "ok",
"state": "inactive",
"query": "messenger:http_error_ratio:rate1m > 0.05 and messenger:http_requests:rate1m > 0.5",
"duration": 30
},
{
"group": "messenger-alerts",
"name": "MessengerQueueLagGrowing",
"type": "alerting",
"health": "ok",
"state": "inactive",
"query": "max(messenger_jobs_oldest_age_seconds) > 60",
"duration": 30
},
{
"group": "messenger-alerts",
"name": "MessengerWorkerAbsent",
"type": "alerting",
"health": "ok",
"state": "inactive",
"query": "absent(messenger_worker_capacity)",
"duration": 60
},
{
"group": "messenger-alerts",
"name": "MessengerTelemetryExportFailing",
"type": "alerting",
"health": "ok",
"state": "inactive",
"query": "(sum(rate({__name__=~\"otelcol_exporter_send_failed_spans(_total)?\"}[1m])) or vector(0)) + (sum(rate({__name__=~\"otelcol_exporter_send_failed_log_records(_total)?\"}[1m])) or vector(0)) > 0",
"duration": 30
},
{
"group": "messenger-sli",
"name": "messenger:http_requests:rate1m",
"type": "recording",
"health": "ok",
"state": null,
"query": "sum(rate(http_server_request_duration_seconds_count{http_route!~\"/health/.*\",job=\"messenger-api\"}[1m]))",
"duration": null
},
{
"group": "messenger-sli",
"name": "messenger:http_errors:rate1m",
"type": "recording",
"health": "ok",
"state": null,
"query": "sum(rate(http_server_request_duration_seconds_count{http_response_status_class=\"5xx\",http_route!~\"/health/.*\",job=\"messenger-api\"}[1m])) or vector(0)",
"duration": null
},
{
"group": "messenger-sli",
"name": "messenger:http_error_ratio:rate1m",
"type": "recording",
"health": "ok",
"state": null,
"query": "messenger:http_errors:rate1m / messenger:http_requests:rate1m",
"duration": null
},
{
"group": "messenger-sli",
"name": "messenger:http_fast_ratio:rate5m",
"type": "recording",
"health": "ok",
"state": null,
"query": "sum(rate(http_server_request_duration_seconds_bucket{http_route!~\"/health/.*\",job=\"messenger-api\",le=\"0.25\"}[5m])) / sum(rate(http_server_request_duration_seconds_count{http_route!~\"/health/.*\",job=\"messenger-api\"}[5m]))",
"duration": null
},
{
"group": "messenger-sli",
"name": "messenger:time_to_ready_fast_ratio:rate5m",
"type": "recording",
"health": "ok",
"state": null,
"query": "sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"30\"}[5m])) / sum(rate(messenger_attachment_time_to_ready_seconds_count[5m]))",
"duration": null
}
],
"alerts_now": {}
}
+124
View File
@@ -0,0 +1,124 @@
{
"scenario": "retry-trace",
"result": {
"request_id": "obs-retry-a9fc5acc8add4f72",
"trace_id": "48230bc7d9dc9e469f654518739a6c8a",
"attempts_in_trace": [
[
"1",
"retry",
"STATUS_CODE_ERROR"
],
[
"2",
"retry",
"STATUS_CODE_ERROR"
],
[
"3",
"retry",
"STATUS_CODE_ERROR"
],
[
"4",
"done",
""
]
],
"publish_spans": 4,
"job_events": "enqueued,claimed,retry_scheduled,claimed,retry_scheduled,claimed,retry_scheduled,claimed,done",
"final_status": "ready",
"tree": [
{
"service": "messenger-api",
"name": "POST /api/v1/chats/{chat_id}/attachments",
"kind": "SPAN_KIND_SERVER",
"ms": 17.78,
"parent": null,
"outcome": null,
"attempt": null,
"status": ""
},
{
"service": "messenger-relay",
"name": "attachment.process publish",
"kind": "SPAN_KIND_PRODUCER",
"ms": 45.89,
"parent": "messenger-api: POST /api/v1/chats/{chat_id}/attachments",
"outcome": null,
"attempt": null,
"status": ""
},
{
"service": "messenger-worker",
"name": "attachment.process process",
"kind": "SPAN_KIND_CONSUMER",
"ms": 384.22,
"parent": "messenger-relay: attachment.process publish",
"outcome": "retry",
"attempt": "1",
"status": "STATUS_CODE_ERROR"
},
{
"service": "messenger-relay",
"name": "attachment.process publish",
"kind": "SPAN_KIND_PRODUCER",
"ms": 0.99,
"parent": "messenger-worker: attachment.process process",
"outcome": null,
"attempt": null,
"status": ""
},
{
"service": "messenger-worker",
"name": "attachment.process process",
"kind": "SPAN_KIND_CONSUMER",
"ms": 245.3,
"parent": "messenger-relay: attachment.process publish",
"outcome": "retry",
"attempt": "2",
"status": "STATUS_CODE_ERROR"
},
{
"service": "messenger-relay",
"name": "attachment.process publish",
"kind": "SPAN_KIND_PRODUCER",
"ms": 44.48,
"parent": "messenger-worker: attachment.process process",
"outcome": null,
"attempt": null,
"status": ""
},
{
"service": "messenger-worker",
"name": "attachment.process process",
"kind": "SPAN_KIND_CONSUMER",
"ms": 271.24,
"parent": "messenger-relay: attachment.process publish",
"outcome": "retry",
"attempt": "3",
"status": "STATUS_CODE_ERROR"
},
{
"service": "messenger-relay",
"name": "attachment.process publish",
"kind": "SPAN_KIND_PRODUCER",
"ms": 2.29,
"parent": "messenger-worker: attachment.process process",
"outcome": null,
"attempt": null,
"status": ""
},
{
"service": "messenger-worker",
"name": "attachment.process process",
"kind": "SPAN_KIND_CONSUMER",
"ms": 34.25,
"parent": "messenger-relay: attachment.process publish",
"outcome": "done",
"attempt": "4",
"status": ""
}
]
}
}
+929
View File
@@ -0,0 +1,929 @@
{
"scenario": "signals",
"result": {
"request": {
"request_id": "obs-msg-249e829f8c1e4049",
"trace_id": "c275fd38f588c08d29aea802ac7c9c83",
"spans": [
{
"service": "messenger-api",
"name": "POST /api/v1/chats/{chat_id}/messages",
"kind": "SPAN_KIND_SERVER",
"ms": 6.55,
"parent": null,
"outcome": null,
"attempt": null,
"status": ""
},
{
"service": "messenger-api",
"name": "get",
"kind": "SPAN_KIND_CLIENT",
"ms": 0.11,
"parent": "messenger-api: POST /api/v1/chats/{chat_id}/messages",
"outcome": null,
"attempt": null,
"status": ""
},
{
"service": "messenger-api",
"name": "ttl",
"kind": "SPAN_KIND_CLIENT",
"ms": 0.06,
"parent": "messenger-api: POST /api/v1/chats/{chat_id}/messages",
"outcome": null,
"attempt": null,
"status": ""
},
{
"service": "messenger-api",
"name": "pool.acquire",
"kind": "SPAN_KIND_CLIENT",
"ms": 0.01,
"parent": "messenger-api: POST /api/v1/chats/{chat_id}/messages",
"outcome": null,
"attempt": null,
"status": ""
},
{
"service": "messenger-api",
"name": "SELECT",
"kind": "SPAN_KIND_CLIENT",
"ms": 0.38,
"parent": "messenger-api: POST /api/v1/chats/{chat_id}/messages",
"outcome": null,
"attempt": null,
"status": ""
},
{
"service": "messenger-api",
"name": "SELECT",
"kind": "SPAN_KIND_CLIENT",
"ms": 0.18,
"parent": "messenger-api: SELECT",
"outcome": null,
"attempt": null,
"status": ""
},
{
"service": "messenger-api",
"name": "pool.acquire",
"kind": "SPAN_KIND_CLIENT",
"ms": 0.0,
"parent": "messenger-api: POST /api/v1/chats/{chat_id}/messages",
"outcome": null,
"attempt": null,
"status": ""
},
{
"service": "messenger-api",
"name": "SELECT",
"kind": "SPAN_KIND_CLIENT",
"ms": 0.79,
"parent": "messenger-api: POST /api/v1/chats/{chat_id}/messages",
"outcome": null,
"attempt": null,
"status": ""
},
{
"service": "messenger-api",
"name": "SELECT",
"kind": "SPAN_KIND_CLIENT",
"ms": 0.29,
"parent": "messenger-api: SELECT",
"outcome": null,
"attempt": null,
"status": ""
},
{
"service": "messenger-api",
"name": "pool.acquire",
"kind": "SPAN_KIND_CLIENT",
"ms": 0.0,
"parent": "messenger-api: POST /api/v1/chats/{chat_id}/messages",
"outcome": null,
"attempt": null,
"status": ""
},
{
"service": "messenger-api",
"name": "BEGIN",
"kind": "SPAN_KIND_CLIENT",
"ms": 0.18,
"parent": "messenger-api: POST /api/v1/chats/{chat_id}/messages",
"outcome": null,
"attempt": null,
"status": ""
},
{
"service": "messenger-api",
"name": "DELETE",
"kind": "SPAN_KIND_CLIENT",
"ms": 1.54,
"parent": "messenger-api: POST /api/v1/chats/{chat_id}/messages",
"outcome": null,
"attempt": null,
"status": ""
}
],
"log": {
"line": "http request",
"labels": {
"detected_level": "info",
"event": "http.request",
"request_id": "obs-msg-249e829f8c1e4049",
"route": "/api/v1/chats/{chat_id}/messages",
"service_name": "messenger-api",
"severity_text": "INFO",
"span_id": "ce40a5e763ec9dd8",
"status": "201",
"trace_id": "c275fd38f588c08d29aea802ac7c9c83"
}
}
},
"upload": {
"request_id": "obs-upload-e0c3f49f30f5446d",
"trace_id": "cf8628e62914dd27bef25fbfca80ba11",
"services": [
"messenger-api",
"messenger-relay",
"messenger-worker"
],
"links": {
"api server → relay producer": true,
"relay producer → worker consumer": true,
"worker consumer → S3 PUT": true,
"worker consumer → image.decode": true
},
"span_count": 65,
"worker_db_spans": [
"BEGIN",
"COMMIT",
"INSERT",
"SELECT",
"UPDATE",
"batch start",
"pool.acquire"
],
"tree": [
{
"service": "messenger-api",
"name": "POST /api/v1/chats/{chat_id}/attachments",
"kind": "SPAN_KIND_SERVER",
"ms": 14.68,
"parent": null,
"outcome": null,
"attempt": null,
"status": ""
},
{
"service": "messenger-api",
"name": "get",
"kind": "SPAN_KIND_CLIENT",
"ms": 0.07,
"parent": "messenger-api: POST /api/v1/chats/{chat_id}/attachments",
"outcome": null,
"attempt": null,
"status": ""
},
{
"service": "messenger-api",
"name": "ttl",
"kind": "SPAN_KIND_CLIENT",
"ms": 0.06,
"parent": "messenger-api: POST /api/v1/chats/{chat_id}/attachments",
"outcome": null,
"attempt": null,
"status": ""
},
{
"service": "messenger-api",
"name": "pool.acquire",
"kind": "SPAN_KIND_CLIENT",
"ms": 0.01,
"parent": "messenger-api: POST /api/v1/chats/{chat_id}/attachments",
"outcome": null,
"attempt": null,
"status": ""
},
{
"service": "messenger-api",
"name": "SELECT",
"kind": "SPAN_KIND_CLIENT",
"ms": 0.2,
"parent": "messenger-api: POST /api/v1/chats/{chat_id}/attachments",
"outcome": null,
"attempt": null,
"status": ""
},
{
"service": "messenger-api",
"name": "pool.acquire",
"kind": "SPAN_KIND_CLIENT",
"ms": 0.0,
"parent": "messenger-api: POST /api/v1/chats/{chat_id}/attachments",
"outcome": null,
"attempt": null,
"status": ""
},
{
"service": "messenger-api",
"name": "SELECT",
"kind": "SPAN_KIND_CLIENT",
"ms": 0.37,
"parent": "messenger-api: POST /api/v1/chats/{chat_id}/attachments",
"outcome": null,
"attempt": null,
"status": ""
},
{
"service": "messenger-api",
"name": "SELECT",
"kind": "SPAN_KIND_CLIENT",
"ms": 0.1,
"parent": "messenger-api: SELECT",
"outcome": null,
"attempt": null,
"status": ""
},
{
"service": "messenger-api",
"name": "pool.acquire",
"kind": "SPAN_KIND_CLIENT",
"ms": 0.0,
"parent": "messenger-api: POST /api/v1/chats/{chat_id}/attachments",
"outcome": null,
"attempt": null,
"status": ""
},
{
"service": "messenger-api",
"name": "INSERT",
"kind": "SPAN_KIND_CLIENT",
"ms": 1.69,
"parent": "messenger-api: POST /api/v1/chats/{chat_id}/attachments",
"outcome": null,
"attempt": null,
"status": ""
},
{
"service": "messenger-api",
"name": "INSERT",
"kind": "SPAN_KIND_CLIENT",
"ms": 0.46,
"parent": "messenger-api: INSERT",
"outcome": null,
"attempt": null,
"status": ""
},
{
"service": "messenger-api",
"name": "S3 PUT",
"kind": "SPAN_KIND_CLIENT",
"ms": 4.93,
"parent": "messenger-api: POST /api/v1/chats/{chat_id}/attachments",
"outcome": null,
"attempt": null,
"status": ""
},
{
"service": "messenger-api",
"name": "pool.acquire",
"kind": "SPAN_KIND_CLIENT",
"ms": 0.01,
"parent": "messenger-api: POST /api/v1/chats/{chat_id}/attachments",
"outcome": null,
"attempt": null,
"status": ""
},
{
"service": "messenger-api",
"name": "BEGIN",
"kind": "SPAN_KIND_CLIENT",
"ms": 0.13,
"parent": "messenger-api: POST /api/v1/chats/{chat_id}/attachments",
"outcome": null,
"attempt": null,
"status": ""
},
{
"service": "messenger-api",
"name": "UPDATE",
"kind": "SPAN_KIND_CLIENT",
"ms": 1.34,
"parent": "messenger-api: POST /api/v1/chats/{chat_id}/attachments",
"outcome": null,
"attempt": null,
"status": ""
},
{
"service": "messenger-api",
"name": "UPDATE",
"kind": "SPAN_KIND_CLIENT",
"ms": 0.26,
"parent": "messenger-api: UPDATE",
"outcome": null,
"attempt": null,
"status": ""
},
{
"service": "messenger-api",
"name": "INSERT",
"kind": "SPAN_KIND_CLIENT",
"ms": 1.56,
"parent": "messenger-api: POST /api/v1/chats/{chat_id}/attachments",
"outcome": null,
"attempt": null,
"status": ""
},
{
"service": "messenger-api",
"name": "INSERT",
"kind": "SPAN_KIND_CLIENT",
"ms": 0.44,
"parent": "messenger-api: INSERT",
"outcome": null,
"attempt": null,
"status": ""
},
{
"service": "messenger-api",
"name": "INSERT",
"kind": "SPAN_KIND_CLIENT",
"ms": 1.48,
"parent": "messenger-api: POST /api/v1/chats/{chat_id}/attachments",
"outcome": null,
"attempt": null,
"status": ""
},
{
"service": "messenger-api",
"name": "INSERT",
"kind": "SPAN_KIND_CLIENT",
"ms": 0.29,
"parent": "messenger-api: INSERT",
"outcome": null,
"attempt": null,
"status": ""
},
{
"service": "messenger-api",
"name": "INSERT",
"kind": "SPAN_KIND_CLIENT",
"ms": 1.82,
"parent": "messenger-api: POST /api/v1/chats/{chat_id}/attachments",
"outcome": null,
"attempt": null,
"status": ""
},
{
"service": "messenger-api",
"name": "INSERT",
"kind": "SPAN_KIND_CLIENT",
"ms": 1.19,
"parent": "messenger-api: INSERT",
"outcome": null,
"attempt": null,
"status": ""
},
{
"service": "messenger-api",
"name": "COMMIT",
"kind": "SPAN_KIND_CLIENT",
"ms": 0.23,
"parent": "messenger-api: POST /api/v1/chats/{chat_id}/attachments",
"outcome": null,
"attempt": null,
"status": ""
},
{
"service": "messenger-relay",
"name": "attachment.process publish",
"kind": "SPAN_KIND_PRODUCER",
"ms": 1.79,
"parent": "messenger-api: POST /api/v1/chats/{chat_id}/attachments",
"outcome": null,
"attempt": null,
"status": ""
},
{
"service": "messenger-worker",
"name": "attachment.process process",
"kind": "SPAN_KIND_CONSUMER",
"ms": 39.52,
"parent": "messenger-relay: attachment.process publish",
"outcome": "done",
"attempt": "1",
"status": ""
},
{
"service": "messenger-worker",
"name": "pool.acquire",
"kind": "SPAN_KIND_CLIENT",
"ms": 0.16,
"parent": "messenger-worker: attachment.process process",
"outcome": null,
"attempt": null,
"status": ""
},
{
"service": "messenger-worker",
"name": "BEGIN",
"kind": "SPAN_KIND_CLIENT",
"ms": 0.12,
"parent": "messenger-worker: attachment.process process",
"outcome": null,
"attempt": null,
"status": ""
},
{
"service": "messenger-worker",
"name": "SELECT",
"kind": "SPAN_KIND_CLIENT",
"ms": 3.24,
"parent": "messenger-worker: attachment.process process",
"outcome": null,
"attempt": null,
"status": ""
},
{
"service": "messenger-worker",
"name": "SELECT",
"kind": "SPAN_KIND_CLIENT",
"ms": 1.66,
"parent": "messenger-worker: SELECT",
"outcome": null,
"attempt": null,
"status": ""
},
{
"service": "messenger-worker",
"name": "SELECT",
"kind": "SPAN_KIND_CLIENT",
"ms": 0.33,
"parent": "messenger-worker: attachment.process process",
"outcome": null,
"attempt": null,
"status": ""
},
{
"service": "messenger-worker",
"name": "SELECT",
"kind": "SPAN_KIND_CLIENT",
"ms": 0.2,
"parent": "messenger-worker: SELECT",
"outcome": null,
"attempt": null,
"status": ""
},
{
"service": "messenger-worker",
"name": "SELECT",
"kind": "SPAN_KIND_CLIENT",
"ms": 1.33,
"parent": "messenger-worker: attachment.process process",
"outcome": null,
"attempt": null,
"status": ""
},
{
"service": "messenger-worker",
"name": "SELECT",
"kind": "SPAN_KIND_CLIENT",
"ms": 0.43,
"parent": "messenger-worker: SELECT",
"outcome": null,
"attempt": null,
"status": ""
},
{
"service": "messenger-worker",
"name": "UPDATE",
"kind": "SPAN_KIND_CLIENT",
"ms": 0.65,
"parent": "messenger-worker: attachment.process process",
"outcome": null,
"attempt": null,
"status": ""
},
{
"service": "messenger-worker",
"name": "UPDATE",
"kind": "SPAN_KIND_CLIENT",
"ms": 0.13,
"parent": "messenger-worker: UPDATE",
"outcome": null,
"attempt": null,
"status": ""
},
{
"service": "messenger-worker",
"name": "UPDATE",
"kind": "SPAN_KIND_CLIENT",
"ms": 0.66,
"parent": "messenger-worker: attachment.process process",
"outcome": null,
"attempt": null,
"status": ""
},
{
"service": "messenger-worker",
"name": "UPDATE",
"kind": "SPAN_KIND_CLIENT",
"ms": 0.11,
"parent": "messenger-worker: UPDATE",
"outcome": null,
"attempt": null,
"status": ""
},
{
"service": "messenger-worker",
"name": "INSERT",
"kind": "SPAN_KIND_CLIENT",
"ms": 1.52,
"parent": "messenger-worker: attachment.process process",
"outcome": null,
"attempt": null,
"status": ""
},
{
"service": "messenger-worker",
"name": "INSERT",
"kind": "SPAN_KIND_CLIENT",
"ms": 1.03,
"parent": "messenger-worker: INSERT",
"outcome": null,
"attempt": null,
"status": ""
},
{
"service": "messenger-worker",
"name": "COMMIT",
"kind": "SPAN_KIND_CLIENT",
"ms": 0.27,
"parent": "messenger-worker: attachment.process process",
"outcome": null,
"attempt": null,
"status": ""
}
],
"worker_log_events": [
"job.claimed",
"job.done"
]
},
"metrics": {
"found": {
"http requests (counter = _count)": {
"series": 11,
"names": [
"http_server_request_duration_seconds_count"
]
},
"http duration histogram (s)": {
"series": 154,
"names": [
"http_server_request_duration_seconds_bucket"
]
},
"jobs processed by outcome": {
"series": 1,
"names": [
"messenger_jobs_processed_total"
]
},
"job duration histogram (s)": {
"series": 14,
"names": [
"messenger_job_duration_seconds_bucket"
]
},
"time to ready histogram (s)": {
"series": 14,
"names": [
"messenger_attachment_time_to_ready_seconds_bucket"
]
},
"queue backlog by state": {
"series": 3,
"names": [
"messenger_jobs_backlog"
]
},
"oldest job age (s)": {
"series": 1,
"names": [
"messenger_jobs_oldest_age_seconds"
]
},
"outbox pending": {
"series": 1,
"names": [
"messenger_outbox_pending"
]
},
"rabbitmq queue messages": {
"series": 2,
"names": [
"rabbitmq_queue_messages_ready"
]
},
"worker busy": {
"series": 1,
"names": [
"messenger_worker_busy"
]
},
"worker capacity": {
"series": 1,
"names": [
"messenger_worker_capacity"
]
},
"db pool acquired": {
"series": 4,
"names": [
"pgxpool_acquired_connections"
]
},
"db pool waits": {
"series": 8,
"names": [
"pgxpool_empty_acquire_total",
"pgxpool_empty_acquire_wait_time_nanoseconds_total"
]
},
"process cpu": {
"series": 8,
"names": [
"process_cpu_time_seconds_total"
]
},
"go memory": {
"series": 8,
"names": [
"go_memory_used_bytes"
]
}
},
"labels_total": 131,
"forbidden_labels": [],
"http_route_values": [
"/api/v1/attachments/{attachment_id}",
"/api/v1/auth/sessions",
"/api/v1/chats",
"/api/v1/chats/{chat_id}/attachments",
"/api/v1/chats/{chat_id}/messages",
"/api/v1/users",
"/health/live",
"/health/ready"
],
"example_series_labels": [
"__name__",
"http_request_method",
"http_response_status_class",
"http_response_status_code",
"http_route",
"instance",
"job"
]
},
"leak_check": {
"loki:password": 0,
"loki:session_token": 0,
"loki:message_text": 0,
"loki:bearer_header": 0,
"tempo:password": 0,
"tempo:session_token": 0,
"tempo:message_text": 0,
"tempo:bearer_header": 0,
"stdout:password": 0,
"stdout:session_token": 0,
"stdout:message_text": 0,
"stdout:bearer_header": 0
},
"grafana": {
"datasources": [
"loki:loki",
"prometheus:prometheus",
"tempo:tempo"
],
"dashboard": "Messenger — API, worker, очередь",
"panels": 30,
"prometheus_targets": 39,
"targets_without_data_now": [
{
"panel": "Ready быстрее 30 s (5m)",
"legend": "",
"series": 0
},
{
"panel": "Доля ошибок 5xx и 4xx",
"legend": "4xx / все (ошибки клиента)",
"series": 0
}
],
"targets": [
{
"panel": "RPS API (бизнес)",
"legend": "",
"series": 1
},
{
"panel": "Доля 5xx (1m)",
"legend": "",
"series": 1
},
{
"panel": "Быстрее 250 ms (5m)",
"legend": "",
"series": 1
},
{
"panel": "Ready быстрее 30 s (5m)",
"legend": "",
"series": 0
},
{
"panel": "Старейшая job",
"legend": "",
"series": 1
},
{
"panel": "Alerts firing",
"legend": "",
"series": 1
},
{
"panel": "RPS по маршруту и классу ответа",
"legend": "{{http_route}} {{http_response_status_class}}",
"series": 6
},
{
"panel": "Задержка API p50 / p95 / p99 (все реплики)",
"legend": "p50",
"series": 1
},
{
"panel": "Задержка API p50 / p95 / p99 (все реплики)",
"legend": "p95",
"series": 1
},
{
"panel": "Задержка API p50 / p95 / p99 (все реплики)",
"legend": "p99",
"series": 1
},
{
"panel": "Доля ошибок 5xx и 4xx",
"legend": "5xx / все",
"series": 1
},
{
"panel": "Доля ошибок 5xx и 4xx",
"legend": "4xx / все (ошибки клиента)",
"series": 0
},
{
"panel": "p95 по маршруту",
"legend": "{{http_route}}",
"series": 6
},
{
"panel": "Запросы в обработке (in-flight) по реплике",
"legend": "{{instance}}",
"series": 2
},
{
"panel": "Jobs по исходу (rate)",
"legend": "{{outcome}}",
"series": 1
},
{
"panel": "Длительность обработки job p50 / p95 / p99",
"legend": "p50",
"series": 1
},
{
"panel": "Длительность обработки job p50 / p95 / p99",
"legend": "p95",
"series": 1
},
{
"panel": "Длительность обработки job p50 / p95 / p99",
"legend": "p99",
"series": 1
},
{
"panel": "Время до ready (202 → ready) p50 / p95",
"legend": "p50",
"series": 1
},
{
"panel": "Время до ready (202 → ready) p50 / p95",
"legend": "p95",
"series": 1
},
{
"panel": "Очередь: jobs по состоянию (PostgreSQL)",
"legend": "{{state}}",
"series": 3
},
{
"panel": "Очередь: jobs по состоянию (PostgreSQL)",
"legend": "outbox не опубликовано",
"series": 1
},
{
"panel": "Возраст старейшей job / outbox",
"legend": "job (queued/processing)",
"series": 1
},
{
"panel": "Возраст старейшей job / outbox",
"legend": "outbox",
"series": 1
},
{
"panel": "RabbitMQ: сообщения в очередях",
"legend": "ready {{queue}}",
"series": 2
},
{
"panel": "RabbitMQ: сообщения в очередях",
"legend": "unacked {{queue}}",
"series": 2
},
{
"panel": "Saturation worker (busy / capacity)",
"legend": "занято слотов",
"series": 1
},
{
"panel": "Publish outbox → broker (confirm) p95",
"legend": "p95",
"series": 1
},
{
"panel": "Publish outbox → broker (confirm) p95",
"legend": "сообщений/с",
"series": 1
},
{
"panel": "Пул БД: занято / ожидания",
"legend": "занято {{job}}",
"series": 3
},
{
"panel": "Пул БД: занято / ожидания",
"legend": "max {{job}}",
"series": 3
},
{
"panel": "Пул БД: занято / ожидания",
"legend": "ожиданий/с {{job}}",
"series": 3
},
{
"panel": "CPU процессов",
"legend": "{{job}} {{instance}}",
"series": 4
},
{
"panel": "Память Go (used)",
"legend": "{{job}} {{instance}}",
"series": 4
},
{
"panel": "Collector: принято / отправлено / потеряно spans",
"legend": "принято",
"series": 1
},
{
"panel": "Collector: принято / отправлено / потеряно spans",
"legend": "отправлено",
"series": 1
},
{
"panel": "Collector: принято / отправлено / потеряно spans",
"legend": "не доставлено",
"series": 1
},
{
"panel": "Collector: принято / отправлено / потеряно spans",
"legend": "отклонено (memory_limiter)",
"series": 1
},
{
"panel": "Collector: принято / отправлено / потеряно spans",
"legend": "очередь экспортёров",
"series": 1
}
]
},
"attachment": {
"id": "ec322ba6-ae46-4757-b0c8-0d79d3ba2b42",
"status": "ready"
}
}
}
+34
View File
@@ -0,0 +1,34 @@
-- до:
mtusi2026lab6-api-1 Up About a minute (healthy)
mtusi2026lab6-api-2 Up 6 seconds (healthy)
mtusi2026lab6-grafana-1 Up About a minute (healthy)
mtusi2026lab6-lb-1 Up About a minute
mtusi2026lab6-loki-1 Up About a minute (healthy)
mtusi2026lab6-otel-collector-1 Up About a minute
mtusi2026lab6-postgres-1 Up About a minute (healthy)
mtusi2026lab6-prometheus-1 Up About a minute (healthy)
mtusi2026lab6-rabbitmq-1 Up About a minute (healthy)
mtusi2026lab6-reconciler-1 Up About a minute (healthy)
mtusi2026lab6-redis-1 Up About a minute (healthy)
mtusi2026lab6-relay-1 Up About a minute (healthy)
mtusi2026lab6-s3-1 Up About a minute (healthy)
mtusi2026lab6-tempo-1 Up About a minute (healthy)
mtusi2026lab6-worker-1 Up About a minute (healthy)
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
-- после:
mtusi2026lab6-api-1 Up 6 seconds (healthy)
mtusi2026lab6-api-2 Up 6 seconds (healthy)
mtusi2026lab6-grafana-1 Up 18 seconds (healthy)
mtusi2026lab6-lb-1 Up Less than a second
mtusi2026lab6-loki-1 Up 17 seconds (health: starting)
mtusi2026lab6-otel-collector-1 Up 18 seconds
mtusi2026lab6-postgres-1 Up 18 seconds (healthy)
mtusi2026lab6-prometheus-1 Up 18 seconds (healthy)
mtusi2026lab6-rabbitmq-1 Up 17 seconds (healthy)
mtusi2026lab6-reconciler-1 Up 6 seconds (healthy)
mtusi2026lab6-redis-1 Up 18 seconds (healthy)
mtusi2026lab6-relay-1 Up 6 seconds (healthy)
mtusi2026lab6-s3-1 Up 18 seconds (healthy)
mtusi2026lab6-tempo-1 Up 17 seconds (health: starting)
mtusi2026lab6-worker-1 Up 6 seconds (healthy)
+22
View File
@@ -0,0 +1,22 @@
{
"scenario": "broker-restart",
"result": {
"attachment_id": "3b4fbbcd-d9e2-4fd8-940f-c8fb30f617ef",
"queue_before_restart": {
"attachment.dead": 0,
"attachment.process": 0
},
"queue_after_restart": {
"attachment.dead": 0,
"attachment.process": 1
},
"outbox_publish_attempts": 1,
"invariants": {
"status": "ready",
"published_version": 1,
"objects_per_version": {
"1": 3
}
}
}
}
+17
View File
@@ -0,0 +1,17 @@
{
"scenario": "crash-commit",
"result": {
"attachment_id": "577b04d0-15fc-440d-be43-c8302d17d1d8",
"job_id": "22dba878-3bc7-46b4-9000-62f4e732243c",
"worker_restarts": 1,
"attempt": 1,
"redelivery_acked_as_noop": true,
"invariants": {
"status": "ready",
"published_version": 1,
"objects_per_version": {
"1": 3
}
}
}
}
+34
View File
@@ -0,0 +1,34 @@
{
"scenario": "crash-effect",
"result": {
"attachment_id": "8ed34b9a-8cd6-43da-a713-3b38d1f0c8ef",
"job_id": "0e6f1a68-ecf9-457c-8d34-c131cb2fb98e",
"worker_restarts": 1,
"attempt": 2,
"seconds_to_ready": 17.7,
"events": "enqueued,claimed,lease_expired_requeued,claimed,done",
"invariants": {
"status": "ready",
"published_version": 1,
"objects_per_version": {
"1": 3
}
},
"variants": {
"thumbnail": [
256,
160
],
"crop": [
128,
128
],
"optimized": [
320,
200
],
"optimized_bytes": 2462,
"original_bytes": 111817
}
}
}
+42
View File
@@ -0,0 +1,42 @@
{
"scenario": "demo",
"result": {
"attachment_id": "40ed2137-afda-4551-a46e-e26af011d234",
"job_id": "56db4cee-5fa9-46a7-9e87-6004bde53d25",
"job_state": "done",
"attempt": 1,
"timeline": [
"worker stopped",
"upload 202 queued id=40ed2137-afda-4551-a46e-e26af011d234",
"message 201 with queued attachment",
"content 409, other messages OK",
"api and rabbitmq restarted (volumes kept), outbox published=1",
"rabbitmq queues after restart: {'attachment.dead': 0, 'attachment.process': 1}",
"worker started → ready in 2.6 s"
],
"variants": {
"thumbnail": [
256,
160
],
"crop": [
128,
128
],
"optimized": [
320,
200
],
"optimized_bytes": 2462,
"original_bytes": 111817
},
"invariants": {
"status": "ready",
"published_version": 1,
"objects_per_version": {
"1": 3
}
},
"events": "enqueued,claimed,done"
}
}
+36
View File
@@ -0,0 +1,36 @@
{
"scenario": "failed-replay",
"result": {
"attachment_id": "8619fa76-2bf8-46a4-a757-07591ca939f3",
"failed_error_code": "checksum_mismatch",
"content_while_failed": 409,
"attach_failed_to_message": 409,
"registry_listed": true,
"replay": {
"attachment_id": "8619fa76-2bf8-46a4-a757-07591ca939f3",
"command": "replay",
"error": "",
"job_id": "7e26e051-dabe-4b1d-b511-85d5f9e2c743",
"ok": true,
"version": 1
},
"replay_after_success": {
"attachment_id": "8619fa76-2bf8-46a4-a757-07591ca939f3",
"command": "replay",
"error": "conflict: job is done, only failed jobs can be replayed",
"job_id": "7e26e051-dabe-4b1d-b511-85d5f9e2c743",
"ok": false,
"version": 1
},
"final_status": "ready",
"events": "enqueued,claimed,failed,replay,claimed,done,replay_refused",
"dlq_messages": 1,
"invariants": {
"status": "ready",
"published_version": 1,
"objects_per_version": {
"1": 3
}
}
}
}
+30
View File
@@ -0,0 +1,30 @@
{
"scenario": "limits",
"result": {
"image": "6000x6000 PNG (119755 bytes)",
"parallel_uploads": 3,
"seconds_all_ready": 12.1,
"worker_anon_memory_max_mib": 574.8,
"worker_memory_peak_incl_page_cache_mib": 580.9,
"oom_kill_events": 0,
"go_memlimit": "560MiB",
"oom_events": 0,
"worker_memory_limit_mib": 768.0,
"pixel_budget": 60000000,
"decoded_rgba_per_image_mib": 137.3,
"variant_sizes": {
"thumbnail": [
256,
256
],
"crop": [
128,
128
],
"optimized": [
1280,
1280
]
}
}
}
+25
View File
@@ -0,0 +1,25 @@
{
"scenario": "outbox-gap",
"result": {
"relay_down_then_up": {
"attachment_id": "0766e424-4356-474b-a409-282566dbbd81",
"unpublished_while_relay_down": 1,
"then": "ready"
},
"relay_crash_after_confirm": {
"attachment_id": "5bb073a8-f9d1-4e64-8226-4ac24792e214",
"relay_restarts": 1,
"outbox_publish_attempts": 1,
"worker_saw_message_twice": true,
"job_attempt": 1,
"invariants": {
"status": "ready",
"published_version": 1,
"objects_per_version": {
"1": 3
}
}
},
"events": "enqueued,claimed,done"
}
}
+17
View File
@@ -0,0 +1,17 @@
{
"scenario": "transient",
"result": {
"attachment_id": "c5d86b25-e26e-42ca-af2e-bed3ee98b538",
"attempts": 3,
"final_status": "ready",
"retry_details": "processing_failed; next_attempt=2 in 1.609s | processing_failed; next_attempt=3 in 2.66s",
"events": "enqueued,claimed,retry_scheduled,claimed,retry_scheduled,claimed,done",
"invariants": {
"status": "ready",
"published_version": 1,
"objects_per_version": {
"1": 3
}
}
}
}
+10
View File
@@ -0,0 +1,10 @@
worker-1 | {"time":"2026-11-12T11:25:53.786929702Z","level":"INFO","msg":"worker starting","event":"worker.start","owner":"d392ccfc07f1:1","concurrency":2,"lease":60000000000,"service.name":"messenger-worker"}
relay-1 | {"time":"2026-11-12T11:25:26.41695347Z","level":"WARN","msg":"expired leases requeued","jobs":1,"service.name":"messenger-relay"}
worker-1 | {"time":"2026-11-12T11:26:27.839005278Z","level":"INFO","msg":"worker starting","event":"worker.start","owner":"d392ccfc07f1:1","concurrency":2,"lease":60000000000,"service.name":"messenger-worker"}
worker-1 | {"time":"2026-11-12T11:26:36.585881442Z","level":"INFO","msg":"worker starting","event":"worker.start","owner":"d392ccfc07f1:1","concurrency":2,"lease":60000000000,"service.name":"messenger-worker"}
worker-1 | {"time":"2026-11-12T11:26:37.150745975Z","level":"WARN","msg":"attempt failed","job_id":"b47271a0-7ba3-4716-b2fe-9f6f03b36194","attachment_id":"c5d86b25-e26e-42ca-af2e-bed3ee98b538","version":1,"event":"job.retry","attempt":1,"code":"processing_failed","final":false,"backoff":1608723537,"trace_id":"a57663e672319236293d4ab274186eae","span_id":"d1c251107bb9c10f","service.name":"messenger-worker"}
worker-1 | {"time":"2026-11-12T11:26:39.012900329Z","level":"WARN","msg":"attempt failed","job_id":"b47271a0-7ba3-4716-b2fe-9f6f03b36194","attachment_id":"c5d86b25-e26e-42ca-af2e-bed3ee98b538","version":1,"event":"job.retry","attempt":2,"code":"processing_failed","final":false,"backoff":2659621162,"trace_id":"a57663e672319236293d4ab274186eae","span_id":"ae8a58278600849a","service.name":"messenger-worker"}
worker-1 | {"time":"2026-11-12T11:26:51.026383949Z","level":"INFO","msg":"worker starting","event":"worker.start","owner":"d392ccfc07f1:1","concurrency":2,"lease":60000000000,"service.name":"messenger-worker"}
worker-1 | {"time":"2026-11-12T11:26:51.087077531Z","level":"WARN","msg":"attempt failed","job_id":"7e26e051-dabe-4b1d-b511-85d5f9e2c743","attachment_id":"8619fa76-2bf8-46a4-a757-07591ca939f3","version":1,"event":"job.failed","attempt":1,"code":"checksum_mismatch","final":true,"backoff":1116187727,"trace_id":"70012050ef241cf92d8f9b19f088cf41","span_id":"191f5c2578ce4819","service.name":"messenger-worker"}
worker-1 | {"time":"2026-11-12T11:26:54.713069545Z","level":"ERROR","msg":"invalid job message, dead-lettering","event":"job.rejected","err":null,"trace_id":"47832279361ecd6b346277a7df057eee","span_id":"ea86222bd1cd28d0","service.name":"messenger-worker"}
worker-1 | {"time":"2026-11-12T11:27:06.108177234Z","level":"INFO","msg":"worker starting","event":"worker.start","owner":"d392ccfc07f1:1","concurrency":2,"lease":60000000000,"service.name":"messenger-worker"}
+3
View File
@@ -0,0 +1,3 @@
PASS: whitespace-only password rejected with 400 (3 variants)
PASS: Redis ACL -incr: message visible on every next read (3 in history)
PASS: Redis ACL -incr -del -unlink -scan: message visible on every next read (4 in history)
+92
View File
@@ -0,0 +1,92 @@
-- до:
mtusi2026lab6-api-1 84c136cad07b Up 2 minutes (healthy)
mtusi2026lab6-api-2 1354d20e1722 Up 2 minutes (healthy)
#1 [internal] load local bake definitions
#1 reading from stdin 380B done
#1 DONE 0.0s
#2 [internal] load build definition from Dockerfile
#2 transferring dockerfile: 2.91kB done
#2 DONE 0.0s
#3 [internal] load metadata for docker.io/library/alpine:3.20@sha256:d9e853e87e55526f6b2917df91a2115c36dd7c696a35be12163d44e6e2a4b6bc
#3 DONE 0.0s
#4 [internal] load metadata for docker.io/library/golang:1.26-alpine@sha256:27f2d3b83cfc5803c8078f8e730af0f7c431bf067afaec8f60a36fb763187cba
#4 DONE 0.0s
#5 [internal] load .dockerignore
#5 transferring context: 212B 0.0s done
#5 DONE 0.0s
#6 [builder 1/6] FROM docker.io/library/golang:1.26-alpine@sha256:27f2d3b83cfc5803c8078f8e730af0f7c431bf067afaec8f60a36fb763187cba
#6 DONE 0.0s
#7 [stage-1 1/7] FROM docker.io/library/alpine:3.20@sha256:d9e853e87e55526f6b2917df91a2115c36dd7c696a35be12163d44e6e2a4b6bc
#7 DONE 0.0s
#8 [internal] load build context
#8 transferring context: 9.04kB 0.0s done
#8 DONE 0.0s
#9 [builder 6/6] RUN CGO_ENABLED=0 GOOS=linux go build -ldflags="-s -w -X github.com/charawein/mtusi2026lab6/internal/telemetry.Version=lab6" -trimpath -o /out/ ./cmd/api ./cmd/reconcile ./cmd/pipeline
#9 CACHED
#10 [builder 3/6] COPY go.mod go.sum ./
#10 CACHED
#11 [builder 2/6] WORKDIR /src
#11 CACHED
#12 [stage-1 5/7] COPY --from=builder --chown=appuser:appgroup /out/api /app/api
#12 CACHED
#13 [builder 5/6] COPY . .
#13 CACHED
#14 [stage-1 6/7] COPY --from=builder --chown=appuser:appgroup /out/reconcile /app/reconcile
#14 CACHED
#15 [stage-1 3/7] RUN addgroup -g 1001 -S appgroup && adduser -u 1001 -S appuser -G appgroup
#15 CACHED
#16 [stage-1 4/7] WORKDIR /app
#16 CACHED
#17 [builder 4/6] RUN go mod download
#17 CACHED
#18 [stage-1 2/7] RUN apk --no-cache add ca-certificates tzdata libwebp-tools
#18 CACHED
#19 [stage-1 7/7] COPY --from=builder --chown=appuser:appgroup /out/pipeline /app/pipeline
#19 CACHED
#20 exporting to image
#20 exporting layers done
#20 writing image sha256:11e3e3f4cb401048102ebf29aa2365118d1cad928fbc0f682535f043ff700ec5 done
#20 naming to docker.io/library/mtusi2026lab6-api:local done
#20 DONE 0.0s
#21 resolving provenance for metadata file
#21 DONE 0.0s
api Built
Container mtusi2026lab6-postgres-1 Running
no change
Container mtusi2026lab6-api-1 Running
Container mtusi2026lab6-api-2 Running
Container mtusi2026lab6-api-3 Creating
Container mtusi2026lab6-api-3 Created
Container mtusi2026lab6-api-3 Starting
Container mtusi2026lab6-api-3 Started
replaced /mtusi2026lab6-api-1
Container mtusi2026lab6-api-2 Running
Container mtusi2026lab6-api-3 Running
Container mtusi2026lab6-api-4 Creating
Container mtusi2026lab6-api-4 Created
Container mtusi2026lab6-api-4 Starting
Container mtusi2026lab6-api-4 Started
replaced /mtusi2026lab6-api-2
mtusi2026lab6-api-3 0cc85552e20a mtusi2026lab6-api:local Up 26 seconds (healthy)
mtusi2026lab6-api-4 2b9b764811a6 mtusi2026lab6-api:local Up 13 seconds (healthy)
-- замена заняла 61 s (внутри 90 s нагрузки)
+17
View File
@@ -0,0 +1,17 @@
{
"lab": 6,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 90.0,
"elapsed_seconds": 90.053,
"concurrency": 4,
"successful_requests": 29983,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 332.947,
"successful_latency_ms": {
"p50": 9.107,
"p95": 24.061,
"p99": 65.137
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
}
+6
View File
@@ -0,0 +1,6 @@
2
total used free shared buff/cache available
Mem: 7932 1968 1612 102 4769 5963
Swap: 3071 0 3071
28.0.4
Docker Compose version v2.38.2
+6
View File
@@ -0,0 +1,6 @@
2
total used free shared buff/cache available
Mem: 7932 2242 1097 83 4991 5690
Swap: 3071 0 3071
28.0.4
Docker Compose version v2.38.2
+6
View File
@@ -0,0 +1,6 @@
2
total used free shared buff/cache available
Mem: 7932 2079 917 147 5397 5853
Swap: 3071 21 3050
28.0.4
Docker Compose version v2.38.2
+6
View File
@@ -0,0 +1,6 @@
2
total used free shared buff/cache available
Mem: 7932 2735 675 83 4920 5196
Swap: 3071 0 3071
28.0.4
Docker Compose version v2.38.2
+6
View File
@@ -0,0 +1,6 @@
2
total used free shared buff/cache available
Mem: 7932 2073 1076 173 5271 5859
Swap: 3071 0 3071
28.0.4
Docker Compose version v2.38.2
+1
View File
@@ -0,0 +1 @@
PASS: lab6 public API scenario; infrastructure and grading evidence remain separate.
+18
View File
@@ -0,0 +1,18 @@
=== RUN TestTracePropagationMapAndAMQP
--- PASS: TestTracePropagationMapAndAMQP (0.00s)
=== RUN TestTraceUnsampledSuppressesBackgroundSpans
--- PASS: TestTraceUnsampledSuppressesBackgroundSpans (0.00s)
=== RUN TestTraceLogCorrelation
--- PASS: TestTraceLogCorrelation (0.00s)
=== RUN TestTelemetrySetupBuildsProviders
--- PASS: TestTelemetrySetupBuildsProviders (0.01s)
PASS
ok github.com/charawein/mtusi2026lab6/internal/telemetry 0.013s
=== RUN TestTelemetryRouteTemplateAndStatus
--- PASS: TestTelemetryRouteTemplateAndStatus (0.00s)
PASS
ok github.com/charawein/mtusi2026lab6/internal/transport/http/middleware 0.007s
=== RUN TestPipelineTracePropagationThroughOutbox
--- PASS: TestPipelineTracePropagationThroughOutbox (0.75s)
PASS
ok github.com/charawein/mtusi2026lab6/internal/worker 0.754s
+999
View File
@@ -0,0 +1,999 @@
{
"method": {
"duration_s": 45,
"warmup_s": 10,
"concurrency": 4,
"locks_at_s": [
12,
28
],
"lock_seconds": 6.5,
"slow_threshold_s": 0.25,
"rounds": 3,
"handler_timeout_s": 5
},
"summary": {
"off": {
"runs": 3,
"ok_rps_median": 247.6,
"p50_ms_median": 10.06,
"p95_ms_median": 20.22,
"p99_ms_median": 31.5,
"errors_total": 24,
"slow_total": 24,
"api_cpu_cores_median": 0.316,
"api_cpu_ms_per_request_median": 1.282,
"api_mem_peak_mib_median": 13.1,
"collector_cpu_cores_median": 0.001,
"collector_mem_peak_mib_median": 30.1,
"spans_received_per_1000_req": 0.0,
"spans_exported_per_1000_req": 0.0,
"tempo_kib_per_1000_req": 0.0,
"loki_kib_per_1000_req": 0.0,
"log_records_per_1000_req": 0.0,
"metric_points_per_1000_req": 0.0,
"traces_kept_errors": {
"kept": 0,
"total": 24,
"ratio": 0.0
},
"traces_kept_slow": {
"kept": 0,
"total": 24,
"ratio": 0.0
},
"traces_kept_normal_sample": {
"kept": 0,
"total": 120,
"ratio": 0.0
}
},
"full": {
"runs": 3,
"ok_rps_median": 197.8,
"p50_ms_median": 12.26,
"p95_ms_median": 27.91,
"p99_ms_median": 46.15,
"errors_total": 24,
"slow_total": 24,
"api_cpu_cores_median": 0.371,
"api_cpu_ms_per_request_median": 1.875,
"api_mem_peak_mib_median": 22.4,
"collector_cpu_cores_median": 0.051,
"collector_mem_peak_mib_median": 55.1,
"spans_received_per_1000_req": 14018.6,
"spans_exported_per_1000_req": 14018.6,
"tempo_kib_per_1000_req": 4547.652,
"loki_kib_per_1000_req": 339.364,
"log_records_per_1000_req": 1018.3,
"metric_points_per_1000_req": 243.9,
"traces_kept_errors": {
"kept": 24,
"total": 24,
"ratio": 1.0
},
"traces_kept_slow": {
"kept": 24,
"total": 24,
"ratio": 1.0
},
"traces_kept_normal_sample": {
"kept": 120,
"total": 120,
"ratio": 1.0
}
},
"head10": {
"runs": 3,
"ok_rps_median": 234.3,
"p50_ms_median": 10.38,
"p95_ms_median": 21.88,
"p99_ms_median": 34.23,
"errors_total": 24,
"slow_total": 24,
"api_cpu_cores_median": 0.332,
"api_cpu_ms_per_request_median": 1.414,
"api_mem_peak_mib_median": 21.3,
"collector_cpu_cores_median": 0.014,
"collector_mem_peak_mib_median": 55.9,
"spans_received_per_1000_req": 1397.7,
"spans_exported_per_1000_req": 1397.7,
"tempo_kib_per_1000_req": 454.021,
"loki_kib_per_1000_req": 334.58,
"log_records_per_1000_req": 1015.5,
"metric_points_per_1000_req": 203.1,
"traces_kept_errors": {
"kept": 2,
"total": 24,
"ratio": 0.083
},
"traces_kept_slow": {
"kept": 2,
"total": 24,
"ratio": 0.083
},
"traces_kept_normal_sample": {
"kept": 14,
"total": 120,
"ratio": 0.117
}
},
"tail": {
"runs": 3,
"ok_rps_median": 195.4,
"p50_ms_median": 12.45,
"p95_ms_median": 27.68,
"p99_ms_median": 41.48,
"errors_total": 24,
"slow_total": 24,
"api_cpu_cores_median": 0.372,
"api_cpu_ms_per_request_median": 1.905,
"api_mem_peak_mib_median": 22.3,
"collector_cpu_cores_median": 0.044,
"collector_mem_peak_mib_median": 106.5,
"spans_received_per_1000_req": 14016.5,
"spans_exported_per_1000_req": 1784.9,
"tempo_kib_per_1000_req": 587.593,
"loki_kib_per_1000_req": 339.584,
"log_records_per_1000_req": 1018.6,
"metric_points_per_1000_req": 246.3,
"traces_kept_errors": {
"kept": 24,
"total": 24,
"ratio": 1.0
},
"traces_kept_slow": {
"kept": 24,
"total": 24,
"ratio": 1.0
},
"traces_kept_normal_sample": {
"kept": 13,
"total": 120,
"ratio": 0.108
}
}
},
"runs": [
{
"config": "off",
"round": 0,
"load": {
"requests": 11155,
"ok": 11147,
"errors": 8,
"error_ratio": 0.00072,
"ok_rps": 247.7,
"seconds": 45.0,
"latency_ms_all_ok": {
"p50": 10.12,
"p95": 20.92,
"p99": 32.92
},
"latency_ms_outside_faults": {
"p50": 10.0,
"p95": 20.32,
"p99": 31.5
}
},
"slow_requests": 8,
"error_requests": 8,
"traces_kept": {
"errors": [
0,
8
],
"slow": [
0,
8
],
"normal_sample": [
0,
40
]
},
"api_cpu_cores": 0.315,
"api_cpu_ms_per_request": 1.271,
"api_mem_peak_mib": [
13.1,
12.7
],
"api_mem_now_mib": [
8.7,
8.3
],
"collector_cpu_cores": 0.002,
"collector_mem_peak_mib": [
42.3
],
"ingest_delta": {
"spans_received": 0,
"spans_exported": 0,
"log_records_received": 0,
"metric_points_received": 0,
"tempo_bytes": 0,
"loki_bytes": 0,
"prometheus_samples": 52263
},
"ingest_per_1000_requests": {
"spans_received": 0.0,
"spans_exported": 0.0,
"log_records_received": 0.0,
"metric_points_received": 0.0,
"tempo_bytes": 0.0,
"loki_bytes": 0.0,
"prometheus_samples": 4688.5
}
},
{
"config": "full",
"round": 0,
"load": {
"requests": 9033,
"ok": 9025,
"errors": 8,
"error_ratio": 0.00089,
"ok_rps": 200.6,
"seconds": 45.0,
"latency_ms_all_ok": {
"p50": 11.94,
"p95": 28.43,
"p99": 48.57
},
"latency_ms_outside_faults": {
"p50": 11.68,
"p95": 26.75,
"p99": 46.8
}
},
"slow_requests": 8,
"error_requests": 8,
"traces_kept": {
"errors": [
8,
8
],
"slow": [
8,
8
],
"normal_sample": [
40,
40
]
},
"api_cpu_cores": 0.372,
"api_cpu_ms_per_request": 1.857,
"api_mem_peak_mib": [
21.9,
21.0
],
"api_mem_now_mib": [
14.5,
14.8
],
"collector_cpu_cores": 0.052,
"collector_mem_peak_mib": [
54.5
],
"ingest_delta": {
"spans_received": 126498,
"spans_exported": 126498,
"log_records_received": 9189,
"metric_points_received": 2170,
"tempo_bytes": 42025732,
"loki_bytes": 3136265,
"prometheus_samples": 53356
},
"ingest_per_1000_requests": {
"spans_received": 14016.4,
"spans_exported": 14016.4,
"log_records_received": 1018.2,
"metric_points_received": 240.4,
"tempo_bytes": 4656590.8,
"loki_bytes": 347508.6,
"prometheus_samples": 5912.0
}
},
{
"config": "head10",
"round": 0,
"load": {
"requests": 10825,
"ok": 10817,
"errors": 8,
"error_ratio": 0.00074,
"ok_rps": 240.3,
"seconds": 45.0,
"latency_ms_all_ok": {
"p50": 10.23,
"p95": 22.32,
"p99": 37.12
},
"latency_ms_outside_faults": {
"p50": 10.14,
"p95": 21.88,
"p99": 36.09
}
},
"slow_requests": 8,
"error_requests": 8,
"traces_kept": {
"errors": [
0,
8
],
"slow": [
0,
8
],
"normal_sample": [
7,
40
]
},
"api_cpu_cores": 0.335,
"api_cpu_ms_per_request": 1.394,
"api_mem_peak_mib": [
21.3,
20.4
],
"api_mem_now_mib": [
15.8,
15.9
],
"collector_cpu_cores": 0.014,
"collector_mem_peak_mib": [
56.5
],
"ingest_delta": {
"spans_received": 15119,
"spans_exported": 15119,
"log_records_received": 10979,
"metric_points_received": 2166,
"tempo_bytes": 5029007,
"loki_bytes": 3703341,
"prometheus_samples": 58820
},
"ingest_per_1000_requests": {
"spans_received": 1397.7,
"spans_exported": 1397.7,
"log_records_received": 1015.0,
"metric_points_received": 200.2,
"tempo_bytes": 464917.0,
"loki_bytes": 342363.0,
"prometheus_samples": 5437.7
}
},
{
"config": "tail",
"round": 0,
"load": {
"requests": 9661,
"ok": 9653,
"errors": 8,
"error_ratio": 0.00083,
"ok_rps": 214.5,
"seconds": 45.0,
"latency_ms_all_ok": {
"p50": 11.51,
"p95": 25.35,
"p99": 42.72
},
"latency_ms_outside_faults": {
"p50": 11.51,
"p95": 24.51,
"p99": 38.18
}
},
"slow_requests": 8,
"error_requests": 8,
"traces_kept": {
"errors": [
8,
8
],
"slow": [
8,
8
],
"normal_sample": [
4,
40
]
},
"api_cpu_cores": 0.398,
"api_cpu_ms_per_request": 1.854,
"api_mem_peak_mib": [
22.3,
20.8
],
"api_mem_now_mib": [
15.9,
14.5
],
"collector_cpu_cores": 0.044,
"collector_mem_peak_mib": [
115.5
],
"ingest_delta": {
"spans_received": 135292,
"spans_exported": 16105,
"log_records_received": 9816,
"metric_points_received": 2157,
"tempo_bytes": 6135814,
"loki_bytes": 3346785,
"prometheus_samples": 56512
},
"ingest_per_1000_requests": {
"spans_received": 14015.5,
"spans_exported": 1668.4,
"log_records_received": 1016.9,
"metric_points_received": 223.5,
"tempo_bytes": 635638.0,
"loki_bytes": 346709.3,
"prometheus_samples": 5854.3
}
},
{
"config": "full",
"round": 1,
"load": {
"requests": 8909,
"ok": 8901,
"errors": 8,
"error_ratio": 0.0009,
"ok_rps": 197.8,
"seconds": 45.0,
"latency_ms_all_ok": {
"p50": 12.28,
"p95": 28.01,
"p99": 46.37
},
"latency_ms_outside_faults": {
"p50": 12.26,
"p95": 27.91,
"p99": 46.15
}
},
"slow_requests": 8,
"error_requests": 8,
"traces_kept": {
"errors": [
8,
8
],
"slow": [
8,
8
],
"normal_sample": [
40,
40
]
},
"api_cpu_cores": 0.371,
"api_cpu_ms_per_request": 1.875,
"api_mem_peak_mib": [
22.5,
21.5
],
"api_mem_now_mib": [
14.9,
16.7
],
"collector_cpu_cores": 0.051,
"collector_mem_peak_mib": [
55.9
],
"ingest_delta": {
"spans_received": 124841,
"spans_exported": 124841,
"log_records_received": 9064,
"metric_points_received": 2171,
"tempo_bytes": 41450142,
"loki_bytes": 3091868,
"prometheus_samples": 55341
},
"ingest_per_1000_requests": {
"spans_received": 14025.5,
"spans_exported": 14025.5,
"log_records_received": 1018.3,
"metric_points_received": 243.9,
"tempo_bytes": 4656796.1,
"loki_bytes": 347361.9,
"prometheus_samples": 6217.4
}
},
{
"config": "head10",
"round": 1,
"load": {
"requests": 10539,
"ok": 10531,
"errors": 8,
"error_ratio": 0.00076,
"ok_rps": 234.0,
"seconds": 45.0,
"latency_ms_all_ok": {
"p50": 10.66,
"p95": 22.47,
"p99": 34.71
},
"latency_ms_outside_faults": {
"p50": 10.68,
"p95": 22.17,
"p99": 34.23
}
},
"slow_requests": 8,
"error_requests": 8,
"traces_kept": {
"errors": [
2,
8
],
"slow": [
1,
8
],
"normal_sample": [
1,
40
]
},
"api_cpu_cores": 0.331,
"api_cpu_ms_per_request": 1.414,
"api_mem_peak_mib": [
20.6,
20.4
],
"api_mem_now_mib": [
14.5,
15.8
],
"collector_cpu_cores": 0.015,
"collector_mem_peak_mib": [
55.9
],
"ingest_delta": {
"spans_received": 14486,
"spans_exported": 14486,
"log_records_received": 10695,
"metric_points_received": 2157,
"tempo_bytes": 4820676,
"loki_bytes": 3608439,
"prometheus_samples": 59603
},
"ingest_per_1000_requests": {
"spans_received": 1375.6,
"spans_exported": 1375.6,
"log_records_received": 1015.6,
"metric_points_received": 204.8,
"tempo_bytes": 457760.5,
"loki_bytes": 342649.2,
"prometheus_samples": 5659.8
}
},
{
"config": "tail",
"round": 1,
"load": {
"requests": 8802,
"ok": 8794,
"errors": 8,
"error_ratio": 0.00091,
"ok_rps": 195.4,
"seconds": 45.0,
"latency_ms_all_ok": {
"p50": 12.42,
"p95": 28.49,
"p99": 48.03
},
"latency_ms_outside_faults": {
"p50": 12.45,
"p95": 27.68,
"p99": 41.48
}
},
"slow_requests": 8,
"error_requests": 8,
"traces_kept": {
"errors": [
8,
8
],
"slow": [
8,
8
],
"normal_sample": [
6,
40
]
},
"api_cpu_cores": 0.372,
"api_cpu_ms_per_request": 1.905,
"api_mem_peak_mib": [
22.3,
22.3
],
"api_mem_now_mib": [
16.4,
15.9
],
"collector_cpu_cores": 0.044,
"collector_mem_peak_mib": [
106.5
],
"ingest_delta": {
"spans_received": 123261,
"spans_exported": 16559,
"log_records_received": 8958,
"metric_points_received": 2166,
"tempo_bytes": 5291306,
"loki_bytes": 3058512,
"prometheus_samples": 57176
},
"ingest_per_1000_requests": {
"spans_received": 14016.5,
"spans_exported": 1883.0,
"log_records_received": 1018.6,
"metric_points_received": 246.3,
"tempo_bytes": 601695.0,
"loki_bytes": 347795.3,
"prometheus_samples": 6501.7
}
},
{
"config": "off",
"round": 1,
"load": {
"requests": 11150,
"ok": 11142,
"errors": 8,
"error_ratio": 0.00072,
"ok_rps": 247.6,
"seconds": 45.0,
"latency_ms_all_ok": {
"p50": 10.08,
"p95": 20.47,
"p99": 33.29
},
"latency_ms_outside_faults": {
"p50": 10.06,
"p95": 19.79,
"p99": 31.2
}
},
"slow_requests": 8,
"error_requests": 8,
"traces_kept": {
"errors": [
0,
8
],
"slow": [
0,
8
],
"normal_sample": [
0,
40
]
},
"api_cpu_cores": 0.317,
"api_cpu_ms_per_request": 1.282,
"api_mem_peak_mib": [
13.1,
13.1
],
"api_mem_now_mib": [
8.6,
8.5
],
"collector_cpu_cores": 0.001,
"collector_mem_peak_mib": [
30.1
],
"ingest_delta": {
"spans_received": 0,
"spans_exported": 0,
"log_records_received": 0,
"metric_points_received": 0,
"tempo_bytes": 0,
"loki_bytes": 0,
"prometheus_samples": 47494
},
"ingest_per_1000_requests": {
"spans_received": 0.0,
"spans_exported": 0.0,
"log_records_received": 0.0,
"metric_points_received": 0.0,
"tempo_bytes": 0.0,
"loki_bytes": 0.0,
"prometheus_samples": 4262.6
}
},
{
"config": "head10",
"round": 2,
"load": {
"requests": 10553,
"ok": 10545,
"errors": 8,
"error_ratio": 0.00076,
"ok_rps": 234.3,
"seconds": 45.0,
"latency_ms_all_ok": {
"p50": 10.59,
"p95": 22.86,
"p99": 36.32
},
"latency_ms_outside_faults": {
"p50": 10.38,
"p95": 21.55,
"p99": 33.46
}
},
"slow_requests": 8,
"error_requests": 8,
"traces_kept": {
"errors": [
0,
8
],
"slow": [
1,
8
],
"normal_sample": [
6,
40
]
},
"api_cpu_cores": 0.332,
"api_cpu_ms_per_request": 1.416,
"api_mem_peak_mib": [
20.6,
21.9
],
"api_mem_now_mib": [
14.4,
15.9
],
"collector_cpu_cores": 0.013,
"collector_mem_peak_mib": [
54.4
],
"ingest_delta": {
"spans_received": 14771,
"spans_exported": 14771,
"log_records_received": 10708,
"metric_points_received": 2142,
"tempo_bytes": 5161389,
"loki_bytes": 3612825,
"prometheus_samples": 55781
},
"ingest_per_1000_requests": {
"spans_received": 1400.8,
"spans_exported": 1400.8,
"log_records_received": 1015.5,
"metric_points_received": 203.1,
"tempo_bytes": 489463.2,
"loki_bytes": 342610.2,
"prometheus_samples": 5289.8
}
},
{
"config": "tail",
"round": 2,
"load": {
"requests": 8738,
"ok": 8730,
"errors": 8,
"error_ratio": 0.00092,
"ok_rps": 194.0,
"seconds": 45.0,
"latency_ms_all_ok": {
"p50": 12.45,
"p95": 29.33,
"p99": 44.71
},
"latency_ms_outside_faults": {
"p50": 12.75,
"p95": 28.68,
"p99": 42.09
}
},
"slow_requests": 8,
"error_requests": 8,
"traces_kept": {
"errors": [
8,
8
],
"slow": [
8,
8
],
"normal_sample": [
3,
40
]
},
"api_cpu_cores": 0.371,
"api_cpu_ms_per_request": 1.913,
"api_mem_peak_mib": [
21.3,
21.7
],
"api_mem_now_mib": [
16.5,
16.0
],
"collector_cpu_cores": 0.043,
"collector_mem_peak_mib": [
104.6
],
"ingest_delta": {
"spans_received": 122371,
"spans_exported": 15582,
"log_records_received": 8892,
"metric_points_received": 2164,
"tempo_bytes": 4960426,
"loki_bytes": 3035717,
"prometheus_samples": 57370
},
"ingest_per_1000_requests": {
"spans_received": 14017.3,
"spans_exported": 1784.9,
"log_records_received": 1018.6,
"metric_points_received": 247.9,
"tempo_bytes": 568204.6,
"loki_bytes": 347733.9,
"prometheus_samples": 6571.6
}
},
{
"config": "off",
"round": 2,
"load": {
"requests": 11068,
"ok": 11060,
"errors": 8,
"error_ratio": 0.00072,
"ok_rps": 245.7,
"seconds": 45.0,
"latency_ms_all_ok": {
"p50": 10.2,
"p95": 20.91,
"p99": 34.15
},
"latency_ms_outside_faults": {
"p50": 10.1,
"p95": 20.22,
"p99": 32.38
}
},
"slow_requests": 8,
"error_requests": 8,
"traces_kept": {
"errors": [
0,
8
],
"slow": [
0,
8
],
"normal_sample": [
0,
40
]
},
"api_cpu_cores": 0.316,
"api_cpu_ms_per_request": 1.284,
"api_mem_peak_mib": [
13.0,
13.0
],
"api_mem_now_mib": [
8.2,
8.3
],
"collector_cpu_cores": 0.001,
"collector_mem_peak_mib": [
28.6
],
"ingest_delta": {
"spans_received": 0,
"spans_exported": 0,
"log_records_received": 0,
"metric_points_received": 0,
"tempo_bytes": 0,
"loki_bytes": 0,
"prometheus_samples": 47826
},
"ingest_per_1000_requests": {
"spans_received": 0.0,
"spans_exported": 0.0,
"log_records_received": 0.0,
"metric_points_received": 0.0,
"tempo_bytes": 0.0,
"loki_bytes": 0.0,
"prometheus_samples": 4324.2
}
},
{
"config": "full",
"round": 2,
"load": {
"requests": 8607,
"ok": 8599,
"errors": 8,
"error_ratio": 0.00093,
"ok_rps": 191.1,
"seconds": 45.0,
"latency_ms_all_ok": {
"p50": 12.57,
"p95": 29.92,
"p99": 46.42
},
"latency_ms_outside_faults": {
"p50": 12.37,
"p95": 28.49,
"p99": 42.89
}
},
"slow_requests": 8,
"error_requests": 8,
"traces_kept": {
"errors": [
8,
8
],
"slow": [
8,
8
],
"normal_sample": [
40,
40
]
},
"api_cpu_cores": 0.364,
"api_cpu_ms_per_request": 1.907,
"api_mem_peak_mib": [
21.3,
22.4
],
"api_mem_now_mib": [
16.4,
16.2
],
"collector_cpu_cores": 0.05,
"collector_mem_peak_mib": [
55.1
],
"ingest_delta": {
"spans_received": 120546,
"spans_exported": 120546,
"log_records_received": 8762,
"metric_points_received": 2166,
"tempo_bytes": 40248112,
"loki_bytes": 2992045,
"prometheus_samples": 55820
},
"ingest_per_1000_requests": {
"spans_received": 14018.6,
"spans_exported": 14018.6,
"log_records_received": 1019.0,
"metric_points_received": 251.9,
"tempo_bytes": 4680557.3,
"loki_bytes": 347952.7,
"prometheus_samples": 6491.5
}
}
]
}
+4
View File
@@ -0,0 +1,4 @@
=== RUN TestOptimizedWebP
--- PASS: TestOptimizedWebP (0.32s)
PASS
ok github.com/charawein/mtusi2026lab6/internal/imaging 0.328s
+74
View File
@@ -0,0 +1,74 @@
module github.com/charawein/mtusi2026lab6
go 1.26.0
require (
github.com/exaring/otelpgx v0.13.0
github.com/google/uuid v1.6.0
github.com/jackc/pgx/v5 v5.11.0
github.com/minio/minio-go/v7 v7.4.1
github.com/rabbitmq/amqp091-go v1.16.0
github.com/redis/go-redis/extra/redisotel/v9 v9.23.0
github.com/redis/go-redis/v9 v9.23.0
go.opentelemetry.io/contrib/bridges/otelslog v0.21.0
go.opentelemetry.io/contrib/instrumentation/host v0.72.0
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.72.0
go.opentelemetry.io/contrib/instrumentation/runtime v0.72.0
go.opentelemetry.io/otel v1.47.0
go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.23.0
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.47.0
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.47.0
go.opentelemetry.io/otel/log v1.47.0
go.opentelemetry.io/otel/metric v1.47.0
go.opentelemetry.io/otel/sdk v1.47.0
go.opentelemetry.io/otel/sdk/log v1.47.0
go.opentelemetry.io/otel/sdk/metric v1.47.0
go.opentelemetry.io/otel/trace v1.47.0
golang.org/x/crypto v0.58.0
golang.org/x/image v0.47.0
golang.org/x/sync v0.25.0
)
require (
github.com/cenkalti/backoff/v5 v5.0.3 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/ebitengine/purego v0.11.1 // indirect
github.com/felixge/httpsnoop v1.1.0 // indirect
github.com/go-logr/logr v1.4.4 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-ole/go-ole v1.3.0 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
github.com/jackc/puddle/v2 v2.2.2 // indirect
github.com/klauspost/compress v1.19.2 // indirect
github.com/klauspost/cpuid/v2 v2.4.0 // indirect
github.com/klauspost/crc32 v1.3.0 // indirect
github.com/lufia/plan9stats v0.0.0-20260802145828-341c2f0c90b5 // indirect
github.com/minio/crc64nvme v1.1.1 // indirect
github.com/minio/md5-simd v1.1.2 // indirect
github.com/philhofer/fwd v1.2.0 // indirect
github.com/power-devops/perfstat v0.0.0-20260916203055-22a1a467d9f0 // indirect
github.com/redis/go-redis/extra/rediscmd/v9 v9.23.0 // indirect
github.com/rs/xid v1.6.0 // indirect
github.com/shirou/gopsutil/v4 v4.26.9 // indirect
github.com/tinylib/msgp v1.6.4 // indirect
github.com/tklauser/go-sysconf v0.4.0 // indirect
github.com/tklauser/numcpus v0.12.0 // indirect
github.com/yusufpapurcu/wmi v1.2.4 // indirect
github.com/zeebo/xxh3 v1.1.0 // indirect
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.47.0 // indirect
go.opentelemetry.io/proto/otlp v1.11.0 // indirect
go.uber.org/atomic v1.12.0 // indirect
go.yaml.in/yaml/v3 v3.0.5 // indirect
golang.org/x/net v0.60.0 // indirect
golang.org/x/sys v0.49.0 // indirect
golang.org/x/text v0.43.0 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20260928230214-8a89bd6388cc // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20260928230214-8a89bd6388cc // indirect
google.golang.org/grpc v1.83.2 // indirect
google.golang.org/protobuf v1.36.12 // indirect
gopkg.in/ini.v1 v1.67.3 // indirect
)
+172
View File
@@ -0,0 +1,172 @@
github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM=
github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/ebitengine/purego v0.11.1 h1:2zpWRSQNVKN4eKsKO9eM1ILDgWfYMY9GwqRmK6XeQ/0=
github.com/ebitengine/purego v0.11.1/go.mod h1:DCHPP08djqhNSoTfImcnHYQRZmd0qhakvrozqaEYhGQ=
github.com/exaring/otelpgx v0.13.0 h1:Vi0I2GWRS3nnVzgr73L/XbSLqthR0h6A1rm4Wt3W4aA=
github.com/exaring/otelpgx v0.13.0/go.mod h1:3OojrUKhhy3lTbYIMBijP3YjMey/jo14eHAW5cXcUdk=
github.com/felixge/httpsnoop v1.1.0 h1:3YtUj32ZZkqZtt3sZZsClsymw/QDuVfpNhoA31zeORc=
github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9vDSi5UZCE=
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8=
github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE=
github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78=
github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 h1:/Tnpcb2E0Pz/tN9s3bfEY2Q8ePCEX9iuS+cneUwncnw=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0/go.mod h1:zOBXOsUaBSjKgmH4OGzV1esUpR3oUSCPYVd2cUBjKYY=
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
github.com/jackc/pgx/v5 v5.11.0 h1:IzBBtyK9AHqf98cctWFifYSci2hgQR/cd56wB4p+ogg=
github.com/jackc/pgx/v5 v5.11.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4=
github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8=
github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw=
github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU=
github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM=
github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw=
github.com/lufia/plan9stats v0.0.0-20260802145828-341c2f0c90b5 h1:eveIIGn4BGM3qknO74omf6HYr30/exH+eVUTuAgwjZ0=
github.com/lufia/plan9stats v0.0.0-20260802145828-341c2f0c90b5/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg=
github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI=
github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg=
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
github.com/minio/minio-go/v7 v7.4.1 h1:PtU/zIOfQRWZ+QydXAZysc+xZYHdG+BBR4Z6TUvy73w=
github.com/minio/minio-go/v7 v7.4.1/go.mod h1:nQnQi7zTfXR4lREtYXZQWqo5aw0Rqv+ZqCGrTORygRk=
github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM=
github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/power-devops/perfstat v0.0.0-20260916203055-22a1a467d9f0 h1:XA01Vk/wv9YikCi1V51yRzIHPMT5of9+cMpZoZvDn/M=
github.com/power-devops/perfstat v0.0.0-20260916203055-22a1a467d9f0/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
github.com/rabbitmq/amqp091-go v1.16.0 h1:1O8oQqnj9hLACoo6CVMgn57z2pLWXvgemhh4GsquV+Y=
github.com/rabbitmq/amqp091-go v1.16.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o=
github.com/redis/go-redis/extra/rediscmd/v9 v9.23.0 h1:YOQCtePRR/cBrxtZB5u2ueYei3zGJm1n0KB2tvth0k0=
github.com/redis/go-redis/extra/rediscmd/v9 v9.23.0/go.mod h1:BzBb5fEjIwijUn8VVkXfQUwFTGyk2vGIVU19zv2GhsM=
github.com/redis/go-redis/extra/redisotel/v9 v9.23.0 h1:EBQ01zUjwarZZ4IoSTQNU9EEc223eB9tpSun6n5EaUU=
github.com/redis/go-redis/extra/redisotel/v9 v9.23.0/go.mod h1:4xBZ7QF+cd90pc58g+cXH1aLJBgOGd4Gp5BBQvNtPn0=
github.com/redis/go-redis/v9 v9.23.0 h1:/TTVdJa8BRNsybwwGGnmiYktHNlUXtbOQxlWpHBPL3s=
github.com/redis/go-redis/v9 v9.23.0/go.mod h1:EWP2UUk+XDaI1s5nQB5mablZN6vccXlmBQTyfnF87D0=
github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
github.com/shirou/gopsutil/v4 v4.26.9 h1:CaBo/hBFqvlJoLvAQEKdHdf5GD97/MN2ACfyv63+2Ig=
github.com/shirou/gopsutil/v4 v4.26.9/go.mod h1:nKH+8wX2zxr/mDbsR+AAy6Qd2z4ZOQ7wfQaEp+yrYmg=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE=
github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg=
github.com/tinylib/msgp v1.6.4 h1:mOwYbyYDLPj35mkA2BjjYejgJk9BuHxDdvRnb6v2ZcQ=
github.com/tinylib/msgp v1.6.4/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA=
github.com/tklauser/go-sysconf v0.4.0 h1:7H0uAN+7RkwWRaxhYXDLqa5V3LPrJeV8wmD9dRUgPQU=
github.com/tklauser/go-sysconf v0.4.0/go.mod h1:8mTNWyog7H+MpKijp4VmKJAd2bbYQ2zuUwkYRbUArPI=
github.com/tklauser/numcpus v0.12.0 h1:NR85qdvHA9pFse3x3weVZ0r0ST8R6l5RHbZrlRaqob4=
github.com/tklauser/numcpus v0.12.0/go.mod h1:ABHeXzJnr/qqwguhClkZKT1/8VABcYrsyUiUGobwWJg=
github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=
github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs=
github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s=
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
go.opentelemetry.io/contrib/bridges/otelslog v0.21.0 h1:/CdgE52i1OEPHhsGUqrggYNRaZy4v4YgVenGd52dRBI=
go.opentelemetry.io/contrib/bridges/otelslog v0.21.0/go.mod h1:9pBdbVbuehIDzxN4accnzzxEUPT8Nq7grbZMo+J1VN4=
go.opentelemetry.io/contrib/instrumentation/host v0.72.0 h1:DqxFkQ1eCzAQNPJNog2+tL9k3OaULuiYHiZaz2Waq54=
go.opentelemetry.io/contrib/instrumentation/host v0.72.0/go.mod h1:Jn5AUVzW8ZWjm4S1uRMlio5gZxVpmXRr/2PmdESRE7Y=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.72.0 h1:LxwW/9ctSCv+QkE/cLR7M91ZIkXNMqJtEMi1vCw9U8s=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.72.0/go.mod h1:tOsftB4SslBwwErVEPaenU2RpThXWPIU8DoJHEC4dyw=
go.opentelemetry.io/contrib/instrumentation/runtime v0.72.0 h1:rv8t3kKjyC2/9D99L1MZt4tJSZO4aYUmHus7L//iKKg=
go.opentelemetry.io/contrib/instrumentation/runtime v0.72.0/go.mod h1:ZzG+JmzLLvKzx6tMeLrw2l7F8ik2GeRZJALoPBqnM0c=
go.opentelemetry.io/otel v1.47.0 h1:j7ALJ/zgkS7Z6aeJW09p8VC9804bC+PpeTfCD4XPnOM=
go.opentelemetry.io/otel v1.47.0/go.mod h1:8wS9O2qfXrYrzp6hIF/HOYJJf/wIhFPhR2xLuP+iXQU=
go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.23.0 h1:5WVp40k2fJHuc5BtNTgKk+EugKFT8Eg3lywFfCPI/nE=
go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.23.0/go.mod h1:59nH0RYCGHRlwt15WyioeSYINBbuObeBpTowPhEKWHY=
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.47.0 h1:7kszRN4Ze5GeAkUnBjBSUIunVBMA+mZIG04auueqguM=
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.47.0/go.mod h1:CLzakYy8jOdkBfcveCWYkeKJxujcF4xPhh2Z7IfE98U=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.47.0 h1:julhjPeUH/q/7hinbSdDdqt5h7Zw9YWmRlWRhI0jd54=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.47.0/go.mod h1:Ao2mz688LH/tFf0yMAenidq6k2YNSx6SIY2q6jDACck=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.47.0 h1:aXZXsZ012wOgVkqaiQv+Z/d8V0xKjmg70F6m4CE94lc=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.47.0/go.mod h1:ziu1gUIJhAaxM3EV1e3Ralk5AAyL1UOYz3lJJ4VSI38=
go.opentelemetry.io/otel/log v1.47.0 h1:cOTS1CcLbSQeZKanGJ+0JpF/+t4PELi3O3bbl2lqCcI=
go.opentelemetry.io/otel/log v1.47.0/go.mod h1:9byitSQ5pLC6PpqwGXjqdMKya6ZTswHRZh2vvXT33nw=
go.opentelemetry.io/otel/metric v1.47.0 h1:4PptaldXx3Eat1XjMZ68pPJEs5wrhlemctZE9a3UdWY=
go.opentelemetry.io/otel/metric v1.47.0/go.mod h1:ADGSXxRrXM6bjbvLo535EstVFlPpPYZm4LBKixjDHwU=
go.opentelemetry.io/otel/metric/x v0.69.0 h1:DjRLr15H83v+hCW7JA9NoJvOkYTtmq5YoDRbe9deYpM=
go.opentelemetry.io/otel/metric/x v0.69.0/go.mod h1:uVvsMPMFFyj/HUQfrUnH3JjnOQ1dwFDorgFLRBasM0k=
go.opentelemetry.io/otel/sdk v1.47.0 h1:zWXEr4j2lFefG87TU6Yg8a7ngfohIKFZHKp0Hf5hC6I=
go.opentelemetry.io/otel/sdk v1.47.0/go.mod h1:VUc24kiOeoGsxG8G9ULx3fWKvB7jMhnGE8Oi607lgR0=
go.opentelemetry.io/otel/sdk/log v1.47.0 h1:W4N/ZfgpkRRnmARnqYoQ6BrWA0mrj8IzVDNLlpmUJZY=
go.opentelemetry.io/otel/sdk/log v1.47.0/go.mod h1:Wu74Z2EmOKXi2FleSNAb0Hyau2G+r5kubSEMJw5aAMs=
go.opentelemetry.io/otel/sdk/log/logtest v0.23.0 h1:j3pm42QCrKYYLLkI6OrV1QHFkUfDA6/4EP+Y1PzggLg=
go.opentelemetry.io/otel/sdk/log/logtest v0.23.0/go.mod h1:zt0gEnuTZgDvqvczHwYswz93eqMaIr6upTTsdV/HUlU=
go.opentelemetry.io/otel/sdk/metric v1.47.0 h1:lfISg2j93VT6yqdk9OfUaZmw/GfcZqCCV3jdXtsPnKw=
go.opentelemetry.io/otel/sdk/metric v1.47.0/go.mod h1:ypLp+mW1Nt2x+Szt3b5/i1syodyts49lMOwxpDI3VGw=
go.opentelemetry.io/otel/trace v1.47.0 h1:JOjX/Oci8K94QHddo+bbfya/Ai/nf6/dt9ZfrFNWSrM=
go.opentelemetry.io/otel/trace v1.47.0/go.mod h1:jNaSLa2PZEYFG6fRjJABAu+bw4FS08uDmPg28lTghu0=
go.opentelemetry.io/proto/otlp v1.11.0 h1:5rrYs0Ykyj50sdU/JU0x8etU+LubXWb+gED6TbEdMIk=
go.opentelemetry.io/proto/otlp v1.11.0/go.mod h1:SmVizdCOAm3XBtG1g1NnOdhW6jtddT72hLMhv8VwA8E=
go.uber.org/atomic v1.12.0 h1:BvcXdFKuviU4fTL/f+SxdQ5qJX/Jix8pAkgdUcb3XOE=
go.uber.org/atomic v1.12.0/go.mod h1:I6c4cg+6HCxRjfjSsYtApoFILnpc0CGUdGkXVqbYVNk=
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
golang.org/x/crypto v0.58.0 h1:COkYLr4k7nDI4r6QtMML2AfzeIswWHAJthLS6K/M/54=
golang.org/x/crypto v0.58.0/go.mod h1:Xh+kl5A+M0gMiWDytH2Hhr5Z7WbNCpv/eB/EFgfSk+w=
golang.org/x/image v0.47.0 h1:za/8gVz++PPZqmZJBNSISICN5Rrj5zixP/8K++fgmqo=
golang.org/x/image v0.47.0/go.mod h1:IkN5o4A7vq55vXN8FWjjnhg9Rs2GOtffqsSi9mWAKYE=
golang.org/x/net v0.60.0 h1:79p50tfZlm0J9YfoDsSi639qSXNGVwEzOPLCxM2FsYU=
golang.org/x/net v0.60.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg=
golang.org/x/sync v0.25.0 h1:dJnifr0nMV8/SBnBvdo4eSGGnrc3O4FFx6IqvtLcmRs=
golang.org/x/sync v0.25.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA=
golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
golang.org/x/text v0.43.0 h1:1QivrlhwAsnMOcqgOdl9my7s9OyZl8O5/UGUTJVXoWc=
golang.org/x/text v0.43.0/go.mod h1:hbSIYA/amXcRXTFSZqh8tWHEoqRvvbKPJ2uz/KU+a+s=
gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=
gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E=
google.golang.org/genproto/googleapis/api v0.0.0-20260928230214-8a89bd6388cc h1:rZHRz0ogq4Pid1IKtA/ivHi8XONVF5tqUQXA2TtgpkE=
google.golang.org/genproto/googleapis/api v0.0.0-20260928230214-8a89bd6388cc/go.mod h1:K/n0XLazCJHjD73YuMyNJsN38BPPy1kfFnKxUXaThFU=
google.golang.org/genproto/googleapis/rpc v0.0.0-20260928230214-8a89bd6388cc h1:4bNTbnb44EqGVy9HaQxSY2jnafSUdgV3qHtedvNpDKg=
google.golang.org/genproto/googleapis/rpc v0.0.0-20260928230214-8a89bd6388cc/go.mod h1:OaIUM3+LpYcK2GXM4FTmhWoIq371Owdr+Cc7/BsYHHc=
google.golang.org/grpc v1.83.2 h1:EManeRomTObA0BU7I8vXgg/78uE5MJ9M8B39EX2WscU=
google.golang.org/grpc v1.83.2/go.mod h1:YPI1hK3kDked6iHvgX3tR0y+nX/qpMFKhPgFsokw1S8=
google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc=
google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/ini.v1 v1.67.3 h1:iM9Lhz5MRSGhHVGGwCuzG9KO8PoirCXj/m/qTmOJJQw=
gopkg.in/ini.v1 v1.67.3/go.mod h1:x/cyOwCgZqOkJoDIJ3c1KNHMo10+nLGAhh+kn3Zizss=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+35
View File
@@ -0,0 +1,35 @@
# Runbooks alert-ов мессенджера
Все шаги выполняются только по сохранённым наблюдениям: Grafana (dashboard «Messenger — API, worker, очередь», Explore → Prometheus / Loki / Tempo). Доступ к контейнерам нужен только для устранения причины, не для поиска.
## MessengerHighErrorRate
**Условие:** доля 5xx среди бизнес-запросов (все маршруты, кроме `/health/*`) за 1 минуту > 5% при ≥ 0.5 запроса/с, 30 s подряд. 4xx — ошибки клиента — не учитываются.
1. Dashboard → «Доля ошибок 5xx и 4xx» и «p95 по маршруту»: один маршрут или все? Если все — общая зависимость (БД, Redis, S3); если один — его обработчик.
2. «Пул БД: занято / ожидания»: `pgxpool_acquired_connections` упёрся в `pgxpool_max_connections` и растут ожидания (`pgxpool_empty_acquire_total`) — БД не отдаёт соединения или запросы висят.
3. Explore → Tempo: `{resource.service.name="messenger-api" && span.http.response.status_code >= 500}`. Открыть трассу и посмотреть, какой дочерний span занимает время:
* длинный `query …` / `acquire` (PostgreSQL) — проблема БД (блокировка, недоступность); подтвердить в БД `pg_stat_activity` (wait_event = Lock);
* длинный `S3 …` — хранилище файлов;
* дочерние spans короткие, а server span длинный — медленный сам обработчик/тело запроса.
4. Explore → Loki: `{service_name="messenger-api"} | detected_level=~"error|ERROR"` за то же окно; по `trace_id` из лога — переход в трассу.
5. Устранить причину; alert перейдёт в resolved, когда окно 1m очистится от ошибок.
## MessengerQueueLagGrowing
**Условие:** старейшая незавершённая job (queued/processing) старше 60 s, 30 s подряд. Возраст считается по PostgreSQL (источник истины), не по брокеру.
1. Dashboard → «Очередь: jobs по состоянию»: растёт `queued` при нулевом `processing` — задания никто не берёт; растёт `processing` — worker берёт, но не завершает.
2. «Jobs по исходу (rate)»: нет `done` — worker стоит; много `retry`/`requeue` — worker работает, но падает на зависимости (S3/БД).
3. `messenger_worker_capacity` есть? Нет серии ≥ 1 мин → worker не запущен (сработает и `MessengerWorkerAbsent`).
4. «RabbitMQ: сообщения в очередях»: ready растёт — сообщения есть, потребителя нет; ready = 0 при queued в БД — relay не публикует outbox (см. «outbox не опубликовано» и возраст outbox).
5. Tempo: последние `attachment.process process` (worker) и `attachment.process publish` (relay) — что выполнялось последним; Loki `{service_name="messenger-worker"}` — последние события (`worker.start`, `job.retry`, `shutdown`).
6. После запуска/починки worker backlog уменьшается (панель возраста падает до 0), alert — resolved.
## MessengerWorkerAbsent
Нет метрик ни одного worker 1 минуту (серии исчезают через 1 мин после остановки процесса: `metric_expiration` Collector). Проверить `docker compose ps worker`, логи запуска (`worker.start`) и доступность Collector (если Collector недоступен, исчезают метрики всех сервисов — смотреть `MessengerTelemetryExportFailing` и метрики самого Collector).
## MessengerTelemetryExportFailing
Collector не может доставить spans/логи в Tempo/Loki (после ограниченной очереди и 60 s повторов данные отбрасываются, счётчики `otelcol_exporter_send_failed_*`). Бизнес-API не затронут. Проверить доступность Tempo/Loki, `otelcol_exporter_queue_size` относительно `otelcol_exporter_queue_capacity`. Потерянные за время отказа трассы/логи не восстанавливаются; метрики приложения при этом продолжают собираться (Prometheus забирает их напрямую из Collector).
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,9 @@
apiVersion: 1
providers:
- name: messenger
folder: Messenger
type: file
disableDeletion: true
allowUiUpdates: false
options:
path: /var/lib/grafana/dashboards
@@ -0,0 +1,42 @@
# Источники Grafana: Grafana сама данные не хранит — только запрашивает.
# Связи: метрика (exemplar) → трасса; лог (trace_id) → трасса; трасса → логи.
apiVersion: 1
datasources:
- name: Prometheus
uid: prometheus
type: prometheus
access: proxy
url: http://prometheus:9090
isDefault: true
jsonData:
timeInterval: 5s
exemplarTraceIdDestinations:
- name: trace_id
datasourceUid: tempo
- name: Loki
uid: loki
type: loki
access: proxy
url: http://loki:3100
jsonData:
derivedFields:
- name: trace
matcherType: label
matcherRegex: trace_id
datasourceUid: tempo
url: "$${__value.raw}"
- name: Tempo
uid: tempo
type: tempo
access: proxy
url: http://tempo:3200
jsonData:
tracesToLogsV2:
datasourceUid: loki
filterByTraceID: true
spanStartTimeShift: "-5m"
spanEndTimeShift: "5m"
serviceMap:
datasourceUid: prometheus
nodeGraph:
enabled: true
+59
View File
@@ -0,0 +1,59 @@
# HAProxy — единственная публичная точка входа (lab3).
# Backend-адреса берутся из встроенного DNS Docker (service name "api"),
# поэтому масштабирование/пересоздание реплик не требует перезапуска LB.
global
log stdout format raw local0 info
maxconn 2048
defaults
mode http
log global
option httplog
option dontlognull
timeout connect 2s
timeout client 30s
timeout server 15s
timeout http-request 10s
timeout queue 5s
# Повтор только при ошибке установки соединения (запрос ещё не отправлен),
# поэтому неидемпотентные POST /users, /chats не дублируются.
retries 2
retry-on conn-failure
option redispatch 1
resolvers docker
nameserver dns 127.0.0.11:53
resolve_retries 3
timeout resolve 1s
timeout retry 1s
# Как долго держать последний ответ DNS при ошибке/пропаже записи.
hold valid 5s
hold other 5s
hold refused 5s
hold nx 2s
hold timeout 5s
hold obsolete 2s
accepted_payload_size 8192
frontend public
bind :8080
default_backend api
backend api
# roundrobin по умолчанию; leastconn — вариант для эксперимента трека 1 (env LB_BALANCE).
balance "${LB_BALANCE}"
# Активная проверка готовности: 503 во время drain или без БД/Redis.
option httpchk
http-check send meth GET uri /health/ready
http-check expect status 200
# Период проверки 1s, исключение после 2 неудач (~2s), возврат после 2 успехов.
default-server check inter 1s fastinter 500ms downinter 1s fall 2 rise 2 resolvers docker resolve-prefer ipv4 init-addr none
# До 8 слотов под реплики сервиса api; лишние слоты остаются в MAINT.
server-template api 8 api:8080
frontend stats
bind :8404
stats enable
stats uri /stats
stats refresh 2s
+39
View File
@@ -0,0 +1,39 @@
# Loki 3 — хранилище логов. Приём OTLP (/otlp) от Collector: service.name и др.
# resource attributes → индексные метки (service_name, …), атрибуты записи
# (event, trace_id, span_id, severity) → structured metadata, без индексации.
auth_enabled: false
server:
http_listen_port: 3100
grpc_listen_port: 9096
log_level: warn
common:
instance_addr: 127.0.0.1
path_prefix: /loki
storage:
filesystem:
chunks_directory: /loki/chunks
rules_directory: /loki/rules
replication_factor: 1
ring:
kvstore:
store: inmemory
schema_config:
configs:
- from: 2024-01-01
store: tsdb
object_store: filesystem
schema: v13
index:
prefix: index_
period: 24h
limits_config:
allow_structured_metadata: true
retention_period: 168h
ingestion_rate_mb: 8
ingestion_burst_size_mb: 16
compactor:
working_directory: /loki/compactor
retention_enabled: true
delete_request_store: filesystem
analytics:
reporting_enabled: false
+125
View File
@@ -0,0 +1,125 @@
# ВАРИАНТ ТРЕКА 1 (tail sampling): SDK отправляет 100% spans, Collector хранит
# только трассы с ошибкой, медленные (≥ 250 ms) и 10% остальных.
#
# OpenTelemetry Collector (contrib) — приём OTLP от API/relay/worker и раздача
# по хранилищам: traces → Tempo, logs → Loki (OTLP/HTTP), metrics → Prometheus (pull).
#
# Ограничение памяти и буферов:
# memory_limiter — первым в каждом pipeline: при превышении soft limit Collector
# отказывает receiver-у (SDK получает ошибку и отбрасывает данные по своей
# ограниченной очереди), вместо бесконечного роста RSS;
# batch — пакеты до 1024 элементов / 2 s;
# sending_queue (in-memory, queue_size пакетов) + retry_on_failure (≤ 60 s) —
# размер подобран так, чтобы очередь целиком (20 пакетов × 1024 spans) помещалась
# ниже soft limit memory_limiter: при queue_size 200 и пакетах до 2048 очередь при
# отказе Tempo упиралась в hard limit, и Collector отвергал НОВЫЕ данные (прогон 4);
# при недоступном Tempo/Loki данные ждут ограниченное время в ограниченной
# очереди, затем отбрасываются и считаются в otelcol_exporter_send_failed_*.
# Метрики не буферизуются вовсе: exporter prometheus хранит только текущее
# cumulative-состояние серий, Prometheus забирает его сам (pull).
receivers:
otlp:
protocols:
grpc:
endpoint: 0.0.0.0:4317
http:
endpoint: 0.0.0.0:4318
processors:
# Решение по трассе целиком через decision_wait после её первого span;
# num_traces ограничивает память буфера решений. decision_wait должен быть больше
# самой длинной трассы: при 5 s терялась часть ошибок по таймауту обработчика (5 s) —
# решение принималось раньше, чем приходил корневой span со статусом ERROR (прогон 3).
tail_sampling:
decision_wait: 10s
num_traces: 20000
expected_new_traces_per_sec: 200
policies:
- name: errors
type: status_code
status_code: { status_codes: [ERROR] }
- name: slow
type: latency
latency: { threshold_ms: 250 }
- name: baseline
type: probabilistic
probabilistic: { sampling_percentage: 10 }
memory_limiter:
check_interval: 1s
limit_mib: 200
spike_limit_mib: 50
batch:
send_batch_size: 1024
send_batch_max_size: 1024
timeout: 2s
# Защита от утечек на случай ошибки инструментирования: такие ключи не должны
# появляться, но если появятся — удаляются до экспорта.
attributes/scrub:
actions:
- { key: http.request.header.authorization, action: delete }
- { key: http.request.header.cookie, action: delete }
- { key: authorization, action: delete }
- { key: password, action: delete }
- { key: token, action: delete }
- { key: db.query.parameters, action: delete }
exporters:
otlp/tempo:
endpoint: tempo:4317
tls:
insecure: true # внутренняя сеть telemetry; TLS — в lab7
sending_queue:
enabled: true
num_consumers: 2
queue_size: 20 # ≤ 20 пакетов × ≤ 1024 элемента: очередь помещается в memory_limiter
retry_on_failure:
enabled: true
initial_interval: 1s
max_interval: 10s
max_elapsed_time: 60s
otlphttp/loki:
endpoint: http://loki:3100/otlp
sending_queue:
enabled: true
num_consumers: 2
queue_size: 20 # ≤ 20 пакетов × ≤ 1024 элемента: очередь помещается в memory_limiter
retry_on_failure:
enabled: true
initial_interval: 1s
max_interval: 10s
max_elapsed_time: 60s
prometheus:
endpoint: 0.0.0.0:8889
enable_open_metrics: true # exemplars (trace_id) в histogram buckets
metric_expiration: 1m # серии остановленного процесса исчезают через 1 мин (absent() в alerts)
extensions:
health_check:
endpoint: 0.0.0.0:13133
service:
extensions: [health_check]
telemetry:
logs:
level: warn
metrics:
level: detailed
readers:
- pull:
exporter:
prometheus:
host: 0.0.0.0
port: 8888
pipelines:
traces:
receivers: [otlp]
processors: [memory_limiter, tail_sampling, attributes/scrub, batch]
exporters: [otlp/tempo]
metrics:
receivers: [otlp]
processors: [memory_limiter, batch]
exporters: [prometheus]
logs:
receivers: [otlp]
processors: [memory_limiter, attributes/scrub, batch]
exporters: [otlphttp/loki]
+104
View File
@@ -0,0 +1,104 @@
# OpenTelemetry Collector (contrib) — приём OTLP от API/relay/worker и раздача
# по хранилищам: traces → Tempo, logs → Loki (OTLP/HTTP), metrics → Prometheus (pull).
#
# Ограничение памяти и буферов:
# memory_limiter — первым в каждом pipeline: при превышении soft limit Collector
# отказывает receiver-у (SDK получает ошибку и отбрасывает данные по своей
# ограниченной очереди), вместо бесконечного роста RSS;
# batch — пакеты до 1024 элементов / 2 s;
# sending_queue (in-memory, queue_size пакетов) + retry_on_failure (≤ 60 s) —
# размер подобран так, чтобы очередь целиком (20 пакетов × 1024 spans) помещалась
# ниже soft limit memory_limiter: при queue_size 200 и пакетах до 2048 очередь при
# отказе Tempo упиралась в hard limit, и Collector отвергал НОВЫЕ данные (прогон 4);
# при недоступном Tempo/Loki данные ждут ограниченное время в ограниченной
# очереди, затем отбрасываются и считаются в otelcol_exporter_send_failed_*.
# Метрики не буферизуются вовсе: exporter prometheus хранит только текущее
# cumulative-состояние серий, Prometheus забирает его сам (pull).
receivers:
otlp:
protocols:
grpc:
endpoint: 0.0.0.0:4317
http:
endpoint: 0.0.0.0:4318
processors:
memory_limiter:
check_interval: 1s
limit_mib: 200
spike_limit_mib: 50
batch:
send_batch_size: 1024
send_batch_max_size: 1024
timeout: 2s
# Защита от утечек на случай ошибки инструментирования: такие ключи не должны
# появляться, но если появятся — удаляются до экспорта.
attributes/scrub:
actions:
- { key: http.request.header.authorization, action: delete }
- { key: http.request.header.cookie, action: delete }
- { key: authorization, action: delete }
- { key: password, action: delete }
- { key: token, action: delete }
- { key: db.query.parameters, action: delete }
exporters:
otlp/tempo:
endpoint: tempo:4317
tls:
insecure: true # внутренняя сеть telemetry; TLS — в lab7
sending_queue:
enabled: true
num_consumers: 2
queue_size: 20 # ≤ 20 пакетов × ≤ 1024 элемента: очередь помещается в memory_limiter
retry_on_failure:
enabled: true
initial_interval: 1s
max_interval: 10s
max_elapsed_time: 60s
otlphttp/loki:
endpoint: http://loki:3100/otlp
sending_queue:
enabled: true
num_consumers: 2
queue_size: 20 # ≤ 20 пакетов × ≤ 1024 элемента: очередь помещается в memory_limiter
retry_on_failure:
enabled: true
initial_interval: 1s
max_interval: 10s
max_elapsed_time: 60s
prometheus:
endpoint: 0.0.0.0:8889
enable_open_metrics: true # exemplars (trace_id) в histogram buckets
metric_expiration: 1m # серии остановленного процесса исчезают через 1 мин (absent() в alerts)
extensions:
health_check:
endpoint: 0.0.0.0:13133
service:
extensions: [health_check]
telemetry:
logs:
level: warn
metrics:
level: detailed
readers:
- pull:
exporter:
prometheus:
host: 0.0.0.0
port: 8888
pipelines:
traces:
receivers: [otlp]
processors: [memory_limiter, attributes/scrub, batch]
exporters: [otlp/tempo]
metrics:
receivers: [otlp]
processors: [memory_limiter, batch]
exporters: [prometheus]
logs:
receivers: [otlp]
processors: [memory_limiter, attributes/scrub, batch]
exporters: [otlphttp/loki]
+33
View File
@@ -0,0 +1,33 @@
# Prometheus — хранилище метрик (TSDB на volume, retention 7d) и вычисление alerts.
# Метрики приложения приходят OTLP → Collector → exporter prometheus (:8889):
# honor_labels сохраняет job=service.name и instance=service.instance.id реплики.
global:
scrape_interval: 5s
evaluation_interval: 5s
external_labels:
deployment_environment: lab
rule_files:
- /etc/prometheus/rules/*.yml
scrape_configs:
- job_name: otel-collector-app
honor_labels: true
static_configs:
- targets: ["otel-collector:8889"]
- job_name: otel-collector
static_configs:
- targets: ["otel-collector:8888"]
- job_name: rabbitmq
metrics_path: /metrics/per-object # глубина очередей attachment.process / attachment.dead
static_configs:
- targets: ["rabbitmq:15692"]
- job_name: loki
static_configs:
- targets: ["loki:3100"]
- job_name: tempo
static_configs:
- targets: ["tempo:3200"]
- job_name: prometheus
static_configs:
- targets: ["localhost:9090"]
+61
View File
@@ -0,0 +1,61 @@
# Recording rules (SLI) и alert rules. Состояние alerts — локально в Prometheus
# (/api/v1/alerts, Grafana → Alerting); доставка наружу не настроена (не требуется).
# Runbooks: infra/RUNBOOKS.md.
#
# SLI/SLO (окно SLO — 28 дней; на стенде проверяются короткие окна):
# доступность API: доля бизнес-запросов (кроме /health/*) без 5xx ≥ 99.5%;
# задержка API: доля бизнес-запросов быстрее 250 ms ≥ 95%;
# время до ready: доля вложений, ставших ready быстрее 30 s после 202, ≥ 95%.
groups:
- name: messenger-sli
interval: 5s
rules:
- record: messenger:http_requests:rate1m
expr: sum(rate(http_server_request_duration_seconds_count{job="messenger-api", http_route!~"/health/.*"}[1m]))
- record: messenger:http_errors:rate1m
expr: sum(rate(http_server_request_duration_seconds_count{job="messenger-api", http_route!~"/health/.*", http_response_status_class="5xx"}[1m])) or vector(0)
- record: messenger:http_error_ratio:rate1m
expr: messenger:http_errors:rate1m / messenger:http_requests:rate1m
- record: messenger:http_fast_ratio:rate5m
expr: |
sum(rate(http_server_request_duration_seconds_bucket{job="messenger-api", http_route!~"/health/.*", le="0.25"}[5m]))
/ sum(rate(http_server_request_duration_seconds_count{job="messenger-api", http_route!~"/health/.*"}[5m]))
- record: messenger:time_to_ready_fast_ratio:rate5m
expr: |
sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="30"}[5m]))
/ sum(rate(messenger_attachment_time_to_ready_seconds_count[5m]))
- name: messenger-alerts
interval: 5s
rules:
- alert: MessengerHighErrorRate
expr: messenger:http_error_ratio:rate1m > 0.05 and messenger:http_requests:rate1m > 0.5
for: 30s
labels: { severity: page }
annotations:
summary: "API 5xx > 5% бизнес-запросов (окно 1m, 30s подряд)"
description: "Доля 5xx {{ $value | humanizePercentage }}. Числитель — 5xx, знаменатель — все запросы кроме /health/*; 4xx (ошибки клиента) не считаются."
runbook: "infra/RUNBOOKS.md#messengerhigherrorrate"
- alert: MessengerQueueLagGrowing
expr: max(messenger_jobs_oldest_age_seconds) > 60
for: 30s
labels: { severity: page }
annotations:
summary: "Старейшая незавершённая job старше 60 s"
description: "Возраст {{ $value | humanizeDuration }}: задания принимаются (202), но не завершаются — worker остановлен/перегружен или broker недоступен."
runbook: "infra/RUNBOOKS.md#messengerqueuelaggrowing"
- alert: MessengerWorkerAbsent
expr: absent(messenger_worker_capacity)
for: 1m
labels: { severity: warning }
annotations:
summary: "Нет метрик ни одного worker 1 минуту"
runbook: "infra/RUNBOOKS.md#messengerworkerabsent"
- alert: MessengerTelemetryExportFailing
expr: (sum(rate({__name__=~"otelcol_exporter_send_failed_spans(_total)?"}[1m])) or vector(0)) + (sum(rate({__name__=~"otelcol_exporter_send_failed_log_records(_total)?"}[1m])) or vector(0)) > 0
for: 30s
labels: { severity: warning }
annotations:
summary: "Collector не может доставить spans/logs в Tempo/Loki"
description: "Данные отбрасываются после ограниченной очереди и 60 s повторов; API не затронут."
runbook: "infra/RUNBOOKS.md#messengertelemetryexportfailing"
+35
View File
@@ -0,0 +1,35 @@
#!/bin/sh
# Идемпотентный bootstrap MinIO: повторный запуск ничего не ломает и не стирает данные.
# 1. приватный bucket (anonymous access: none);
# 2. политика messenger-api: только Get/Put/Delete объектов и List этого bucket;
# 3. отдельный пользователь API с этой политикой (root-ключ API не получает).
# Секреты читаются из /run/secrets и не печатаются.
set -eu
: "${S3_BUCKET:?}"
ROOT_USER=$(cat /run/secrets/s3_root_user)
ROOT_PASS=$(cat /run/secrets/s3_root_password)
API_KEY=$(cat /run/secrets/s3_access_key_id)
API_SECRET=$(cat /run/secrets/s3_secret_access_key)
mc alias set local http://s3:9000 "$ROOT_USER" "$ROOT_PASS" >/dev/null
mc mb --ignore-existing "local/$S3_BUCKET"
mc anonymous set none "local/$S3_BUCKET" >/dev/null
cat > /tmp/messenger-api.json <<EOF
{
"Version": "2012-10-17",
"Statement": [
{"Effect": "Allow", "Action": ["s3:GetBucketLocation", "s3:ListBucket"],
"Resource": ["arn:aws:s3:::$S3_BUCKET"]},
{"Effect": "Allow", "Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"],
"Resource": ["arn:aws:s3:::$S3_BUCKET/*"]}
]
}
EOF
mc admin policy create local messenger-api /tmp/messenger-api.json >/dev/null
# user add для существующего пользователя обновляет секрет — повтор безопасен.
mc admin user add local "$API_KEY" "$API_SECRET" >/dev/null
if ! mc admin policy entities local --user "$API_KEY" 2>/dev/null | grep -q messenger-api; then
mc admin policy attach local messenger-api --user "$API_KEY" >/dev/null
fi
echo "s3 bootstrap ok: bucket=$S3_BUCKET private, api user with messenger-api policy"
+31
View File
@@ -0,0 +1,31 @@
# Tempo 2 — хранилище трасс (локальный backend на volume). Приём OTLP gRPC от
# Collector; поиск TraceQL и выдача трассы по ID для Grafana.
stream_over_http_enabled: true
server:
http_listen_port: 3200
log_level: warn
distributor:
receivers:
otlp:
protocols:
grpc:
endpoint: 0.0.0.0:4317
# Явный адрес frontend для querier в single-binary: автоматическая настройка после
# рестарта Tempo оставляла запросы без ответа (прогон 4, отказ backend).
querier:
frontend_worker:
frontend_address: 127.0.0.1:9095
ingester:
max_block_duration: 5m
compactor:
compaction:
block_retention: 168h
storage:
trace:
backend: local
wal:
path: /var/tempo/wal
local:
path: /var/tempo/blocks
usage_report:
reporting_enabled: false
+189
View File
@@ -0,0 +1,189 @@
// Package broker — RabbitMQ для заданий обработки вложений.
//
// Топология (объявляется идемпотентно при каждом подключении):
//
// exchange attachments (direct, durable) ──process──► queue attachment.process
// quorum-очередь (реплицируемый журнал на диске), x-delivery-limit=10:
// сообщение, которое 10 раз вернулось без ack (poison), уходит в DLX
// exchange attachments.dlx (fanout, durable) ──► queue attachment.dead (реестр «мёртвых»)
//
// Publisher confirm подтверждает, что broker устойчиво ПРИНЯЛ сообщение (записал
// в quorum-журнал); consumer ack подтверждает, что worker устойчиво ОБРАБОТАЛ его.
// Это разные границы: между ними сообщение может быть доставлено несколько раз.
package broker
import (
"context"
"errors"
"fmt"
"time"
amqp "github.com/rabbitmq/amqp091-go"
"go.opentelemetry.io/otel"
)
// HeaderCarrier — заголовки AMQP как TextMapCarrier (traceparent, tracestate).
type HeaderCarrier amqp.Table
func (c HeaderCarrier) Get(key string) string {
if v, ok := c[key].(string); ok {
return v
}
return ""
}
func (c HeaderCarrier) Set(key, value string) { c[key] = value }
func (c HeaderCarrier) Keys() []string {
keys := make([]string, 0, len(c))
for k := range c {
keys = append(keys, k)
}
return keys
}
const (
Exchange = "attachments"
RoutingKey = "process"
Queue = "attachment.process"
DeadExchange = "attachments.dlx"
DeadQueue = "attachment.dead"
DeliveryLimit = 10
)
// Declare объявляет топологию; повторный вызов с теми же аргументами — no-op.
func Declare(ch *amqp.Channel) error {
if err := ch.ExchangeDeclare(Exchange, "direct", true, false, false, false, nil); err != nil {
return fmt.Errorf("declare exchange: %w", err)
}
if err := ch.ExchangeDeclare(DeadExchange, "fanout", true, false, false, false, nil); err != nil {
return fmt.Errorf("declare dlx: %w", err)
}
if _, err := ch.QueueDeclare(DeadQueue, true, false, false, false, amqp.Table{"x-queue-type": "quorum"}); err != nil {
return fmt.Errorf("declare dead queue: %w", err)
}
if err := ch.QueueBind(DeadQueue, "", DeadExchange, false, nil); err != nil {
return fmt.Errorf("bind dead queue: %w", err)
}
args := amqp.Table{
"x-queue-type": "quorum",
"x-dead-letter-exchange": DeadExchange,
"x-delivery-limit": int32(DeliveryLimit),
}
if _, err := ch.QueueDeclare(Queue, true, false, false, false, args); err != nil {
return fmt.Errorf("declare queue: %w", err)
}
if err := ch.QueueBind(Queue, RoutingKey, Exchange, false, nil); err != nil {
return fmt.Errorf("bind queue: %w", err)
}
return nil
}
// Publisher публикует с publisher confirms.
type Publisher struct {
conn *amqp.Connection
ch *amqp.Channel
}
func NewPublisher(url string) (*Publisher, error) {
conn, err := amqp.DialConfig(url, amqp.Config{Heartbeat: 10 * time.Second, Properties: amqp.Table{"connection_name": "relay"}})
if err != nil {
return nil, fmt.Errorf("dial broker: %w", err)
}
ch, err := conn.Channel()
if err != nil {
conn.Close()
return nil, err
}
if err := Declare(ch); err != nil {
conn.Close()
return nil, err
}
if err := ch.Confirm(false); err != nil {
conn.Close()
return nil, fmt.Errorf("enable confirms: %w", err)
}
return &Publisher{conn: conn, ch: ch}, nil
}
func (p *Publisher) Closed() bool { return p.conn.IsClosed() || p.ch.IsClosed() }
func (p *Publisher) Close() { _ = p.conn.Close() }
// Publish отправляет persistent-сообщение и ждёт confirm (ack от broker).
// nack или таймаут — ошибка: outbox не помечается опубликованным.
// W3C trace context из ctx (producer span) уходит в заголовки AMQP traceparent/tracestate.
func (p *Publisher) Publish(ctx context.Context, messageID string, body []byte) error {
headers := amqp.Table{}
otel.GetTextMapPropagator().Inject(ctx, HeaderCarrier(headers))
ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
dc, err := p.ch.PublishWithDeferredConfirmWithContext(ctx, Exchange, RoutingKey, true, false, amqp.Publishing{
ContentType: "application/json",
DeliveryMode: amqp.Persistent,
MessageId: messageID,
Timestamp: time.Now(),
Type: "attachment.process.v1",
Headers: headers,
Body: body,
})
if err != nil {
return fmt.Errorf("publish: %w", err)
}
ok, err := dc.WaitContext(ctx)
if err != nil {
return fmt.Errorf("publish confirm: %w", err)
}
if !ok {
return errors.New("publish nacked by broker")
}
return nil
}
// Consumer — подписка с ручным ack и prefetch.
type Consumer struct {
conn *amqp.Connection
ch *amqp.Channel
}
func NewConsumer(url string, prefetch int, name string) (*Consumer, <-chan amqp.Delivery, error) {
conn, err := amqp.DialConfig(url, amqp.Config{Heartbeat: 10 * time.Second, Properties: amqp.Table{"connection_name": name}})
if err != nil {
return nil, nil, fmt.Errorf("dial broker: %w", err)
}
ch, err := conn.Channel()
if err != nil {
conn.Close()
return nil, nil, err
}
if err := Declare(ch); err != nil {
conn.Close()
return nil, nil, err
}
// prefetch = максимум неподтверждённых сообщений у этого worker:
// ограничивает конкуренцию и не даёт одному worker забрать всю очередь.
if err := ch.Qos(prefetch, 0, false); err != nil {
conn.Close()
return nil, nil, fmt.Errorf("qos: %w", err)
}
deliveries, err := ch.Consume(Queue, name, false, false, false, false, nil)
if err != nil {
conn.Close()
return nil, nil, fmt.Errorf("consume: %w", err)
}
return &Consumer{conn: conn, ch: ch}, deliveries, nil
}
// Cancel прекращает получение новых сообщений; неподтверждённые вернутся в очередь.
func (c *Consumer) Cancel(name string) { _ = c.ch.Cancel(name, false) }
func (c *Consumer) Close() { _ = c.conn.Close() }
// Ping проверяет соединение (для readiness/heartbeat-проверок).
func Ping(url string) error {
conn, err := amqp.DialConfig(url, amqp.Config{Heartbeat: 10 * time.Second})
if err != nil {
return err
}
return conn.Close()
}
+148
View File
@@ -0,0 +1,148 @@
package config
import (
"log/slog"
"os"
"strconv"
"strings"
"time"
)
type Config struct {
Host string
Port string
Env string
SessionTTLSeconds int
DatabaseURL string
CacheURL string
SessionStore string
LogLevel slog.Level
ShutdownTimeout time.Duration
CacheEnabled bool
InstanceID string
IdempotencyTTL time.Duration
DrainDelay time.Duration
DBPoolMax int32
CacheInvalidation string
InternalAddr string
S3Endpoint string
S3Region string
S3Bucket string
S3AccessKey string
S3SecretKey string
MaxUploadBytes int64
MaxImagePixels int64
// Pipeline (lab5)
BrokerURL string
PipelineVersion int
JobMaxAttempts int
JobTimeout time.Duration
WorkerConcurrency int
StagingRetention time.Duration
WebPQuality int
PixelBudget int64
RetryBaseDelay time.Duration
RetryMaxDelay time.Duration
}
func Load() Config {
return Config{
Host: getEnv("APP_HOST", "0.0.0.0"),
Port: getEnv("APP_PORT", "8080"),
Env: getEnv("APP_ENV", "lab"),
SessionTTLSeconds: getInt("SESSION_TTL_SECONDS", 3600),
DatabaseURL: readSecretFile("DATABASE_URL_FILE"),
CacheURL: readSecretFile("CACHE_URL_FILE"),
SessionStore: getEnv("SESSION_STORE", "postgres"),
LogLevel: parseLevel(getEnv("LOG_LEVEL", "info")),
ShutdownTimeout: parseDuration(getEnv("SHUTDOWN_TIMEOUT", "8s")),
CacheEnabled: getEnv("CACHE_ENABLED", "true") == "true",
InstanceID: instanceID(),
IdempotencyTTL: time.Duration(getInt("IDEMPOTENCY_TTL_SECONDS", 86400)) * time.Second,
DrainDelay: parseDuration(getEnv("DRAIN_DELAY", "3s")),
DBPoolMax: int32(max(1, getInt("DB_POOL_MAX", 20))),
CacheInvalidation: getEnv("CACHE_INVALIDATION", "version"),
InternalAddr: getEnv("INTERNAL_ADDR", ":9090"),
S3Endpoint: getEnv("S3_ENDPOINT", "http://s3:9000"),
S3Region: getEnv("S3_REGION", "us-east-1"),
S3Bucket: getEnv("S3_BUCKET", "messenger-attachments"),
S3AccessKey: readSecretFile("S3_ACCESS_KEY_ID_FILE"),
S3SecretKey: readSecretFile("S3_SECRET_ACCESS_KEY_FILE"),
MaxUploadBytes: int64(getInt("MAX_UPLOAD_BYTES", 10*1024*1024)),
MaxImagePixels: int64(getInt("MAX_IMAGE_PIXELS", 40_000_000)),
BrokerURL: readSecretFile("BROKER_URL_FILE"),
PipelineVersion: max(1, getInt("PIPELINE_VERSION", 1)),
JobMaxAttempts: max(1, getInt("JOB_MAX_ATTEMPTS", 5)),
JobTimeout: time.Duration(max(5, getInt("JOB_TIMEOUT_SECONDS", 60))) * time.Second,
WorkerConcurrency: max(1, getInt("WORKER_CONCURRENCY", 2)),
StagingRetention: time.Duration(max(60, getInt("STAGING_RETENTION_SECONDS", 86400))) * time.Second,
WebPQuality: min(100, max(1, getInt("WEBP_QUALITY", 80))),
PixelBudget: int64(max(1_000_000, getInt("WORKER_PIXEL_BUDGET", 60_000_000))),
RetryBaseDelay: parseDuration(getEnv("RETRY_BASE_DELAY", "2s")),
RetryMaxDelay: parseDuration(getEnv("RETRY_MAX_DELAY", "60s")),
}
}
// instanceID — идентификатор реплики для X-Instance-Id. По умолчанию hostname,
// который в Docker равен короткому ID контейнера: так stop-one.sh однозначно
// находит контейнер. В lab7 его заменит непрозрачный alias.
func instanceID() string {
if v := os.Getenv("INSTANCE_ID"); v != "" {
return v
}
if h, err := os.Hostname(); err == nil && h != "" {
return h
}
return "unknown"
}
func (c Config) SessionTTL() time.Duration {
return time.Duration(c.SessionTTLSeconds) * time.Second
}
func (c Config) Addr() string { return c.Host + ":" + c.Port }
func getEnv(key, def string) string {
if v := os.Getenv(key); v != "" {
return v
}
return def
}
func getInt(key string, def int) int {
if v := os.Getenv(key); v != "" {
if n, err := strconv.Atoi(v); err == nil {
return n
}
}
return def
}
func readSecretFile(envKey string) string {
path := os.Getenv(envKey)
if path == "" {
return ""
}
b, err := os.ReadFile(path)
if err != nil {
slog.Warn("secret file not readable", "path", path, "err", err)
return ""
}
return strings.TrimSpace(string(b))
}
func parseLevel(s string) slog.Level {
var l slog.Level
if err := l.UnmarshalText([]byte(s)); err != nil {
return slog.LevelInfo
}
return l
}
func parseDuration(s string) time.Duration {
d, err := time.ParseDuration(s)
if err != nil {
return 8 * time.Second
}
return d
}
+95
View File
@@ -0,0 +1,95 @@
package domain
import (
"encoding/json"
"time"
)
const (
AttachmentUploading = "uploading" // внутреннее намерение до 202, клиенту не видно
AttachmentQueued = "queued"
AttachmentProcessing = "processing"
AttachmentReady = "ready"
AttachmentFailed = "failed"
AttachmentMissing = "missing" // карантин reconciler
)
// Variant — результат обработки одной версии pipeline.
type Variant struct {
Name string `json:"name"`
ContentType string `json:"content_type"`
SizeBytes int64 `json:"size_bytes"`
Width int `json:"width"`
Height int `json:"height"`
SHA256 string `json:"sha256"`
ObjectKey string `json:"-"`
PipelineVersion int `json:"-"`
}
// Attachment — метаданные вложения. Ключи S3 никогда не уходят клиенту.
type Attachment struct {
ID string
ChatID string
OwnerID string
Filename string
ObjectKey string // текущее место оригинала: staging до ready, конечный prefix после
StagingKey string // staging-объект, пока он не удалён после публикации
Status string
ContentType string
SizeBytes int64
SHA256 string
ErrorCode string
PipelineVersion int // версия, чей результат опубликован (0 — ещё нет)
Variants []Variant
CreatedAt time.Time
UpdatedAt time.Time
}
// MarshalJSON — представление по контракту: queued/processing/ready/failed,
// варианты текущей опубликованной версии, error_code только у failed.
func (a Attachment) MarshalJSON() ([]byte, error) {
variants := a.Variants
if variants == nil {
variants = []Variant{}
}
var errCode *string
if a.Status == AttachmentFailed && a.ErrorCode != "" {
ec := a.ErrorCode
errCode = &ec
}
return json.Marshal(struct {
ID string `json:"id"`
ChatID string `json:"chat_id"`
OwnerID string `json:"owner_id"`
Filename string `json:"filename"`
ContentType string `json:"content_type"`
SizeBytes int64 `json:"size_bytes"`
SHA256 string `json:"sha256"`
Status string `json:"status"`
Variants []Variant `json:"variants"`
ErrorCode *string `json:"error_code"`
CreatedAt string `json:"created_at"`
UpdatedAt string `json:"updated_at"`
}{
ID: a.ID, ChatID: a.ChatID, OwnerID: a.OwnerID, Filename: a.Filename,
ContentType: a.ContentType, SizeBytes: a.SizeBytes, SHA256: a.SHA256,
Status: a.Status, Variants: variants, ErrorCode: errCode,
CreatedAt: a.CreatedAt.UTC().Truncate(time.Second).Format(time.RFC3339),
UpdatedAt: a.UpdatedAt.UTC().Truncate(time.Second).Format(time.RFC3339),
})
}
// Job — сообщение broker по contracts/jobs.schema.json. Только ID и служебные
// поля: ни байтов, ни URL, ни credentials.
type Job struct {
SchemaVersion int `json:"schema_version"`
JobID string `json:"job_id"`
Type string `json:"type"`
AttachmentID string `json:"attachment_id"`
PipelineVersion int `json:"pipeline_version"`
Attempt int `json:"attempt"`
CreatedAt string `json:"created_at"`
Traceparent string `json:"traceparent,omitempty"`
}
const JobType = "attachment.process.v1"
+30
View File
@@ -0,0 +1,30 @@
package domain
import (
"encoding/json"
"time"
)
type Chat struct {
ID string `json:"id"`
Title string `json:"title"`
MemberIDs []string `json:"member_ids"`
CreatedAt time.Time `json:"created_at"`
}
func (c Chat) MarshalJSON() ([]byte, error) {
type Alias Chat
members := c.MemberIDs
if members == nil {
members = []string{}
}
return json.Marshal(&struct {
Alias
MemberIDs []string `json:"member_ids"`
CreatedAt string `json:"created_at"`
}{
Alias: Alias(c),
MemberIDs: members,
CreatedAt: c.CreatedAt.UTC().Truncate(time.Second).Format(time.RFC3339),
})
}
+30
View File
@@ -0,0 +1,30 @@
package domain
import (
"encoding/json"
"time"
)
type Message struct {
ID string `json:"id"`
ChatID string `json:"chat_id"`
SenderID string `json:"sender_id"`
Text string `json:"text"`
CreatedAt time.Time `json:"created_at"`
// AttachmentIDs — в порядке отправки; всегда массив (пустой, если вложений нет).
AttachmentIDs []string `json:"attachment_ids"`
}
func (m Message) MarshalJSON() ([]byte, error) {
type Alias Message
if m.AttachmentIDs == nil {
m.AttachmentIDs = []string{}
}
return json.Marshal(&struct {
Alias
CreatedAt string `json:"created_at"`
}{
Alias: Alias(m),
CreatedAt: m.CreatedAt.UTC().Truncate(time.Second).Format(time.RFC3339),
})
}
+10
View File
@@ -0,0 +1,10 @@
package domain
import "time"
type Session struct {
Token string `json:"token"`
UserID string `json:"-"`
ExpiresAt time.Time `json:"expires_at"`
CreatedAt time.Time `json:"-"`
}
+25
View File
@@ -0,0 +1,25 @@
package domain
import (
"encoding/json"
"time"
)
type User struct {
ID string `json:"id"`
Username string `json:"username"`
DisplayName string `json:"display_name"`
CreatedAt time.Time `json:"created_at"`
}
// MarshalJSON форматирует CreatedAt в RFC3339 без микросекунд.
func (u User) MarshalJSON() ([]byte, error) {
type Alias User
return json.Marshal(&struct {
Alias
CreatedAt string `json:"created_at"`
}{
Alias: Alias(u),
CreatedAt: u.CreatedAt.UTC().Truncate(time.Second).Format(time.RFC3339),
})
}
+592
View File
@@ -0,0 +1,592 @@
// Package imagecheck потоково проверяет PNG/JPEG без декодирования пикселей.
//
// Проверка идёт по мере чтения (используется параллельно с PUT в S3), память
// O(1) от размера изображения: размеры берутся из заголовка (IHDR/SOF) и
// сравниваются с лимитом до чтения тела; PNG — CRC каждого chunk, длина
// распакованного IDAT-потока (zlib → io.Discard) и байт фильтра каждой строки,
// JPEG — структура сегментов и маркер EOI. «Заведомо повреждённые» файлы
// (обрыв, битый CRC, мусор вместо данных, чужой формат) дают ErrInvalid,
// слишком большие по пикселям — ErrTooLarge.
package imagecheck
import (
"bufio"
"bytes"
"compress/zlib"
"encoding/binary"
"errors"
"fmt"
"hash/crc32"
"io"
)
const (
PNG = "image/png"
JPEG = "image/jpeg"
)
var (
ErrInvalid = errors.New("invalid or unsupported image")
ErrTooLarge = errors.New("image dimensions exceed limit")
)
var pngSignature = []byte("\x89PNG\r\n\x1a\n")
// Sniff определяет формат по первым байтам (не по заголовку Content-Type клиента).
func Sniff(head []byte) (string, bool) {
switch {
case bytes.HasPrefix(head, pngSignature):
return PNG, true
case len(head) >= 3 && head[0] == 0xFF && head[1] == 0xD8 && head[2] == 0xFF:
return JPEG, true
}
return "", false
}
// Result — что удалось узнать при проверке.
type Result struct {
Format string
Width int
Height int
}
// Validate читает r до конца и проверяет изображение формата format.
// maxPixels ограничивает width*height. Хвост после конца изображения
// дочитывается и игнорируется, чтобы не блокировать писателя.
func Validate(r io.Reader, format string, maxPixels int64) (Result, error) {
br := bufio.NewReaderSize(r, 32*1024)
var res Result
var err error
switch format {
case PNG:
res, err = validatePNG(br, maxPixels)
case JPEG:
res, err = validateJPEG(br, maxPixels)
default:
return Result{}, ErrInvalid
}
if err != nil {
return res, err
}
_, _ = io.Copy(io.Discard, br)
return res, nil
}
func invalid(format string, args ...any) error {
return fmt.Errorf("%w: %s", ErrInvalid, fmt.Sprintf(format, args...))
}
// --- PNG ---
func channels(colorType byte) (int, bool) {
switch colorType {
case 0:
return 1, true // grayscale
case 2:
return 3, true // RGB
case 3:
return 1, true // palette
case 4:
return 2, true // gray + alpha
case 6:
return 4, true // RGBA
}
return 0, false
}
func validBitDepth(colorType, depth byte) bool {
switch colorType {
case 0:
return depth == 1 || depth == 2 || depth == 4 || depth == 8 || depth == 16
case 3:
return depth == 1 || depth == 2 || depth == 4 || depth == 8
case 2, 4, 6:
return depth == 8 || depth == 16
}
return false
}
// pngRowLen — длина одной отсканированной строки прохода: 1 байт фильтра +
// данные пикселей.
func pngRowLen(pw, bitsPerPixel int64) int64 {
return 1 + (pw*bitsPerPixel+7)/8
}
// expectedRaw — длина распакованного потока IDAT: строки с байтом фильтра.
func expectedRaw(w, h int64, bitsPerPixel int64, interlaced bool) int64 {
if !interlaced {
return h * pngRowLen(w, bitsPerPixel)
}
passes := [7][4]int64{{0, 0, 8, 8}, {4, 0, 8, 8}, {0, 4, 4, 8}, {2, 0, 4, 4}, {0, 2, 2, 4}, {1, 0, 2, 2}, {0, 1, 1, 2}}
var total int64
for _, p := range passes {
if w <= p[0] || h <= p[1] {
continue
}
pw := (w - p[0] + p[2] - 1) / p[2]
ph := (h - p[1] + p[3] - 1) / p[3]
total += ph * pngRowLen(pw, bitsPerPixel)
}
return total
}
// pngRows — итератор по (pw, ph) проходов Adam7 или единственного прохода.
// Используется, чтобы валидировать байт фильтра каждой строки, не зная
// заранее, как разбивается распакованный поток на проходы.
type pngPass struct {
pw, ph int64
}
func pngPasses(w, h int64, interlaced bool) []pngPass {
if !interlaced {
return []pngPass{{w, h}}
}
coords := [7][4]int64{{0, 0, 8, 8}, {4, 0, 8, 8}, {0, 4, 4, 8}, {2, 0, 4, 4}, {0, 2, 2, 4}, {1, 0, 2, 2}, {0, 1, 1, 2}}
out := make([]pngPass, 0, len(coords))
for _, p := range coords {
if w <= p[0] || h <= p[1] {
continue
}
pw := (w - p[0] + p[2] - 1) / p[2]
ph := (h - p[1] + p[3] - 1) / p[3]
out = append(out, pngPass{pw, ph})
}
return out
}
type pngChunk struct {
length uint32
typ [4]byte
}
func readChunkHeader(br *bufio.Reader) (pngChunk, error) {
var hdr [8]byte
if _, err := io.ReadFull(br, hdr[:]); err != nil {
return pngChunk{}, invalid("truncated chunk header")
}
c := pngChunk{length: binary.BigEndian.Uint32(hdr[:4])}
copy(c.typ[:], hdr[4:])
if c.length > 1<<31-1 {
return c, invalid("chunk too long")
}
for _, b := range c.typ {
if !((b >= 'A' && b <= 'Z') || (b >= 'a' && b <= 'z')) {
return c, invalid("bad chunk type")
}
}
return c, nil
}
// readChunkData читает тело chunk в dst (если не nil) и проверяет CRC.
func readChunkData(br *bufio.Reader, c pngChunk, dst io.Writer) error {
h := crc32.NewIEEE()
h.Write(c.typ[:])
w := io.Writer(h)
if dst != nil {
w = io.MultiWriter(h, dst)
}
if _, err := io.CopyN(w, br, int64(c.length)); err != nil {
return invalid("truncated %s chunk", c.typ[:])
}
var crc [4]byte
if _, err := io.ReadFull(br, crc[:]); err != nil {
return invalid("truncated CRC")
}
if binary.BigEndian.Uint32(crc[:]) != h.Sum32() {
return invalid("CRC mismatch in %s", c.typ[:])
}
return nil
}
// idatStream отдаёт данные подряд идущих IDAT-chunk как один поток.
type idatStream struct {
br *bufio.Reader
remaining uint32
crc uint32
next *pngChunk // первый chunk после последнего IDAT
err error
}
func (s *idatStream) Read(p []byte) (int, error) {
for s.remaining == 0 {
if s.err != nil || s.next != nil {
if s.err != nil {
return 0, s.err
}
return 0, io.EOF
}
if err := s.finishChunk(); err != nil {
s.err = err
return 0, err
}
c, err := readChunkHeader(s.br)
if err != nil {
s.err = err
return 0, err
}
if string(c.typ[:]) != "IDAT" {
s.next = &c
return 0, io.EOF
}
s.startChunk(c)
}
if uint32(len(p)) > s.remaining {
p = p[:s.remaining]
}
n, err := s.br.Read(p)
s.crc = crc32.Update(s.crc, crc32.IEEETable, p[:n])
s.remaining -= uint32(n)
if err != nil && err != io.EOF {
s.err = err
return n, err
}
if n == 0 && err == io.EOF {
s.err = invalid("truncated IDAT")
return 0, s.err
}
return n, nil
}
func (s *idatStream) startChunk(c pngChunk) {
s.remaining = c.length
s.crc = crc32.Update(0, crc32.IEEETable, c.typ[:])
}
func (s *idatStream) finishChunk() error {
var crc [4]byte
if _, err := io.ReadFull(s.br, crc[:]); err != nil {
return invalid("truncated CRC")
}
if binary.BigEndian.Uint32(crc[:]) != s.crc {
return invalid("CRC mismatch in IDAT")
}
return nil
}
// validatePNGStream проверяет распакованный поток: длину, кратность строкам
// и байты фильтра (0..4) в начале каждой строки каждого прохода.
func validatePNGStream(zr io.Reader, passes []pngPass, bitsPerPixel, want int64) error {
total := int64(0)
buf := make([]byte, 32*1024)
for _, pass := range passes {
rowLen := pngRowLen(pass.pw, bitsPerPixel)
for y := int64(0); y < pass.ph; y++ {
// читаем ровно одну строку, но чанками, чтобы не аллоцировать
// rowLen (может быть большой для широких изображений)
left := rowLen
first := true
for left > 0 {
chunk := int64(len(buf))
if left < chunk {
chunk = left
}
n, err := io.ReadFull(zr, buf[:chunk])
if err != nil {
if errors.Is(err, io.ErrUnexpectedEOF) || errors.Is(err, io.EOF) {
return invalid("truncated IDAT (got %d of %d)", total, want)
}
return invalid("corrupt image data")
}
if first {
if buf[0] > 4 {
return invalid("bad row filter %d", buf[0])
}
first = false
}
total += int64(n)
left -= int64(n)
}
}
}
// Проверяем, что распакованный поток ровно want байт (ни больше, ни меньше).
var extra [1]byte
n, err := zr.Read(extra[:])
if n != 0 {
return invalid("image data longer than expected")
}
if err != nil && !errors.Is(err, io.EOF) {
return invalid("corrupt image data")
}
if total != want {
return invalid("image data length %d, want %d", total, want)
}
return nil
}
func validatePNG(br *bufio.Reader, maxPixels int64) (Result, error) {
res := Result{Format: PNG}
sig := make([]byte, 8)
if _, err := io.ReadFull(br, sig); err != nil || !bytes.Equal(sig, pngSignature) {
return res, invalid("not a PNG")
}
c, err := readChunkHeader(br)
if err != nil {
return res, err
}
if string(c.typ[:]) != "IHDR" || c.length != 13 {
return res, invalid("IHDR must be first")
}
var ihdr bytes.Buffer
if err := readChunkData(br, c, &ihdr); err != nil {
return res, err
}
b := ihdr.Bytes()
w := int64(binary.BigEndian.Uint32(b[0:4]))
h := int64(binary.BigEndian.Uint32(b[4:8]))
depth, colorType, compression, filter, interlace := b[8], b[9], b[10], b[11], b[12]
res.Width, res.Height = int(w), int(h)
if w < 1 || h < 1 || w > 1<<31-1 || h > 1<<31-1 {
return res, invalid("bad dimensions")
}
// Лимит пикселей — до чтения и распаковки данных изображения.
if w*h > maxPixels {
return res, ErrTooLarge
}
ch, ok := channels(colorType)
if !ok || !validBitDepth(colorType, depth) || compression != 0 || filter != 0 || interlace > 1 {
return res, invalid("unsupported IHDR")
}
bpp := int64(ch) * int64(depth)
want := expectedRaw(w, h, bpp, interlace == 1)
passes := pngPasses(w, h, interlace == 1)
sawIDAT := false
for {
c, err := readChunkHeader(br)
if err != nil {
return res, err
}
switch string(c.typ[:]) {
case "IDAT":
if sawIDAT {
return res, invalid("non-consecutive IDAT")
}
sawIDAT = true
s := &idatStream{br: br}
s.startChunk(c)
zr, err := zlib.NewReader(s)
if err != nil {
return res, invalid("bad zlib header")
}
// Распаковка и проверка фильтров строк — без хранения пикселей.
if err := validatePNGStream(zr, passes, bpp, want); err != nil {
// Закрываем zlib, чтобы дочитать хвост CRC в idatStream, но
// игнорируем ошибку закрытия: она не важна для результата.
_ = zr.Close()
return res, err
}
if err := zr.Close(); err != nil && !errors.Is(err, io.EOF) {
return res, invalid("bad zlib trailer")
}
// Остаток IDAT (например, пустые chunk) дочитываем с проверкой CRC.
if _, err := io.Copy(io.Discard, s); err != nil {
return res, err
}
if s.next == nil {
return res, invalid("missing IEND")
}
c = *s.next
if string(c.typ[:]) == "IEND" {
if err := readChunkData(br, c, nil); err != nil {
return res, err
}
return res, nil
}
if err := readChunkData(br, c, nil); err != nil {
return res, err
}
case "IEND":
if !sawIDAT {
return res, invalid("no image data")
}
if err := readChunkData(br, c, nil); err != nil {
return res, err
}
return res, nil
case "IHDR":
return res, invalid("duplicate IHDR")
default:
if err := readChunkData(br, c, nil); err != nil {
return res, err
}
}
}
}
// --- JPEG ---
func isSOF(m byte) bool {
switch m {
case 0xC0, 0xC1, 0xC2, 0xC3, 0xC5, 0xC6, 0xC7, 0xC9, 0xCA, 0xCB, 0xCD, 0xCE, 0xCF:
return true
}
return false
}
// readMarker читает 0xFF (с заполнителями) и код маркера.
func readMarker(br *bufio.Reader) (byte, error) {
b, err := br.ReadByte()
if err != nil {
return 0, invalid("truncated JPEG")
}
if b != 0xFF {
return 0, invalid("expected JPEG marker")
}
for b == 0xFF {
if b, err = br.ReadByte(); err != nil {
return 0, invalid("truncated JPEG")
}
}
return b, nil
}
func segmentLength(br *bufio.Reader) (int, error) {
var l [2]byte
if _, err := io.ReadFull(br, l[:]); err != nil {
return 0, invalid("truncated JPEG segment")
}
n := int(binary.BigEndian.Uint16(l[:]))
if n < 2 {
return 0, invalid("bad JPEG segment length")
}
return n - 2, nil
}
// skipEntropy пропускает сжатые данные скана и возвращает следующий маркер.
func skipEntropy(br *bufio.Reader) (byte, error) {
for {
b, err := br.ReadByte()
if err != nil {
return 0, invalid("truncated JPEG scan")
}
if b != 0xFF {
continue
}
for b == 0xFF {
if b, err = br.ReadByte(); err != nil {
return 0, invalid("truncated JPEG scan")
}
}
if b == 0x00 || (b >= 0xD0 && b <= 0xD7) {
continue // байт-заполнитель или RSTn внутри скана
}
return b, nil
}
}
// readSOF читает сегмент SOF и проверяет его структуру (ITU T.81, B.2.2):
// P(1) Y(2) X(2) Nf(1), затем Nf компонент по 3 байта. n — длина сегмента
// без двух байт поля длины. Возвращает (width, height).
func readSOF(br *bufio.Reader, n int) (int64, int64, error) {
if n < 6 {
return 0, 0, invalid("short SOF")
}
hdr := make([]byte, 6)
if _, err := io.ReadFull(br, hdr); err != nil {
return 0, 0, invalid("truncated SOF")
}
// hdr[0] — точность выборки (precision), не число компонент.
h := int64(binary.BigEndian.Uint16(hdr[1:3]))
w := int64(binary.BigEndian.Uint16(hdr[3:5]))
nf := int(hdr[5])
if nf == 0 {
return 0, 0, invalid("SOF without components")
}
if n != 6+3*nf {
return 0, 0, invalid("SOF length %d, want %d", n, 6+3*nf)
}
if _, err := br.Discard(3 * nf); err != nil {
return 0, 0, invalid("truncated SOF")
}
return w, h, nil
}
func validateJPEG(br *bufio.Reader, maxPixels int64) (Result, error) {
res := Result{Format: JPEG}
var soi [2]byte
if _, err := io.ReadFull(br, soi[:]); err != nil || soi != [2]byte{0xFF, 0xD8} {
return res, invalid("not a JPEG")
}
sawSOF, sawSOS := false, false
marker, err := readMarker(br)
if err != nil {
return res, err
}
for {
switch {
case marker == 0xD9: // EOI
if !sawSOF || !sawSOS {
return res, invalid("JPEG without frame or scan")
}
return res, nil
case marker == 0xD8:
return res, invalid("unexpected SOI")
case marker == 0x01 || (marker >= 0xD0 && marker <= 0xD7):
// маркеры без длины
case isSOF(marker):
n, err := segmentLength(br)
if err != nil {
return res, err
}
if sawSOF {
// читаем сегмент, но так и так вернём ошибку
_, _, _ = readSOF(br, n)
return res, invalid("multiple frames")
}
w, h, err := readSOF(br, n)
if err != nil {
return res, err
}
res.Width, res.Height = int(w), int(h)
if w < 1 || h < 1 {
return res, invalid("bad JPEG dimensions")
}
if w*h > maxPixels {
return res, ErrTooLarge
}
sawSOF = true
case marker == 0xDA: // SOS
if !sawSOF {
return res, invalid("scan before frame")
}
n, err := segmentLength(br)
if err != nil {
return res, err
}
// SOS: Ns + Ss + Se + Ah/Al + 2*Ns = 1 + 3 + 2*Ns = 4 + 2*Ns
if n < 4 {
return res, invalid("short SOS")
}
nsBuf := make([]byte, 1)
if _, err := io.ReadFull(br, nsBuf); err != nil {
return res, invalid("truncated SOS")
}
ns := int(nsBuf[0])
if ns < 1 {
return res, invalid("SOS without components")
}
if n != 4+2*ns {
return res, invalid("SOS length %d, want %d", n, 4+2*ns)
}
if _, err := br.Discard(n - 1); err != nil {
return res, invalid("truncated SOS")
}
sawSOS = true
if marker, err = skipEntropy(br); err != nil {
return res, err
}
continue
default:
n, err := segmentLength(br)
if err != nil {
return res, err
}
if _, err := br.Discard(n); err != nil {
return res, invalid("truncated JPEG segment")
}
}
if marker, err = readMarker(br); err != nil {
return res, err
}
}
}
+187
View File
@@ -0,0 +1,187 @@
package imagecheck
import (
"bytes"
"compress/zlib"
"encoding/binary"
"errors"
"hash/crc32"
"image"
"image/color"
"image/jpeg"
"image/png"
"os"
"testing"
)
const limit = 40_000_000
func testImage(w, h int) *image.RGBA {
img := image.NewRGBA(image.Rect(0, 0, w, h))
for y := 0; y < h; y++ {
for x := 0; x < w; x++ {
img.Set(x, y, color.RGBA{uint8(x * 7), uint8(y * 3), uint8(x ^ y), 255})
}
}
return img
}
func encodePNG(t *testing.T, w, h int) []byte {
var b bytes.Buffer
if err := png.Encode(&b, testImage(w, h)); err != nil {
t.Fatal(err)
}
return b.Bytes()
}
func encodeJPEG(t *testing.T, w, h int, progressive bool) []byte {
var b bytes.Buffer
if err := jpeg.Encode(&b, testImage(w, h), &jpeg.Options{Quality: 80}); err != nil {
t.Fatal(err)
}
return b.Bytes()
}
func chunk(typ string, data []byte) []byte {
var b bytes.Buffer
binary.Write(&b, binary.BigEndian, uint32(len(data)))
b.WriteString(typ)
b.Write(data)
crc := crc32.NewIEEE()
crc.Write([]byte(typ))
crc.Write(data)
binary.Write(&b, binary.BigEndian, crc.Sum32())
return b.Bytes()
}
// craftPNG собирает PNG вручную: так можно задать любые IHDR и поток данных.
func craftPNG(w, h uint32, colorType, depth, interlace byte, raw []byte) []byte {
var b bytes.Buffer
b.Write(pngSignature)
ihdr := make([]byte, 13)
binary.BigEndian.PutUint32(ihdr[0:], w)
binary.BigEndian.PutUint32(ihdr[4:], h)
ihdr[8], ihdr[9], ihdr[12] = depth, colorType, interlace
b.Write(chunk("IHDR", ihdr))
var z bytes.Buffer
zw := zlib.NewWriter(&z)
zw.Write(raw)
zw.Close()
b.Write(chunk("IDAT", z.Bytes()))
b.Write(chunk("IEND", nil))
return b.Bytes()
}
func check(t *testing.T, name string, data []byte, format string, want error) {
t.Helper()
_, err := Validate(bytes.NewReader(data), format, limit)
if want == nil && err != nil {
t.Errorf("%s: unexpected error %v", name, err)
}
if want != nil && !errors.Is(err, want) {
t.Errorf("%s: got %v, want %v", name, err, want)
}
}
func TestValidPNGAndJPEG(t *testing.T) {
p := encodePNG(t, 320, 200)
res, err := Validate(bytes.NewReader(p), PNG, limit)
if err != nil || res.Width != 320 || res.Height != 200 {
t.Fatalf("png: %+v %v", res, err)
}
j := encodeJPEG(t, 321, 199, false)
res, err = Validate(bytes.NewReader(j), JPEG, limit)
if err != nil || res.Width != 321 || res.Height != 199 {
t.Fatalf("jpeg: %+v %v", res, err)
}
check(t, "png with trailing bytes", append(p, []byte("trailer")...), PNG, nil)
}
func TestCourseFixture(t *testing.T) {
data, err := os.ReadFile("../../tests/fixtures/sample.png")
if err != nil {
t.Skip("fixture not found")
}
res, err := Validate(bytes.NewReader(data), PNG, limit)
if err != nil || res.Width != 320 || res.Height != 200 {
t.Fatalf("fixture: %+v %v", res, err)
}
}
func TestInterlacedPNG(t *testing.T) {
// 10×7 RGB, Adam7: длина сырых данных по семи проходам.
w, h := int64(10), int64(7)
raw := make([]byte, expectedRaw(w, h, 24, true))
check(t, "interlaced", craftPNG(uint32(w), uint32(h), 2, 8, 1, raw), PNG, nil)
check(t, "interlaced short data", craftPNG(uint32(w), uint32(h), 2, 8, 1, raw[:len(raw)-1]), PNG, ErrInvalid)
}
func TestBrokenImages(t *testing.T) {
p := encodePNG(t, 64, 64)
j := encodeJPEG(t, 64, 64, false)
check(t, "png truncated", p[:len(p)/2], PNG, ErrInvalid)
check(t, "png without IEND", p[:len(p)-12], PNG, ErrInvalid)
bad := bytes.Clone(p)
bad[len(bad)/2] ^= 0xFF
check(t, "png corrupted byte", bad, PNG, ErrInvalid)
check(t, "jpeg truncated", j[:len(j)/2], JPEG, ErrInvalid)
check(t, "jpeg without EOI", j[:len(j)-2], JPEG, ErrInvalid)
check(t, "png declared as jpeg", p, JPEG, ErrInvalid)
check(t, "text as png", []byte("hello, not an image at all"), PNG, ErrInvalid)
check(t, "empty", nil, PNG, ErrInvalid)
check(t, "png short raw data", craftPNG(8, 8, 2, 8, 0, make([]byte, 8*(1+24)-1)), PNG, ErrInvalid)
check(t, "png bad bit depth", craftPNG(8, 8, 2, 4, 0, make([]byte, 8*(1+12))), PNG, ErrInvalid)
}
func TestPixelLimitBeforeData(t *testing.T) {
// «Бомба»: IHDR на 100000×100000, данных почти нет. Должно быть ErrTooLarge,
// а не попытка распаковать 30 ГБ.
bomb := craftPNG(100000, 100000, 2, 8, 0, []byte{0})
check(t, "png bomb", bomb, PNG, ErrTooLarge)
// Ровно на границе лимита — допустимо по пикселям (данных не хватает → invalid).
check(t, "png at limit", craftPNG(8000, 5000, 0, 1, 0, nil), PNG, ErrInvalid)
check(t, "png over limit", craftPNG(8000, 5001, 0, 1, 0, nil), PNG, ErrTooLarge)
// JPEG: подменяем размеры в SOF0 на 60000×60000.
j := encodeJPEG(t, 16, 16, false)
i := bytes.Index(j, []byte{0xFF, 0xC0})
if i < 0 {
t.Fatal("no SOF0")
}
big := bytes.Clone(j)
binary.BigEndian.PutUint16(big[i+5:], 60000)
binary.BigEndian.PutUint16(big[i+7:], 60000)
check(t, "jpeg bomb", big, JPEG, ErrTooLarge)
}
func TestJPEGSOFStructure(t *testing.T) {
// Серый JPEG: одна компонента (Nf=1), длина SOF = 6 + 3*1.
var g bytes.Buffer
gray := image.NewGray(image.Rect(0, 0, 33, 17))
if err := jpeg.Encode(&g, gray, nil); err != nil {
t.Fatal(err)
}
res, err := Validate(bytes.NewReader(g.Bytes()), JPEG, limit)
if err != nil || res.Width != 33 || res.Height != 17 {
t.Fatalf("grayscale jpeg: %+v %v", res, err)
}
// Цветной JPEG с испорченным Nf (длина сегмента больше не сходится).
j := encodeJPEG(t, 16, 16, false)
i := bytes.Index(j, []byte{0xFF, 0xC0})
bad := bytes.Clone(j)
bad[i+9] = 7 // Nf: было 3
check(t, "jpeg SOF length mismatch", bad, JPEG, ErrInvalid)
}
func TestSniff(t *testing.T) {
if f, ok := Sniff(encodePNG(t, 2, 2)); !ok || f != PNG {
t.Error("png sniff")
}
if f, ok := Sniff(encodeJPEG(t, 2, 2, false)); !ok || f != JPEG {
t.Error("jpeg sniff")
}
if _, ok := Sniff([]byte("GIF89a")); ok {
t.Error("gif must not be accepted")
}
}
+291
View File
@@ -0,0 +1,291 @@
// Package imaging строит варианты вложения по контракту lab5.
//
// thumbnail JPEG, вписан в 256×256 с сохранением пропорций, без увеличения,
// EXIF orientation применён, alpha компонуется на белый,
// размеры округляются вниз, минимум 1 px (320×200 → 256×160)
// crop JPEG, центрированный квадрат 128×128 (маленькие увеличиваются)
// optimized WebP, вписан в 1280×1280 без увеличения; lossy, качество WEBP_QUALITY,
// метаданные удалены (cwebp -metadata none; пиксели и так
// перекодированы из декодированного изображения, EXIF не переносится)
//
// Все кодировщики детерминированы: одинаковый вход даёт одинаковые байты, поэтому
// повтор job перезаписывает объект тем же содержимым.
package imaging
import (
"bytes"
"context"
"crypto/sha256"
"encoding/binary"
"encoding/hex"
"errors"
"fmt"
"image"
"image/color"
"image/draw"
"image/jpeg"
"image/png"
"os"
"os/exec"
"path/filepath"
xdraw "golang.org/x/image/draw"
)
// ErrDecode — изображение не декодируется: постоянная ошибка (повтор не поможет).
var ErrDecode = errors.New("cannot decode image")
// Params — параметры версии pipeline. Новая версия может менять их, не трогая
// уже опубликованные результаты старой (ключ эффекта включает версию).
type Params struct {
ThumbQuality int
CropQuality int
WebPQuality int
}
// ParamsFor — параметры по номеру версии pipeline.
func ParamsFor(version, webpQuality int) Params {
p := Params{ThumbQuality: 85, CropQuality: 85, WebPQuality: webpQuality}
if version >= 2 {
// v2 (трек 5): чуть выше качество превью — другой, но детерминированный результат.
p.ThumbQuality, p.CropQuality = 92, 92
}
return p
}
// Output — закодированный вариант.
type Output struct {
Name string
ContentType string
Data []byte
Width int
Height int
SHA256 string
}
func newOutput(name, ctype string, data []byte, w, h int) Output {
sum := sha256.Sum256(data)
return Output{Name: name, ContentType: ctype, Data: data, Width: w, Height: h, SHA256: hex.EncodeToString(sum[:])}
}
// Decode декодирует оригинал, применяет EXIF orientation (JPEG) и компонует
// прозрачность на белый фон.
func Decode(data []byte) (*image.RGBA, error) {
src, _, err := image.Decode(bytes.NewReader(data))
if err != nil {
return nil, fmt.Errorf("%w: %v", ErrDecode, err)
}
b := src.Bounds()
// Непрозрачный RGBA (типичный PNG RGB) уже готов: вторая полноразмерная копия
// (4 байта × пиксель) не нужна — для 36 Мпикс это 137 MiB памяти worker.
if rgba, ok := src.(*image.RGBA); ok && b.Min == (image.Point{}) && rgba.Opaque() {
return Orient(rgba, ExifOrientation(data)), nil
}
out := image.NewRGBA(image.Rect(0, 0, b.Dx(), b.Dy()))
draw.Draw(out, out.Bounds(), image.NewUniform(color.White), image.Point{}, draw.Src)
draw.Draw(out, out.Bounds(), src, b.Min, draw.Over) // alpha → на белый
return Orient(out, ExifOrientation(data)), nil
}
// FitSize — вписать w×h в box×box с сохранением пропорций; без увеличения,
// если allowUpscale=false; округление вниз, минимум 1.
func FitSize(w, h, box int, allowUpscale bool) (int, int) {
if !allowUpscale && w <= box && h <= box {
return w, h
}
if w >= h {
nh := h * box / w
return box, max(1, nh)
}
nw := w * box / h
return max(1, nw), box
}
func resize(src image.Image, w, h int) *image.RGBA {
dst := image.NewRGBA(image.Rect(0, 0, w, h))
xdraw.CatmullRom.Scale(dst, dst.Bounds(), src, src.Bounds(), xdraw.Src, nil)
return dst
}
func encodeJPEG(img image.Image, quality int) ([]byte, error) {
var b bytes.Buffer
if err := jpeg.Encode(&b, img, &jpeg.Options{Quality: quality}); err != nil {
return nil, err
}
return b.Bytes(), nil
}
// Thumbnail — JPEG, вписан в 256×256 без увеличения.
func Thumbnail(img *image.RGBA, p Params) (Output, error) {
w, h := FitSize(img.Bounds().Dx(), img.Bounds().Dy(), 256, false)
var src image.Image = img
if w != img.Bounds().Dx() || h != img.Bounds().Dy() {
src = resize(img, w, h)
}
data, err := encodeJPEG(src, p.ThumbQuality)
if err != nil {
return Output{}, err
}
return newOutput("thumbnail", "image/jpeg", data, w, h), nil
}
// Crop — JPEG, центрированный квадрат 128×128 (маленькие исходники увеличиваются).
func Crop(img *image.RGBA, p Params) (Output, error) {
b := img.Bounds()
side := min(b.Dx(), b.Dy())
x0 := b.Min.X + (b.Dx()-side)/2
y0 := b.Min.Y + (b.Dy()-side)/2
sq := img.SubImage(image.Rect(x0, y0, x0+side, y0+side))
data, err := encodeJPEG(resize(sq, 128, 128), p.CropQuality)
if err != nil {
return Output{}, err
}
return newOutput("crop", "image/jpeg", data, 128, 128), nil
}
// WebPEncoder кодирует RGBA в WebP.
type WebPEncoder func(ctx context.Context, img image.Image, quality int) ([]byte, error)
// CWebP — кодировщик через cwebp (libwebp-tools): lossy, -metadata none.
// Вход — PNG во временном каталоге процесса (эфемерный, удаляется сразу).
func CWebP(ctx context.Context, img image.Image, quality int) ([]byte, error) {
dir, err := os.MkdirTemp("", "webp-")
if err != nil {
return nil, err
}
defer os.RemoveAll(dir)
in, out := filepath.Join(dir, "in.png"), filepath.Join(dir, "out.webp")
f, err := os.Create(in)
if err != nil {
return nil, err
}
enc := png.Encoder{CompressionLevel: png.BestSpeed}
if err := enc.Encode(f, img); err != nil {
f.Close()
return nil, err
}
f.Close()
cmd := exec.CommandContext(ctx, "cwebp", "-quiet", "-q", fmt.Sprint(quality), "-m", "4", "-metadata", "none", in, "-o", out)
if msg, err := cmd.CombinedOutput(); err != nil {
return nil, fmt.Errorf("cwebp: %v: %s", err, bytes.TrimSpace(msg))
}
return os.ReadFile(out)
}
// Optimized — WebP, вписан в 1280×1280 без увеличения.
func Optimized(ctx context.Context, img *image.RGBA, p Params, enc WebPEncoder) (Output, error) {
w, h := FitSize(img.Bounds().Dx(), img.Bounds().Dy(), 1280, false)
var src image.Image = img
if w != img.Bounds().Dx() || h != img.Bounds().Dy() {
src = resize(img, w, h)
}
data, err := enc(ctx, src, p.WebPQuality)
if err != nil {
return Output{}, err
}
if len(data) < 12 || string(data[0:4]) != "RIFF" || string(data[8:12]) != "WEBP" {
return Output{}, errors.New("encoder produced non-WebP data")
}
return newOutput("optimized", "image/webp", data, w, h), nil
}
// --- EXIF orientation ---
// ExifOrientation возвращает тег 0x0112 из APP1 Exif JPEG (1 — нет поворота).
func ExifOrientation(data []byte) int {
if len(data) < 4 || data[0] != 0xFF || data[1] != 0xD8 {
return 1
}
i := 2
for i+4 <= len(data) && data[i] == 0xFF {
marker := data[i+1]
if marker == 0xDA || marker == 0xD9 {
break
}
segLen := int(binary.BigEndian.Uint16(data[i+2 : i+4]))
if segLen < 2 || i+2+segLen > len(data) {
break
}
seg := data[i+4 : i+2+segLen]
if marker == 0xE1 && len(seg) > 14 && string(seg[:6]) == "Exif\x00\x00" {
if o := tiffOrientation(seg[6:]); o >= 1 && o <= 8 {
return o
}
}
i += 2 + segLen
}
return 1
}
func tiffOrientation(t []byte) int {
if len(t) < 8 {
return 1
}
var bo binary.ByteOrder
switch string(t[:2]) {
case "II":
bo = binary.LittleEndian
case "MM":
bo = binary.BigEndian
default:
return 1
}
off := int(bo.Uint32(t[4:8]))
if off+2 > len(t) {
return 1
}
n := int(bo.Uint16(t[off : off+2]))
for k := 0; k < n; k++ {
e := off + 2 + 12*k
if e+12 > len(t) {
return 1
}
if bo.Uint16(t[e:e+2]) == 0x0112 {
return int(bo.Uint16(t[e+8 : e+10]))
}
}
return 1
}
// Orient применяет EXIF orientation 1..8 (поворот/отражение).
func Orient(src *image.RGBA, o int) *image.RGBA {
if o <= 1 || o > 8 {
return src
}
b := src.Bounds()
w, h := b.Dx(), b.Dy()
dw, dh := w, h
if o >= 5 {
dw, dh = h, w
}
dst := image.NewRGBA(image.Rect(0, 0, dw, dh))
for y := 0; y < h; y++ {
for x := 0; x < w; x++ {
var dx, dy int
switch o {
case 2:
dx, dy = w-1-x, y
case 3:
dx, dy = w-1-x, h-1-y
case 4:
dx, dy = x, h-1-y
case 5:
dx, dy = y, x
case 6:
dx, dy = h-1-y, x
case 7:
dx, dy = h-1-y, w-1-x
case 8:
dx, dy = y, w-1-x
}
dst.SetRGBA(dx, dy, src.RGBAAt(b.Min.X+x, b.Min.Y+y))
}
}
return dst
}
// SHA256Hex — lowercase hex SHA-256.
func SHA256Hex(data []byte) string {
sum := sha256.Sum256(data)
return hex.EncodeToString(sum[:])
}
+208
View File
@@ -0,0 +1,208 @@
package imaging
import (
"bytes"
"context"
"encoding/binary"
"image"
"image/color"
"image/jpeg"
"image/png"
"os"
"os/exec"
"testing"
)
func solid(w, h int, c color.Color) *image.NRGBA {
m := image.NewNRGBA(image.Rect(0, 0, w, h))
for y := 0; y < h; y++ {
for x := 0; x < w; x++ {
m.Set(x, y, c)
}
}
return m
}
func encPNG(t *testing.T, img image.Image) []byte {
var b bytes.Buffer
if err := png.Encode(&b, img); err != nil {
t.Fatal(err)
}
return b.Bytes()
}
func TestFitSize(t *testing.T) {
cases := []struct {
w, h, box int
up bool
ww, wh int
}{
{320, 200, 256, false, 256, 160}, // fixture курса
{200, 320, 256, false, 160, 256},
{100, 50, 256, false, 100, 50}, // без увеличения
{5000, 1, 256, false, 256, 1}, // минимум 1 px
{1, 5000, 256, false, 1, 256},
{257, 255, 256, false, 256, 254}, // округление вниз
{40, 30, 128, true, 128, 96},
{3000, 2000, 1280, false, 1280, 853},
}
for _, c := range cases {
if w, h := FitSize(c.w, c.h, c.box, c.up); w != c.ww || h != c.wh {
t.Errorf("FitSize(%d,%d,%d,%v)=%d×%d want %d×%d", c.w, c.h, c.box, c.up, w, h, c.ww, c.wh)
}
}
}
func jpegSize(t *testing.T, data []byte) (int, int) {
cfg, err := jpeg.DecodeConfig(bytes.NewReader(data))
if err != nil {
t.Fatal(err)
}
return cfg.Width, cfg.Height
}
func TestCourseFixtureThumbnail(t *testing.T) {
data, err := os.ReadFile("../../tests/fixtures/sample.png")
if err != nil {
t.Skip("fixture not found")
}
img, err := Decode(data)
if err != nil {
t.Fatal(err)
}
th, err := Thumbnail(img, ParamsFor(1, 80))
if err != nil {
t.Fatal(err)
}
if w, h := jpegSize(t, th.Data); w != 256 || h != 160 || th.Width != 256 || th.Height != 160 {
t.Fatalf("thumbnail %d×%d (meta %d×%d), want 256×160", w, h, th.Width, th.Height)
}
again, _ := Thumbnail(img, ParamsFor(1, 80))
if !bytes.Equal(th.Data, again.Data) {
t.Error("thumbnail encoding is not deterministic")
}
cr, err := Crop(img, ParamsFor(1, 80))
if err != nil {
t.Fatal(err)
}
if w, h := jpegSize(t, cr.Data); w != 128 || h != 128 {
t.Fatalf("crop %d×%d", w, h)
}
v2, _ := Thumbnail(img, ParamsFor(2, 80))
if bytes.Equal(v2.Data, th.Data) {
t.Error("pipeline v2 must produce a different thumbnail")
}
}
func TestSmallImageNotUpscaledButCropIs(t *testing.T) {
img, err := Decode(encPNG(t, solid(40, 30, color.RGBA{10, 20, 30, 255})))
if err != nil {
t.Fatal(err)
}
th, _ := Thumbnail(img, ParamsFor(1, 80))
if th.Width != 40 || th.Height != 30 {
t.Errorf("small thumbnail upscaled to %d×%d", th.Width, th.Height)
}
cr, _ := Crop(img, ParamsFor(1, 80))
if w, h := jpegSize(t, cr.Data); w != 128 || h != 128 {
t.Errorf("crop of small image %d×%d", w, h)
}
}
func TestAlphaOnWhite(t *testing.T) {
img, err := Decode(encPNG(t, solid(10, 10, color.NRGBA{0, 0, 0, 0})))
if err != nil {
t.Fatal(err)
}
if c := img.RGBAAt(5, 5); c.R != 255 || c.G != 255 || c.B != 255 {
t.Errorf("transparent pixel composited to %v, want white", c)
}
half, _ := Decode(encPNG(t, solid(4, 4, color.NRGBA{0, 0, 0, 128})))
if c := half.RGBAAt(1, 1); c.R < 120 || c.R > 135 {
t.Errorf("50%% black over white = %v, want ~127", c)
}
}
// withExif вставляет APP1 Exif с orientation сразу после SOI.
func withExif(jpg []byte, orientation uint16) []byte {
var tiff bytes.Buffer
tiff.WriteString("MM\x00\x2a")
binary.Write(&tiff, binary.BigEndian, uint32(8))
binary.Write(&tiff, binary.BigEndian, uint16(1))
binary.Write(&tiff, binary.BigEndian, uint16(0x0112))
binary.Write(&tiff, binary.BigEndian, uint16(3))
binary.Write(&tiff, binary.BigEndian, uint32(1))
binary.Write(&tiff, binary.BigEndian, orientation)
tiff.Write([]byte{0, 0})
binary.Write(&tiff, binary.BigEndian, uint32(0))
seg := append([]byte("Exif\x00\x00"), tiff.Bytes()...)
var out bytes.Buffer
out.Write(jpg[:2])
out.Write([]byte{0xFF, 0xE1})
binary.Write(&out, binary.BigEndian, uint16(len(seg)+2))
out.Write(seg)
out.Write(jpg[2:])
return out.Bytes()
}
func TestExifOrientation(t *testing.T) {
// 300×100: слева красный, справа синий.
src := image.NewRGBA(image.Rect(0, 0, 300, 100))
for y := 0; y < 100; y++ {
for x := 0; x < 300; x++ {
c := color.RGBA{255, 0, 0, 255}
if x >= 150 {
c = color.RGBA{0, 0, 255, 255}
}
src.Set(x, y, c)
}
}
var b bytes.Buffer
jpeg.Encode(&b, src, &jpeg.Options{Quality: 95})
data := withExif(b.Bytes(), 6) // 90° по часовой
if o := ExifOrientation(data); o != 6 {
t.Fatalf("orientation %d, want 6", o)
}
img, err := Decode(data)
if err != nil {
t.Fatal(err)
}
if img.Bounds().Dx() != 100 || img.Bounds().Dy() != 300 {
t.Fatalf("rotated size %v, want 100×300", img.Bounds())
}
// После поворота на 90° по часовой левая (красная) половина оказывается сверху.
if c := img.RGBAAt(50, 20); c.R < 200 || c.B > 60 {
t.Errorf("top after rotation = %v, want red", c)
}
th, _ := Thumbnail(img, ParamsFor(1, 80))
if th.Width != 85 || th.Height != 256 {
t.Errorf("thumbnail of rotated %d×%d, want 85×256", th.Width, th.Height)
}
if ExifOrientation(b.Bytes()) != 1 {
t.Error("no exif must mean orientation 1")
}
}
func TestDecodeGarbage(t *testing.T) {
if _, err := Decode([]byte("not an image")); err == nil {
t.Fatal("garbage decoded")
}
}
func TestOptimizedWebP(t *testing.T) {
if _, err := exec.LookPath("cwebp"); err != nil {
t.Skip("cwebp not installed (worker image has libwebp-tools)")
}
img, _ := Decode(encPNG(t, solid(2000, 1000, color.RGBA{200, 100, 50, 255})))
out, err := Optimized(context.Background(), img, ParamsFor(1, 80), CWebP)
if err != nil {
t.Fatal(err)
}
if out.Width != 1280 || out.Height != 640 || out.ContentType != "image/webp" {
t.Fatalf("optimized %+v", out)
}
again, _ := Optimized(context.Background(), img, ParamsFor(1, 80), CWebP)
if !bytes.Equal(out.Data, again.Data) {
t.Error("cwebp output not deterministic")
}
}
+404
View File
@@ -0,0 +1,404 @@
// Package reconcile сверяет метаданные вложений (PostgreSQL) и объекты (S3).
//
// Обход — один merge-проход по двум потокам, отсортированным одинаково
// (object_key COLLATE "C" и лексикографический листинг S3), поэтому память
// O(1) от числа вложений, а стоимость — O(N) строк + ceil(N/1000) LIST-запросов.
//
// Расхождения и безопасные действия:
//
// stale_intent строка uploading старше grace (брошенная загрузка) → удалить объект, затем строку
// orphan_object объект без строки, старше grace → удалить объект (после перепроверки строки)
// missing_object строка ready без объекта → карантин: status=missing (после перепроверки Stat)
// size_mismatch строка ready, размер объекта другой → карантин: status=missing
// restored строка missing, объект снова есть и размер совпадает → status=ready
// purge_missing строка missing старше grace, ни одно сообщение не ссылается → удалить строку
// missing_variant у ready-варианта нет объекта → только отчёт (без автоисправления)
//
// С lab5 вложения в queued/processing/failed и их staging принадлежат pipeline
// (lease, retry, retention): reconciler их не трогает. Объекты без строки
// (например, частичные результаты попытки, закончившейся failed) удаляются как
// orphan после grace — так же, как в lab4.
//
// Свежие объекты и намерения (моложе grace) и вложения, на которые ссылаются
// сообщения, не удаляются никогда. Режим по умолчанию — dry-run (только отчёт).
package reconcile
import (
"context"
"errors"
"fmt"
"runtime"
"time"
"github.com/charawein/mtusi2026lab6/internal/domain"
"github.com/charawein/mtusi2026lab6/internal/storage"
)
// Repo — нужные reconciler операции над метаданными.
type Repo interface {
StreamByKey(ctx context.Context, prefix string, fn func(*domain.Attachment) error) error
HasKey(ctx context.Context, key string) (bool, error)
DeleteStaleIntent(ctx context.Context, id string, cutoff time.Time) (bool, error)
SetStatus(ctx context.Context, id, from, to string) (bool, error)
DeleteUnreferencedMissing(ctx context.Context, id string, cutoff time.Time) (bool, error)
IsReferenced(ctx context.Context, id string) (bool, error)
}
type Options struct {
Prefix string
Grace time.Duration
Apply bool
Now time.Time // общий «сейчас» для строк и объектов (время БД)
// MaxItems ограничивает список примеров в отчёте на каждый вид расхождения.
MaxItems int
}
type Item struct {
Kind string `json:"kind"`
Key string `json:"key"`
AttachmentID string `json:"attachment_id,omitempty"`
Action string `json:"action"`
Detail string `json:"detail,omitempty"`
}
type Report struct {
DryRun bool `json:"dry_run"`
GraceSeconds float64 `json:"grace_seconds"`
Cutoff time.Time `json:"cutoff"`
DurationMS int64 `json:"duration_ms"`
RowsScanned int `json:"rows_scanned"`
ObjectsScanned int `json:"objects_scanned"`
Consistent int `json:"consistent"`
Found map[string]int `json:"found"`
Fixed map[string]int `json:"fixed"`
Skipped map[string]int `json:"skipped"`
Calls map[string]int `json:"calls"`
HeapAllocMB float64 `json:"heap_alloc_mb"`
// WaitMS — сколько merge-цикл простоял в ожидании очередного элемента каждого
// потока: показывает, кто (S3-листинг или запрос к БД) задаёт время обхода.
WaitMS map[string]int64 `json:"wait_ms"`
Errors []string `json:"errors"`
Items []Item `json:"items"`
}
type reconciler struct {
repo Repo
store storage.Store
opt Options
rep *Report
items map[string]int
}
func (r *reconciler) note(kind string, it Item) {
r.rep.Found[kind]++
if r.items[kind] < r.opt.MaxItems {
r.items[kind]++
r.rep.Items = append(r.rep.Items, it)
}
}
func (r *reconciler) fail(format string, args ...any) {
if len(r.rep.Errors) < 100 {
r.rep.Errors = append(r.rep.Errors, fmt.Sprintf(format, args...))
}
}
// Run выполняет один проход. Ошибки отдельных действий попадают в отчёт и не
// прерывают обход; ошибка листинга/запроса прерывает проход целиком.
func Run(ctx context.Context, repo Repo, store storage.Store, opt Options) (*Report, error) {
start := time.Now()
if opt.MaxItems == 0 {
opt.MaxItems = 50
}
cutoff := opt.Now.Add(-opt.Grace)
rep := &Report{
DryRun: !opt.Apply, GraceSeconds: opt.Grace.Seconds(), Cutoff: cutoff,
Found: map[string]int{}, Fixed: map[string]int{}, Skipped: map[string]int{}, Calls: map[string]int{},
Errors: []string{}, Items: []Item{}, WaitMS: map[string]int64{},
}
rc := &reconciler{repo: repo, store: store, opt: opt, rep: rep, items: map[string]int{}}
ctx, cancel := context.WithCancel(ctx)
defer cancel()
objs := make(chan storage.Object, 512)
objErr := make(chan error, 1)
go func() {
defer close(objs)
objErr <- store.List(ctx, opt.Prefix, func(o storage.Object) error {
select {
case objs <- o:
return nil
case <-ctx.Done():
return ctx.Err()
}
})
}()
rows := make(chan *domain.Attachment, 512)
rowErr := make(chan error, 1)
go func() {
defer close(rows)
rowErr <- repo.StreamByKey(ctx, opt.Prefix, func(a *domain.Attachment) error {
select {
case rows <- a:
return nil
case <-ctx.Done():
return ctx.Err()
}
})
}()
var waitS3, waitDB time.Duration
nextObj := func() (storage.Object, bool) {
t := time.Now()
o, ok := <-objs
waitS3 += time.Since(t)
return o, ok
}
nextRow := func() (*domain.Attachment, bool) {
t := time.Now()
a, ok := <-rows
waitDB += time.Since(t)
return a, ok
}
o, okO := nextObj()
a, okA := nextRow()
for okO || okA {
switch {
case okA && (!okO || a.ObjectKey < o.Key):
rep.RowsScanned++
rc.rowWithoutObject(ctx, a, cutoff)
a, okA = nextRow()
case okO && (!okA || o.Key < a.ObjectKey):
rep.ObjectsScanned++
rc.objectWithoutRow(ctx, o, cutoff)
o, okO = nextObj()
default:
rep.RowsScanned++
rep.ObjectsScanned++
rc.both(ctx, a, o, cutoff)
a, okA = nextRow()
o, okO = nextObj()
}
}
rep.WaitMS["s3_list"] = waitS3.Milliseconds()
rep.WaitMS["db_stream"] = waitDB.Milliseconds()
if err := <-objErr; err != nil {
return rep, fmt.Errorf("list objects: %w", err)
}
if err := <-rowErr; err != nil {
return rep, fmt.Errorf("stream rows: %w", err)
}
rep.Calls["s3_list_pages"] = (rep.ObjectsScanned + 999) / 1000
var ms runtime.MemStats
runtime.ReadMemStats(&ms)
rep.HeapAllocMB = float64(ms.HeapAlloc) / (1 << 20)
rep.DurationMS = time.Since(start).Milliseconds()
return rep, nil
}
func (r *reconciler) rowWithoutObject(ctx context.Context, a *domain.Attachment, cutoff time.Time) {
switch a.Status {
case "variant":
if r.objectExists(ctx, a.ObjectKey) {
r.rep.Skipped["recheck_object_present"]++
return
}
r.note("missing_variant", Item{Kind: "missing_variant", Key: a.ObjectKey, AttachmentID: a.ID, Action: "report"})
case "staging", domain.AttachmentQueued, domain.AttachmentProcessing, domain.AttachmentFailed:
r.rep.Skipped["owned_by_pipeline"]++
case domain.AttachmentUploading:
if !a.UpdatedAt.Before(cutoff) {
r.rep.Skipped["fresh_intent"]++ // загрузка может быть в процессе
return
}
r.staleIntent(ctx, a, false)
case domain.AttachmentReady:
// Листинг мог начаться раньше, чем объект записан: перепроверяем точечно.
if r.objectExists(ctx, a.ObjectKey) {
r.rep.Skipped["recheck_object_present"]++
return
}
r.quarantine(ctx, a, "missing_object", "")
case domain.AttachmentMissing:
r.purgeMissing(ctx, a, cutoff)
}
}
func (r *reconciler) objectWithoutRow(ctx context.Context, o storage.Object, cutoff time.Time) {
if !o.LastModified.Before(cutoff) {
r.rep.Skipped["fresh_object"]++
return
}
// Строка могла появиться после начала обхода — перепроверяем перед удалением.
r.rep.Calls["db_haskey"]++
has, err := r.repo.HasKey(ctx, o.Key)
if err != nil {
r.fail("haskey %s: %v", o.Key, err)
return
}
if has {
r.rep.Skipped["recheck_row_present"]++
return
}
it := Item{Kind: "orphan_object", Key: o.Key, Action: "delete_object",
Detail: fmt.Sprintf("size=%d modified=%s", o.Size, o.LastModified.UTC().Format(time.RFC3339))}
if !r.opt.Apply {
it.Action = "would_delete_object"
r.note("orphan_object", it)
return
}
r.rep.Calls["s3_delete"]++
if err := r.store.Delete(ctx, o.Key); err != nil {
r.fail("delete orphan %s: %v", o.Key, err)
return
}
r.note("orphan_object", it)
r.rep.Fixed["orphan_object"]++
}
func (r *reconciler) both(ctx context.Context, a *domain.Attachment, o storage.Object, cutoff time.Time) {
switch a.Status {
case "variant":
if o.Size != a.SizeBytes {
r.note("variant_size_mismatch", Item{Kind: "variant_size_mismatch", Key: a.ObjectKey, AttachmentID: a.ID,
Action: "report", Detail: fmt.Sprintf("db=%d s3=%d", a.SizeBytes, o.Size)})
return
}
r.rep.Consistent++
case "staging", domain.AttachmentQueued, domain.AttachmentProcessing, domain.AttachmentFailed:
r.rep.Consistent++
case domain.AttachmentUploading:
if !a.UpdatedAt.Before(cutoff) {
r.rep.Skipped["fresh_intent"]++
return
}
r.staleIntent(ctx, a, true)
case domain.AttachmentReady:
if o.Size != a.SizeBytes {
r.quarantine(ctx, a, "size_mismatch", fmt.Sprintf("db=%d s3=%d", a.SizeBytes, o.Size))
return
}
r.rep.Consistent++
case domain.AttachmentMissing:
if o.Size != a.SizeBytes {
r.purgeMissing(ctx, a, cutoff)
return
}
it := Item{Kind: "restored", Key: a.ObjectKey, AttachmentID: a.ID, Action: "set_ready"}
if !r.opt.Apply {
it.Action = "would_set_ready"
r.note("restored", it)
return
}
r.rep.Calls["db_update"]++
ok, err := r.repo.SetStatus(ctx, a.ID, domain.AttachmentMissing, domain.AttachmentReady)
if err != nil {
r.fail("restore %s: %v", a.ID, err)
return
}
r.note("restored", it)
if ok {
r.rep.Fixed["restored"]++
}
}
}
func (r *reconciler) objectExists(ctx context.Context, key string) bool {
r.rep.Calls["s3_stat"]++
_, err := r.store.Stat(ctx, key)
if err == nil {
return true
}
if !errors.Is(err, storage.ErrNotFound) {
r.fail("stat %s: %v", key, err)
return true // при ошибке хранилища ничего не трогаем
}
return false
}
func (r *reconciler) staleIntent(ctx context.Context, a *domain.Attachment, hasObject bool) {
it := Item{Kind: "stale_intent", Key: a.ObjectKey, AttachmentID: a.ID, Action: "delete_intent",
Detail: "updated_at=" + a.UpdatedAt.UTC().Format(time.RFC3339)}
if hasObject {
it.Action = "delete_object_and_intent"
}
if !r.opt.Apply {
it.Action = "would_" + it.Action
r.note("stale_intent", it)
return
}
// Сначала объект: если упадём между шагами, останется намерение без объекта,
// и следующий проход его доберёт (обратный порядок оставил бы сироту без учёта).
if hasObject {
r.rep.Calls["s3_delete"]++
if err := r.store.Delete(ctx, a.ObjectKey); err != nil {
r.fail("delete intent object %s: %v", a.ObjectKey, err)
return
}
}
r.rep.Calls["db_delete"]++
ok, err := r.repo.DeleteStaleIntent(ctx, a.ID, r.opt.Now.Add(-r.opt.Grace))
if err != nil {
r.fail("delete intent %s: %v", a.ID, err)
return
}
r.note("stale_intent", it)
if ok {
r.rep.Fixed["stale_intent"]++
}
}
func (r *reconciler) quarantine(ctx context.Context, a *domain.Attachment, kind, detail string) {
it := Item{Kind: kind, Key: a.ObjectKey, AttachmentID: a.ID, Action: "quarantine", Detail: detail}
if !r.opt.Apply {
it.Action = "would_quarantine"
r.note(kind, it)
return
}
r.rep.Calls["db_update"]++
ok, err := r.repo.SetStatus(ctx, a.ID, domain.AttachmentReady, domain.AttachmentMissing)
if err != nil {
r.fail("quarantine %s: %v", a.ID, err)
return
}
r.note(kind, it)
if ok {
r.rep.Fixed[kind]++
}
}
// purgeMissing удаляет запись карантина, только если она старше grace и на неё
// не ссылается ни одно сообщение. Используемые вложения остаются (их история цела).
func (r *reconciler) purgeMissing(ctx context.Context, a *domain.Attachment, cutoff time.Time) {
if !a.UpdatedAt.Before(cutoff) {
r.rep.Skipped["fresh_missing"]++
return
}
r.rep.Calls["db_referenced"]++
ref, err := r.repo.IsReferenced(ctx, a.ID)
if err != nil {
r.fail("referenced %s: %v", a.ID, err)
return
}
if ref {
r.rep.Skipped["referenced_missing_kept"]++
return
}
it := Item{Kind: "purge_missing", Key: a.ObjectKey, AttachmentID: a.ID, Action: "delete_row"}
if !r.opt.Apply {
it.Action = "would_delete_row"
r.note("purge_missing", it)
return
}
r.rep.Calls["db_delete"]++
ok, err := r.repo.DeleteUnreferencedMissing(ctx, a.ID, cutoff)
if err != nil {
r.fail("purge %s: %v", a.ID, err)
return
}
r.note("purge_missing", it)
if ok {
r.rep.Fixed["purge_missing"]++
}
}
+241
View File
@@ -0,0 +1,241 @@
package reconcile
import (
"bytes"
"context"
"io"
"sort"
"testing"
"time"
"github.com/charawein/mtusi2026lab6/internal/domain"
"github.com/charawein/mtusi2026lab6/internal/storage"
)
type fakeStore struct{ objs map[string]storage.Object }
func (f *fakeStore) Put(_ context.Context, key string, r io.Reader, size int64, _ string) error {
f.objs[key] = storage.Object{Key: key, Size: size, LastModified: time.Now()}
return nil
}
func (f *fakeStore) Get(context.Context, string) (io.ReadCloser, int64, error) {
return io.NopCloser(bytes.NewReader(nil)), 0, nil
}
func (f *fakeStore) Stat(_ context.Context, key string) (storage.Object, error) {
o, ok := f.objs[key]
if !ok {
return o, storage.ErrNotFound
}
return o, nil
}
func (f *fakeStore) Delete(_ context.Context, key string) error { delete(f.objs, key); return nil }
func (f *fakeStore) Copy(_ context.Context, src, dst string) error {
o, ok := f.objs[src]
if !ok {
return storage.ErrNotFound
}
o.Key = dst
f.objs[dst] = o
return nil
}
func (f *fakeStore) List(_ context.Context, prefix string, fn func(storage.Object) error) error {
keys := make([]string, 0, len(f.objs))
for k := range f.objs {
keys = append(keys, k)
}
sort.Strings(keys)
for _, k := range keys {
if err := fn(f.objs[k]); err != nil {
return err
}
}
return nil
}
type fakeRepo struct {
rows map[string]*domain.Attachment // by id
refs map[string]bool
}
func (f *fakeRepo) StreamByKey(_ context.Context, _ string, fn func(*domain.Attachment) error) error {
list := make([]*domain.Attachment, 0, len(f.rows))
for _, a := range f.rows {
c := *a
list = append(list, &c)
}
sort.Slice(list, func(i, j int) bool { return list[i].ObjectKey < list[j].ObjectKey })
for _, a := range list {
if err := fn(a); err != nil {
return err
}
}
return nil
}
func (f *fakeRepo) HasKey(_ context.Context, key string) (bool, error) {
for _, a := range f.rows {
if a.ObjectKey == key {
return true, nil
}
}
return false, nil
}
func (f *fakeRepo) DeleteStaleIntent(_ context.Context, id string, cutoff time.Time) (bool, error) {
a, ok := f.rows[id]
if !ok || a.Status != domain.AttachmentUploading || !a.UpdatedAt.Before(cutoff) {
return false, nil
}
delete(f.rows, id)
return true, nil
}
func (f *fakeRepo) SetStatus(_ context.Context, id, from, to string) (bool, error) {
a, ok := f.rows[id]
if !ok || a.Status != from {
return false, nil
}
a.Status = to
a.UpdatedAt = now
return true, nil
}
func (f *fakeRepo) DeleteUnreferencedMissing(_ context.Context, id string, cutoff time.Time) (bool, error) {
a, ok := f.rows[id]
if !ok || a.Status != domain.AttachmentMissing || !a.UpdatedAt.Before(cutoff) || f.refs[id] {
return false, nil
}
delete(f.rows, id)
return true, nil
}
func (f *fakeRepo) IsReferenced(_ context.Context, id string) (bool, error) { return f.refs[id], nil }
var now = time.Date(2026, 10, 30, 12, 0, 0, 0, time.UTC)
const grace = time.Hour
func old() time.Time { return now.Add(-2 * grace) }
func fresh() time.Time { return now.Add(-time.Minute) }
func setup() (*fakeRepo, *fakeStore) {
r := &fakeRepo{rows: map[string]*domain.Attachment{}, refs: map[string]bool{}}
s := &fakeStore{objs: map[string]storage.Object{}}
row := func(id, status string, size int64, at time.Time) {
r.rows[id] = &domain.Attachment{ID: id, ObjectKey: "a/c/" + id, Status: status, SizeBytes: size, UpdatedAt: at}
}
obj := func(id string, size int64, at time.Time) {
s.objs["a/c/"+id] = storage.Object{Key: "a/c/" + id, Size: size, LastModified: at}
}
// согласованное
row("ok", domain.AttachmentReady, 10, old())
obj("ok", 10, old())
// брошенная загрузка с объектом и без
row("stale1", domain.AttachmentUploading, 0, old())
obj("stale1", 7, old())
row("stale2", domain.AttachmentUploading, 0, old())
// загрузка в процессе — не трогать
row("inflight", domain.AttachmentUploading, 0, fresh())
obj("inflight", 5, fresh())
// сирота старая и свежая
obj("orphan", 3, old())
obj("orphan-fresh", 3, fresh())
// ready без объекта: используемое и нет
row("lost-used", domain.AttachmentReady, 9, old())
r.refs["lost-used"] = true
row("lost-free", domain.AttachmentReady, 9, old())
// подменённый размер
row("resized", domain.AttachmentReady, 10, old())
obj("resized", 11, old())
return r, s
}
func run(t *testing.T, r *fakeRepo, s *fakeStore, apply bool, at time.Time) *Report {
t.Helper()
rep, err := Run(context.Background(), r, s, Options{Prefix: "a/", Grace: grace, Apply: apply, Now: at})
if err != nil {
t.Fatal(err)
}
if len(rep.Errors) > 0 {
t.Fatalf("errors: %v", rep.Errors)
}
return rep
}
func TestDryRunChangesNothing(t *testing.T) {
r, s := setup()
nRows, nObjs := len(r.rows), len(s.objs)
rep := run(t, r, s, false, now)
if len(r.rows) != nRows || len(s.objs) != nObjs || r.rows["lost-free"].Status != domain.AttachmentReady {
t.Fatal("dry-run mutated state")
}
want := map[string]int{"stale_intent": 2, "orphan_object": 1, "missing_object": 2, "size_mismatch": 1}
for k, v := range want {
if rep.Found[k] != v {
t.Errorf("found[%s]=%d want %d (%v)", k, rep.Found[k], v, rep.Found)
}
}
if len(rep.Fixed) != 0 {
t.Errorf("dry-run fixed %v", rep.Fixed)
}
}
func TestApplyFixesAndIsIdempotent(t *testing.T) {
r, s := setup()
run(t, r, s, true, now)
for _, id := range []string{"stale1", "stale2"} {
if _, ok := r.rows[id]; ok {
t.Errorf("stale intent %s not removed", id)
}
}
if _, ok := s.objs["a/c/stale1"]; ok {
t.Error("stale intent object not removed")
}
if _, ok := s.objs["a/c/orphan"]; ok {
t.Error("old orphan not removed")
}
// Свежие — никогда.
if _, ok := s.objs["a/c/orphan-fresh"]; !ok {
t.Error("fresh orphan deleted")
}
if _, ok := r.rows["inflight"]; !ok {
t.Error("in-flight intent deleted")
}
if _, ok := s.objs["a/c/inflight"]; !ok {
t.Error("in-flight object deleted")
}
for _, id := range []string{"lost-used", "lost-free", "resized"} {
if r.rows[id].Status != domain.AttachmentMissing {
t.Errorf("%s not quarantined", id)
}
}
if r.rows["ok"].Status != domain.AttachmentReady {
t.Error("consistent row touched")
}
// Повтор сразу: исправлять нечего, карантин ещё моложе grace.
rep := run(t, r, s, true, now)
if len(rep.Fixed) != 0 {
t.Errorf("second pass fixed %v", rep.Fixed)
}
// Через grace: неиспользуемая запись карантина удаляется, используемая остаётся.
later := now.Add(2 * grace)
run(t, r, s, true, later)
if _, ok := r.rows["lost-free"]; ok {
t.Error("unreferenced missing row not purged")
}
if _, ok := r.rows["lost-used"]; !ok {
t.Error("referenced missing row deleted")
}
if _, ok := s.objs["a/c/orphan-fresh"]; ok {
t.Error("orphan not removed after grace")
}
}
func TestRestoredObjectLeavesQuarantine(t *testing.T) {
r, s := setup()
run(t, r, s, true, now)
// Объект вернули (например, из бэкапа) с правильным размером.
s.objs["a/c/lost-used"] = storage.Object{Key: "a/c/lost-used", Size: 9, LastModified: now}
rep := run(t, r, s, true, now)
if rep.Fixed["restored"] != 1 || r.rows["lost-used"].Status != domain.AttachmentReady {
t.Errorf("restore failed: %v %s", rep.Fixed, r.rows["lost-used"].Status)
}
}
+242
View File
@@ -0,0 +1,242 @@
package postgres
import (
"context"
"errors"
"fmt"
"time"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool"
"github.com/charawein/mtusi2026lab6/internal/domain"
"github.com/charawein/mtusi2026lab6/internal/repository"
)
type AttachmentRepository struct {
pool *pgxpool.Pool
}
func NewAttachmentRepository(pool *pgxpool.Pool) *AttachmentRepository {
return &AttachmentRepository{pool: pool}
}
const attachmentColumns = `id, chat_id, owner_id, filename, object_key, status,
COALESCE(content_type, ''), COALESCE(size_bytes, 0), COALESCE(sha256, ''), created_at, updated_at,
COALESCE(error_code, ''), pipeline_version, COALESCE(staging_key, '')`
func scanAttachment(row pgx.Row) (*domain.Attachment, error) {
var a domain.Attachment
err := row.Scan(&a.ID, &a.ChatID, &a.OwnerID, &a.Filename, &a.ObjectKey, &a.Status,
&a.ContentType, &a.SizeBytes, &a.SHA256, &a.CreatedAt, &a.UpdatedAt,
&a.ErrorCode, &a.PipelineVersion, &a.StagingKey)
if err != nil {
if errors.Is(err, pgx.ErrNoRows) {
return nil, repository.ErrNotFound
}
return nil, err
}
return &a, nil
}
// CreateIntent записывает намерение загрузки (status=uploading) ДО записи в S3.
func (r *AttachmentRepository) CreateIntent(ctx context.Context, a *domain.Attachment) error {
const q = `
INSERT INTO attachments (id, chat_id, owner_id, filename, object_key, staging_key, status, created_at, updated_at)
VALUES ($1::uuid, $2::uuid, $3::uuid, $4, $5, $5, 'uploading', $6, $6)
`
if _, err := r.pool.Exec(ctx, q, a.ID, a.ChatID, a.OwnerID, a.Filename, a.ObjectKey, a.CreatedAt); err != nil {
return fmt.Errorf("insert attachment intent: %w", err)
}
return nil
}
// MarkQueued — точка 202. В ОДНОЙ транзакции: метаданные проверенного оригинала
// и status=queued, логическое задание (jobs) и сообщение для broker (outbox).
// После COMMIT работа не теряется: даже если broker недоступен или relay упал,
// outbox будет опубликован позже. Если строки uploading уже нет — ErrNotFound.
func (r *AttachmentRepository) MarkQueued(ctx context.Context, a *domain.Attachment, pipelineVersion, maxAttempts int) (string, error) {
tx, err := r.pool.Begin(ctx)
if err != nil {
return "", fmt.Errorf("begin tx: %w", err)
}
defer func() { _ = tx.Rollback(ctx) }()
const q = `
UPDATE attachments
SET status = 'queued', content_type = $2, size_bytes = $3, sha256 = $4, updated_at = $5
WHERE id = $1::uuid AND status = 'uploading'
`
tag, err := tx.Exec(ctx, q, a.ID, a.ContentType, a.SizeBytes, a.SHA256, a.UpdatedAt)
if err != nil {
return "", fmt.Errorf("mark attachment queued: %w", err)
}
if tag.RowsAffected() != 1 {
return "", repository.ErrNotFound
}
jobID, err := EnqueueJobTx(ctx, tx, a.ID, pipelineVersion, maxAttempts, "upload")
if err != nil {
return "", err
}
if err := tx.Commit(ctx); err != nil {
return "", fmt.Errorf("commit tx: %w", err)
}
return jobID, nil
}
// Variants — варианты указанной версии pipeline.
func (r *AttachmentRepository) Variants(ctx context.Context, attachmentID string, version int) ([]domain.Variant, error) {
const q = `
SELECT name, content_type, size_bytes, width, height, sha256, object_key, pipeline_version
FROM attachment_variants
WHERE attachment_id = $1::uuid AND pipeline_version = $2
ORDER BY CASE name WHEN 'thumbnail' THEN 1 WHEN 'crop' THEN 2 ELSE 3 END
`
rows, err := r.pool.Query(ctx, q, attachmentID, version)
if err != nil {
return nil, fmt.Errorf("select variants: %w", err)
}
defer rows.Close()
out := []domain.Variant{}
for rows.Next() {
var v domain.Variant
if err := rows.Scan(&v.Name, &v.ContentType, &v.SizeBytes, &v.Width, &v.Height, &v.SHA256, &v.ObjectKey, &v.PipelineVersion); err != nil {
return nil, fmt.Errorf("scan variant: %w", err)
}
out = append(out, v)
}
return out, rows.Err()
}
// DeleteIntent убирает намерение, если загрузка не удалась (компенсация).
func (r *AttachmentRepository) DeleteIntent(ctx context.Context, id string) error {
const q = `DELETE FROM attachments WHERE id = $1::uuid AND status = 'uploading'`
if _, err := r.pool.Exec(ctx, q, id); err != nil {
return fmt.Errorf("delete attachment intent: %w", err)
}
return nil
}
// GetForMember возвращает вложение, если пользователь — участник его чата.
// Чужое и несуществующее вложение неразличимы (ErrNotFound).
func (r *AttachmentRepository) GetForMember(ctx context.Context, id, userID string) (*domain.Attachment, error) {
q := `SELECT ` + attachmentColumns + `
FROM attachments a
WHERE a.id = $1::uuid
AND EXISTS (SELECT 1 FROM chat_members m WHERE m.chat_id = a.chat_id AND m.user_id = $2::uuid)`
a, err := scanAttachment(r.pool.QueryRow(ctx, q, id, userID))
if err != nil && !errors.Is(err, repository.ErrNotFound) {
return nil, fmt.Errorf("select attachment: %w", err)
}
return a, err
}
// --- для reconciler ---
// StreamByKey отдаёт все известные БД ключи объектов в байтовом порядке (как
// ListObjectsV2): оригинал вложения (object_key), отдельный staging-объект и
// объекты вариантов. Для staging и вариантов Status — «staging»/«variant», чтобы
// reconciler не путал их с оригиналом.
func (r *AttachmentRepository) StreamByKey(ctx context.Context, prefix string, fn func(*domain.Attachment) error) error {
const q = `
SELECT key, id::text, status, size_bytes, updated_at FROM (
SELECT object_key AS key, id, status, COALESCE(size_bytes, 0) AS size_bytes, updated_at FROM attachments
UNION ALL
SELECT staging_key, id, 'staging', COALESCE(size_bytes, 0), updated_at FROM attachments
WHERE staging_key IS NOT NULL AND staging_key <> object_key
UNION ALL
SELECT object_key, attachment_id, 'variant', size_bytes, created_at FROM attachment_variants
) k
WHERE starts_with(key, $1)
ORDER BY key COLLATE "C"`
rows, err := r.pool.Query(ctx, q, prefix)
if err != nil {
return fmt.Errorf("stream object keys: %w", err)
}
defer rows.Close()
for rows.Next() {
var a domain.Attachment
if err := rows.Scan(&a.ObjectKey, &a.ID, &a.Status, &a.SizeBytes, &a.UpdatedAt); err != nil {
return fmt.Errorf("scan object key: %w", err)
}
if err := fn(&a); err != nil {
return err
}
}
return rows.Err()
}
// HasKey — есть ли строка с этим object_key (повторная проверка перед удалением объекта).
func (r *AttachmentRepository) HasKey(ctx context.Context, key string) (bool, error) {
var ok bool
err := r.pool.QueryRow(ctx, `SELECT EXISTS (SELECT 1 FROM attachments WHERE object_key = $1 OR staging_key = $1)
OR EXISTS (SELECT 1 FROM attachment_variants WHERE object_key = $1)`, key).Scan(&ok)
return ok, err
}
// DeleteStaleIntent удаляет намерение, только если оно всё ещё uploading и старше cutoff.
func (r *AttachmentRepository) DeleteStaleIntent(ctx context.Context, id string, cutoff time.Time) (bool, error) {
const q = `DELETE FROM attachments WHERE id = $1::uuid AND status = 'uploading' AND updated_at < $2`
tag, err := r.pool.Exec(ctx, q, id, cutoff)
return tag.RowsAffected() == 1, err
}
// SetStatus меняет статус ready↔missing (карантин и восстановление).
func (r *AttachmentRepository) SetStatus(ctx context.Context, id, from, to string) (bool, error) {
const q = `UPDATE attachments SET status = $3, updated_at = now() WHERE id = $1::uuid AND status = $2`
tag, err := r.pool.Exec(ctx, q, id, from, to)
return tag.RowsAffected() == 1, err
}
// DeleteUnreferencedMissing удаляет запись карантина старше cutoff, если на неё
// не ссылается ни одно сообщение. Блокировка строки (FOR UPDATE) конфликтует с
// FOR SHARE при создании сообщения, поэтому гонки «удалили, пока прикрепляли» нет.
func (r *AttachmentRepository) DeleteUnreferencedMissing(ctx context.Context, id string, cutoff time.Time) (bool, error) {
tx, err := r.pool.Begin(ctx)
if err != nil {
return false, err
}
defer func() { _ = tx.Rollback(ctx) }()
var locked string
err = tx.QueryRow(ctx, `SELECT id FROM attachments WHERE id = $1::uuid AND status = 'missing' AND updated_at < $2 FOR UPDATE`, id, cutoff).Scan(&locked)
if errors.Is(err, pgx.ErrNoRows) {
return false, nil
}
if err != nil {
return false, err
}
var referenced bool
if err := tx.QueryRow(ctx, `SELECT EXISTS (SELECT 1 FROM message_attachments WHERE attachment_id = $1::uuid)`, id).Scan(&referenced); err != nil {
return false, err
}
if referenced {
return false, nil
}
if _, err := tx.Exec(ctx, `DELETE FROM attachments WHERE id = $1::uuid`, id); err != nil {
return false, err
}
return true, tx.Commit(ctx)
}
// IsReferenced — используется ли вложение каким-либо сообщением.
func (r *AttachmentRepository) IsReferenced(ctx context.Context, id string) (bool, error) {
var ok bool
err := r.pool.QueryRow(ctx, `SELECT EXISTS (SELECT 1 FROM message_attachments WHERE attachment_id = $1::uuid)`, id).Scan(&ok)
return ok, err
}
// TryLock — advisory lock, чтобы одновременно работал один reconciler.
func (r *AttachmentRepository) TryLock(ctx context.Context) (func(), bool, error) {
conn, err := r.pool.Acquire(ctx)
if err != nil {
return nil, false, err
}
var ok bool
if err := conn.QueryRow(ctx, `SELECT pg_try_advisory_lock(4242004)`).Scan(&ok); err != nil || !ok {
conn.Release()
return nil, false, err
}
return func() {
_, _ = conn.Exec(context.Background(), `SELECT pg_advisory_unlock(4242004)`)
conn.Release()
}, true, nil
}
+180
View File
@@ -0,0 +1,180 @@
package postgres
import (
"context"
"errors"
"fmt"
"time"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool"
"github.com/charawein/mtusi2026lab6/internal/domain"
"github.com/charawein/mtusi2026lab6/internal/repository"
)
type ChatRepository struct {
pool *pgxpool.Pool
}
func NewChatRepository(pool *pgxpool.Pool) *ChatRepository {
return &ChatRepository{pool: pool}
}
func (r *ChatRepository) Create(ctx context.Context, chat *domain.Chat) error {
tx, err := r.pool.Begin(ctx)
if err != nil {
return fmt.Errorf("begin tx: %w", err)
}
defer func() { _ = tx.Rollback(ctx) }()
const insertChat = `INSERT INTO chats (id, title, created_at) VALUES ($1::uuid, $2, $3)`
if _, err := tx.Exec(ctx, insertChat, chat.ID, chat.Title, chat.CreatedAt); err != nil {
return fmt.Errorf("insert chat: %w", err)
}
const insertMember = `INSERT INTO chat_members (chat_id, user_id) VALUES ($1::uuid, $2::uuid)`
for _, uid := range chat.MemberIDs {
if _, err := tx.Exec(ctx, insertMember, chat.ID, uid); err != nil {
return fmt.Errorf("insert member %s: %w", uid, err)
}
}
if err := tx.Commit(ctx); err != nil {
return fmt.Errorf("commit tx: %w", err)
}
return nil
}
func (r *ChatRepository) GetByID(ctx context.Context, id string) (*domain.Chat, error) {
const q = `SELECT id, title, created_at FROM chats WHERE id = $1::uuid`
var c domain.Chat
err := r.pool.QueryRow(ctx, q, id).Scan(&c.ID, &c.Title, &c.CreatedAt)
if err != nil {
if errors.Is(err, pgx.ErrNoRows) {
return nil, repository.ErrNotFound
}
return nil, fmt.Errorf("select chat: %w", err)
}
const qMembers = `SELECT user_id FROM chat_members WHERE chat_id = $1::uuid ORDER BY user_id`
rows, err := r.pool.Query(ctx, qMembers, id)
if err != nil {
return nil, fmt.Errorf("select members: %w", err)
}
defer rows.Close()
c.MemberIDs = []string{}
for rows.Next() {
var uid string
if err := rows.Scan(&uid); err != nil {
return nil, fmt.Errorf("scan member: %w", err)
}
c.MemberIDs = append(c.MemberIDs, uid)
}
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("iterate members: %w", err)
}
return &c, nil
}
func (r *ChatRepository) IsMember(ctx context.Context, chatID, userID string) (bool, error) {
const q = `SELECT 1 FROM chat_members WHERE chat_id = $1::uuid AND user_id = $2::uuid`
var one int
err := r.pool.QueryRow(ctx, q, chatID, userID).Scan(&one)
if err == nil {
return true, nil
}
if !errors.Is(err, pgx.ErrNoRows) {
return false, fmt.Errorf("check membership: %w", err)
}
const qChat = `SELECT 1 FROM chats WHERE id = $1::uuid`
err = r.pool.QueryRow(ctx, qChat, chatID).Scan(&one)
if err == nil {
return false, nil
}
if errors.Is(err, pgx.ErrNoRows) {
return false, repository.ErrNotFound
}
return false, fmt.Errorf("check chat exists: %w", err)
}
// MemberVersion одним запросом возвращает версию истории чата и признак членства.
// Несуществующий чат — ErrNotFound.
func (r *ChatRepository) MemberVersion(ctx context.Context, chatID, userID string) (int64, bool, error) {
const q = `
SELECT c.version,
EXISTS (SELECT 1 FROM chat_members m WHERE m.chat_id = c.id AND m.user_id = $2::uuid)
FROM chats c
WHERE c.id = $1::uuid
`
var version int64
var member bool
err := r.pool.QueryRow(ctx, q, chatID, userID).Scan(&version, &member)
if err != nil {
if errors.Is(err, pgx.ErrNoRows) {
return 0, false, repository.ErrNotFound
}
return 0, false, fmt.Errorf("select chat version: %w", err)
}
return version, member, nil
}
func (r *ChatRepository) ListByMember(ctx context.Context, userID string, cursor time.Time, cursorID string, limit int) ([]*domain.Chat, error) {
const qChats = `
SELECT c.id, c.title, c.created_at
FROM chats c
JOIN chat_members m ON m.chat_id = c.id
WHERE m.user_id = $1::uuid
AND (c.created_at, c.id) > ($2::timestamptz, $3::uuid)
ORDER BY c.created_at, c.id
LIMIT $4
`
rows, err := r.pool.Query(ctx, qChats, userID, cursor, cursorID, limit)
if err != nil {
return nil, fmt.Errorf("list chats: %w", err)
}
defer rows.Close()
chats := []*domain.Chat{}
var chatIDs []string
for rows.Next() {
var c domain.Chat
if err := rows.Scan(&c.ID, &c.Title, &c.CreatedAt); err != nil {
return nil, fmt.Errorf("scan chat: %w", err)
}
c.MemberIDs = []string{}
chats = append(chats, &c)
chatIDs = append(chatIDs, c.ID)
}
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("iterate chats: %w", err)
}
if len(chats) == 0 {
return chats, nil
}
const qMembers = `SELECT chat_id, user_id FROM chat_members WHERE chat_id = ANY($1::uuid[]) ORDER BY chat_id, user_id`
memberRows, err := r.pool.Query(ctx, qMembers, chatIDs)
if err != nil {
return nil, fmt.Errorf("list members: %w", err)
}
defer memberRows.Close()
byChat := make(map[string][]string)
for memberRows.Next() {
var chatID, uid string
if err := memberRows.Scan(&chatID, &uid); err != nil {
return nil, fmt.Errorf("scan member: %w", err)
}
byChat[chatID] = append(byChat[chatID], uid)
}
if err := memberRows.Err(); err != nil {
return nil, fmt.Errorf("iterate members: %w", err)
}
for _, c := range chats {
if members, ok := byChat[c.ID]; ok {
c.MemberIDs = members
}
}
return chats, nil
}
+201
View File
@@ -0,0 +1,201 @@
package postgres
import (
"context"
"fmt"
"time"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool"
"github.com/charawein/mtusi2026lab6/internal/domain"
"github.com/charawein/mtusi2026lab6/internal/repository"
)
type MessageRepository struct {
pool *pgxpool.Pool
}
func NewMessageRepository(pool *pgxpool.Pool) *MessageRepository {
return &MessageRepository{pool: pool}
}
func (r *MessageRepository) Create(ctx context.Context, msg *domain.Message) error {
const q = `
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
VALUES ($1::uuid, $2::uuid, $3::uuid, $4, $5)
`
if _, err := r.pool.Exec(ctx, q, msg.ID, msg.ChatID, msg.SenderID, msg.Text, msg.CreatedAt); err != nil {
return fmt.Errorf("insert message: %w", err)
}
return nil
}
// CreateIdempotent вставляет сообщение и ключ в одной транзакции.
// Конкурентные запросы с одним ключом сериализуются уникальным индексом
// idempotency_keys: второй INSERT ждёт завершения первой транзакции, после
// COMMIT получает конфликт (DO NOTHING), откатывает своё сообщение и читает
// уже сохранённое. Если первая транзакция откатилась (транзиентная ошибка),
// вторая просто продолжает — неуспешный результат не фиксируется.
func (r *MessageRepository) CreateIdempotent(ctx context.Context, msg *domain.Message, key, requestHash string, ttl time.Duration) (*domain.Message, bool, error) {
tx, err := r.pool.Begin(ctx)
if err != nil {
return nil, false, fmt.Errorf("begin tx: %w", err)
}
defer func() { _ = tx.Rollback(ctx) }()
// Ключ старше TTL больше не защищает от повтора — освобождаем его.
const deleteExpired = `
DELETE FROM idempotency_keys
WHERE user_id = $1::uuid AND chat_id = $2::uuid AND key = $3
AND created_at < now() - make_interval(secs => $4)
`
if _, err := tx.Exec(ctx, deleteExpired, msg.SenderID, msg.ChatID, key, ttl.Seconds()); err != nil {
return nil, false, fmt.Errorf("delete expired key: %w", err)
}
const insertMsg = `
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
VALUES ($1::uuid, $2::uuid, $3::uuid, $4, $5)
`
if _, err := tx.Exec(ctx, insertMsg, msg.ID, msg.ChatID, msg.SenderID, msg.Text, msg.CreatedAt); err != nil {
return nil, false, fmt.Errorf("insert message: %w", err)
}
if len(msg.AttachmentIDs) > 0 {
if err := attachToMessage(ctx, tx, msg); err != nil {
return nil, false, err
}
}
const insertKey = `
INSERT INTO idempotency_keys (user_id, chat_id, key, request_hash, message_id)
VALUES ($1::uuid, $2::uuid, $3, $4, $5::uuid)
ON CONFLICT (user_id, chat_id, key) DO NOTHING
`
tag, err := tx.Exec(ctx, insertKey, msg.SenderID, msg.ChatID, key, requestHash, msg.ID)
if err != nil {
return nil, false, fmt.Errorf("insert idempotency key: %w", err)
}
if tag.RowsAffected() == 1 {
// Версия истории растёт атомарно с сообщением: после COMMIT любое чтение
// увидит новую версию и не попадёт в кэш, записанный до этого сообщения.
const bumpVersion = `UPDATE chats SET version = version + 1 WHERE id = $1::uuid`
if _, err := tx.Exec(ctx, bumpVersion, msg.ChatID); err != nil {
return nil, false, fmt.Errorf("bump chat version: %w", err)
}
if err := tx.Commit(ctx); err != nil {
return nil, false, fmt.Errorf("commit tx: %w", err)
}
return msg, true, nil
}
// Ключ уже занят подтверждённой записью: наше сообщение не сохраняем.
_ = tx.Rollback(ctx)
const selectStored = `
SELECT k.request_hash, m.id, m.chat_id, m.sender_id, m.text, m.created_at,
` + attachmentIDsExpr + `
FROM idempotency_keys k
JOIN messages m ON m.id = k.message_id
WHERE k.user_id = $1::uuid AND k.chat_id = $2::uuid AND k.key = $3
`
var storedHash string
var m domain.Message
err = r.pool.QueryRow(ctx, selectStored, msg.SenderID, msg.ChatID, key).
Scan(&storedHash, &m.ID, &m.ChatID, &m.SenderID, &m.Text, &m.CreatedAt, &m.AttachmentIDs)
if err != nil {
return nil, false, fmt.Errorf("select stored message: %w", err)
}
if storedHash != requestHash {
return nil, false, repository.ErrConflict
}
return &m, false, nil
}
// DeleteExpiredKeys удаляет ключи старше TTL. Безопасно выполнять с любой реплики.
func (r *MessageRepository) DeleteExpiredKeys(ctx context.Context, ttl time.Duration) (int64, error) {
const q = `DELETE FROM idempotency_keys WHERE created_at < now() - make_interval(secs => $1)`
tag, err := r.pool.Exec(ctx, q, ttl.Seconds())
if err != nil {
return 0, fmt.Errorf("delete expired keys: %w", err)
}
return tag.RowsAffected(), nil
}
func (r *MessageRepository) ListByChat(ctx context.Context, chatID string, cursor time.Time, cursorID string, limit int) ([]*domain.Message, error) {
q := `
SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at, ` + attachmentIDsExpr + `
FROM messages m
WHERE m.chat_id = $1::uuid
AND (m.created_at, m.id) > ($2::timestamptz, $3::uuid)
ORDER BY m.created_at, m.id
LIMIT $4
`
rows, err := r.pool.Query(ctx, q, chatID, cursor, cursorID, limit)
if err != nil {
return nil, fmt.Errorf("list messages: %w", err)
}
defer rows.Close()
msgs := []*domain.Message{}
for rows.Next() {
var m domain.Message
if err := rows.Scan(&m.ID, &m.ChatID, &m.SenderID, &m.Text, &m.CreatedAt, &m.AttachmentIDs); err != nil {
return nil, fmt.Errorf("scan message: %w", err)
}
msgs = append(msgs, &m)
}
return msgs, rows.Err()
}
// attachmentIDsExpr — id вложений сообщения m в порядке отправки (всегда массив).
const attachmentIDsExpr = `ARRAY(SELECT ma.attachment_id::text FROM message_attachments ma
WHERE ma.message_id = m.id ORDER BY ma.position)`
// attachToMessage проверяет и привязывает вложения в транзакции сообщения.
// FOR SHARE блокирует строки вложений до COMMIT: reconciler (FOR UPDATE) не
// удалит вложение, которое сейчас прикрепляют. Вложение должно быть из того же
// чата и в статусе ready; чужое, неизвестное и незавершённое — ErrNotFound.
func attachToMessage(ctx context.Context, tx pgx.Tx, msg *domain.Message) error {
const q = `
SELECT id::text, status FROM attachments
WHERE id = ANY($1::uuid[]) AND chat_id = $2::uuid
FOR SHARE
`
rows, err := tx.Query(ctx, q, msg.AttachmentIDs, msg.ChatID)
if err != nil {
return fmt.Errorf("select attachments: %w", err)
}
status := map[string]string{}
for rows.Next() {
var id, st string
if err := rows.Scan(&id, &st); err != nil {
rows.Close()
return fmt.Errorf("scan attachment: %w", err)
}
status[id] = st
}
rows.Close()
if err := rows.Err(); err != nil {
return fmt.Errorf("select attachments: %w", err)
}
for _, id := range msg.AttachmentIDs {
switch status[id] {
case domain.AttachmentReady, domain.AttachmentQueued, domain.AttachmentProcessing:
// с lab5 можно ссылаться на ещё обрабатываемое вложение
case domain.AttachmentFailed, domain.AttachmentMissing:
return repository.ErrConflict
default:
return repository.ErrNotFound // чужое, неизвестное, не принятое (uploading)
}
}
batch := &pgx.Batch{}
for i, id := range msg.AttachmentIDs {
batch.Queue(`INSERT INTO message_attachments (message_id, attachment_id, position) VALUES ($1::uuid, $2::uuid, $3)`,
msg.ID, id, i)
}
if err := tx.SendBatch(ctx, batch).Close(); err != nil {
return fmt.Errorf("insert message attachments: %w", err)
}
return nil
}

Some files were not shown because too many files have changed in this diff Show More