Author SHA1 Message Date
Grbisba 5af8d75fc7 Обновил REPORT.md 2026-11-17 13:08:54 +03:00
Grbisba 60f48c0337 Реализация на 5 2026-11-17 13:08:13 +03:00
229 changed files with 74928 additions and 47 deletions
+3
View File
@@ -18,3 +18,6 @@ __pycache__
node_modules
evidence
.DS_Store
.lab6-faults
bin
+17 -15
View File
@@ -1,18 +1,20 @@
## Лабораторная 6
## Лабораторная 6 — БПИ2303 — Поликарпов — на 5
Автор / группа: TODO
Целевая оценка (3/4/5): TODO
Трек 5, если заявлен: TODO
Автор: **Поликарпов Даниил Александрович**, группа **БПИ2303**.
Трек 5: **SLO и обнаружение инцидентов**.
Что реализовано и что перенесено: TODO
Команды запуска из чистого clone: TODO
Результаты проверок и ссылка на REPORT.md: TODO
Ограничения / известные ошибки: TODO
Источники и использование ИИ: TODO
Из собственной lab5 перенесены Go API, миграции, transactional outbox, RabbitMQ worker, S3 pipeline и существующие проверки. Добавлены OTLP metrics/logs/traces, durable W3C context для retry/recovery, Collector, Prometheus/Loki/Tempo, Grafana provisioning/dashboard и burn-rate alerts. Controller выполняет parsing/validation, service — бизнес-операции/SLO, storage — сохранение; интеграция OTel находится в pkg/telemetry.
- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
- [ ] Контракт и публичные тесты не ослаблены.
- [ ] make check и make test выполнены; приложен реальный вывод.
- [ ] Сценарии отказов/безопасности своего уровня воспроизведены.
- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
Запуск: `make up`, затем `make test test-go prove-correlation`. Полные prerequisites, конфигурация и воспроизводимые команды — в [REPORT.md](../REPORT.md).
Результаты: 13/13 pipeline-сценариев PASS, persistence всех stores с сохранёнными volumes, API failover и privacy/trace checks PASS. В research: 74 094 запроса, 3529 вложений ready, 6334 подтверждённых сообщения сохранены; 0 firing на 6 коротких воздействиях, все ожидаемые alerts firing/resolved на 6 длительных. При 300s отказа Tempo бизнес-запросы продолжались; buffer/RAM и потери telemetry измерены.
Ограничения: один host и единичные stores, учебный часовой SLO, best-effort telemetry; production HA/TLS/mTLS не заявляются. Источники, роль ИИ и два незасчитанных промежуточных прогона описаны в отчёте. Новые unit-тесты не добавлялись; запускались существующие проверки и новые сценарии измерений.
- [x] Обязательные возможности сохранены; контракт и публичные tests не ослаблены.
- [x] make check/validate, build/vet и smoke выполнены; реальные результаты в evidence/.
- [x] Отказы выбранного уровня воспроизведены; есть raw observations, summary и runbook.
- [x] Секреты/приватные ключи вне Git; в доказательствах нет session tokens.
- [x] README задания сохранён; REPORT.md содержит инструкцию решения.
Это заполненный шаблон описания; публикация PR в рамках этой работы не выполнялась.
+28
View File
@@ -0,0 +1,28 @@
name: Application
on: [push, pull_request]
permissions:
contents: read
jobs:
application:
runs-on: ubuntu-latest
timeout-minutes: 20
steps:
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
with:
go-version: '1.27.2'
cache-dependency-path: service/go.sum
- uses: actions/setup-python@v5
with:
python-version: '3.12'
- run: pip install -r requirements-dev.txt
- run: make check validate build vet
- run: make up
- run: make test test-go prove-correlation observe
- if: always()
uses: actions/upload-artifact@v4
with:
name: sanitized-evidence
path: evidence/
- if: always()
run: make down
+2
View File
@@ -30,3 +30,5 @@ coverage/
.vscode/
# Keep small, sanitized evidence summaries in evidence/ under version control.
# Inspect git diff --cached: ignore rules are not a secret scanner.
.lab6-faults/
+35
View File
@@ -0,0 +1,35 @@
# syntax=docker/dockerfile:1
FROM --platform=$BUILDPLATFORM golang:1.27.2-alpine3.23@sha256:9e45f0eb4a63ed37ad6e604950407558b7c738e34e015ae77a5d4ad369cde079 AS build
WORKDIR /src
COPY service/go.mod service/go.sum ./
RUN go mod download
COPY service/ ./
ARG TARGETOS
ARG TARGETARCH
RUN CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH \
go build -mod=readonly -trimpath -ldflags="-s -w" -o /out/messenger ./cmd/app
FROM alpine:3.23.6@sha256:85fe1e81d6758c208f3e1eed4338a1997e19d4be002d4dd32d3100c9a8c010a0
RUN apk add --no-cache imagemagick=7.1.2.35-r0 imagemagick-webp=7.1.2.35-r0 imagemagick-jpeg=7.1.2.35-r0
COPY infra/imagemagick-policy.xml /etc/ImageMagick-7/policy.xml
COPY --from=build --chmod=0555 /out/messenger /usr/local/bin/messenger
ENV APP_HOST=0.0.0.0 \
APP_PORT=8080 \
APP_SHUTDOWN_TIMEOUT=8s
USER 65532:65532
EXPOSE 8080
STOPSIGNAL SIGTERM
HEALTHCHECK --interval=10s --timeout=3s --start-period=5s --retries=3 \
CMD wget -q -O /dev/null "http://127.0.0.1:${APP_PORT}/health/ready" || exit 1
ENTRYPOINT ["/usr/local/bin/messenger"]
+111 -3
View File
@@ -1,13 +1,25 @@
SHELL := /bin/bash
GO ?= go
PYTHON ?= python3
BASE_URL ?= http://localhost:8080
BASE_URL ?= http://localhost:18086
CA_FILE ?=
ACTION_SCRIPT ?=
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
OPENAPI_SPEC := contracts/openapi.json
GEN_OUTPUT := service/internal/gen/contracts/types.gen.go
GEN_PACKAGE := contracts
OAPI_CODEGEN ?= $(shell $(GO) env GOPATH)/bin/oapi-codegen
.PHONY: help check validate test load test-persistence test-failover
.PHONY: help check validate test load test-persistence test-failover openapi-generate certs key-gen create-dir gen-private-key gen-public-key gen-key gen-crt
help:
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
@echo "observe/prove-correlation/experiment-slo: telemetry evidence and SLO research"
@echo "up/down: lab6 async image pipeline; build/vet: Go checks"
@echo "test: lab6 API smoke; test-go/test-storage: Go checks"
@echo "restart/restore/rolling-update: infrastructure actions; benchmark/load: synthetic traffic"
@echo "check/validate: lab6 contracts; test-persistence/test-failover: explicit ACTION_SCRIPT hooks"
@echo "reconcile/reconcile-apply: storage consistency"
@echo "openapi-generate: generate types from lab6 contract; key-gen/certs: local keys"
check:
$(PYTHON) scripts/check_contract.py
@@ -28,3 +40,99 @@ test-persistence:
test-failover:
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
openapi-generate:
@mkdir -p $(dir $(GEN_OUTPUT))
@echo "Generating Go types from $(OPENAPI_SPEC)..."
@set -o pipefail; \
$(PYTHON) scripts/generate-types.py | \
$(OAPI_CODEGEN) -generate types -package $(GEN_PACKAGE) -o $(GEN_OUTPUT) /dev/stdin
key-gen: gen-public-key
create-dir:
@mkdir -p certs
gen-private-key: | create-dir
@openssl genrsa -out certs/private_key.pem 2048
gen-public-key: gen-private-key
@openssl rsa -in certs/private_key.pem -pubout -outform PEM -out certs/public_key.pem
certs: gen-crt
gen-key: | create-dir
@openssl genrsa -out ./certs/server.key 2048
gen-crt: gen-key
@openssl req -new -x509 -key ./certs/server.key -out ./certs/server.crt -days 365
.PHONY: up down test-go
up:
./scripts/bootstrap.sh
./scripts/compose.sh up -d --build --wait --wait-timeout 120
$(PYTHON) scripts/wait-ready.py
down:
./scripts/compose.sh down
test-go:
cd service && TEST_BASE_URL="$(BASE_URL)" $(GO) test -race -count=1 -v ./...
RATE ?= 100
DURATION ?= 30s
BENCH_ARGS ?=
.PHONY: build vet restart restore rolling-update benchmark test-storage
build:
cd service && $(GO) build ./...
@mkdir -p bin
cd service && $(GO) build -o ../bin/benchmark ./cmd/benchmark
vet:
cd service && $(GO) vet ./...
restart:
./scripts/restart.sh
restore:
./scripts/compose.sh up -d --wait --wait-timeout 90 --no-recreate api lb
rolling-update:
./scripts/rolling-update.sh
benchmark: build
./bin/benchmark -base-url "$(BASE_URL)" -rate "$(RATE)" -duration "$(DURATION)" $(BENCH_ARGS)
test-storage:
./scripts/test-storage.sh
.PHONY: reconcile reconcile-apply
reconcile:
./scripts/compose.sh run --rm -T operator storage reconcile
reconcile-apply:
./scripts/compose.sh run --rm -T operator storage reconcile --apply
.PHONY: test-pipeline
test-pipeline:
$(PYTHON) student-tests/pipeline.py --base-url "$(BASE_URL)" --repeats 3 --output evidence/pipeline.json
.PHONY: test-images test-staging
test-images:
$(PYTHON) student-tests/images.py
test-staging:
$(PYTHON) student-tests/storage.py
.PHONY: test-broker-persistence
test-broker-persistence:
$(PYTHON) student-tests/broker_persistence.py
.PHONY: observe prove-correlation experiment-slo
observe:
$(PYTHON) scripts/observations.py
prove-correlation:
$(PYTHON) scripts/prove-correlation.py
experiment-slo:
$(PYTHON) scripts/experiment-slo.py
+203 -21
View File
@@ -1,37 +1,219 @@
# Отчёт — лабораторная 6
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
- Автор: **Поликарпов Даниил Александрович**.
- Группа: **БПИ2303**.
- Целевая оценка: **5**, трек 2 — SLO и обнаружение инцидентов.
- Ветка: `solution/bpi2303-polikarpov`. В репозитории есть commit `60f48c03375dbdf953cc9ffb0ccd2c385e069b6c` («Реализация на 5»); завершившиеся после него измерения и итоговый отчёт дополнены в рабочем дереве. Базовая ревизия, fingerprint исходников и image IDs фиксируются в `evidence/environment.json` и результатах исследования.
- Основа: собственная реализация соседней lab5. Использованы Go, pgx, AWS SDK, RabbitMQ amqp091-go, ImageMagick, OpenTelemetry Go и Grafana stack. Перенос, подготовка отчета и проверка выполнены с помощью Codex.
## Запуск из чистого clone
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
Нужны Docker с Compose, Python 3.9+ и для локальной сборки Go 1.27+. Контейнерная сборка закреплена на Go 1.27.2 / Alpine 3.23.6; ImageMagick 7.1.2.35-r0. Образы и библиотеки имеют фиксированные версии; фактические image IDs и ресурсы приведены в `evidence/environment.json`.
## Архитектура и решения
```sh
make check build vet
make up
make test test-go
make prove-correlation observe
python3 scripts/prove-signals.py
make test-storage
make test-images test-staging test-broker-persistence
make test-persistence ACTION_SCRIPT=scripts/restart.sh
make test-failover ACTION_SCRIPT=scripts/stop-one.sh
make restore
python3 student-tests/pipeline.py --repeats 1 --output evidence/pipeline.json
make experiment-slo
python3 scripts/summarize-slo.py
python3 scripts/analyze-backend.py
python3 scripts/prove-persistence.py
python3 scripts/capture-environment.py
make down
```
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
Все сценарии отказов запускаются последовательно, отдельно от исследования. Полное исследование длительное: реальное ожидание окон, три повтора, без ускорения часов и подмены метрик.
Полная валидация контракта:
```sh
python3 -m venv .venv
.venv/bin/python -m pip install -r requirements-dev.txt
make validate PYTHON=.venv/bin/python
```
`make up` идемпотентно создаёт локальные синтетические credentials в `.env.lab6` и `secrets/lab6`, запускает отдельный migrator, bootstrap приватного bucket и provisioning Grafana. Существующие credentials не ротируются. Новая конфигурация задаёт SESSION_TTL_SECONDS=14400 для длительного исследования; стандартный контрактный default без настройки остаётся 3600. Пароли и ключи находятся вне Git и не входят в Docker build context.
API: http://127.0.0.1:18086. Grafana: http://127.0.0.1:13006/d/messenger-lab6, login `admin`; пароль читается локально из `secrets/lab6/grafana_password` и не включается в доказательства.
Обычная остановка `make down` сохраняет данные. Намеренное удаление **только учебных данных lab6**: `./scripts/compose.sh down --volumes`; автоматически эта команда не выполняется. Стенд lab5 на 18085 имеет отдельные проект, volumes и секреты. Первый локальный запуск перенесённого Go suite ещё использовал старый default URL и создал синтетические записи в lab5; адрес исправлен, результат не засчитывался, проверки повторены на lab6.
## Архитектура и границы
```mermaid
flowchart LR
Client --> LB[HAProxy]
LB --> API[API x2]
API --> S3[Private S3 staging/results]
API --> DB[PostgreSQL metadata/jobs/outbox/SLO]
DB --> Dispatcher
Dispatcher --> Rabbit[RabbitMQ]
Rabbit --> Worker
Worker --> S3
Worker --> DB
API --> Collector[OTel Collector]
Worker --> Collector
Dispatcher --> Collector
Collector --> Prometheus
Collector --> Loki
Collector --> Tempo
Prometheus --> Grafana
Loki --> Grafana
Tempo --> Grafana
```
Handler разбирает HTTP, проверяет поля и ограничивает входной поток. Service проверяет доступ, управляет бизнес-состояниями, транзакциями, retries и решением good/bad для SLO. Repo/storage выполняет чтение и сохранение фактов; SQL-агрегаты формируют снимок состояния. `pkg/postgres` содержит только настройку доступа/пула, timeout и подключение безопасного tracer. Новая интеграция SDK расположена в `internal/pkg/telemetry`; обработка изображений и S3 также остаются в своих pkg.
Из lab5 перенесены API, сессии, cache, идемпотентность сообщений, durable staging, transactional outbox, RabbitMQ, worker, lease/retry/recovery, versioned manifest и операторские команды. Бизнес-контракт lab6 совпадает с lab5; сохранены контракт и публичные проверки текущего skeleton. Старые README/REPORT, секреты и evidence не переносились как результаты lab6.
До 202 проверенный оригинал находится в приватном S3 staging, а metadata/job/outbox фиксируются одной транзакцией. Worker публикует оригинал и варианты в S3 и переводит job в ready одной DB-транзакцией. At-least-once доставка и идемпотентный эффект сохранены; exactly-once доставка не заявляется. S3 staging и results — префиксы одного сервиса, а не независимые отказоустойчивые хранилища.
Миграция 010 добавляет durable trace carrier и таблицу однократных SLO-наблюдений; 011 — факт terminal failure. Метаданные готовности и terminal error записываются вместе с соответствующим бизнес-переходом. Observer в dispatcher оценивает deadline даже при остановленном worker. Повторная доставка/replay не отменяет уже зафиксированное нарушение.
### Компоненты и эксплуатация
| Роль → service | Версия | Readiness / persistent state / доступ |
| --- | --- | --- |
| API → api, две реплики | Go 1.27.2, собственный image | `/health/ready`: DB + S3; без durable local state, порты реплик закрыты |
| LB → lb | HAProxy 3.2-alpine, digest в Compose | HTTP readiness API; localhost:18086 |
| DB → postgres | PostgreSQL 17.11, digest | pg_isready; postgres_data; локальный пароль env |
| Cache → redis | Redis 8.2.11, digest | authenticated PING; redis_data/AOF; необязателен для корректности |
| S3 → s3 | SeaweedFS 4.52, digest | bootstrap проверяет запись; s3_data/internal network; credentials files |
| Broker → broker | RabbitMQ 4.2.0, digest | check_running; broker_data; durable queue/persistent delivery/confirms/ack |
| Worker / dispatcher | собственный image | отдельные процессы/reconnect; прогресс по jobs/observations, без HTTP healthcheck |
| Collector → otel-collector | image tag 0.140.0, binary сообщает 0.140.1 | :13133; ограниченные недолговечные очереди, не хранилище истории |
| Metrics → prometheus | 3.16.0 | /-/ready; prometheus_data; retention 7d / 1 GiB |
| Logs → loki | 3.7.10 | /ready; loki_data, TSDB v13/filesystem; retention 7d |
| Traces → tempo | 2.9.0 | /ready; tempo_data, WAL/local blocks; retention 7d |
| UI → grafana | 13.3.1 | /api/health; grafana_data + provisioning; localhost:13006, пароль file |
Внутренние OTLP/metrics/storage/admin порты не публикуются. `scripts/wait-ready.py` проверяет backend endpoints через внутреннюю сеть, а не только состояние running. `scripts/restart.sh` перезапускает API/worker/dispatcher и все бизнес- и telemetry-хранилища с volumes; жизненные события сохранены отдельно. Grafana — интерфейс, Collector — обработчик/экспортёр; долговременные сигналы хранятся в Prometheus/Loki/Tempo.
`.env.example` содержит логические рекомендации skeleton. Реальная конфигурация использует APP_*, POSTGRES_*, REDIS_*, SESSION_TTL_SECONDS, S3_* и *_FILE, BROKER_URL_FILE, WORKER_*, JOB_*, STAGING_RETENTION и OTEL_EXPORTER_OTLP_ENDPOINT/PROTOCOL. DATABASE_URL_FILE/CACHE_URL_FILE/SESSION_STORE из примера не используются: PostgreSQL настраивается отдельными полями, сессии хранятся в DB, Redis — кэш. Секундные JOB_TIMEOUT_SECONDS/STAGING_RETENTION_SECONDS сопоставлены duration-настройкам JOB_TIMEOUT/STAGING_RETENTION. Имена OTel services задаются ролью процесса, resource attributes — кодом Init.
### Сигналы и связь процессов
SDK: OTel Go 1.47.0, OTLP logs exporter 0.23.0, slog bridge 0.21.0; собственная схема instrumentation `messenger`; выбранное подмножество стандартных имён атрибутов закреплено по Semantic Conventions **1.37.0** (входит в закреплённый OTel Go SDK). Использованы имена атрибутов OTel для ресурсов/HTTP/DB; публичные метрики приложения имеют документированную собственную схему и не дублируют стандартные инструменты.
| Сигнал | Реальное имя в Prometheus / поля | Единицы и labels |
| --- | --- | --- |
| HTTP count / duration | messenger_http_requests_total / messenger_http_duration_seconds_bucket | counter: method/route/status_class; duration: method/route, секунды |
| Jobs | messenger_jobs_total | job_type, outcome=ready/retry/failed/duplicate |
| Processing duration | messenger_job_duration_seconds_bucket | job_type, outcome; секунды |
| Time to ready | messenger_job_ready_duration_seconds_bucket | job_type; секунды, после COMMIT |
| Queue | messenger_queue_depth / messenger_queue_oldest_age_seconds | queue; Rabbit ready count / возраст логической незавершённой job |
| Logical backlog | messenger_jobs_pending / messenger_outbox_pending | число jobs / публикаций, не взаимозаменяемы с broker depth |
| DB pool | messenger_db_pool_used/capacity/waiting; messenger_db_acquire_duration_seconds_bucket; messenger_db_locks_waiting | соединения, ожидающие acquire / секунды |
| Worker saturation | messenger_worker_busy/capacity | активные attempts / доступные slots |
| Process resources | messenger_process_memory / messenger_process_cpu | RSS bytes / накопленные CPU seconds |
| Ready SLO / freshness | messenger_ready_slo_total / messenger_observation_timestamp | outcome good/bad / Unix seconds |
| Export health | otelcol_* и messenger_telemetry_export_errors | accepted/sent/failed, queue size/capacity / SDK errors |
| Logs | timestamp, severity, service.name, event, trace_id, span_id, request_id | технические IDs только вне labels метрик |
Histograms: buckets 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2, 5, 10, 20, 30, 60, 120, 300 seconds. Пример p95: `histogram_quantile(0.95, sum by(le)(rate(messenger_http_duration_seconds_bucket[1m])))`. Resource attributes: service.name/version/instance.id, deployment.environment.name=lab. Default sampling — ParentBased(AlwaysSample).
W3C context сохраняется вместе с job/outbox. Dispatcher создаёт producer span из durable context и помещает его carrier в сообщение, worker создаёт дочерний consumer span. Retry наследует consumer context; recovery извлекает его из job. SQL-параметры, raw URL, Authorization, body и object keys не записываются. Error values редактируются до безопасной категории. В Loki service.instance.id и version хранятся как structured metadata; индексируются service.name и environment.
## Доказательства критериев
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
| Критерий | Команда / наблюдение | Доказательство |
| --- | --- | --- |
| Целостность и схемы | make check validate | evidence/contracts.txt |
| Сборка и статические проверки | make build vet | evidence/build.txt |
| Публичный API и прежние Go-проверки с telemetry | make test test-go | evidence/final-checks.txt |
| Изображения, staging, broker persistence | make test-images test-staging test-broker-persistence | evidence/previous-lab-checks.txt и соответствующие JSON |
| Storage transactions | make test-storage | evidence/storage-checks.txt |
| Persistence с прежним токеном | make test-persistence | evidence/persistence.txt, evidence/persistence-processes.json |
| API failover | make test-failover; затем make restore | evidence/failover.txt, evidence/failover-action.jsonl |
| API → dispatcher → worker, retry, S3/DB, logs | make prove-correlation | evidence/correlation.json, correlation-check.txt |
| Реальные метрики, безопасные labels, canary, dashboard queries | python3 scripts/prove-signals.py | evidence/signals.json, signal-checks.txt |
| Config/provisioning | promtool, Collector validate, Loki verify; readiness | evidence/prometheus-config.txt, collector-config.txt, loki-config.txt; infra/otel/ |
| SLI/SLO и alerts | реальные recording/alert rules, общий dashboard | infra/otel/alerts.yaml, RUNBOOK.md |
| Два инцидента и исследование | make experiment-slo; summarize-slo.py | evidence/slo-experiment.json, slo-summary.json |
| Отказ telemetry backend | остановка Tempo под той же нагрузкой | evidence/backend-outage.json, backend-checks.txt |
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
Итоговый прогон завершён успешно. Сводная проверка полноты: `evidence/research-validation.json`; исходные наблюдения не заменены расчётными или искусственными сигналами. В `evidence/pipeline.json` подтверждены 13/13 сценариев. После них повторно прошли smoke, существующие Go-проверки с race detector, correlation и privacy/dashboard checks. Проверка failover восстановила запись за 1.148s; 133 записи подтверждены, один переходный сбой учтён. Persistence подтвердил перезапуск всех восьми хранилищ с прежними volumes и сохранение старой сессии.
## Эксперименты и отказы
## Методика исследования
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
Гипотеза: правила burn-rate с двумя окнами отсекают краткий всплеск и обнаруживают устойчивую деградацию; независимый deadline observer выявляет нарушение ready-SLO при полном отсутствии завершённых jobs.
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
SLO: 99% бизнес-ответов без 5xx и 99% принятых jobs до ready ≤30s, окно 1h. Выбор 99% — учебный бюджет 1% событий за час: при постоянном профиле это соответствует примерно 36 секундам полного отказа. Он допускает единичный 5s всплеск, но не 240s деградацию. Для маленького fixture нормальная обработка существенно короче 30s; такой ready-бюджет оставляет запас на краткий backlog и несколько быстрых retries. Это целевой SLO для данного профиля, а не гарантия для любого изображения/нагрузки; публичный timeout smoke 60s остаётся отдельным условием.
## Ограничения и следующие шаги
Полные формулы, minimum traffic, timestamps и пределы интерпретации — в [runbook](infra/otel/RUNBOOK.md). Fast: 1m/5m, >6×, for20s. Slow: 2m/10m, >3×, for30s. Отдельно QueueLag >30s for20s и stale observations. Уведомления наружу не отправляются.
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
Нагрузка: open-loop 10 message requests/s, 90% GET history / 10% POST message, плюс fixture PNG 320×200 каждые 2s, in-flight ≤64, HTTP timeout6s, без retries. Перцентили успешных ответов включают scheduling delay; неуспешные статусы и пропуски генератора учитываются отдельно. RPS не является поиском максимальной пропускной способности.
После обнаружения ограничения S3 первый прогон остановлен и сохранён как `evidence/aborted-capacity-*`; причины и исправление описаны в `evidence/aborted-capacity-diagnosis.md`. Второй промежуточный прогон выявил потерю первого всплеска в rate для новых counter series; причина и исправление: `evidence/aborted-counter-diagnosis.md`. Итоговые counters заранее публикуют нули, duration histogram не разделяется по status class; pool=16 на процесс.
Для итогового прогона S3 допускает 16 volume slots с лимитом volume 1024 MiB, без удаления существующих данных.
Прогрев 15 минут. Три повтора остановки worker и блокировки messages отдельной ограниченной DB-транзакцией: короткое воздействие 5s, длительное 240s, после длительного — 660s восстановления с трафиком, чтобы очистить 10m окно. Короткие воздействия разделены 60s. Затем Tempo останавливается на 300s при работающих metrics/logs pipelines. Состояния alerts и метрики снимаются каждые ~5s, Docker CPU/RAM — каждые ~30s. Цикл включает время сбора, поэтому точность обнаружения ограничена фактическим интервалом наблюдений, SDK export/scrape/evaluation и планированием процессов.
Причины сначала устанавливаются по сохранённым сигналам, затем сопоставляются с отдельными timestamps инъекции. Для worker ожидается рост backlog/age без ухудшения HTTP; для DB — 5xx и длительные DB spans при работающем worker; для Tempo — экспортные ошибки/потери при сохранении бизнес-функций.
## Фактические результаты исследования
Прогон выполнен **16 ноября 2026, 08:17:58–10:16:11 UTC**, длительность **7092.82s (1ч 58мин 13с)**. Итоговая сверка артефактов выполнена 17 ноября. Аппаратная конфигурация: Apple M4 Pro, 14 CPU, host RAM 24 GiB; Docker VM — 14 CPU, 7.75 GiB. Лимиты приложены в `evidence/environment.json`. У генератора не было пропусков, у сборщика — ошибок чтения наблюдений.
Всего **74 094** запроса: **66 860** успешных и **7 234** ответа 503 при намеренных воздействиях на messages. Таймаутов клиента, 401 и 429 в итоговом прогоне нет. Все **6 334** подтверждённых сообщения найдены в PostgreSQL; все **3 529** принятых вложений в конце достигли ready. Исходные PNG — по 111 817 bytes; принятый объём оригиналов **394 602 193 bytes**. История сообщений росла в ходе прогона; профиль читает первую страницу, а не доказывает максимальную пропускную способность на произвольном объёме данных.
| Интервал | Успешные RPS | Доля ошибок | p50 / p95 / p99 успешных ответов, ms |
| --- | ---: | ---: | --- |
| Прогрев, 900s | 10.50 | 0% | 8.37 / 16.35 / 25.75 |
| Worker остановлен, повтор 1, 240s | 10.50 | 0% | 8.79 / 17.55 / 28.45 |
| DB lock, повтор 1, 240s | 0.50 | 95.2% | 27.06 / 41.26 / 48.94 |
| Tempo остановлен, 300s | 10.50 | 0% | 6.96 / 16.46 / 25.76 |
Во время DB lock успешными оставались главным образом upload; низкий percentile **успешных** ответов не означает здоровую работу messages. Медленные ошибки видны в HTTP histogram и DB spans. Для всего прогона p50/p95/p99 успешных ответов: 7.84/17.53/29.31ms. Интервальные статусы классифицируются по моменту завершения ответа; обращения на границах фаз могут завершиться между операторскими действиями. Все ответы входят в общий итог.
### Обнаружение и восстановление, три повтора
Время ниже — первое сохранённое наблюдение состояния, а не точный момент пересечения порога. Точность ограничена экспортом, scrape/evaluation и опросом; дробные исходные значения находятся в `evidence/slo-summary.json`.
| Alert | До первого firing, s: минимум–максимум | До resolved после восстановления, s: минимум–максимум |
| --- | ---: | ---: |
| QueueLag | 59.91–64.88 | 12.54–17.60 |
| ReadyBurnRateFast | 72.20–81.44 | 64.20–72.46 |
| ReadyBurnRateSlow | 87.29–91.54 | 123.94–129.35 |
| ApiBurnRateFast | 44.78–53.87 | 55.14–64.97 |
| ApiBurnRateSlow | 54.89–63.98 | 119.53–124.81 |
На **6 коротких воздействиях** firing не наблюдался; на **6 длительных** все ожидаемые alerts сработали и затем разрешились. Это результат данного профиля и трёх повторов, а не доказательство нулевой вероятности ложных срабатываний. Отдельное сравнение с правилами без второго окна/без for не проводилось, поэтому вклад каждого механизма фильтрации из этого эксперимента не выделяется. Целевой SLO во время намеренных деградаций нарушается — обнаружение этого нарушения и проверяется.
### Диагноз по сохранённым сигналам
- **Worker:** на каждом длительном интервале растут возраст и backlog, новых `job_received` нет, upload продолжает получать 202, API burn rate равен нулю. В первой сохранённой выборке — 35 успешных upload и 0 событий потребления. После запуска появляются consumer/S3/DB spans, очередь убывает, jobs становятся ready. Ready counter учитывал bad по deadline ещё до завершения jobs.
- **DB:** одновременно наблюдаются около 20 ожиданий Lock, HTTP 5xx и DB client spans длительностью около двух секунд (первая сохранённая трасса — `postgresql SELECT` 1995.4ms). При этом ready burn rate остаётся нулевым. Это указывает на ожидание БД, а не на преобразование изображений или медленный код HTTP handler. Конкретная операторская блокировка messages сверена с журналом воздействия.
Девять компактных наборов фактов (worker stop, recovery и DB wait для каждого повтора) сохранены в `evidence/incident-signals.json`; ошибок их сбора нет. [Runbook](infra/otel/RUNBOOK.md) задаёт те же шаги диагностики для другого студента.
### Отказ Tempo и границы буферизации
За 300s недоступности Tempo генератор получил 3150 успешных ответов: 2700 чтений, 300 сообщений и 150 upload; ошибок нет. После запуска Tempo вновь доступны traces. Очередь exporter достигла **32/32 запросов** и после восстановления стала **0**. Максимальная наблюдавшаяся память Collector — **93.84 MiB** при лимите контейнера **256 MiB**; в этот интервал его CPU составлял медианно 0.61%, максимум 3.51% одного ядра по выборкам Docker. API-реплики использовали примерно 70–77 MiB, dispatcher — 15 MiB, worker — 16 MiB в последних выборках этого интервала.
По счётчикам за outage/recovery/drain: принято 51 598 spans, отправлено 13 416, разница — **около 38 182**. Зафиксированы 35 676 enqueue-failed spans и 2524 dropped items в отдельных записях exporter. Баланс приблизительный: остаются фоновые health/observer spans и несколько in-flight batches. Он не включает возможные потери до Collector; send-failed нельзя механически считать уникальными потерями. Telemetry в этом отказе терялась, бизнес-данные сохранились. Подробности: `evidence/backend-outage.json`.
Ограничение очереди подтверждается одновременно конфигурацией и наблюдаемым плато; пятиминутная выборка RAM не является доказательством отсутствия любого возможного memory leak на произвольной длительности. CPU/RAM снимались примерно раз в 30s и могли пропустить короткие пики.
### Версии артефактов
Research записал OCI descriptor `d661ef49…`, повторный pipeline — `c7021c3f…`. Compose добавляет разные service labels и attestations при сборке одного Dockerfile для ролей, поэтому descriptor не является хешем бинарника. Платформенный manifest текущего dispatcher совпал с manifest в сохранённом логе сборки перед research; SHA-256 исполняемого файла — `33c6b59d…`. Доказательство: `evidence/image-provenance.json`; полный fingerprint исходников и фактические container images — в `evidence/environment.json`.
## Ограничения
Один host, PostgreSQL, S3, broker и LB остаются точками отказа. HA хоста/БД и потеря volumes не проверяются. CPU/RAM ограничены Compose; рядом работал старый стенд lab5, поэтому результаты относятся к этой конфигурации и не являются универсальным benchmark.
HTTP SLI считает ответы приложения: сетевые отказы/LB 503 измеряются генератором отдельно, а 4xx входят в denominator. Ready-SLI относится к серверной публикации результата, не скачиванию файла клиентом. Пока DB недоступна, observer не может зафиксировать новые результаты, что должно быть видно по freshness. Клиентские timestamps и серверные timestamps имеют разные границы измерения. accepted_at и ready_at записываются внутри соответствующих транзакций; классификация непосредственно возле 30s зависит от длительности COMMIT. Histogram percentiles также являются интерполяцией внутри заданных buckets, а не точными клиентскими квантилями.
Telemetry best effort: ограниченные SDK/Collector очереди допускают потерю сигналов при долгом отказе; sent-failed считает попытки, а не уникальные потерянные spans. Рестарт теряет недолговечные очереди. Retention ограничивает срок хранения; для Loki/Tempo нет отдельной гарантии дисковой квоты/backup. Credentials учебные, HTTP допускается только на localhost. TLS/mTLS и управление секретами относятся к lab7.
Источники: [OpenTelemetry Go](https://opentelemetry.io/docs/languages/go/), [Collector configuration](https://opentelemetry.io/docs/collector/configuration/), [Loki native OTLP](https://grafana.com/docs/loki/latest/send-data/otel/), [Grafana provisioning](https://grafana.com/docs/grafana/latest/administration/provisioning/), [Prometheus histograms](https://prometheus.io/docs/practices/histograms/), [RabbitMQ reliability](https://www.rabbitmq.com/docs/reliability).
+22
View File
@@ -0,0 +1,22 @@
# Explicit isolated experiment profile. Never enabled by make up.
services:
api:
environment:
LAB5_FAULTS: "1"
volumes:
- ./.lab6-faults:/faults
worker:
environment:
LAB5_FAULTS: "1"
JOB_TIMEOUT: 5s
JOB_LEASE: 8s
volumes:
- ./.lab6-faults:/faults
dispatcher:
environment:
LAB5_FAULTS: "1"
volumes:
- ./.lab6-faults:/faults
operator:
environment:
LAB5_FAULTS: "1"
+297
View File
@@ -0,0 +1,297 @@
name: mtusi2026lab6
x-api: &api
image: ${API_IMAGE:-mtusi-messenger:lab6}
build: .
restart: unless-stopped
cpus: ${API_CPUS:-1}
mem_limit: ${API_MEMORY:-512m}
environment: &environment
APP_ENABLED: "true"
APP_HOST: 0.0.0.0
APP_PORT: 8080
APP_READ_TIMEOUT: 125s
APP_WRITE_TIMEOUT: 125s
APP_IDLE_TIMEOUT: 60s
APP_SHUTDOWN_TIMEOUT: 8s
APPLICATION_LOG_LEVEL: ${APPLICATION_LOG_LEVEL:-info}
OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318
OTEL_EXPORTER_OTLP_PROTOCOL: http/protobuf
OTEL_SDK_DISABLED: ${OTEL_SDK_DISABLED:-false}
APPLICATION_ENVIRONMENT: prod
APPLICATION_SHUTDOWN_TIMEOUT: 8s
POSTGRES_HOST: postgres
POSTGRES_PORT: 5432
POSTGRES_USER: messenger
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Run scripts/bootstrap.sh}
POSTGRES_DATABASE: messenger
POSTGRES_VERSION_TABLE: versions
POSTGRES_RUN_MIGRATIONS: "false"
POSTGRES_MAX_CONNECTIONS: ${POSTGRES_MAX_CONNECTIONS:-16}
POSTGRES_CONNECT_TIMEOUT: 500ms
POSTGRES_QUERY_TIMEOUT: 2s
REDIS_HOST: redis
REDIS_PORT: 6379
REDIS_DB: 0
REDIS_USERNAME: default
REDIS_PASSWORD: ${REDIS_PASSWORD:?Run scripts/bootstrap.sh}
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
JWT_ACCESS_TOKEN_LIFETIME: 15m
JWT_PRIVATE_KEY_PATH: /run/certs/private_key.pem
JWT_PUBLIC_KEY_PATH: /run/certs/public_key.pem
S3_ENDPOINT: http://s3:9000
S3_REGION: us-east-1
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret
BROKER_URL_FILE: /run/secrets/broker_url
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
WORKER_PREFETCH: ${WORKER_PREFETCH:-2}
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
JOB_TIMEOUT: ${JOB_TIMEOUT:-60s}
JOB_LEASE: ${JOB_LEASE:-90s}
STAGING_RETENTION: ${STAGING_RETENTION:-24h}
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
UPLOAD_TIMEOUT: ${UPLOAD_TIMEOUT:-120s}
RECONCILE_GRACE_PERIOD: ${RECONCILE_GRACE_PERIOD:-1h}
GOMEMLIMIT: 384MiB
networks: [default, storage]
volumes:
- ./secrets/lab6/broker_url:/run/secrets/broker_url:ro
- ./secrets/lab6/private_key.pem:/run/certs/private_key.pem:ro
- ./secrets/lab6/public_key.pem:/run/certs/public_key.pem:ro
- ./secrets/lab6/s3_api_access:/run/secrets/s3_access:ro
- ./secrets/lab6/s3_api_secret:/run/secrets/s3_secret:ro
stop_grace_period: 10s
services:
api:
<<: *api
deploy:
replicas: ${API_REPLICAS:-2}
depends_on:
migrate:
condition: service_completed_successfully
s3-init:
condition: service_completed_successfully
migrate:
<<: *api
# The migration command uses only PostgreSQL: no signing keys or Redis credentials.
volumes: []
environment:
<<: *environment
REDIS_PASSWORD: ""
restart: "no"
command: [migrate]
cpus: 0.5
mem_limit: 128m
healthcheck:
disable: true
depends_on:
postgres:
condition: service_healthy
broker:
hostname: broker
image: rabbitmq:4.2.0-alpine@sha256:32962eea9bf99928c2fc541a7defb78c7feca9df0434c5dfb7b539d61e1af524
restart: unless-stopped
cpus: 0.5
mem_limit: 512m
environment:
RABBITMQ_DEFAULT_USER: messenger
RABBITMQ_DEFAULT_PASS: ${BROKER_PASSWORD:?Run scripts/bootstrap.sh}
volumes:
- broker_data:/var/lib/rabbitmq
healthcheck:
test: [CMD, rabbitmq-diagnostics, -q, check_running]
interval: 3s
timeout: 5s
retries: 30
stop_grace_period: 30s
worker:
<<: *api
command: [worker]
cpus: 1
mem_limit: 1g
stop_grace_period: 10s
healthcheck:
disable: true
depends_on:
migrate:
condition: service_completed_successfully
s3-init:
condition: service_completed_successfully
broker:
condition: service_healthy
dispatcher:
<<: *api
command: [dispatcher]
cpus: 0.25
mem_limit: 128m
healthcheck:
disable: true
depends_on:
migrate:
condition: service_completed_successfully
broker:
condition: service_healthy
lb:
image: ${HAPROXY_IMAGE:-haproxy:3.2-alpine@sha256:96ec3c75a9264ba94e733eeacb44051ed6e7b79bb77a73c2c36f236b6423f1f0}
restart: unless-stopped
cpus: 0.5
mem_limit: 128m
ports:
- "127.0.0.1:${HOST_PORT:-18086}:8080"
volumes:
- ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
depends_on:
api:
condition: service_started
healthcheck:
test: [CMD, wget, -q, -O, /dev/null, http://127.0.0.1:8080/health/ready]
interval: 2s
timeout: 1s
retries: 5
stop_grace_period: 10s
postgres:
image: postgres:17.11-alpine3.23@sha256:05aa77262db9277fde6374bb2f901c9604b8a9373029e1a7993440c6aae48736
restart: unless-stopped
cpus: 2
mem_limit: 1g
environment:
POSTGRES_USER: messenger
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Run scripts/bootstrap.sh}
POSTGRES_DB: messenger
volumes:
- postgres_data:/var/lib/postgresql/data
healthcheck:
test: [CMD-SHELL, 'pg_isready -U messenger -d messenger']
interval: 2s
timeout: 1s
retries: 30
stop_grace_period: 30s
redis:
image: redis:8.2.11-alpine3.22@sha256:489a769e0a54dc245896132a15c72d2bc7296719954da99398f43a3877e5f8eb
restart: unless-stopped
cpus: 0.25
mem_limit: 128m
environment:
REDIS_PASSWORD: ${REDIS_PASSWORD:?Run scripts/bootstrap.sh}
command: [sh, -c, 'exec docker-entrypoint.sh redis-server --appendonly yes --maxmemory 64mb --maxmemory-policy allkeys-lru --requirepass "$$REDIS_PASSWORD"']
volumes:
- redis_data:/data
healthcheck:
test: [CMD-SHELL, 'REDISCLI_AUTH="$$REDIS_PASSWORD" redis-cli --no-auth-warning ping | grep -qx PONG']
interval: 2s
timeout: 1s
retries: 20
stop_grace_period: 10s
s3:
image: chrislusf/seaweedfs:4.52@sha256:57877e32bb90073b37dc109b438ba66e84fd5ed9ced5f5c04f56444ef243ce59
restart: unless-stopped
cpus: 1
mem_limit: 512m
networks: [storage]
command: [server, '-dir=/data', '-ip=s3', '-ip.bind=0.0.0.0', '-volume.max=16', '-master.volumeSizeLimitMB=1024', '-filer', '-s3', '-s3.port=9000', '-s3.config=/etc/seaweedfs/s3.json']
volumes:
- s3_data:/data
- ./secrets/lab6/s3.json:/etc/seaweedfs/s3.json:ro
healthcheck:
test: [CMD, wget, -q, -O, /dev/null, http://127.0.0.1:9333/cluster/status]
interval: 2s
timeout: 1s
retries: 30
stop_grace_period: 30s
s3-init:
<<: *api
restart: "no"
cpus: 0.25
mem_limit: 128m
command: [storage, bootstrap]
volumes:
- ./secrets/lab6/s3_admin_access:/run/secrets/s3_access:ro
- ./secrets/lab6/s3_admin_secret:/run/secrets/s3_secret:ro
healthcheck:
disable: true
depends_on:
s3:
condition: service_healthy
operator:
<<: *api
profiles: [tools]
restart: "no"
command: [storage, reconcile]
volumes:
- ./secrets/lab6/s3_admin_access:/run/secrets/s3_access:ro
- ./secrets/lab6/s3_admin_secret:/run/secrets/s3_secret:ro
- ./tests/fixtures:/fixtures:ro
- ./secrets/lab6/broker_url:/run/secrets/broker_url:ro
healthcheck:
disable: true
depends_on: {}
otel-collector:
image: otel/opentelemetry-collector-contrib:0.140.0
restart: unless-stopped
command: [--config=/etc/otelcol/config.yaml]
cpus: 0.5
mem_limit: 256m
volumes:
- ./infra/otel/collector.yaml:/etc/otelcol/config.yaml:ro
prometheus:
image: prom/prometheus:v3.16.0
restart: unless-stopped
command: [--config.file=/etc/prometheus/prometheus.yaml, --storage.tsdb.retention.time=7d, --storage.tsdb.retention.size=1GB, --web.enable-lifecycle]
cpus: 0.5
mem_limit: 384m
volumes:
- ./infra/otel/prometheus.yaml:/etc/prometheus/prometheus.yaml:ro
- ./infra/otel/alerts.yaml:/etc/prometheus/alerts.yaml:ro
- prometheus_data:/prometheus
loki:
image: grafana/loki:3.7.10
restart: unless-stopped
command: [-config.file=/etc/loki/config.yaml]
cpus: 0.5
mem_limit: 384m
volumes:
- ./infra/otel/loki.yaml:/etc/loki/config.yaml:ro
- loki_data:/loki
tempo:
image: grafana/tempo:2.9.0
restart: unless-stopped
command: [-config.file=/etc/tempo.yaml]
user: '0:0'
cpus: 0.5
mem_limit: 384m
volumes:
- ./infra/otel/tempo.yaml:/etc/tempo.yaml:ro
- tempo_data:/var/tempo
grafana:
image: grafana/grafana:13.3.1
restart: unless-stopped
cpus: 0.5
mem_limit: 384m
ports: ['127.0.0.1:13006:3000']
environment:
GF_SECURITY_ADMIN_PASSWORD__FILE: /run/secrets/grafana_password
GF_AUTH_ANONYMOUS_ENABLED: 'false'
GF_ANALYTICS_REPORTING_ENABLED: 'false'
GF_ANALYTICS_CHECK_FOR_UPDATES: 'false'
volumes:
- grafana_data:/var/lib/grafana
- ./secrets/lab6/grafana_password:/run/secrets/grafana_password:ro
- ./infra/otel/grafana:/etc/grafana/provisioning:ro
- ./infra/otel/dashboards:/var/lib/grafana/dashboards:ro
volumes:
prometheus_data:
loki_data:
tempo_data:
grafana_data:
postgres_data:
redis_data:
s3_data:
broker_data:
networks:
storage:
internal: true
+14
View File
@@ -3,3 +3,17 @@
Храните небольшие обезличенные результаты: команды и stdout проверок, JSON/CSV агрегатов нагрузки, EXPLAIN, dashboard JSON, текст разбора отказа. Укажите дату, commit, аппаратные лимиты и профиль нагрузки.
Не добавляйте .env, session tokens, cookies, приватные ключи, credentials, реальные сообщения, raw dumps и signed URLs. Для скриншотов скрывайте секреты. Большие логи/бинарные traces сохраняйте отдельно по правилам преподавателя; одного скриншота без методики недостаточно.
## Итоговый набор lab6
- `research-validation.json`, `slo-summary.json` — полнота и времена обнаружения/восстановления.
- `slo-experiment.json`, `slo-run.txt` — исходные наблюдения, нагрузка и ресурсы; прогон 16.11.2026, 08:17–10:16 UTC.
- `incident-signals.json`, `correlation.json` — реальные trace/log связи и диагнозы.
- `backend-outage.json` — ограниченная очередь, RAM и потери при отказе Tempo.
- `persistence-processes.json`, `failover-action.jsonl`, `pipeline.json` — реальные воздействия на процессы/volumes и 13 сценариев pipeline.
- `final-checks.txt`, `contracts.txt`, `build.txt`, `signal-checks.txt` — результаты проверок.
- `environment.json`, `image-provenance.json` — среда, fingerprint исходников и образы.
- `aborted-capacity-*`, `aborted-counter-*` — незавершённые промежуточные прогоны; они не считаются успешным исследованием.
Методика, команды, результаты и ограничения: [REPORT.md](../REPORT.md). Даты измерений сохранены; итоговая проверка отчёта выполнена 17.11.2026. Секреты и локальные абсолютные пути не являются частью доказательств.
+16
View File
@@ -0,0 +1,16 @@
# Незавершённый первый прогон
2026-11-16 UTC. См. `aborted-capacity-slo-experiment.json` и `aborted-capacity-slo-run.txt`.
За 996.6 s генератор получил 437 подтверждённых upload, 30 transport/timeouts и 31 ответ 429; после короткой остановки worker восстановление завершилось timeout запуска зависимого s3-init. Этот прогон не является доказательством успешно завершённого исследования.
Наблюдавшийся native log SeaweedFS около 07:49 UTC:
```text
No writable volumes and no free volumes left
create 7 volume, created 0: Not enough data nodes found!
```
В перенесённой конфигурации были `-volume.max=4` и `-master.volumeSizeLimitMB=128`. Ограничение роста writable volumes мешало записи данных/служебного журнала filer. API upload зависал до timeout, затем заполнялись upload slots и возвращался 429. Нулевой HTTP 5xx burn rate не означал отсутствие клиентских ошибок: timeout и 429 не входят в numerator серверного SLI.
Исправлено без удаления volumes: volume.max=16, volumeSizeLimitMB=1024; worker/Tempo в эксперименте восстанавливаются прямым docker start существующего контейнера, без повторного запуска одноразовых зависимостей. После восстановления все существовавшие jobs были ready (487 на момент проверки), публичный smoke PASS. Затем основной эксперимент запущен заново с полным прогревом.
@@ -0,0 +1,4 @@
{
"evidence": {},
"collection_errors": []
}
File diff suppressed because it is too large Load Diff
+44
View File
@@ -0,0 +1,44 @@
baseline 900 seconds
baseline 2 s 24 requests
baseline 66 s 697 requests
baseline 131 s 1375 requests
baseline 195 s 2050 requests
baseline 260 s 2727 requests
baseline 324 s 3408 requests
baseline 389 s 4081 requests
baseline 453 s 4761 requests
baseline 517 s 5432 requests
baseline 582 s 6113 requests
baseline 646 s 6787 requests
baseline 711 s 7467 requests
baseline 775 s 8142 requests
baseline 840 s 8817 requests
worker_short_1 5 seconds
worker_short_1 2 s 9477 requests
Traceback (most recent call last):
File "<repo>/scripts/experiment-slo.py", line 143, in <module>
if __name__=='__main__':main()
^^^^^^
File "<repo>/scripts/experiment-slo.py", line 113, in main
worker(5,f'worker_short_{i}');phase(f'worker_short_recovery_{i}',60)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "<repo>/scripts/experiment-slo.py", line 96, in worker
finally:compose('start','worker')
^^^^^^^^^^^^^^^^^^^^^^^^^
File "<repo>/scripts/experiment-slo.py", line 12, in compose
def compose(*a):return subprocess.run(C+list(a),cwd=ROOT,check=True,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL,timeout=90)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/subprocess.py", line 550, in run
stdout, stderr = process.communicate(input, timeout=timeout)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/subprocess.py", line 1209, in communicate
stdout, stderr = self._communicate(input, endtime, timeout)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/subprocess.py", line 2141, in _communicate
self.wait(timeout=self._remaining_time(endtime))
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/subprocess.py", line 1264, in wait
return self._wait(timeout=timeout)
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/subprocess.py", line 2045, in _wait
raise TimeoutExpired(self.args, timeout)
subprocess.TimeoutExpired: Command '['<repo>/scripts/compose.sh', 'start', 'worker']' timed out after 89.99991420796141 seconds
+15
View File
@@ -0,0 +1,15 @@
# Промежуточная проверка учёта первого всплеска
После исправления ёмкости S3 прогрев дал 9452 успешных ответа без ошибок. Короткая DB-блокировка дала 38 ответов 503 в генераторе. В native counters API было 30 business 5xx; часть остальных ответов создал LB при исчерпании DB pool и временной неготовности API.
У новых 5xx time series первая точка Prometheus уже содержала весь короткий всплеск. Поэтому rate/increase показывали 0, несмотря на ненулевые raw counters. Этот прогон остановлен, а отсутствие alert не засчитано как успешное подавление короткого всплеска.
Исправления:
- При регистрации каждого маршрута HTTP counters публикуются с нулевыми значениями всех status classes. Первый всплеск имеет предшествующие нулевые samples.
- HTTP duration histogram объединяет status classes, сохраняя method/route: percentiles включают реальные медленные ошибки в уже существующих сериях без искусственных нулевых измерений.
- Jobs counters также инициализируются нулями известных outcomes.
- Добавлена gauge ожиданий Lock из pg_stat_activity, без SQL text/IDs; root cause можно отличить по сохранённым сигналам.
- DB pool в учебном профиле увеличен с 8 до 16 на процесс: при 10 message requests/s и timeout2s остаётся место для health. Это ограничивает побочное переключение LB, не снимая блокировку messages.
Полное исследование затем запускается заново на исправленном образе. Промежуточные артефакты: `aborted-counter-*`.
@@ -0,0 +1,62 @@
{
"evidence": {
"worker_long_1": {
"captured": 1794816601.2943192,
"observed": {
"at": 1794816600.582247,
"phase": "worker_long_1",
"alerts": [
{
"labels": {
"alertname": "ReadyBurnRateFast",
"severity": "warning",
"slo": "ready"
},
"annotations": {
"runbook": "infra/otel/RUNBOOK.md",
"summary": "ready SLO budget burning above 6x"
},
"state": "firing",
"activeAt": "2026-11-16T08:09:39.758725496Z",
"value": "1e+00"
},
{
"labels": {
"alertname": "ReadyBurnRateSlow",
"severity": "warning",
"slo": "ready"
},
"annotations": {
"runbook": "infra/otel/RUNBOOK.md",
"summary": "ready SLO budget burning above 3x"
},
"state": "pending",
"activeAt": "2026-11-16T08:09:44.758725496Z",
"value": "4.0476190476190477e-01"
},
{
"labels": {
"alertname": "QueueLag",
"severity": "warning"
},
"annotations": {
"runbook": "infra/otel/RUNBOOK.md",
"summary": "Unfinished attachment age exceeds 30s"
},
"state": "firing",
"activeAt": "2026-11-16T08:09:29.758725496Z",
"value": "6.3674393e+01"
}
],
"pending": 32.0,
"queue_age": 63.674393,
"api_burn": 0.0,
"ready_burn": 100.0
},
"worker_received_events": 0,
"accepted_upload_events": 35,
"diagnosis": "Queue age and ready burn increase while API accepts uploads and no worker consumption is observed"
}
},
"collection_errors": []
}
File diff suppressed because it is too large Load Diff
+45
View File
@@ -0,0 +1,45 @@
baseline 900 seconds
baseline 2 s 24 requests
baseline 66 s 698 requests
baseline 131 s 1378 requests
baseline 195 s 2049 requests
baseline 260 s 2728 requests
baseline 324 s 3404 requests
baseline 389 s 4083 requests
baseline 453 s 4758 requests
baseline 517 s 5434 requests
baseline 582 s 6116 requests
baseline 646 s 6789 requests
baseline 711 s 7469 requests
baseline 775 s 8140 requests
baseline 840 s 8819 requests
worker_short_1 5 seconds
worker_short_1 2 s 9479 requests
worker_short_recovery_1 60 seconds
worker_short_recovery_1 2 s 9532 requests
db_short_1 5 seconds
db_short_1 2 s 10144 requests
db_short_recovery_1 60 seconds
db_short_recovery_1 2 s 10219 requests
worker_long_1 240 seconds
worker_long_1 2 s 10849 requests
worker_long_1 66 s 11525 requests
worker_long_1 131 s 12207 requests
worker_long_1 194 s 12869 requests
Traceback (most recent call last):
File "<repo>/scripts/experiment-slo.py", line 147, in <module>
if __name__=='__main__':main()
^^^^^^
File "<repo>/scripts/experiment-slo.py", line 119, in main
worker(a.long,f'worker_long_{i}');phase(f'worker_recovery_{i}',a.cooldown)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "<repo>/scripts/experiment-slo.py", line 99, in worker
try:phase(name,duration)
^^^^^^^^^^^^^^^^^^^^
File "<repo>/scripts/experiment-slo.py", line 92, in phase
save();time.sleep(min(5,max(0,at+seconds-time.time())))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "<repo>/scripts/experiment-slo.py", line 67, in interrupted
def interrupted(*_):raise KeyboardInterrupt()
^^^^^^^^^^^^^^^^^^^^^^^^^
KeyboardInterrupt
+1
View File
@@ -0,0 +1 @@
Saved backend outage balance, queue, memory and business persistence evidence
File diff suppressed because it is too large Load Diff
+10
View File
@@ -0,0 +1,10 @@
{
"result": "PASS",
"attachment_id": "6e909ff4-fb98-496d-a0bf-adc4ac734463",
"broker_hostname": "broker",
"old_container_id": "4ad7e02f4e35c2a6907afafb3a4bcb48c13c4ddefb09322b46bba644984b3e04",
"new_container_id": "56a8639818cd4c10d3c6f521f03a72132c0535824f5bd815906cefc717b6893a",
"message_present_before_and_after": true,
"outbox_published_before_replacement": true,
"dispatcher_stopped_during_replacement": true
}
+3
View File
@@ -0,0 +1,3 @@
cd service && go build ./...
cd service && go build -o ../bin/benchmark ./cmd/benchmark
cd service && go vet ./...
+2
View File
@@ -0,0 +1,2 @@
Container mtusi2026lab6-otel-collector-run-63fc05410c49 Creating
Container mtusi2026lab6-otel-collector-run-63fc05410c49 Created
+4
View File
@@ -0,0 +1,4 @@
.venv/bin/python scripts/check_contract.py
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
.venv/bin/python scripts/validate_openapi.py
PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.
+3
View File
@@ -0,0 +1,3 @@
python3 scripts/prove-correlation.py
PASS: correlated upload, retry=False
PASS: correlated upload, retry=True
File diff suppressed because it is too large Load Diff
+10
View File
@@ -0,0 +1,10 @@
{
"verified_at_utc": "2026-11-17T09:13:49.849327+00:00",
"checked_files": 259,
"generated_secret_values_absent": true,
"secret_files_ignored": true,
"absolute_repository_paths_removed_from_text_evidence": true,
"current_smoke_passed": true,
"current_correlation_and_privacy_checks_passed": true,
"dashboard_pool_worker_backlog_exporter_have_separate_queries": true
}
+194
View File
@@ -0,0 +1,194 @@
{
"timestamp": 1794906731.3549702,
"revision": "8b381c115209cf7ac143df4826c468bf38128d71",
"working_tree_dirty": true,
"source_sha256": "efaad10cbbe055be8252463aaab6258fcc6b10b2e20211b4dcd56195bcecc991",
"platform": "macOS-26.5.1-arm64-arm-64bit",
"go": "go version go1.27.0 darwin/arm64",
"docker": {
"server_version": "29.7.2",
"cpus": 14,
"memory": 8320954368,
"architecture": "aarch64"
},
"containers": [
{
"service": "api",
"image": "mtusi-messenger:lab6",
"image_id": "sha256:c7021c3fed2700b5234fedad71dd0478acb126a697aa25ec5ac1a2c99020962b",
"started_at": "2026-11-16T10:20:53.477815219Z",
"memory_limit": 536870912,
"nano_cpus": 1000000000,
"mounts": []
},
{
"service": "api",
"image": "mtusi-messenger:lab6",
"image_id": "sha256:c7021c3fed2700b5234fedad71dd0478acb126a697aa25ec5ac1a2c99020962b",
"started_at": "2026-11-16T10:20:53.332318052Z",
"memory_limit": 536870912,
"nano_cpus": 1000000000,
"mounts": []
},
{
"service": "broker",
"image": "rabbitmq:4.2.0-alpine@sha256:32962eea9bf99928c2fc541a7defb78c7feca9df0434c5dfb7b539d61e1af524",
"image_id": "sha256:32962eea9bf99928c2fc541a7defb78c7feca9df0434c5dfb7b539d61e1af524",
"started_at": "2026-11-16T10:17:58.357427221Z",
"memory_limit": 536870912,
"nano_cpus": 500000000,
"mounts": [
{
"type": "volume",
"name": "mtusi2026lab6_broker_data",
"destination": "/var/lib/rabbitmq"
}
]
},
{
"service": "dispatcher",
"image": "mtusi-messenger:lab6",
"image_id": "sha256:c7021c3fed2700b5234fedad71dd0478acb126a697aa25ec5ac1a2c99020962b",
"started_at": "2026-11-16T10:20:53.325688052Z",
"memory_limit": 134217728,
"nano_cpus": 250000000,
"mounts": []
},
{
"service": "grafana",
"image": "grafana/grafana:13.3.1",
"image_id": "sha256:0615ec4fbd4072004b7d86b46ded63013a74b7d6eb1119273dd89e8fa25c13cf",
"started_at": "2026-11-16T10:16:16.370235216Z",
"memory_limit": 402653184,
"nano_cpus": 500000000,
"mounts": [
{
"type": "volume",
"name": "mtusi2026lab6_grafana_data",
"destination": "/var/lib/grafana"
}
]
},
{
"service": "lb",
"image": "haproxy:3.2-alpine@sha256:96ec3c75a9264ba94e733eeacb44051ed6e7b79bb77a73c2c36f236b6423f1f0",
"image_id": "sha256:96ec3c75a9264ba94e733eeacb44051ed6e7b79bb77a73c2c36f236b6423f1f0",
"started_at": "2026-11-16T10:17:15.316601507Z",
"memory_limit": 134217728,
"nano_cpus": 500000000,
"mounts": []
},
{
"service": "loki",
"image": "grafana/loki:3.7.10",
"image_id": "sha256:b43c257ccfc925d4a50b8717cac61a3b51df8c48d01847936057e5c4075a24a7",
"started_at": "2026-11-16T10:16:18.125747508Z",
"memory_limit": 402653184,
"nano_cpus": 500000000,
"mounts": [
{
"type": "volume",
"name": "mtusi2026lab6_loki_data",
"destination": "/loki"
}
]
},
{
"service": "otel-collector",
"image": "otel/opentelemetry-collector-contrib:0.140.0",
"image_id": "sha256:ffc818ee108d0b934fd14207fd87ff247c9b64f344ed349d0b66c166c18a2312",
"started_at": "2026-11-16T10:16:17.9785623Z",
"memory_limit": 268435456,
"nano_cpus": 500000000,
"mounts": []
},
{
"service": "postgres",
"image": "postgres:17.11-alpine3.23@sha256:05aa77262db9277fde6374bb2f901c9604b8a9373029e1a7993440c6aae48736",
"image_id": "sha256:05aa77262db9277fde6374bb2f901c9604b8a9373029e1a7993440c6aae48736",
"started_at": "2026-11-16T10:17:57.426626804Z",
"memory_limit": 1073741824,
"nano_cpus": 2000000000,
"mounts": [
{
"type": "volume",
"name": "mtusi2026lab6_postgres_data",
"destination": "/var/lib/postgresql/data"
}
]
},
{
"service": "prometheus",
"image": "prom/prometheus:v3.16.0",
"image_id": "sha256:e04987641b57704f408e31f6373ba4138ecf27cb4a165d24aabb6c38fddd0f06",
"started_at": "2026-11-16T10:16:15.930341549Z",
"memory_limit": 402653184,
"nano_cpus": 500000000,
"mounts": [
{
"type": "volume",
"name": "mtusi2026lab6_prometheus_data",
"destination": "/prometheus"
}
]
},
{
"service": "redis",
"image": "redis:8.2.11-alpine3.22@sha256:489a769e0a54dc245896132a15c72d2bc7296719954da99398f43a3877e5f8eb",
"image_id": "sha256:489a769e0a54dc245896132a15c72d2bc7296719954da99398f43a3877e5f8eb",
"started_at": "2026-11-16T10:17:57.309233929Z",
"memory_limit": 134217728,
"nano_cpus": 250000000,
"mounts": [
{
"type": "volume",
"name": "mtusi2026lab6_redis_data",
"destination": "/data"
}
]
},
{
"service": "s3",
"image": "chrislusf/seaweedfs:4.52@sha256:57877e32bb90073b37dc109b438ba66e84fd5ed9ced5f5c04f56444ef243ce59",
"image_id": "sha256:57877e32bb90073b37dc109b438ba66e84fd5ed9ced5f5c04f56444ef243ce59",
"started_at": "2026-11-16T10:18:22.319458969Z",
"memory_limit": 536870912,
"nano_cpus": 1000000000,
"mounts": [
{
"type": "volume",
"name": "mtusi2026lab6_s3_data",
"destination": "/data"
}
]
},
{
"service": "tempo",
"image": "grafana/tempo:2.9.0",
"image_id": "sha256:65a5789759435f1ef696f1953258b9bbdb18eb571d5ce711ff812d2e128288a4",
"started_at": "2026-11-17T09:01:50.420850382Z",
"memory_limit": 402653184,
"nano_cpus": 500000000,
"mounts": [
{
"type": "volume",
"name": "mtusi2026lab6_tempo_data",
"destination": "/var/tempo"
}
]
},
{
"service": "worker",
"image": "mtusi-messenger:lab6",
"image_id": "sha256:c7021c3fed2700b5234fedad71dd0478acb126a697aa25ec5ac1a2c99020962b",
"started_at": "2026-11-17T09:11:55.452974759Z",
"memory_limit": 1073741824,
"nano_cpus": 1000000000,
"mounts": []
}
],
"host": {
"cpu": "Apple M4 Pro",
"memory": "25769803776"
}
}
+1
View File
@@ -0,0 +1 @@
{"at": 1794824242.3058822, "container": "747d3207a8824277939ee3d3727b9528eae03ae9f38e9d7a9d8c7c660b0e5cf8", "instance_id": "78f4a480-202b-41b7-a1a1-ab499d7be4c6", "action": "docker stop", "project": "mtusi2026lab6"}
+4
View File
@@ -0,0 +1,4 @@
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:18086"
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
{"observed_instances_before": 2, "successful_writes": 133, "failed_attempts": 1, "first_recovery_seconds": 1.148}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
+94
View File
@@ -0,0 +1,94 @@
python3 tests/smoke.py --base-url "http://localhost:18086"
PASS: lab6 public API scenario; infrastructure and grading evidence remain separate.
cd service && TEST_BASE_URL="http://localhost:18086" go test -race -count=1 -v ./...
? gitea.privcloud.tech/mtusi2026/cmd/app [no test files]
=== RUN TestLostPostResponseIsNotRetried
--- PASS: TestLostPostResponseIsNotRetried (0.00s)
PASS
ok gitea.privcloud.tech/mtusi2026/cmd/benchmark 1.439s
? gitea.privcloud.tech/mtusi2026/internal/adapter/broker [no test files]
? gitea.privcloud.tech/mtusi2026/internal/adapter/controller/http [no test files]
? gitea.privcloud.tech/mtusi2026/internal/adapter/repository/cache/outside [no test files]
=== RUN TestIdempotencyTransactionAndExpiry
idempotency_test.go:27: run make test-storage for an isolated PostgreSQL database
--- SKIP: TestIdempotencyTransactionAndExpiry (0.00s)
PASS
ok gitea.privcloud.tech/mtusi2026/internal/adapter/repository/storage 1.715s
? gitea.privcloud.tech/mtusi2026/internal/config [no test files]
? gitea.privcloud.tech/mtusi2026/internal/dto [no test files]
? gitea.privcloud.tech/mtusi2026/internal/gen/contracts [no test files]
? gitea.privcloud.tech/mtusi2026/internal/pkg/apperr [no test files]
? gitea.privcloud.tech/mtusi2026/internal/pkg/auth/jwtprovider [no test files]
? gitea.privcloud.tech/mtusi2026/internal/pkg/cache [no test files]
=== RUN TestSessionFingerprints
--- PASS: TestSessionFingerprints (0.03s)
PASS
ok gitea.privcloud.tech/mtusi2026/internal/pkg/encrypting 1.341s
? gitea.privcloud.tech/mtusi2026/internal/pkg/fault [no test files]
? gitea.privcloud.tech/mtusi2026/internal/pkg/imageprocessor [no test files]
? gitea.privcloud.tech/mtusi2026/internal/pkg/migrator [no test files]
? gitea.privcloud.tech/mtusi2026/internal/pkg/objectstore [no test files]
? gitea.privcloud.tech/mtusi2026/internal/pkg/postgres [no test files]
? gitea.privcloud.tech/mtusi2026/internal/pkg/postgres/txmanager [no test files]
? gitea.privcloud.tech/mtusi2026/internal/pkg/rest/framework [no test files]
? gitea.privcloud.tech/mtusi2026/internal/pkg/rest/headers [no test files]
=== RUN TestVerifiedSessionIsRequestScoped
2026/11/16 13:21:03 ERROR http_error error_code=CodeUnauthorized status=401 request_id=a407e1e0-b4ee-4006-b34f-8256cb26951b
--- PASS: TestVerifiedSessionIsRequestScoped (0.00s)
=== RUN TestMissingTokenDoesNotCallSessionStore
2026/11/16 13:21:03 ERROR http_error error_code=CodeUnauthorized status=401 request_id=12e5acbe-dc1b-4b12-ab1a-d7bf23c5b0c2
--- PASS: TestMissingTokenDoesNotCallSessionStore (0.00s)
PASS
ok gitea.privcloud.tech/mtusi2026/internal/pkg/rest/middleware 1.992s
? gitea.privcloud.tech/mtusi2026/internal/pkg/rest/router [no test files]
? gitea.privcloud.tech/mtusi2026/internal/pkg/telemetry [no test files]
? gitea.privcloud.tech/mtusi2026/internal/pkg/types [no test files]
? gitea.privcloud.tech/mtusi2026/internal/pkg/types/authprovider [no test files]
? gitea.privcloud.tech/mtusi2026/internal/pkg/types/environment [no test files]
? gitea.privcloud.tech/mtusi2026/internal/pkg/types/role [no test files]
? gitea.privcloud.tech/mtusi2026/internal/pkg/validator [no test files]
? gitea.privcloud.tech/mtusi2026/internal/service/attachment [no test files]
? gitea.privcloud.tech/mtusi2026/internal/service/auth [no test files]
? gitea.privcloud.tech/mtusi2026/internal/service/chat [no test files]
? gitea.privcloud.tech/mtusi2026/internal/service/observation [no test files]
? gitea.privcloud.tech/mtusi2026/internal/service/outbox [no test files]
? gitea.privcloud.tech/mtusi2026/internal/service/user [no test files]
? gitea.privcloud.tech/mtusi2026/migrations [no test files]
=== RUN TestInvalidJSON
=== RUN TestInvalidJSON/malformed
=== RUN TestInvalidJSON/wrong_type
=== RUN TestInvalidJSON/unknown_field
--- PASS: TestInvalidJSON (0.01s)
--- PASS: TestInvalidJSON/malformed (0.01s)
--- PASS: TestInvalidJSON/wrong_type (0.00s)
--- PASS: TestInvalidJSON/unknown_field (0.00s)
=== RUN TestForeignChatIsHidden
--- PASS: TestForeignChatIsHidden (0.57s)
=== RUN TestEmptyMessage
--- PASS: TestEmptyMessage (0.19s)
=== RUN TestDuplicateUsername
--- PASS: TestDuplicateUsername (0.22s)
=== RUN TestMessagePagination
--- PASS: TestMessagePagination (0.20s)
=== RUN TestAttachmentContractAndErrors
=== RUN TestAttachmentContractAndErrors/fake
=== RUN TestAttachmentContractAndErrors/mime_mismatch
=== RUN TestAttachmentContractAndErrors/wrong_type
=== RUN TestAttachmentContractAndErrors/truncated
--- PASS: TestAttachmentContractAndErrors (1.05s)
--- PASS: TestAttachmentContractAndErrors/fake (0.00s)
--- PASS: TestAttachmentContractAndErrors/mime_mismatch (0.00s)
--- PASS: TestAttachmentContractAndErrors/wrong_type (0.00s)
--- PASS: TestAttachmentContractAndErrors/truncated (0.00s)
=== RUN TestConcurrentAttachmentMessage
--- PASS: TestConcurrentAttachmentMessage (0.20s)
=== RUN TestConcurrentMessages
concurrency_test.go:160: 10 clients sent 100 messages; history contains 100 unique messages across 6 pages
--- PASS: TestConcurrentMessages (1.46s)
=== RUN TestIdempotencyContract
--- PASS: TestIdempotencyContract (0.53s)
=== RUN TestConcurrentIdempotencyAcrossReplicas
idempotency_test.go:103: 32 requests across 2 replicas created exactly one message
--- PASS: TestConcurrentIdempotencyAcrossReplicas (0.22s)
PASS
ok gitea.privcloud.tech/mtusi2026/tests 6.932s
+1
View File
@@ -0,0 +1 @@
PASS: first 5xx spike appears in Prometheus increase: 31.11
+349
View File
@@ -0,0 +1,349 @@
{
"at": 1794817011.404624,
"before": [
{
"metric": {
"__name__": "messenger_http_requests_total",
"deployment_environment_name": "lab",
"exported_instance": "715b9cfb-8dd2-451b-89a4-5e2cbc4b4332",
"exported_job": "messenger-api",
"instance": "otel-collector:8889",
"job": "messenger",
"method": "GET",
"otel_scope_name": "messenger",
"route": "/api/v1/chats/{chat_id}/messages",
"service_instance_id": "715b9cfb-8dd2-451b-89a4-5e2cbc4b4332",
"service_name": "messenger-api",
"service_version": "lab6",
"status_class": "5xx"
},
"value": [
1794816991.111,
"15"
]
},
{
"metric": {
"__name__": "messenger_http_requests_total",
"deployment_environment_name": "lab",
"exported_instance": "715b9cfb-8dd2-451b-89a4-5e2cbc4b4332",
"exported_job": "messenger-api",
"instance": "otel-collector:8889",
"job": "messenger",
"method": "POST",
"otel_scope_name": "messenger",
"route": "/api/v1/chats/{chat_id}/messages",
"service_instance_id": "715b9cfb-8dd2-451b-89a4-5e2cbc4b4332",
"service_name": "messenger-api",
"service_version": "lab6",
"status_class": "5xx"
},
"value": [
1794816991.111,
"1"
]
},
{
"metric": {
"__name__": "messenger_http_requests_total",
"deployment_environment_name": "lab",
"exported_instance": "de655f64-de54-45b8-923b-ace3dde6c38e",
"exported_job": "messenger-api",
"instance": "otel-collector:8889",
"job": "messenger",
"method": "GET",
"otel_scope_name": "messenger",
"route": "/api/v1/chats/{chat_id}/messages",
"service_instance_id": "de655f64-de54-45b8-923b-ace3dde6c38e",
"service_name": "messenger-api",
"service_version": "lab6",
"status_class": "5xx"
},
"value": [
1794816991.111,
"12"
]
},
{
"metric": {
"__name__": "messenger_http_requests_total",
"deployment_environment_name": "lab",
"exported_instance": "de655f64-de54-45b8-923b-ace3dde6c38e",
"exported_job": "messenger-api",
"instance": "otel-collector:8889",
"job": "messenger",
"method": "POST",
"otel_scope_name": "messenger",
"route": "/api/v1/chats/{chat_id}/messages",
"service_instance_id": "de655f64-de54-45b8-923b-ace3dde6c38e",
"service_name": "messenger-api",
"service_version": "lab6",
"status_class": "5xx"
},
"value": [
1794816991.111,
"2"
]
},
{
"metric": {
"__name__": "messenger_http_requests_total",
"deployment_environment_name": "lab",
"exported_instance": "44064e41-06b3-4298-850f-67948dc8dd4c",
"exported_job": "messenger-api",
"instance": "otel-collector:8889",
"job": "messenger",
"method": "GET",
"otel_scope_name": "messenger",
"route": "/api/v1/chats/{chat_id}/messages",
"service_instance_id": "44064e41-06b3-4298-850f-67948dc8dd4c",
"service_name": "messenger-api",
"service_version": "lab6",
"status_class": "5xx"
},
"value": [
1794816991.111,
"0"
]
},
{
"metric": {
"__name__": "messenger_http_requests_total",
"deployment_environment_name": "lab",
"exported_instance": "44064e41-06b3-4298-850f-67948dc8dd4c",
"exported_job": "messenger-api",
"instance": "otel-collector:8889",
"job": "messenger",
"method": "POST",
"otel_scope_name": "messenger",
"route": "/api/v1/chats/{chat_id}/messages",
"service_instance_id": "44064e41-06b3-4298-850f-67948dc8dd4c",
"service_name": "messenger-api",
"service_version": "lab6",
"status_class": "5xx"
},
"value": [
1794816991.111,
"0"
]
},
{
"metric": {
"__name__": "messenger_http_requests_total",
"deployment_environment_name": "lab",
"exported_instance": "e57133fe-8e06-48a4-8984-b23d3e6bda08",
"exported_job": "messenger-api",
"instance": "otel-collector:8889",
"job": "messenger",
"method": "GET",
"otel_scope_name": "messenger",
"route": "/api/v1/chats/{chat_id}/messages",
"service_instance_id": "e57133fe-8e06-48a4-8984-b23d3e6bda08",
"service_name": "messenger-api",
"service_version": "lab6",
"status_class": "5xx"
},
"value": [
1794816991.111,
"0"
]
},
{
"metric": {
"__name__": "messenger_http_requests_total",
"deployment_environment_name": "lab",
"exported_instance": "e57133fe-8e06-48a4-8984-b23d3e6bda08",
"exported_job": "messenger-api",
"instance": "otel-collector:8889",
"job": "messenger",
"method": "POST",
"otel_scope_name": "messenger",
"route": "/api/v1/chats/{chat_id}/messages",
"service_instance_id": "e57133fe-8e06-48a4-8984-b23d3e6bda08",
"service_name": "messenger-api",
"service_version": "lab6",
"status_class": "5xx"
},
"value": [
1794816991.111,
"0"
]
}
],
"after": [
{
"metric": {
"__name__": "messenger_http_requests_total",
"deployment_environment_name": "lab",
"exported_instance": "715b9cfb-8dd2-451b-89a4-5e2cbc4b4332",
"exported_job": "messenger-api",
"instance": "otel-collector:8889",
"job": "messenger",
"method": "GET",
"otel_scope_name": "messenger",
"route": "/api/v1/chats/{chat_id}/messages",
"service_instance_id": "715b9cfb-8dd2-451b-89a4-5e2cbc4b4332",
"service_name": "messenger-api",
"service_version": "lab6",
"status_class": "5xx"
},
"value": [
1794817011.408,
"15"
]
},
{
"metric": {
"__name__": "messenger_http_requests_total",
"deployment_environment_name": "lab",
"exported_instance": "715b9cfb-8dd2-451b-89a4-5e2cbc4b4332",
"exported_job": "messenger-api",
"instance": "otel-collector:8889",
"job": "messenger",
"method": "POST",
"otel_scope_name": "messenger",
"route": "/api/v1/chats/{chat_id}/messages",
"service_instance_id": "715b9cfb-8dd2-451b-89a4-5e2cbc4b4332",
"service_name": "messenger-api",
"service_version": "lab6",
"status_class": "5xx"
},
"value": [
1794817011.408,
"1"
]
},
{
"metric": {
"__name__": "messenger_http_requests_total",
"deployment_environment_name": "lab",
"exported_instance": "de655f64-de54-45b8-923b-ace3dde6c38e",
"exported_job": "messenger-api",
"instance": "otel-collector:8889",
"job": "messenger",
"method": "GET",
"otel_scope_name": "messenger",
"route": "/api/v1/chats/{chat_id}/messages",
"service_instance_id": "de655f64-de54-45b8-923b-ace3dde6c38e",
"service_name": "messenger-api",
"service_version": "lab6",
"status_class": "5xx"
},
"value": [
1794817011.408,
"12"
]
},
{
"metric": {
"__name__": "messenger_http_requests_total",
"deployment_environment_name": "lab",
"exported_instance": "de655f64-de54-45b8-923b-ace3dde6c38e",
"exported_job": "messenger-api",
"instance": "otel-collector:8889",
"job": "messenger",
"method": "POST",
"otel_scope_name": "messenger",
"route": "/api/v1/chats/{chat_id}/messages",
"service_instance_id": "de655f64-de54-45b8-923b-ace3dde6c38e",
"service_name": "messenger-api",
"service_version": "lab6",
"status_class": "5xx"
},
"value": [
1794817011.408,
"2"
]
},
{
"metric": {
"__name__": "messenger_http_requests_total",
"deployment_environment_name": "lab",
"exported_instance": "44064e41-06b3-4298-850f-67948dc8dd4c",
"exported_job": "messenger-api",
"instance": "otel-collector:8889",
"job": "messenger",
"method": "GET",
"otel_scope_name": "messenger",
"route": "/api/v1/chats/{chat_id}/messages",
"service_instance_id": "44064e41-06b3-4298-850f-67948dc8dd4c",
"service_name": "messenger-api",
"service_version": "lab6",
"status_class": "5xx"
},
"value": [
1794817011.408,
"15"
]
},
{
"metric": {
"__name__": "messenger_http_requests_total",
"deployment_environment_name": "lab",
"exported_instance": "44064e41-06b3-4298-850f-67948dc8dd4c",
"exported_job": "messenger-api",
"instance": "otel-collector:8889",
"job": "messenger",
"method": "POST",
"otel_scope_name": "messenger",
"route": "/api/v1/chats/{chat_id}/messages",
"service_instance_id": "44064e41-06b3-4298-850f-67948dc8dd4c",
"service_name": "messenger-api",
"service_version": "lab6",
"status_class": "5xx"
},
"value": [
1794817011.408,
"0"
]
},
{
"metric": {
"__name__": "messenger_http_requests_total",
"deployment_environment_name": "lab",
"exported_instance": "e57133fe-8e06-48a4-8984-b23d3e6bda08",
"exported_job": "messenger-api",
"instance": "otel-collector:8889",
"job": "messenger",
"method": "GET",
"otel_scope_name": "messenger",
"route": "/api/v1/chats/{chat_id}/messages",
"service_instance_id": "e57133fe-8e06-48a4-8984-b23d3e6bda08",
"service_name": "messenger-api",
"service_version": "lab6",
"status_class": "5xx"
},
"value": [
1794817011.408,
"15"
]
},
{
"metric": {
"__name__": "messenger_http_requests_total",
"deployment_environment_name": "lab",
"exported_instance": "e57133fe-8e06-48a4-8984-b23d3e6bda08",
"exported_job": "messenger-api",
"instance": "otel-collector:8889",
"job": "messenger",
"method": "POST",
"otel_scope_name": "messenger",
"route": "/api/v1/chats/{chat_id}/messages",
"service_instance_id": "e57133fe-8e06-48a4-8984-b23d3e6bda08",
"service_name": "messenger-api",
"service_version": "lab6",
"status_class": "5xx"
},
"value": [
1794817011.408,
"0"
]
}
],
"increase_1m": 31.11488882020327,
"statuses": {
"200": 20,
"503": 30
},
"passed": true
}
+92
View File
@@ -0,0 +1,92 @@
cd service && TEST_BASE_URL="http://localhost:18086" go test -race -count=1 -v ./...
? gitea.privcloud.tech/mtusi2026/cmd/app [no test files]
=== RUN TestLostPostResponseIsNotRetried
--- PASS: TestLostPostResponseIsNotRetried (0.00s)
PASS
ok gitea.privcloud.tech/mtusi2026/cmd/benchmark 1.364s
? gitea.privcloud.tech/mtusi2026/internal/adapter/broker [no test files]
? gitea.privcloud.tech/mtusi2026/internal/adapter/controller/http [no test files]
? gitea.privcloud.tech/mtusi2026/internal/adapter/repository/cache/outside [no test files]
=== RUN TestIdempotencyTransactionAndExpiry
idempotency_test.go:27: run make test-storage for an isolated PostgreSQL database
--- SKIP: TestIdempotencyTransactionAndExpiry (0.00s)
PASS
ok gitea.privcloud.tech/mtusi2026/internal/adapter/repository/storage 1.350s
? gitea.privcloud.tech/mtusi2026/internal/config [no test files]
? gitea.privcloud.tech/mtusi2026/internal/dto [no test files]
? gitea.privcloud.tech/mtusi2026/internal/gen/contracts [no test files]
? gitea.privcloud.tech/mtusi2026/internal/pkg/apperr [no test files]
? gitea.privcloud.tech/mtusi2026/internal/pkg/auth/jwtprovider [no test files]
? gitea.privcloud.tech/mtusi2026/internal/pkg/cache [no test files]
=== RUN TestSessionFingerprints
--- PASS: TestSessionFingerprints (0.02s)
PASS
ok gitea.privcloud.tech/mtusi2026/internal/pkg/encrypting 1.493s
? gitea.privcloud.tech/mtusi2026/internal/pkg/fault [no test files]
? gitea.privcloud.tech/mtusi2026/internal/pkg/imageprocessor [no test files]
? gitea.privcloud.tech/mtusi2026/internal/pkg/migrator [no test files]
? gitea.privcloud.tech/mtusi2026/internal/pkg/objectstore [no test files]
? gitea.privcloud.tech/mtusi2026/internal/pkg/postgres [no test files]
? gitea.privcloud.tech/mtusi2026/internal/pkg/postgres/txmanager [no test files]
? gitea.privcloud.tech/mtusi2026/internal/pkg/rest/framework [no test files]
? gitea.privcloud.tech/mtusi2026/internal/pkg/rest/headers [no test files]
=== RUN TestVerifiedSessionIsRequestScoped
2026/11/16 10:13:58 ERROR response error="revoked session" status=401 requestId=5fc00aba-9637-4083-a41e-e825350baf99
--- PASS: TestVerifiedSessionIsRequestScoped (0.00s)
=== RUN TestMissingTokenDoesNotCallSessionStore
2026/11/16 10:13:58 ERROR response error="missing or invalid bearer token" status=401 requestId=4fb2abe7-293f-4814-96dd-be8dc4a1b768
--- PASS: TestMissingTokenDoesNotCallSessionStore (0.00s)
PASS
ok gitea.privcloud.tech/mtusi2026/internal/pkg/rest/middleware 1.655s
? gitea.privcloud.tech/mtusi2026/internal/pkg/rest/router [no test files]
? gitea.privcloud.tech/mtusi2026/internal/pkg/telemetry [no test files]
? gitea.privcloud.tech/mtusi2026/internal/pkg/types [no test files]
? gitea.privcloud.tech/mtusi2026/internal/pkg/types/authprovider [no test files]
? gitea.privcloud.tech/mtusi2026/internal/pkg/types/environment [no test files]
? gitea.privcloud.tech/mtusi2026/internal/pkg/types/role [no test files]
? gitea.privcloud.tech/mtusi2026/internal/pkg/validator [no test files]
? gitea.privcloud.tech/mtusi2026/internal/service/attachment [no test files]
? gitea.privcloud.tech/mtusi2026/internal/service/auth [no test files]
? gitea.privcloud.tech/mtusi2026/internal/service/chat [no test files]
? gitea.privcloud.tech/mtusi2026/internal/service/observation [no test files]
? gitea.privcloud.tech/mtusi2026/internal/service/outbox [no test files]
? gitea.privcloud.tech/mtusi2026/internal/service/user [no test files]
? gitea.privcloud.tech/mtusi2026/migrations [no test files]
=== RUN TestInvalidJSON
=== RUN TestInvalidJSON/malformed
=== RUN TestInvalidJSON/wrong_type
=== RUN TestInvalidJSON/unknown_field
--- PASS: TestInvalidJSON (0.02s)
--- PASS: TestInvalidJSON/malformed (0.01s)
--- PASS: TestInvalidJSON/wrong_type (0.00s)
--- PASS: TestInvalidJSON/unknown_field (0.00s)
=== RUN TestForeignChatIsHidden
--- PASS: TestForeignChatIsHidden (0.61s)
=== RUN TestEmptyMessage
--- PASS: TestEmptyMessage (0.20s)
=== RUN TestDuplicateUsername
--- PASS: TestDuplicateUsername (0.29s)
=== RUN TestMessagePagination
--- PASS: TestMessagePagination (0.22s)
=== RUN TestAttachmentContractAndErrors
=== RUN TestAttachmentContractAndErrors/fake
=== RUN TestAttachmentContractAndErrors/mime_mismatch
=== RUN TestAttachmentContractAndErrors/wrong_type
=== RUN TestAttachmentContractAndErrors/truncated
--- PASS: TestAttachmentContractAndErrors (13.12s)
--- PASS: TestAttachmentContractAndErrors/fake (0.00s)
--- PASS: TestAttachmentContractAndErrors/mime_mismatch (0.00s)
--- PASS: TestAttachmentContractAndErrors/wrong_type (0.00s)
--- PASS: TestAttachmentContractAndErrors/truncated (0.01s)
=== RUN TestConcurrentAttachmentMessage
--- PASS: TestConcurrentAttachmentMessage (0.23s)
=== RUN TestConcurrentMessages
concurrency_test.go:160: 10 clients sent 100 messages; history contains 100 unique messages across 6 pages
--- PASS: TestConcurrentMessages (2.09s)
=== RUN TestIdempotencyContract
--- PASS: TestIdempotencyContract (0.56s)
=== RUN TestConcurrentIdempotencyAcrossReplicas
idempotency_test.go:103: 32 requests across 2 replicas created exactly one message
--- PASS: TestConcurrentIdempotencyAcrossReplicas (0.21s)
PASS
ok gitea.privcloud.tech/mtusi2026/tests 19.538s
+47
View File
@@ -0,0 +1,47 @@
{
"research_tag_descriptor": "sha256:d661ef49e464a482dcae899acf1513fa1cf57fae01664445e2dbfb323e65c28a",
"pipeline_tag_descriptor": "sha256:c7021c3fed2700b5234fedad71dd0478acb126a697aa25ec5ac1a2c99020962b",
"runtime_platform_manifest": "sha256:cdbc329b56054f2a3effd832e70dcc50e8eff9b8b601515e2cf70f9de1e6d763",
"runtime_config": "sha256:90726e8699e10a67adece45ae424631076793a340b603ed12cadae16de4d6a45",
"runtime_manifest_present_in_research_build_log": true,
"compose_image_labels": {
"com.docker.compose.project": "mtusi2026lab6",
"com.docker.compose.service": "dispatcher",
"com.docker.compose.version": "5.3.1"
},
"rootfs_layers": [
"sha256:18932007cc273c0541920263e57c02d40013c4730d7db35437a4ffcc31422eda",
"sha256:3e2a611abc7aa98be9c8ce72079d284cc151fd38da2bcf878ba671bf756e2e7f",
"sha256:276547fba5cc9516e768b719cdff99136849d9059b464c0d7bb7322d26d4e9a6",
"sha256:e0fa85f3fe36e74b66b4b4dce01473ebabd7b983b3aa57800581c3901774b4a5"
],
"messenger_binary_sha256": "33c6b59dd87a04c05cf8ff8ad0628fb302c361d32ef125204c4225c1f8337d62",
"note": "Compose builds the same Dockerfile for multiple roles and adds different service labels; OCI index/attestation digests differ. The runtime dispatcher platform manifest matches the research build export.",
"research_build_exports": {
"api": {
"platform_manifest": "sha256:fb582ac957d339c0e15c75ffdd5a469f037565dce4e04ff3eae9593e1a00ae59",
"config": "sha256:f0c85496a5195527417b80b23ac4764efa4c322ae8b655ffba02c14676428452",
"index": "sha256:daa0107d0ba7c7b03e5defcb1e0af9a768384eb3b56e3cb7df597287b06fc32f"
},
"dispatcher": {
"platform_manifest": "sha256:cdbc329b56054f2a3effd832e70dcc50e8eff9b8b601515e2cf70f9de1e6d763",
"config": "sha256:90726e8699e10a67adece45ae424631076793a340b603ed12cadae16de4d6a45",
"index": "sha256:bd4d9ad2bf79cc347d125dbda02a4ddafc804196f6f9f0e2f1b68cdb9963e0a5"
},
"worker": {
"platform_manifest": "sha256:6f8740eb7bdcd57abb07582d25c02b6b509f184c92ddc2a93ed1f21be1791f34",
"config": "sha256:e1220128a009c3c3031d20b771d5685ae8a51a1fd36ac049e6a9a62c351ea1e3",
"index": "sha256:9c8899b0323070c8d73943d0b782ae161ed42bf312ce3759bac0e65b737cad1c"
},
"migrate": {
"platform_manifest": "sha256:dfcdf8937da1d219f4383c6f1f1057bc38713c17102e136321d371a2c5a888b9",
"config": "sha256:8afe86149f38a380ee2e3bbdd2cd8ec9c9437d3a401218191462a7708530fb2c",
"index": "sha256:00b2fe15bc327b038deeeb471cad000ad01e4c32c6f73039e3b5f57ab46d5963"
},
"s3-init": {
"platform_manifest": "sha256:0374cc3c70e64749b51918452b598aad596a8a83896196f6a5b66eb954fc644e",
"config": "sha256:99c7245ce4f99b2f48429a11d1b80fb62604400d5e6feb3c3972219901076a6d",
"index": "sha256:d661ef49e464a482dcae899acf1513fa1cf57fae01664445e2dbfb323e65c28a"
}
}
}
+53
View File
@@ -0,0 +1,53 @@
{
"results": [
{
"case": "small",
"result": "PASS",
"thumbnail": [
16,
8
]
},
{
"case": "floor",
"result": "PASS",
"thumbnail": [
256,
76
]
},
{
"case": "alpha",
"result": "PASS",
"thumbnail": [
40,
20
]
},
{
"case": "orientation",
"result": "PASS",
"thumbnail": [
128,
256
]
},
{
"case": "thin",
"result": "PASS",
"thumbnail": [
1,
256
]
},
{
"case": "center_crop",
"result": "PASS",
"thumbnail": [
256,
85
]
}
],
"negative_cases": 4
}
File diff suppressed because it is too large Load Diff
+3
View File
@@ -0,0 +1,3 @@
Container mtusi2026lab6-loki-run-e631f6d38697 Creating
Container mtusi2026lab6-loki-run-e631f6d38697 Created
level=info ts=2026-11-16T07:28:10.950532589Z caller=main.go:109 msg="config is valid"
+1
View File
@@ -0,0 +1 @@
PASS: persistence, real restart of all stores, unchanged volume identities
+220
View File
@@ -0,0 +1,220 @@
{
"started": 1794824173.232328,
"finished": 1794824240.646864,
"command": [
"make",
"test-persistence",
"ACTION_SCRIPT=scripts/restart.sh"
],
"returncode": 0,
"before": {
"mtusi2026lab6-api-1": {
"service": "api",
"started_at": "2026-11-16T08:13:21.61203226Z",
"volumes": []
},
"mtusi2026lab6-api-2": {
"service": "api",
"started_at": "2026-11-16T08:13:21.47365851Z",
"volumes": []
},
"mtusi2026lab6-broker-1": {
"service": "broker",
"started_at": "2026-11-16T07:26:13.19346959Z",
"volumes": [
"mtusi2026lab6_broker_data"
]
},
"mtusi2026lab6-dispatcher-1": {
"service": "dispatcher",
"started_at": "2026-11-16T08:13:21.48129851Z",
"volumes": []
},
"mtusi2026lab6-grafana-1": {
"service": "grafana",
"started_at": "2026-11-16T07:17:01.172943543Z",
"volumes": [
"mtusi2026lab6_grafana_data"
]
},
"mtusi2026lab6-lb-1": {
"service": "lb",
"started_at": "2026-11-16T07:18:04.666655461Z",
"volumes": []
},
"mtusi2026lab6-loki-1": {
"service": "loki",
"started_at": "2026-11-16T07:28:11.851867589Z",
"volumes": [
"mtusi2026lab6_loki_data"
]
},
"mtusi2026lab6-otel-collector-1": {
"service": "otel-collector",
"started_at": "2026-11-16T07:17:00.733510959Z",
"volumes": []
},
"mtusi2026lab6-postgres-1": {
"service": "postgres",
"started_at": "2026-11-16T07:26:11.526101381Z",
"volumes": [
"mtusi2026lab6_postgres_data"
]
},
"mtusi2026lab6-prometheus-1": {
"service": "prometheus",
"started_at": "2026-11-16T07:17:00.662441876Z",
"volumes": [
"mtusi2026lab6_prometheus_data"
]
},
"mtusi2026lab6-redis-1": {
"service": "redis",
"started_at": "2026-11-16T07:26:11.457109339Z",
"volumes": [
"mtusi2026lab6_redis_data"
]
},
"mtusi2026lab6-s3-1": {
"service": "s3",
"started_at": "2026-11-16T07:50:16.345342466Z",
"volumes": [
"mtusi2026lab6_s3_data"
]
},
"mtusi2026lab6-tempo-1": {
"service": "tempo",
"started_at": "2026-11-16T10:14:35.984164211Z",
"volumes": [
"mtusi2026lab6_tempo_data"
]
},
"mtusi2026lab6-worker-1": {
"service": "worker",
"started_at": "2026-11-16T09:43:34.761728669Z",
"volumes": []
}
},
"after": {
"mtusi2026lab6-api-1": {
"service": "api",
"started_at": "2026-11-16T10:17:15.204445174Z",
"volumes": []
},
"mtusi2026lab6-api-2": {
"service": "api",
"started_at": "2026-11-16T10:17:15.085021257Z",
"volumes": []
},
"mtusi2026lab6-broker-1": {
"service": "broker",
"started_at": "2026-11-16T10:16:17.026791924Z",
"volumes": [
"mtusi2026lab6_broker_data"
]
},
"mtusi2026lab6-dispatcher-1": {
"service": "dispatcher",
"started_at": "2026-11-16T10:17:15.074397715Z",
"volumes": []
},
"mtusi2026lab6-grafana-1": {
"service": "grafana",
"started_at": "2026-11-16T10:16:16.370235216Z",
"volumes": [
"mtusi2026lab6_grafana_data"
]
},
"mtusi2026lab6-lb-1": {
"service": "lb",
"started_at": "2026-11-16T10:17:15.316601507Z",
"volumes": []
},
"mtusi2026lab6-loki-1": {
"service": "loki",
"started_at": "2026-11-16T10:16:18.125747508Z",
"volumes": [
"mtusi2026lab6_loki_data"
]
},
"mtusi2026lab6-otel-collector-1": {
"service": "otel-collector",
"started_at": "2026-11-16T10:16:17.9785623Z",
"volumes": []
},
"mtusi2026lab6-postgres-1": {
"service": "postgres",
"started_at": "2026-11-16T10:16:16.097257841Z",
"volumes": [
"mtusi2026lab6_postgres_data"
]
},
"mtusi2026lab6-prometheus-1": {
"service": "prometheus",
"started_at": "2026-11-16T10:16:15.930341549Z",
"volumes": [
"mtusi2026lab6_prometheus_data"
]
},
"mtusi2026lab6-redis-1": {
"service": "redis",
"started_at": "2026-11-16T10:16:16.023978757Z",
"volumes": [
"mtusi2026lab6_redis_data"
]
},
"mtusi2026lab6-s3-1": {
"service": "s3",
"started_at": "2026-11-16T10:16:41.23677863Z",
"volumes": [
"mtusi2026lab6_s3_data"
]
},
"mtusi2026lab6-tempo-1": {
"service": "tempo",
"started_at": "2026-11-16T10:16:16.187121549Z",
"volumes": [
"mtusi2026lab6_tempo_data"
]
},
"mtusi2026lab6-worker-1": {
"service": "worker",
"started_at": "2026-11-16T10:17:15.092656715Z",
"volumes": []
}
},
"checks": {
"postgres": {
"restarted": true,
"same_volumes": true
},
"redis": {
"restarted": true,
"same_volumes": true
},
"s3": {
"restarted": true,
"same_volumes": true
},
"broker": {
"restarted": true,
"same_volumes": true
},
"prometheus": {
"restarted": true,
"same_volumes": true
},
"loki": {
"restarted": true,
"same_volumes": true
},
"tempo": {
"restarted": true,
"same_volumes": true
},
"grafana": {
"restarted": true,
"same_volumes": true
}
}
}
+3
View File
@@ -0,0 +1,3 @@
python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://localhost:18086"
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
+13
View File
@@ -0,0 +1,13 @@
{"scenario": "worker_down_and_store_restart", "run": 1, "result": "PASS", "seconds": 66.526, "attachment_id": "2c3b8f0c-8104-4619-a1fe-8041c09a3beb", "job_id": "668e23d6-b840-4577-ba49-3b90791fc3ec", "postgres_restarted": true}
{"scenario": "crash_before_objects", "run": 1, "result": "PASS", "seconds": 11.223, "job_id": "41e04582-a1bc-4c99-be90-1f927df9903b", "manifest_rows": 4}
{"scenario": "crash_after_original_put", "run": 1, "result": "PASS", "seconds": 10.999, "job_id": "515ac833-e1e4-46dc-96ab-fdcb89e7ed40", "manifest_rows": 4}
{"scenario": "crash_after_objects", "run": 1, "result": "PASS", "seconds": 10.992, "job_id": "aceb0f8b-3084-4569-b562-d80cc11f58e2", "manifest_rows": 4}
{"scenario": "crash_after_result_commit", "run": 1, "result": "PASS", "seconds": 1.992, "job_id": "f0694c6f-9b0b-41e1-98f4-5816d52c860e", "manifest_rows": 4}
{"scenario": "crash_before_ack", "run": 1, "result": "PASS", "seconds": 2.052, "job_id": "8609eafe-72b6-4640-bf8d-447e8fa8eb89", "manifest_rows": 4}
{"scenario": "ready_redelivery", "run": 1, "result": "PASS", "seconds": 2.585, "job_id": "c741a727-32ed-4846-878c-be719aa5bc51", "copies": 3}
{"scenario": "crash_confirm_before_outbox_mark", "run": 1, "result": "PASS", "seconds": 34.324, "job_id": "22b742c2-f101-4f6d-965a-0dad4bc4a287"}
{"scenario": "lease_reorder_and_stale_version", "run": 1, "result": "PASS", "seconds": 14.869, "attachment_id": "bcd403eb-854f-4ea9-b0f2-272731468061", "old_job_id": "e930a58f-f6f0-43cd-b2c0-860cec77ff33", "pipeline_version": 2, "manifest_rows": 8}
{"scenario": "bounded_retry_and_explicit_replay", "run": 1, "result": "PASS", "seconds": 13.693, "job_id": "dd8d36b6-387f-453e-a898-e08b7674523e", "failed_attempt": 5, "replayed_attempt": 6}
{"scenario": "permanent_error", "run": 1, "result": "PASS", "seconds": 2.667, "job_id": "83a5bdb8-7e1a-4b33-ba40-557ec119c534"}
{"scenario": "api_crash_after_staging", "run": 1, "result": "PASS", "seconds": 0.149, "attachment_id": "328ad979-0ee9-4d41-8b16-7c420d4173f0", "accepted_commit": false}
{"scenario": "api_crash_after_upload_commit", "run": 1, "result": "PASS", "seconds": 2.262, "attachment_id": "cc0a1cf1-862e-4e81-a276-449b075f5c48", "accepted_commit": true}
+123
View File
@@ -0,0 +1,123 @@
{
"date_utc": "2026-11-16T10:20:50.742339+00:00",
"commit": "8b381c115209cf7ac143df4826c468bf38128d71",
"working_tree": "implementation changes uncommitted",
"repeats": 1,
"worker": {
"concurrency": 2,
"prefetch": 2,
"timeout_seconds": 5,
"lease_seconds": 8,
"memory": "1GiB",
"cpus": 1
},
"results": [
{
"scenario": "worker_down_and_store_restart",
"run": 1,
"result": "PASS",
"seconds": 66.526,
"attachment_id": "2c3b8f0c-8104-4619-a1fe-8041c09a3beb",
"job_id": "668e23d6-b840-4577-ba49-3b90791fc3ec",
"postgres_restarted": true
},
{
"scenario": "crash_before_objects",
"run": 1,
"result": "PASS",
"seconds": 11.223,
"job_id": "41e04582-a1bc-4c99-be90-1f927df9903b",
"manifest_rows": 4
},
{
"scenario": "crash_after_original_put",
"run": 1,
"result": "PASS",
"seconds": 10.999,
"job_id": "515ac833-e1e4-46dc-96ab-fdcb89e7ed40",
"manifest_rows": 4
},
{
"scenario": "crash_after_objects",
"run": 1,
"result": "PASS",
"seconds": 10.992,
"job_id": "aceb0f8b-3084-4569-b562-d80cc11f58e2",
"manifest_rows": 4
},
{
"scenario": "crash_after_result_commit",
"run": 1,
"result": "PASS",
"seconds": 1.992,
"job_id": "f0694c6f-9b0b-41e1-98f4-5816d52c860e",
"manifest_rows": 4
},
{
"scenario": "crash_before_ack",
"run": 1,
"result": "PASS",
"seconds": 2.052,
"job_id": "8609eafe-72b6-4640-bf8d-447e8fa8eb89",
"manifest_rows": 4
},
{
"scenario": "ready_redelivery",
"run": 1,
"result": "PASS",
"seconds": 2.585,
"job_id": "c741a727-32ed-4846-878c-be719aa5bc51",
"copies": 3
},
{
"scenario": "crash_confirm_before_outbox_mark",
"run": 1,
"result": "PASS",
"seconds": 34.324,
"job_id": "22b742c2-f101-4f6d-965a-0dad4bc4a287"
},
{
"scenario": "lease_reorder_and_stale_version",
"run": 1,
"result": "PASS",
"seconds": 14.869,
"attachment_id": "bcd403eb-854f-4ea9-b0f2-272731468061",
"old_job_id": "e930a58f-f6f0-43cd-b2c0-860cec77ff33",
"pipeline_version": 2,
"manifest_rows": 8
},
{
"scenario": "bounded_retry_and_explicit_replay",
"run": 1,
"result": "PASS",
"seconds": 13.693,
"job_id": "dd8d36b6-387f-453e-a898-e08b7674523e",
"failed_attempt": 5,
"replayed_attempt": 6
},
{
"scenario": "permanent_error",
"run": 1,
"result": "PASS",
"seconds": 2.667,
"job_id": "83a5bdb8-7e1a-4b33-ba40-557ec119c534"
},
{
"scenario": "api_crash_after_staging",
"run": 1,
"result": "PASS",
"seconds": 0.149,
"attachment_id": "328ad979-0ee9-4d41-8b16-7c420d4173f0",
"accepted_commit": false
},
{
"scenario": "api_crash_after_upload_commit",
"run": 1,
"result": "PASS",
"seconds": 2.262,
"attachment_id": "cc0a1cf1-862e-4e81-a276-449b075f5c48",
"accepted_commit": true
}
],
"image_id": "sha256:c7021c3fed2700b5234fedad71dd0478acb126a697aa25ec5ac1a2c99020962b"
}
+6
View File
@@ -0,0 +1,6 @@
python3 student-tests/images.py
PASS: image geometry, EXIF, alpha, real formats, checksums, ACL and negative inputs
python3 student-tests/storage.py
PASS: active source protected, ready staging removed, failed retention enforced, restore/replay verified
python3 student-tests/broker_persistence.py
PASS: confirmed message survived broker container replacement with the same volume and stable node identity
+65
View File
@@ -0,0 +1,65 @@
[
{
"time": 1794814090673707380,
"action": "create",
"service": "loki",
"container": "c2c4608c32287a2666869a56b5d11cab57bbcc23268d46b5927b3df895b1a4c0",
"exit_code": null
},
{
"time": 1794814090753412339,
"action": "start",
"service": "loki",
"container": "c2c4608c32287a2666869a56b5d11cab57bbcc23268d46b5927b3df895b1a4c0",
"exit_code": null
},
{
"time": 1794814091040505797,
"action": "die",
"service": "loki",
"container": "c2c4608c32287a2666869a56b5d11cab57bbcc23268d46b5927b3df895b1a4c0",
"exit_code": "0"
},
{
"time": 1794814091079755672,
"action": "destroy",
"service": "loki",
"container": "c2c4608c32287a2666869a56b5d11cab57bbcc23268d46b5927b3df895b1a4c0",
"exit_code": null
},
{
"time": 1794814091245157881,
"action": "kill",
"service": "loki",
"container": "421f16b0cf6f897f24ffc81e9f95e16919b0364882475923c15de7b8bbfa8e41",
"exit_code": null
},
{
"time": 1794814091841150214,
"action": "stop",
"service": "loki",
"container": "421f16b0cf6f897f24ffc81e9f95e16919b0364882475923c15de7b8bbfa8e41",
"exit_code": null
},
{
"time": 1794814091842610839,
"action": "die",
"service": "loki",
"container": "421f16b0cf6f897f24ffc81e9f95e16919b0364882475923c15de7b8bbfa8e41",
"exit_code": "0"
},
{
"time": 1794814091940223131,
"action": "start",
"service": "loki",
"container": "421f16b0cf6f897f24ffc81e9f95e16919b0364882475923c15de7b8bbfa8e41",
"exit_code": null
},
{
"time": 1794814091940331339,
"action": "restart",
"service": "loki",
"container": "421f16b0cf6f897f24ffc81e9f95e16919b0364882475923c15de7b8bbfa8e41",
"exit_code": null
}
]
+7
View File
@@ -0,0 +1,7 @@
Checking /etc/prometheus/prometheus.yaml
SUCCESS: 1 rule files found
SUCCESS: /etc/prometheus/prometheus.yaml is valid prometheus config file syntax
Checking /etc/prometheus/alerts.yaml
SUCCESS: 16 rules found
+52
View File
@@ -0,0 +1,52 @@
{
"dashboard_uid": "messenger-lab6",
"panels": 22,
"provisioned": true,
"datasources": [
{
"uid": "loki",
"type": "loki",
"jsonData": {
"derivedFields": [
{
"datasourceUid": "tempo",
"matcherRegex": "trace_id",
"matcherType": "label",
"name": "TraceID",
"url": "${__value.raw}"
}
],
"maxLines": 1000
}
},
{
"uid": "prometheus",
"type": "prometheus",
"jsonData": {
"timeInterval": "5s"
}
},
{
"uid": "tempo",
"type": "tempo",
"jsonData": {
"nodeGraph": {
"enabled": true
},
"tracesToLogsV2": {
"datasourceUid": "loki",
"filterBySpanID": false,
"filterByTraceID": true,
"spanEndTimeShift": "1m",
"spanStartTimeShift": "-1m",
"tags": [
{
"key": "service.name",
"value": "service_name"
}
]
}
}
}
]
}
+12
View File
@@ -0,0 +1,12 @@
{
"research_completed": true,
"short_incidents": 6,
"short_incidents_with_firing": 0,
"long_incidents": 6,
"long_incidents_all_expected_alerts_fired_and_resolved": true,
"collection_errors": 0,
"generator_dropped": 0,
"confirmed_messages_missing": 0,
"attachments_ready": 3529,
"attachments_accepted": 3529
}
+33
View File
@@ -0,0 +1,33 @@
./scripts/compose.sh up -d --wait --wait-timeout 90 --no-recreate api lb
Container mtusi2026lab6-s3-1 Running
Container mtusi2026lab6-postgres-1 Running
Container mtusi2026lab6-api-1 Running
Container mtusi2026lab6-lb-1 Running
Container mtusi2026lab6-s3-1 Waiting
Container mtusi2026lab6-postgres-1 Waiting
Container mtusi2026lab6-postgres-1 Healthy
Container mtusi2026lab6-s3-1 Healthy
Container mtusi2026lab6-migrate-1 Starting
Container mtusi2026lab6-s3-init-1 Starting
Container mtusi2026lab6-s3-init-1 Started
Container mtusi2026lab6-migrate-1 Started
Container mtusi2026lab6-migrate-1 Waiting
Container mtusi2026lab6-s3-init-1 Waiting
Container mtusi2026lab6-s3-init-1 Exited
Container mtusi2026lab6-migrate-1 Exited
Container mtusi2026lab6-api-2 Starting
Container mtusi2026lab6-api-2 Started
Container mtusi2026lab6-lb-1 Waiting
Container mtusi2026lab6-s3-1 Waiting
Container mtusi2026lab6-postgres-1 Waiting
Container mtusi2026lab6-migrate-1 Waiting
Container mtusi2026lab6-api-2 Waiting
Container mtusi2026lab6-api-1 Waiting
Container mtusi2026lab6-s3-init-1 Waiting
Container mtusi2026lab6-s3-init-1 Exited
Container mtusi2026lab6-postgres-1 Healthy
Container mtusi2026lab6-s3-1 Healthy
Container mtusi2026lab6-migrate-1 Exited
Container mtusi2026lab6-lb-1 Healthy
Container mtusi2026lab6-api-2 Healthy
Container mtusi2026lab6-api-1 Healthy
+1
View File
@@ -0,0 +1 @@
PASS: stored metrics, safe labels, privacy canaries, trace search and dashboard queries
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
+151
View File
@@ -0,0 +1,151 @@
baseline 900 seconds
baseline 2 s 17 requests
baseline 66 s 693 requests
baseline 131 s 1374 requests
baseline 195 s 2048 requests
baseline 260 s 2728 requests
baseline 324 s 3398 requests
baseline 388 s 4079 requests
baseline 453 s 4753 requests
baseline 517 s 5433 requests
baseline 582 s 6108 requests
baseline 646 s 6785 requests
baseline 711 s 7465 requests
baseline 775 s 8140 requests
baseline 840 s 8820 requests
worker_short_1 5 seconds
worker_short_1 1 s 9468 requests
worker_short_recovery_1 60 seconds
worker_short_recovery_1 2 s 9532 requests
db_short_1 5 seconds
db_short_1 2 s 10143 requests
db_short_recovery_1 60 seconds
db_short_recovery_1 2 s 10215 requests
worker_long_1 240 seconds
worker_long_1 2 s 10844 requests
worker_long_1 66 s 11525 requests
worker_long_1 131 s 12199 requests
worker_long_1 195 s 12876 requests
worker_recovery_1 660 seconds
worker_recovery_1 1 s 13365 requests
worker_recovery_1 66 s 14045 requests
worker_recovery_1 130 s 14719 requests
worker_recovery_1 195 s 15397 requests
worker_recovery_1 260 s 16078 requests
worker_recovery_1 324 s 16751 requests
worker_recovery_1 389 s 17433 requests
worker_recovery_1 452 s 18101 requests
worker_recovery_1 517 s 18782 requests
worker_recovery_1 581 s 19456 requests
worker_recovery_1 646 s 20136 requests
db_long_1 240 seconds
db_long_1 1 s 20283 requests
db_long_1 66 s 20957 requests
db_long_1 130 s 21630 requests
db_long_1 195 s 22307 requests
db_recovery_1 660 seconds
db_recovery_1 2 s 22827 requests
db_recovery_1 67 s 23508 requests
db_recovery_1 131 s 24181 requests
db_recovery_1 196 s 24863 requests
db_recovery_1 260 s 25535 requests
db_recovery_1 325 s 26214 requests
db_recovery_1 390 s 26895 requests
db_recovery_1 454 s 27567 requests
db_recovery_1 518 s 28248 requests
db_recovery_1 582 s 28917 requests
db_recovery_1 647 s 29599 requests
worker_short_2 5 seconds
worker_short_2 2 s 29760 requests
worker_short_recovery_2 60 seconds
worker_short_recovery_2 2 s 29816 requests
db_short_2 5 seconds
db_short_2 2 s 30427 requests
db_short_recovery_2 60 seconds
db_short_recovery_2 2 s 30502 requests
worker_long_2 240 seconds
worker_long_2 2 s 31135 requests
worker_long_2 67 s 31817 requests
worker_long_2 131 s 32488 requests
worker_long_2 196 s 33170 requests
worker_recovery_2 660 seconds
worker_recovery_2 2 s 33658 requests
worker_recovery_2 67 s 34341 requests
worker_recovery_2 131 s 35009 requests
worker_recovery_2 196 s 35690 requests
worker_recovery_2 260 s 36363 requests
worker_recovery_2 325 s 37044 requests
worker_recovery_2 389 s 37718 requests
worker_recovery_2 453 s 38395 requests
worker_recovery_2 518 s 39077 requests
worker_recovery_2 582 s 39749 requests
worker_recovery_2 647 s 40431 requests
db_long_2 240 seconds
db_long_2 2 s 40569 requests
db_long_2 67 s 41251 requests
db_long_2 131 s 41920 requests
db_long_2 196 s 42601 requests
db_recovery_2 660 seconds
db_recovery_2 2 s 43111 requests
db_recovery_2 67 s 43793 requests
db_recovery_2 132 s 44475 requests
db_recovery_2 197 s 45157 requests
db_recovery_2 261 s 45825 requests
db_recovery_2 326 s 46506 requests
db_recovery_2 390 s 47180 requests
db_recovery_2 455 s 47862 requests
db_recovery_2 519 s 48535 requests
db_recovery_2 583 s 49212 requests
db_recovery_2 648 s 49894 requests
worker_short_3 5 seconds
worker_short_3 2 s 50045 requests
worker_short_recovery_3 60 seconds
worker_short_recovery_3 2 s 50100 requests
db_short_3 5 seconds
db_short_3 2 s 50712 requests
db_short_recovery_3 60 seconds
db_short_recovery_3 2 s 50786 requests
worker_long_3 240 seconds
worker_long_3 2 s 51420 requests
worker_long_3 67 s 52103 requests
worker_long_3 132 s 52782 requests
worker_long_3 197 s 53464 requests
worker_recovery_3 660 seconds
worker_recovery_3 2 s 53943 requests
worker_recovery_3 67 s 54626 requests
worker_recovery_3 132 s 55302 requests
worker_recovery_3 197 s 55984 requests
worker_recovery_3 261 s 56656 requests
worker_recovery_3 326 s 57339 requests
worker_recovery_3 389 s 58007 requests
worker_recovery_3 454 s 58689 requests
worker_recovery_3 518 s 59361 requests
worker_recovery_3 583 s 60043 requests
worker_recovery_3 647 s 60716 requests
db_long_3 240 seconds
db_long_3 2 s 60854 requests
db_long_3 67 s 61536 requests
db_long_3 132 s 62213 requests
db_long_3 197 s 62896 requests
db_recovery_3 660 seconds
db_recovery_3 2 s 63396 requests
db_recovery_3 60 s 64004 requests
db_recovery_3 124 s 64680 requests
db_recovery_3 185 s 65319 requests
db_recovery_3 249 s 65991 requests
db_recovery_3 309 s 66622 requests
db_recovery_3 374 s 67303 requests
db_recovery_3 435 s 67938 requests
db_recovery_3 499 s 68615 requests
db_recovery_3 559 s 69246 requests
db_recovery_3 624 s 69926 requests
tempo_unavailable 300 seconds
tempo_unavailable 2 s 70333 requests
tempo_unavailable 60 s 70941 requests
tempo_unavailable 124 s 71613 requests
tempo_unavailable 184 s 72245 requests
tempo_unavailable 249 s 72921 requests
tempo_recovery 60 seconds
tempo_recovery 2 s 73486 requests
final_drain 30 seconds
final_drain 2 s 74094 requests
+342
View File
@@ -0,0 +1,342 @@
{
"completed": true,
"duration_seconds": 7092.821509838104,
"parameters": {
"warmup": 900,
"long": 240,
"cooldown": 660,
"repeats": 3,
"backend": 300,
"output": "evidence/slo-experiment.json"
},
"incidents": [
{
"phase": "worker_short_1",
"duration": 5.004392862319946,
"alerts": {
"QueueLag": {
"detection_seconds": null,
"resolved_after_restore_seconds": null,
"fired": false
},
"ReadyBurnRateFast": {
"detection_seconds": null,
"resolved_after_restore_seconds": null,
"fired": false
},
"ReadyBurnRateSlow": {
"detection_seconds": null,
"resolved_after_restore_seconds": null,
"fired": false
}
},
"successful_rps": 10.3908971041871,
"error_ratio": 0.0,
"latency_success_ms": {
"0.5": 7.937416899949312,
"0.95": 16.20679209008813,
"0.99": 17.71804178133607
}
},
{
"phase": "db_short_1",
"duration": 5.0065529346466064,
"alerts": {
"ApiBurnRateFast": {
"detection_seconds": null,
"resolved_after_restore_seconds": null,
"fired": false
},
"ApiBurnRateSlow": {
"detection_seconds": null,
"resolved_after_restore_seconds": null,
"fired": false
}
},
"successful_rps": 0.5992161038489773,
"error_ratio": 0.9090909090909091,
"latency_success_ms": {
"0.5": 23.7681670114398,
"0.95": 27.764666825532913,
"0.99": 27.764666825532913
}
},
{
"phase": "worker_long_1",
"duration": 240.00321888923645,
"alerts": {
"QueueLag": {
"detection_seconds": 64.16309976577759,
"resolved_after_restore_seconds": 16.647381067276,
"fired": true
},
"ReadyBurnRateFast": {
"detection_seconds": 81.4437928199768,
"resolved_after_restore_seconds": 64.19608902931213,
"fired": true
},
"ReadyBurnRateSlow": {
"detection_seconds": 91.5375108718872,
"resolved_after_restore_seconds": 123.94137406349182,
"fired": true
}
},
"successful_rps": 10.499865558974752,
"error_ratio": 0.0,
"latency_success_ms": {
"0.5": 8.786541875451803,
"0.95": 17.55095785483718,
"0.99": 28.451249934732914
}
},
{
"phase": "db_long_1",
"duration": 240.002925157547,
"alerts": {
"ApiBurnRateFast": {
"detection_seconds": 53.87334394454956,
"resolved_after_restore_seconds": 64.97496175765991,
"fired": true
},
"ApiBurnRateSlow": {
"detection_seconds": 63.97737908363342,
"resolved_after_restore_seconds": 124.8059549331665,
"fired": true
}
},
"successful_rps": 0.49999409771972575,
"error_ratio": 0.952,
"latency_success_ms": {
"0.5": 27.061082888394594,
"0.95": 41.25562496483326,
"0.99": 48.944166861474514
}
},
{
"phase": "worker_short_2",
"duration": 5.0105671882629395,
"alerts": {
"QueueLag": {
"detection_seconds": null,
"resolved_after_restore_seconds": null,
"fired": false
},
"ReadyBurnRateFast": {
"detection_seconds": null,
"resolved_after_restore_seconds": null,
"fired": false
},
"ReadyBurnRateSlow": {
"detection_seconds": null,
"resolved_after_restore_seconds": null,
"fired": false
}
},
"successful_rps": 10.37807944264453,
"error_ratio": 0.0,
"latency_success_ms": {
"0.5": 6.861874833703041,
"0.95": 17.471542116254568,
"0.99": 27.467207983136177
}
},
{
"phase": "db_short_2",
"duration": 5.0033698081970215,
"alerts": {
"ApiBurnRateFast": {
"detection_seconds": null,
"resolved_after_restore_seconds": null,
"fired": false
},
"ApiBurnRateSlow": {
"detection_seconds": null,
"resolved_after_restore_seconds": null,
"fired": false
}
},
"successful_rps": 0.5995964668020037,
"error_ratio": 0.9090909090909091,
"latency_success_ms": {
"0.5": 31.136707868427038,
"0.95": 34.025250002741814,
"0.99": 34.025250002741814
}
},
{
"phase": "worker_long_2",
"duration": 240.0071098804474,
"alerts": {
"QueueLag": {
"detection_seconds": 64.87712001800537,
"resolved_after_restore_seconds": 17.59833526611328,
"fired": true
},
"ReadyBurnRateFast": {
"detection_seconds": 77.18091893196106,
"resolved_after_restore_seconds": 72.45875120162964,
"fired": true
},
"ReadyBurnRateSlow": {
"detection_seconds": 87.29317092895508,
"resolved_after_restore_seconds": 129.3502321243286,
"fired": true
}
},
"successful_rps": 10.499693843708734,
"error_ratio": 0.0,
"latency_success_ms": {
"0.5": 8.250332903116941,
"0.95": 16.793041955679655,
"0.99": 24.823333136737347
}
},
{
"phase": "db_long_2",
"duration": 240.00172305107117,
"alerts": {
"ApiBurnRateFast": {
"detection_seconds": 49.7854859828949,
"resolved_after_restore_seconds": 60.06502294540405,
"fired": true
},
"ApiBurnRateSlow": {
"detection_seconds": 59.878856897354126,
"resolved_after_restore_seconds": 119.95286202430725,
"fired": true
}
},
"successful_rps": 0.4999965896441693,
"error_ratio": 0.952,
"latency_success_ms": {
"0.5": 28.64320809021592,
"0.95": 42.32870787382126,
"0.99": 52.682749927043915
}
},
{
"phase": "worker_short_3",
"duration": 5.0045859813690186,
"alerts": {
"QueueLag": {
"detection_seconds": null,
"resolved_after_restore_seconds": null,
"fired": false
},
"ReadyBurnRateFast": {
"detection_seconds": null,
"resolved_after_restore_seconds": null,
"fired": false
},
"ReadyBurnRateSlow": {
"detection_seconds": null,
"resolved_after_restore_seconds": null,
"fired": false
}
},
"successful_rps": 10.59029924161761,
"error_ratio": 0.0,
"latency_success_ms": {
"0.5": 8.69549997150898,
"0.95": 24.411499965935946,
"0.99": 26.277707889676094
}
},
{
"phase": "db_short_3",
"duration": 5.008757829666138,
"alerts": {
"ApiBurnRateFast": {
"detection_seconds": null,
"resolved_after_restore_seconds": null,
"fired": false
},
"ApiBurnRateSlow": {
"detection_seconds": null,
"resolved_after_restore_seconds": null,
"fired": false
}
},
"successful_rps": 4.591960382015873,
"error_ratio": 0.5660377358490566,
"latency_success_ms": {
"0.5": 904.9617499113083,
"0.95": 1902.7691250666976,
"0.99": 2003.8470001891255
}
},
{
"phase": "worker_long_3",
"duration": 240.00332617759705,
"alerts": {
"QueueLag": {
"detection_seconds": 59.91160011291504,
"resolved_after_restore_seconds": 12.543959856033325,
"fired": true
},
"ReadyBurnRateFast": {
"detection_seconds": 72.20107531547546,
"resolved_after_restore_seconds": 65.21866607666016,
"fired": true
},
"ReadyBurnRateSlow": {
"detection_seconds": 87.36326217651367,
"resolved_after_restore_seconds": 124.68797492980957,
"fired": true
}
},
"successful_rps": 10.499858507930577,
"error_ratio": 0.0,
"latency_success_ms": {
"0.5": 8.27891705557704,
"0.95": 17.318916972726583,
"0.99": 25.771250016987324
}
},
{
"phase": "db_long_3",
"duration": 240.00675511360168,
"alerts": {
"ApiBurnRateFast": {
"detection_seconds": 44.78353834152222,
"resolved_after_restore_seconds": 55.135868310928345,
"fired": true
},
"ApiBurnRateSlow": {
"detection_seconds": 54.89339232444763,
"resolved_after_restore_seconds": 119.52835726737976,
"fired": true
}
},
"successful_rps": 0.49998607674270545,
"error_ratio": 0.952,
"latency_success_ms": {
"0.5": 27.898625005036592,
"0.95": 41.145917028188705,
"0.99": 74.70795791596174
}
}
],
"traffic": {
"requests": 74094,
"success": 66860,
"errors": 7234,
"generator_dropped": 0,
"uploads_accepted": 3529,
"latency_success_ms": {
"0.5": 7.835667114704847,
"0.95": 17.5331667996943,
"0.99": 29.314542189240456
},
"statuses": {
"200": 56997,
"201": 6334,
"202": 3529,
"503": 7234
}
},
"attachment_states": {
"ready": 3529
},
"image_id": "sha256:d661ef49e464a482dcae899acf1513fa1cf57fae01664445e2dbfb323e65c28a"
}
+342
View File
@@ -0,0 +1,342 @@
{
"completed": true,
"duration_seconds": 7092.821509838104,
"parameters": {
"warmup": 900,
"long": 240,
"cooldown": 660,
"repeats": 3,
"backend": 300,
"output": "evidence/slo-experiment.json"
},
"incidents": [
{
"phase": "worker_short_1",
"duration": 5.004392862319946,
"alerts": {
"QueueLag": {
"detection_seconds": null,
"resolved_after_restore_seconds": null,
"fired": false
},
"ReadyBurnRateFast": {
"detection_seconds": null,
"resolved_after_restore_seconds": null,
"fired": false
},
"ReadyBurnRateSlow": {
"detection_seconds": null,
"resolved_after_restore_seconds": null,
"fired": false
}
},
"successful_rps": 10.3908971041871,
"error_ratio": 0.0,
"latency_success_ms": {
"0.5": 7.937416899949312,
"0.95": 16.20679209008813,
"0.99": 17.71804178133607
}
},
{
"phase": "db_short_1",
"duration": 5.0065529346466064,
"alerts": {
"ApiBurnRateFast": {
"detection_seconds": null,
"resolved_after_restore_seconds": null,
"fired": false
},
"ApiBurnRateSlow": {
"detection_seconds": null,
"resolved_after_restore_seconds": null,
"fired": false
}
},
"successful_rps": 0.5992161038489773,
"error_ratio": 0.9090909090909091,
"latency_success_ms": {
"0.5": 23.7681670114398,
"0.95": 27.764666825532913,
"0.99": 27.764666825532913
}
},
{
"phase": "worker_long_1",
"duration": 240.00321888923645,
"alerts": {
"QueueLag": {
"detection_seconds": 64.16309976577759,
"resolved_after_restore_seconds": 16.647381067276,
"fired": true
},
"ReadyBurnRateFast": {
"detection_seconds": 81.4437928199768,
"resolved_after_restore_seconds": 64.19608902931213,
"fired": true
},
"ReadyBurnRateSlow": {
"detection_seconds": 91.5375108718872,
"resolved_after_restore_seconds": 123.94137406349182,
"fired": true
}
},
"successful_rps": 10.499865558974752,
"error_ratio": 0.0,
"latency_success_ms": {
"0.5": 8.786541875451803,
"0.95": 17.55095785483718,
"0.99": 28.451249934732914
}
},
{
"phase": "db_long_1",
"duration": 240.002925157547,
"alerts": {
"ApiBurnRateFast": {
"detection_seconds": 53.87334394454956,
"resolved_after_restore_seconds": 64.97496175765991,
"fired": true
},
"ApiBurnRateSlow": {
"detection_seconds": 63.97737908363342,
"resolved_after_restore_seconds": 124.8059549331665,
"fired": true
}
},
"successful_rps": 0.49999409771972575,
"error_ratio": 0.952,
"latency_success_ms": {
"0.5": 27.061082888394594,
"0.95": 41.25562496483326,
"0.99": 48.944166861474514
}
},
{
"phase": "worker_short_2",
"duration": 5.0105671882629395,
"alerts": {
"QueueLag": {
"detection_seconds": null,
"resolved_after_restore_seconds": null,
"fired": false
},
"ReadyBurnRateFast": {
"detection_seconds": null,
"resolved_after_restore_seconds": null,
"fired": false
},
"ReadyBurnRateSlow": {
"detection_seconds": null,
"resolved_after_restore_seconds": null,
"fired": false
}
},
"successful_rps": 10.37807944264453,
"error_ratio": 0.0,
"latency_success_ms": {
"0.5": 6.861874833703041,
"0.95": 17.471542116254568,
"0.99": 27.467207983136177
}
},
{
"phase": "db_short_2",
"duration": 5.0033698081970215,
"alerts": {
"ApiBurnRateFast": {
"detection_seconds": null,
"resolved_after_restore_seconds": null,
"fired": false
},
"ApiBurnRateSlow": {
"detection_seconds": null,
"resolved_after_restore_seconds": null,
"fired": false
}
},
"successful_rps": 0.5995964668020037,
"error_ratio": 0.9090909090909091,
"latency_success_ms": {
"0.5": 31.136707868427038,
"0.95": 34.025250002741814,
"0.99": 34.025250002741814
}
},
{
"phase": "worker_long_2",
"duration": 240.0071098804474,
"alerts": {
"QueueLag": {
"detection_seconds": 64.87712001800537,
"resolved_after_restore_seconds": 17.59833526611328,
"fired": true
},
"ReadyBurnRateFast": {
"detection_seconds": 77.18091893196106,
"resolved_after_restore_seconds": 72.45875120162964,
"fired": true
},
"ReadyBurnRateSlow": {
"detection_seconds": 87.29317092895508,
"resolved_after_restore_seconds": 129.3502321243286,
"fired": true
}
},
"successful_rps": 10.499693843708734,
"error_ratio": 0.0,
"latency_success_ms": {
"0.5": 8.250332903116941,
"0.95": 16.793041955679655,
"0.99": 24.823333136737347
}
},
{
"phase": "db_long_2",
"duration": 240.00172305107117,
"alerts": {
"ApiBurnRateFast": {
"detection_seconds": 49.7854859828949,
"resolved_after_restore_seconds": 60.06502294540405,
"fired": true
},
"ApiBurnRateSlow": {
"detection_seconds": 59.878856897354126,
"resolved_after_restore_seconds": 119.95286202430725,
"fired": true
}
},
"successful_rps": 0.4999965896441693,
"error_ratio": 0.952,
"latency_success_ms": {
"0.5": 28.64320809021592,
"0.95": 42.32870787382126,
"0.99": 52.682749927043915
}
},
{
"phase": "worker_short_3",
"duration": 5.0045859813690186,
"alerts": {
"QueueLag": {
"detection_seconds": null,
"resolved_after_restore_seconds": null,
"fired": false
},
"ReadyBurnRateFast": {
"detection_seconds": null,
"resolved_after_restore_seconds": null,
"fired": false
},
"ReadyBurnRateSlow": {
"detection_seconds": null,
"resolved_after_restore_seconds": null,
"fired": false
}
},
"successful_rps": 10.59029924161761,
"error_ratio": 0.0,
"latency_success_ms": {
"0.5": 8.69549997150898,
"0.95": 24.411499965935946,
"0.99": 26.277707889676094
}
},
{
"phase": "db_short_3",
"duration": 5.008757829666138,
"alerts": {
"ApiBurnRateFast": {
"detection_seconds": null,
"resolved_after_restore_seconds": null,
"fired": false
},
"ApiBurnRateSlow": {
"detection_seconds": null,
"resolved_after_restore_seconds": null,
"fired": false
}
},
"successful_rps": 4.591960382015873,
"error_ratio": 0.5660377358490566,
"latency_success_ms": {
"0.5": 904.9617499113083,
"0.95": 1902.7691250666976,
"0.99": 2003.8470001891255
}
},
{
"phase": "worker_long_3",
"duration": 240.00332617759705,
"alerts": {
"QueueLag": {
"detection_seconds": 59.91160011291504,
"resolved_after_restore_seconds": 12.543959856033325,
"fired": true
},
"ReadyBurnRateFast": {
"detection_seconds": 72.20107531547546,
"resolved_after_restore_seconds": 65.21866607666016,
"fired": true
},
"ReadyBurnRateSlow": {
"detection_seconds": 87.36326217651367,
"resolved_after_restore_seconds": 124.68797492980957,
"fired": true
}
},
"successful_rps": 10.499858507930577,
"error_ratio": 0.0,
"latency_success_ms": {
"0.5": 8.27891705557704,
"0.95": 17.318916972726583,
"0.99": 25.771250016987324
}
},
{
"phase": "db_long_3",
"duration": 240.00675511360168,
"alerts": {
"ApiBurnRateFast": {
"detection_seconds": 44.78353834152222,
"resolved_after_restore_seconds": 55.135868310928345,
"fired": true
},
"ApiBurnRateSlow": {
"detection_seconds": 54.89339232444763,
"resolved_after_restore_seconds": 119.52835726737976,
"fired": true
}
},
"successful_rps": 0.49998607674270545,
"error_ratio": 0.952,
"latency_success_ms": {
"0.5": 27.898625005036592,
"0.95": 41.145917028188705,
"0.99": 74.70795791596174
}
}
],
"traffic": {
"requests": 74094,
"success": 66860,
"errors": 7234,
"generator_dropped": 0,
"uploads_accepted": 3529,
"latency_success_ms": {
"0.5": 7.835667114704847,
"0.95": 17.5331667996943,
"0.99": 29.314542189240456
},
"statuses": {
"200": 56997,
"201": 6334,
"202": 3529,
"503": 7234
}
},
"attachment_states": {
"ready": 3529
},
"image_id": "sha256:d661ef49e464a482dcae899acf1513fa1cf57fae01664445e2dbfb323e65c28a"
}
+2
View File
@@ -0,0 +1,2 @@
python3 tests/smoke.py --base-url "http://localhost:18086"
PASS: lab6 public API scenario; infrastructure and grading evidence remain separate.
+109
View File
@@ -0,0 +1,109 @@
./scripts/test-storage.sh
=== RUN TestIdempotencyTransactionAndExpiry
2026/11/16 07:14:23 INFO connected to database db=lab5_atomic_1794813262_55454
2026/11/16 07:14:23 INFO successfully initialized migrator
2026/11/16 07:14:23 INFO successfully loaded migrations error=<nil>
2026/11/16 07:14:23 INFO postgres_pool_config max_conns=8 connect_timeout=500ms statement_timeout_ms=2000
2026/11/16 07:14:23 INFO successfully connected to postgres pool
=== RUN TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules
=== RUN TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/empty
2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test"
2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test"
2026/11/16 07:14:23 INFO chat_db_lookup chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90
=== RUN TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/whitespace
2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test"
2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test"
2026/11/16 07:14:23 INFO chat_db_lookup chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90
=== RUN TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/invalid_UTF8
2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test"
2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test"
2026/11/16 07:14:23 INFO chat_db_lookup chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90
=== RUN TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/nul
2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test"
2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test"
2026/11/16 07:14:23 INFO chat_db_lookup chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90
=== RUN TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/too_long
2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test"
2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test"
2026/11/16 07:14:23 INFO chat_db_lookup chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90
=== RUN TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/duplicate
2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test"
2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test"
2026/11/16 07:14:23 INFO chat_db_lookup chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90
=== RUN TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/too_many
2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test"
2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test"
2026/11/16 07:14:23 INFO chat_db_lookup chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90
=== RUN TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/nil_id
2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test"
2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test"
2026/11/16 07:14:23 INFO chat_db_lookup chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90
=== RUN TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/unknown
2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test"
2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test"
2026/11/16 07:14:23 INFO chat_db_lookup chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90
=== RUN TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/foreign_chat
2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test"
2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test"
2026/11/16 07:14:23 INFO chat_db_lookup chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90
=== RUN TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/missing_object
2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test"
2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test"
2026/11/16 07:14:23 INFO chat_db_lookup chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90
2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test"
2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test"
2026/11/16 07:14:23 INFO chat_db_lookup chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90
2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test"
2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test"
2026/11/16 07:14:23 INFO chat_db_lookup chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90
2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test"
2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test"
2026/11/16 07:14:23 INFO chat_db_lookup chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90
2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test"
2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test"
2026/11/16 07:14:23 INFO chat_db_lookup chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90
2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test"
2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test"
2026/11/16 07:14:23 INFO chat_db_lookup chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90
2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test"
2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test"
2026/11/16 07:14:23 INFO chat_db_lookup chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90
2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test"
2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test"
2026/11/16 07:14:23 INFO chat_db_lookup chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90
2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test"
2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test"
2026/11/16 07:14:23 INFO chat_db_lookup chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90
2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test"
2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test"
2026/11/16 07:14:23 INFO chat_db_lookup chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90
2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test"
2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test"
2026/11/16 07:14:23 INFO chat_db_lookup chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90
2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test"
2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test"
2026/11/16 07:14:23 INFO chat_db_lookup chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90
2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test"
2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test"
2026/11/16 07:14:23 INFO chat_db_lookup chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90
2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test"
2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test"
2026/11/16 07:14:23 INFO chat_db_lookup chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90
=== RUN TestIdempotencyTransactionAndExpiry/object_lock_cancellation_does_not_leak_a_pooled_lock
=== NAME TestIdempotencyTransactionAndExpiry
idempotency_test.go:295: rollback removed both side effect and key; expired key created one new message
--- PASS: TestIdempotencyTransactionAndExpiry (0.11s)
--- PASS: TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules (0.01s)
--- PASS: TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/empty (0.00s)
--- PASS: TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/whitespace (0.00s)
--- PASS: TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/invalid_UTF8 (0.00s)
--- PASS: TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/nul (0.00s)
--- PASS: TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/too_long (0.00s)
--- PASS: TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/duplicate (0.00s)
--- PASS: TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/too_many (0.00s)
--- PASS: TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/nil_id (0.00s)
--- PASS: TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/unknown (0.00s)
--- PASS: TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/foreign_chat (0.00s)
--- PASS: TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/missing_object (0.00s)
--- PASS: TestIdempotencyTransactionAndExpiry/object_lock_cancellation_does_not_leak_a_pooled_lock (0.05s)
PASS
+24
View File
@@ -0,0 +1,24 @@
{
"results": [
{
"scenario": "queued_staging_protected",
"result": "PASS",
"attachment_id": "34aa2186-8c54-49ff-b517-03af038fc35c"
},
{
"scenario": "staging_deleted_after_ready",
"result": "PASS",
"attachment_id": "34aa2186-8c54-49ff-b517-03af038fc35c"
},
{
"scenario": "failed_retention_and_missing_source_replay",
"result": "PASS",
"age_simulated_hours": 25
},
{
"scenario": "verified_restore_then_replay",
"result": "PASS",
"job_id": "c425d43e-a61c-4501-b900-8fb75fe0add1"
}
]
}
+37
View File
@@ -0,0 +1,37 @@
[
{
"reference": "otel/opentelemetry-collector-contrib:0.140.0",
"id": "sha256:ffc818ee108d0b934fd14207fd87ff247c9b64f344ed349d0b66c166c18a2312",
"repo_digests": [
"otel/opentelemetry-collector-contrib@sha256:ffc818ee108d0b934fd14207fd87ff247c9b64f344ed349d0b66c166c18a2312"
]
},
{
"reference": "prom/prometheus:v3.16.0",
"id": "sha256:e04987641b57704f408e31f6373ba4138ecf27cb4a165d24aabb6c38fddd0f06",
"repo_digests": [
"prom/prometheus@sha256:e04987641b57704f408e31f6373ba4138ecf27cb4a165d24aabb6c38fddd0f06"
]
},
{
"reference": "grafana/loki:3.7.10",
"id": "sha256:b43c257ccfc925d4a50b8717cac61a3b51df8c48d01847936057e5c4075a24a7",
"repo_digests": [
"grafana/loki@sha256:b43c257ccfc925d4a50b8717cac61a3b51df8c48d01847936057e5c4075a24a7"
]
},
{
"reference": "grafana/tempo:2.9.0",
"id": "sha256:65a5789759435f1ef696f1953258b9bbdb18eb571d5ce711ff812d2e128288a4",
"repo_digests": [
"grafana/tempo@sha256:65a5789759435f1ef696f1953258b9bbdb18eb571d5ce711ff812d2e128288a4"
]
},
{
"reference": "grafana/grafana:13.3.1",
"id": "sha256:0615ec4fbd4072004b7d86b46ded63013a74b7d6eb1119273dd89e8fa25c13cf",
"repo_digests": [
"grafana/grafana@sha256:0615ec4fbd4072004b7d86b46ded63013a74b7d6eb1119273dd89e8fa25c13cf"
]
}
]
+4 -4
View File
@@ -1,7 +1,7 @@
# Инфраструктура студента
# Стенд lab6
Здесь разместите конфигурации выбранных сервисов; основную топологию — в корневом compose.yaml/stack.yaml. `compose.example.yaml` — только worksheet ролей и намеренно не запускается. `contracts/infrastructure.json` задаёт минимальные роли, не готовые образы/пароли.
Основная топология — `compose.yaml`; запуск `make up`, обычная остановка `make down` сохраняет volumes. Bootstrap создаёт отдельные синтетические credentials в игнорируемых `.env.lab6` и `secrets/lab6`, не меняя существующие значения. Источники приложения собираются из `service/`.
В REPORT.md сопоставьте роль → service name → версия/digest → health/readiness → volume → сеть/порт → способ передачи credentials. Приведите bootstrap (schema, bucket, broker topology, а с lab6 provisioning) и безопасное повторение этих команд.
HAProxy публикует только `127.0.0.1:18086`; Grafana — `127.0.0.1:13006`. PostgreSQL, Redis, S3, RabbitMQ, Collector и хранилища telemetry не публикуют порты на host. API-реплик две; приватный S3 находится в отдельной internal storage network. Проект и volumes не совпадают со стендом lab5.
Приложение строится из исходников. Не используйте заранее существующий образ преподавательского сервера вместо своей реализации. Restart hook перезапускает все соответствующие stores с volumes; failover hook останавливает только заданную API-реплику.
Роли, версии, health/readiness, лимиты и переменные описаны в [отчёте](../REPORT.md). Конфигурация [наблюдаемости](otel/README.md), [runbook](otel/RUNBOOK.md). Конфигурация ImageMagick ограничивает ресурсы преобразования изображений. `compose.faults.yaml` включает только явно запускаемые учебные барьеры; обычный запуск их не включает.
+39
View File
@@ -0,0 +1,39 @@
global
maxconn 4096
stats socket /tmp/haproxy.sock mode 600 level admin
stats socket ipv4@127.0.0.1:9999 level admin
defaults
mode http
timeout connect 500ms
timeout client 125s
timeout server 125s
timeout check 500ms
retries 0
resolvers docker
nameserver dns 127.0.0.11:53
resolve_retries 2
timeout resolve 1s
timeout retry 1s
hold valid 1s
hold obsolete 1s
hold nx 1s
frontend public
bind :8080
http-request deny deny_status 404 if { path_beg /internal }
default_backend api
backend api
balance roundrobin
http-reuse always
option httpchk
http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host api
http-check expect status 200
server-template replica 1-4 api:8080 check inter 1s fall 2 rise 1 resolvers docker init-addr none resolve-prefer ipv4
listen statistics
bind :8404
stats enable
stats uri /stats
+13
View File
@@ -0,0 +1,13 @@
<policymap>
<policy domain="delegate" rights="none" pattern="*"/>
<policy domain="filter" rights="none" pattern="*"/>
<policy domain="coder" rights="none" pattern="*"/>
<policy domain="coder" rights="read|write" pattern="{PNG,JPEG,WEBP,NULL}"/>
<policy domain="resource" name="memory" value="256MiB"/>
<policy domain="resource" name="map" value="0"/>
<policy domain="resource" name="disk" value="256MiB"/>
<policy domain="resource" name="thread" value="1"/>
<policy domain="resource" name="time" value="60"/>
<policy domain="resource" name="list-length" value="4"/>
<policy domain="path" rights="none" pattern="@*"/>
</policymap>
+17 -4
View File
@@ -1,7 +1,20 @@
# Конфигурация наблюдаемости — реализовать
# Наблюдаемость lab6
Добавьте `collector.yaml` с OTLP receivers, processors и exporters выбранной версии Collector. Укажите явные bind endpoints, внутренние сети, memory limits/batch и поведение при backpressure по уровню. Сопоставьте все три pipeline с реально развёрнутыми backend.
API / worker / dispatcher → OTLP HTTP/protobuf :4318 → отдельный Collector.
Добавьте конфигурации storage, Grafana provisioning/datasources, provisioning/dashboards и импортируемые dashboard JSON. На 4 — alert rules и runbooks. Пустой dashboard или debug exporter, который только печатает данные в stdout, не заменяют хранилища/доказательства из `contracts/observability.md`.
- Metrics: Collector Prometheus exporter :8889 → Prometheus, TSDB volume, 7 дней / 1 GiB.
- Logs: Collector `otlphttp/loki` → Loki native `/otlp`, TSDB v13, filesystem volume, retention 7 дней.
- Traces: Collector `otlphttp/tempo` → Tempo OTLP :4318, WAL + local blocks volume, retention 7 дней.
- Grafana хранит UI-настройки в собственном volume; datasource/dashboard provisioning находится здесь в Git.
Не публикуйте OTLP/metrics/административные ports в общую сеть без необходимости. В lab7 реализуйте TLS/mTLS по trust matrix; экспорт Authorization и message bodies запрещён.
Административные/OTLP порты не публикуются на host; Grafana доступна только через localhost:13006. Образы закреплены явными версиями в Compose; фактические image IDs сохраняет `scripts/capture-environment.py`.
Collector: memory limiter: hard limit 180 MiB, soft limit 140 MiB (spike 40 MiB), контейнер 256 MiB; batch до 256 элементов, timeout 1s; по 32 запроса на exporter и два consumers, retry до 30s. SDK traces/logs: очереди по 1024 элементов, batch 128, timeout экспорта 1s, без блокирования бизнес-запроса; метрики экспортируются раз в 5s. RAM и количество consumers конечны; очереди telemetry не durable.
Имена стандартных атрибутов выбраны по Semantic Conventions 1.37.0; собственные имена инструментов и их соответствие контракту перечислены в REPORT.md.
Resource attributes: service.name, service.version=lab6, service.instance.id и deployment.environment.name=lab. Реплика использует один ID и для OTel resource, и для X-Instance-Id. Default trace sampler — ParentBased(AlwaysSample), то есть 100% новых корней; входящий sampled flag сохраняется. Sampling не применяется к метрикам.
Политика данных: SQL-параметры, HTTP body/Authorization/URL и object keys не инструментируются. Логгер заменяет исходные error values безопасной категорией. Технические job/request IDs допустимы только в логах; labels метрик содержат route template, method/status class, job type/outcome, имя очереди и resource attributes. Уровень info, шумные cache-hit/miss события оставлены на debug.
[Runbook и SLO](RUNBOOK.md), [отчёт](../../REPORT.md), [dashboard JSON](dashboards/messenger.json).
+48
View File
@@ -0,0 +1,48 @@
# Диагностика lab6
Открыть Grafana: http://127.0.0.1:13006/d/messenger-lab6, пользователь `admin`.
Локальный пароль находится в `secrets/lab6/grafana_password`; не прикладывать его к отчёту.
Dashboard, datasource и alerts загружаются из Git. Внешняя доставка уведомлений не настроена.
## Порядок действий
1. Выбрать интервал инцидента. Проверить свежесть `messenger_observation_timestamp`, состояние scrape `up` и ошибки Collector. Пропуск данных не означает нулевую очередь или отсутствие ошибок.
2. Сопоставить HTTP RPS/5xx и p95 с backlog, возрастом jobs, worker busy/capacity и DB pool. Процентили вычисляются из суммарных buckets, а не усредняются между репликами.
3. В Explore → Tempo найти trace по времени/service.name; затем открыть связанные логи. Для конкретного upload взять trace ID из `http_request` или `job_received`. В Loki фильтр: `{service_name=~"messenger-.+"} | trace_id="TRACE_ID"`.
4. Сформулировать причину по сигналам. После этого сверить её с журналом воздействия. Проверить восстановление записи, выход jobs в ready, снижение backlog и исчезновение firing alerts.
## Worker остановлен
Признаки: растут `messenger_queue_depth`, `messenger_jobs_pending` и `messenger_queue_oldest_age_seconds`; HTTP upload продолжает возвращать 202. Свежие `job_received`/`job_ready` отсутствуют, новые consumer spans не появляются. Последняя worker gauge может оставаться в Prometheus некоторое время: нулевой busy сам по себе не доказывает остановку процесса.
Alerts: `QueueLag`, затем `ReadyBurnRateFast`/`ReadyBurnRateSlow`. Даже незавершённые jobs учитываются как bad после 30 секунд: оценку выполняет dispatcher.
Восстановление: `./scripts/compose.sh start worker`. Проверить новые consumer spans, ready, опустошение очереди. Ready-SLO уже нарушенных jobs остаётся bad после восстановления; burn-rate снижается только по мере выхода ошибок из окон.
## PostgreSQL замедлена
Признаки: HTTP 5xx и histogram duration растут; в trace длительность находится внутри `postgresql SELECT/INSERT`, заканчивается безопасным error status. `messenger_db_locks_waiting` подтверждает ожидание DB lock. Pool used/waiting помогает отличить ожидание соединения от выполняющегося запроса. Upload и worker могут работать, если заблокирована только таблица messages. Это не общий отказ PostgreSQL и не задержка преобразования изображения.
Alerts: `ApiBurnRateFast`/`ApiBurnRateSlow`. В эксперименте блокировка имеет ограничение времени и отдельный `application_name=lab6_slo_injection`. Скрипт всегда завершает эту DB-сессию в finally. При аварийном завершении внешнего процесса оператор может выполнить:
```sh
./scripts/compose.sh exec -T postgres psql -U messenger -d messenger -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE application_name='lab6_slo_injection'"
```
Затем проверить успешные запросы и завершение alerts. SELECT 1 не блокируется таблицей messages, но исчерпание пула может временно нарушить readiness и добавить ответы LB. Учебный профиль использует 16 соединений на процесс, чтобы при данной нагрузке сохранить запас для health. Ответы самого LB всё равно учитываются генератором отдельно.
## Telemetry backend недоступен
Признаки: растут `otelcol_exporter_send_failed_spans_total`, появляются retry и bounded queue; после исчерпания бюджета растёт разница accepted/sent spans. Сопоставить `otelcol_exporter_queue_size` с capacity и RSS Collector. CPU/RAM процесса приложения доступны через оставшийся metrics pipeline.
Восстановление: `./scripts/compose.sh start tempo`. Потерянные spans не восстанавливаются из бизнес-БД; это допустимая потеря telemetry, не потеря сообщений. Проверить новые traces и опустошение exporter queue. Ошибки отправки включают повторные попытки, поэтому их число нельзя приравнивать числу потерянных spans.
## SLI и правила
Учебное окно SLO — 1 час; бюджет ошибок — 1%. HTTP good = ответ бизнес-API без 5xx, denominator включает все `/api/v1/*`, в том числе 4xx. Health/internal исключены. Поэтому это серверный SLI, а не полный клиентский availability SLI: сетевые отказы и ответы самого LB отдельно учитываются генератором.
Ready good = durable job первой обработки/новой версии достигла ready за ≤30 секунд от записанного accepted_at; terminal failure или deadline без ready = bad. Replay сохраняет первый SLO-результат. accepted_at фиксируется в транзакции durable handoff непосредственно перед COMMIT; небольшой интервал до подтверждения COMMIT входит в измерение. ready_at также фиксируется в транзакции публикации результата. Это серверные timestamps, не время полного скачивания клиентом.
`error_ratio = bad / total`, `burn_rate = error_ratio / 0.01`. Fast: оба окна 1m и 5m >6×, for=20s. Slow: оба окна 2m и 10m >3×, for=30s. Минимум событий в коротком окне: API 20, ready 5. При отсутствии событий ratio не определена; отсутствие firing при малом трафике не доказывает SLO.
Окна намеренно подходят для часового учебного стенда; это не стандартный 30-дневный production SLO. Scrape/evaluation/export — 5s, observer — примерно 1s. Регистрация результата во время отказа DB задерживается до восстановления; `ObservationsStale` показывает этот пробел.
+126
View File
@@ -0,0 +1,126 @@
{
"groups": [
{
"name": "messenger-slo",
"interval": "5s",
"rules": [
{
"record": "messenger:api_error_ratio:1m",
"expr": "(sum(rate(messenger_http_requests_total{route=~\"/api/v1/.*\",status_class=\"5xx\"}[1m])) or vector(0)) / sum(rate(messenger_http_requests_total{route=~\"/api/v1/.*\"}[1m]))"
},
{
"record": "messenger:api_error_ratio:2m",
"expr": "(sum(rate(messenger_http_requests_total{route=~\"/api/v1/.*\",status_class=\"5xx\"}[2m])) or vector(0)) / sum(rate(messenger_http_requests_total{route=~\"/api/v1/.*\"}[2m]))"
},
{
"record": "messenger:api_error_ratio:5m",
"expr": "(sum(rate(messenger_http_requests_total{route=~\"/api/v1/.*\",status_class=\"5xx\"}[5m])) or vector(0)) / sum(rate(messenger_http_requests_total{route=~\"/api/v1/.*\"}[5m]))"
},
{
"record": "messenger:api_error_ratio:10m",
"expr": "(sum(rate(messenger_http_requests_total{route=~\"/api/v1/.*\",status_class=\"5xx\"}[10m])) or vector(0)) / sum(rate(messenger_http_requests_total{route=~\"/api/v1/.*\"}[10m]))"
},
{
"alert": "ApiBurnRateFast",
"expr": "(messenger:api_error_ratio:1m > 0.06) and (messenger:api_error_ratio:5m > 0.06) and (sum(increase(messenger_http_requests_total{route=~\"/api/v1/.*\"}[1m])) >= 20)",
"for": "20s",
"labels": {
"severity": "warning",
"slo": "api"
},
"annotations": {
"summary": "api SLO budget burning above 6x",
"runbook": "infra/otel/RUNBOOK.md"
}
},
{
"alert": "ApiBurnRateSlow",
"expr": "(messenger:api_error_ratio:2m > 0.03) and (messenger:api_error_ratio:10m > 0.03) and (sum(increase(messenger_http_requests_total{route=~\"/api/v1/.*\"}[2m])) >= 20)",
"for": "30s",
"labels": {
"severity": "warning",
"slo": "api"
},
"annotations": {
"summary": "api SLO budget burning above 3x",
"runbook": "infra/otel/RUNBOOK.md"
}
},
{
"record": "messenger:ready_error_ratio:1m",
"expr": "sum(rate(messenger_ready_slo_total{outcome=\"bad\"}[1m])) / sum(rate(messenger_ready_slo_total[1m]))"
},
{
"record": "messenger:ready_error_ratio:2m",
"expr": "sum(rate(messenger_ready_slo_total{outcome=\"bad\"}[2m])) / sum(rate(messenger_ready_slo_total[2m]))"
},
{
"record": "messenger:ready_error_ratio:5m",
"expr": "sum(rate(messenger_ready_slo_total{outcome=\"bad\"}[5m])) / sum(rate(messenger_ready_slo_total[5m]))"
},
{
"record": "messenger:ready_error_ratio:10m",
"expr": "sum(rate(messenger_ready_slo_total{outcome=\"bad\"}[10m])) / sum(rate(messenger_ready_slo_total[10m]))"
},
{
"alert": "ReadyBurnRateFast",
"expr": "(messenger:ready_error_ratio:1m > 0.06) and (messenger:ready_error_ratio:5m > 0.06) and (sum(increase(messenger_ready_slo_total[1m])) >= 5)",
"for": "20s",
"labels": {
"severity": "warning",
"slo": "ready"
},
"annotations": {
"summary": "ready SLO budget burning above 6x",
"runbook": "infra/otel/RUNBOOK.md"
}
},
{
"alert": "ReadyBurnRateSlow",
"expr": "(messenger:ready_error_ratio:2m > 0.03) and (messenger:ready_error_ratio:10m > 0.03) and (sum(increase(messenger_ready_slo_total[2m])) >= 5)",
"for": "30s",
"labels": {
"severity": "warning",
"slo": "ready"
},
"annotations": {
"summary": "ready SLO budget burning above 3x",
"runbook": "infra/otel/RUNBOOK.md"
}
},
{
"alert": "QueueLag",
"expr": "max(messenger_queue_oldest_age_seconds) > 30",
"for": "20s",
"labels": {
"severity": "warning"
},
"annotations": {
"summary": "Unfinished attachment age exceeds 30s",
"runbook": "infra/otel/RUNBOOK.md"
}
},
{
"alert": "ObservationsStale",
"expr": "time() - max(messenger_observation_timestamp) > 20 or absent(messenger_observation_timestamp)",
"for": "20s",
"labels": {
"severity": "warning"
},
"annotations": {
"summary": "No fresh dispatcher observations",
"runbook": "infra/otel/RUNBOOK.md"
}
},
{
"record": "messenger:api_error_ratio:1h",
"expr": "(sum(rate(messenger_http_requests_total{route=~\"/api/v1/.*\",status_class=\"5xx\"}[1h])) or vector(0)) / sum(rate(messenger_http_requests_total{route=~\"/api/v1/.*\"}[1h]))"
},
{
"record": "messenger:ready_error_ratio:1h",
"expr": "sum(rate(messenger_ready_slo_total{outcome=\"bad\"}[1h])) / sum(rate(messenger_ready_slo_total[1h]))"
}
]
}
]
}
+65
View File
@@ -0,0 +1,65 @@
receivers:
otlp:
protocols:
http:
endpoint: 0.0.0.0:4318
processors:
memory_limiter:
check_interval: 1s
limit_mib: 180
spike_limit_mib: 40
batch:
timeout: 1s
send_batch_size: 128
send_batch_max_size: 256
exporters:
prometheus:
endpoint: 0.0.0.0:8889
resource_to_telemetry_conversion:
enabled: true
otlphttp/tempo:
endpoint: http://tempo:4318
timeout: 2s
retry_on_failure:
initial_interval: 1s
max_interval: 5s
max_elapsed_time: 30s
sending_queue:
queue_size: 32
num_consumers: 2
otlphttp/loki:
endpoint: http://loki:3100/otlp
timeout: 2s
retry_on_failure:
initial_interval: 1s
max_interval: 5s
max_elapsed_time: 30s
sending_queue:
queue_size: 32
num_consumers: 2
extensions:
health_check:
endpoint: 0.0.0.0:13133
service:
telemetry:
metrics:
readers:
- pull:
exporter:
prometheus:
host: 0.0.0.0
port: 8888
extensions: [health_check]
pipelines:
metrics:
receivers: [otlp]
processors: [memory_limiter, batch]
exporters: [prometheus]
traces:
receivers: [otlp]
processors: [memory_limiter, batch]
exporters: [otlphttp/tempo]
logs:
receivers: [otlp]
processors: [memory_limiter, batch]
exporters: [otlphttp/loki]
+770
View File
@@ -0,0 +1,770 @@
{
"uid": "messenger-lab6",
"title": "Messenger \u2014 lab6",
"schemaVersion": 39,
"version": 2,
"refresh": "5s",
"time": {
"from": "now-30m",
"to": "now"
},
"panels": [
{
"id": 1,
"title": "HTTP RPS",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 0,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "sum by (route) (rate(messenger_http_requests_total[1m]))",
"legendFormat": "{{service_name}} {{route}} {{outcome}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "reqps"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 2,
"title": "HTTP 5xx ratio",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 0,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "messenger:api_error_ratio:1m",
"legendFormat": "{{service_name}} {{route}} {{outcome}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "percentunit"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 3,
"title": "HTTP p50/p95/p99",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 8,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "0.5",
"expr": "histogram_quantile(0.5,sum by (le)(rate(messenger_http_duration_seconds_bucket{route=~\"/api/v1/.*\"}[1m])))",
"legendFormat": "0.5"
},
{
"refId": "0.95",
"expr": "histogram_quantile(0.95,sum by (le)(rate(messenger_http_duration_seconds_bucket{route=~\"/api/v1/.*\"}[1m])))",
"legendFormat": "0.95"
},
{
"refId": "0.99",
"expr": "histogram_quantile(0.99,sum by (le)(rate(messenger_http_duration_seconds_bucket{route=~\"/api/v1/.*\"}[1m])))",
"legendFormat": "0.99"
}
],
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
},
"description": "Business routes only; histogram quantiles across all API replicas. Bucket interpolation is approximate."
},
{
"id": 4,
"title": "Job outcomes",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 8,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "sum by (outcome) (rate(messenger_jobs_total[1m]))",
"legendFormat": "{{service_name}} {{route}} {{outcome}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "ops"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 5,
"title": "Processing p95",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 16,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "histogram_quantile(0.95,sum by(le)(rate(messenger_job_duration_seconds_bucket[1m])))",
"legendFormat": "{{service_name}} {{route}} {{outcome}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 6,
"title": "Time to ready p95",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 16,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "histogram_quantile(0.95,sum by(le)(rate(messenger_job_ready_duration_seconds_bucket[1m])))",
"legendFormat": "{{service_name}} {{route}} {{outcome}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 7,
"title": "RabbitMQ ready messages",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 24,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "messenger_queue_depth",
"legendFormat": "{{service_name}} {{route}} {{outcome}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 8,
"title": "Logical pending jobs / outbox",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 24,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "messenger_jobs_pending",
"legendFormat": "jobs"
},
{
"refId": "B",
"expr": "messenger_outbox_pending",
"legendFormat": "outbox"
}
],
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 9,
"title": "Oldest unfinished job",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 32,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "messenger_queue_oldest_age_seconds",
"legendFormat": "{{service_name}} {{route}} {{outcome}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 10,
"title": "DB connections used/capacity/waiting",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 32,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "sum by (service_name)(messenger_db_pool_used)",
"legendFormat": "used {{service_name}}"
},
{
"refId": "B",
"expr": "sum by (service_name)(messenger_db_pool_capacity)",
"legendFormat": "capacity {{service_name}}"
},
{
"refId": "C",
"expr": "sum by (service_name)(messenger_db_pool_waiting)",
"legendFormat": "waiting {{service_name}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 11,
"title": "Worker busy/capacity",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 40,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "messenger_worker_busy{service_name=\"messenger-worker\"}",
"legendFormat": "busy"
},
{
"refId": "B",
"expr": "messenger_worker_capacity{service_name=\"messenger-worker\"}",
"legendFormat": "capacity"
}
],
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 12,
"title": "Process memory RSS",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 40,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "messenger_process_memory",
"legendFormat": "{{service_name}} {{route}} {{outcome}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "bytes"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 13,
"title": "CPU by service (% of one core)",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 48,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "100 * sum by(service_name)(rate(messenger_process_cpu[1m]))",
"legendFormat": "{{service_name}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "percent"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 14,
"title": "API burn rate",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 48,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "messenger:api_error_ratio:1m / 0.01",
"legendFormat": "{{service_name}} {{route}} {{outcome}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 15,
"title": "Ready burn rate",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 56,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "messenger:ready_error_ratio:1m / 0.01",
"legendFormat": "{{service_name}} {{route}} {{outcome}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 16,
"title": "Alerts",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 56,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "ALERTS{alertstate=\"firing\"}",
"legendFormat": "{{service_name}} {{route}} {{outcome}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 17,
"title": "Collector export failures",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 64,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "rate(otelcol_exporter_send_failed_spans_total[1m])",
"legendFormat": "{{service_name}} {{route}} {{outcome}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "ops"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 18,
"title": "Observation freshness",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 64,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "time() - messenger_observation_timestamp",
"legendFormat": "{{service_name}} {{route}} {{outcome}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 19,
"title": "API SLI \u2014 1 hour",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 72,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "1 - messenger:api_error_ratio:1h"
}
],
"fieldConfig": {
"defaults": {
"unit": "percentunit"
},
"overrides": []
}
},
{
"id": 20,
"title": "Ready SLI \u2014 1 hour",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 72,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "1 - messenger:ready_error_ratio:1h"
}
],
"fieldConfig": {
"defaults": {
"unit": "percentunit"
},
"overrides": []
}
},
{
"id": 21,
"title": "Exporter bounded queue",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 80,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "otelcol_exporter_queue_size",
"legendFormat": "size {{exporter}}"
},
{
"refId": "B",
"expr": "otelcol_exporter_queue_capacity",
"legendFormat": "capacity {{exporter}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
}
},
{
"id": 22,
"title": "DB lock waiters",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 80,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "messenger_db_locks_waiting"
}
],
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
}
}
]
}
@@ -0,0 +1,7 @@
apiVersion: 1
providers:
- name: Messenger
folder: Lab6
type: file
options:
path: /var/lib/grafana/dashboards
@@ -0,0 +1,35 @@
apiVersion: 1
datasources:
- name: Prometheus
uid: prometheus
type: prometheus
url: http://prometheus:9090
isDefault: true
jsonData:
timeInterval: 5s
- name: Loki
uid: loki
type: loki
url: http://loki:3100
jsonData:
derivedFields:
- name: TraceID
matcherRegex: trace_id
matcherType: label
datasourceUid: tempo
url: '$${__value.raw}'
maxLines: 1000
- name: Tempo
uid: tempo
type: tempo
url: http://tempo:3200
jsonData:
tracesToLogsV2:
datasourceUid: loki
tags: [{key: 'service.name', value: 'service_name'}]
spanStartTimeShift: '-1m'
spanEndTimeShift: '1m'
filterByTraceID: true
filterBySpanID: false
nodeGraph:
enabled: true
+37
View File
@@ -0,0 +1,37 @@
auth_enabled: false
server:
http_listen_port: 3100
common:
path_prefix: /loki
replication_factor: 1
ring:
kvstore:
store: inmemory
storage:
filesystem:
chunks_directory: /loki/chunks
rules_directory: /loki/rules
schema_config:
configs:
- from: 2024-01-01
store: tsdb
object_store: filesystem
schema: v13
index:
prefix: index_
period: 24h
limits_config:
allow_structured_metadata: true
retention_period: 168h
otlp_config:
resource_attributes:
ignore_defaults: true
attributes_config:
- action: index_label
attributes: [service.name, deployment.environment.name]
- action: structured_metadata
attributes: [service.instance.id, service.version]
compactor:
working_directory: /loki/compactor
retention_enabled: true
delete_request_store: filesystem
+15
View File
@@ -0,0 +1,15 @@
global:
scrape_interval: 5s
evaluation_interval: 5s
rule_files:
- /etc/prometheus/alerts.yaml
scrape_configs:
- job_name: messenger
static_configs:
- targets: ['otel-collector:8889']
- job_name: collector
static_configs:
- targets: ['otel-collector:8888']
- job_name: prometheus
static_configs:
- targets: ['localhost:9090']
+22
View File
@@ -0,0 +1,22 @@
server:
http_listen_port: 3200
distributor:
receivers:
otlp:
protocols:
http:
endpoint: 0.0.0.0:4318
ingester:
max_block_duration: 5m
compactor:
compaction:
block_retention: 168h
storage:
trace:
backend: local
wal:
path: /var/tempo/wal
local:
path: /var/tempo/blocks
usage_report:
reporting_enabled: false
+34
View File
@@ -0,0 +1,34 @@
#!/usr/bin/env python3
"""Save bounded-buffer and loss observations for the actual Tempo outage."""
import datetime as dt,json,re,subprocess,sys
from pathlib import Path
ROOT=Path(__file__).resolve().parents[1];sys.path.insert(0,str(ROOT/'scripts'))
from observations import get
x=json.loads((ROOT/'evidence/slo-experiment.json').read_text())
if not x.get('completed'):raise SystemExit('Experiment has not completed successfully')
p=next(p for p in x['phases'] if p['name']=='tempo_unavailable');finish=next(p['end'] for p in x['phases'] if p['name']=='final_drain')
metrics=['otelcol_receiver_accepted_spans_total','otelcol_exporter_sent_spans_total','otelcol_exporter_send_failed_spans_total','otelcol_exporter_enqueue_failed_spans_total','otelcol_receiver_refused_spans_total','otelcol_exporter_queue_size','otelcol_exporter_queue_capacity']
def at(name,t):
rows=get('prometheus','/api/v1/query',{'query':name,'time':str(t)})['data']['result']
return [r for r in rows if name.startswith('otelcol_receiver') or r['metric'].get('exporter')=='otlphttp/tempo']
before={n:at(n,p['start']) for n in metrics};after={n:at(n,finish) for n in metrics}
def total(rows):return sum(float(r['value'][1]) for r in rows)
deltas={n:total(after[n])-total(before[n]) for n in metrics if n.endswith('_total')}
samples=get('prometheus','/api/v1/query_range',{'query':'otelcol_exporter_queue_size{exporter="otlphttp/tempo"}','start':str(p['start']),'end':str(finish),'step':'5'})['data']['result']
peak=max((float(v[1]) for r in samples for v in r['values']),default=0)
raw=subprocess.check_output([str(ROOT/'scripts/compose.sh'),'logs','--no-log-prefix','--since',dt.datetime.fromtimestamp(p['start'],dt.timezone.utc).isoformat(),'otel-collector'],text=True)
drops=[]
for line in raw.splitlines():
try:d=json.loads(line[line.index('{'):])
except (ValueError,KeyError):continue
if d.get('otelcol.component.id')=='otlphttp/tempo' and ('dropped_items' in d or 'rejected_items' in d):
drops.append({k:d[k] for k in ['dropped_items','rejected_items','otelcol.component.id','otelcol.signal'] if k in d})
mem=[]
for record in x['resources']:
if p['start']<=record['at']<=finish:
for c in record['containers']:
if 'otel-collector' in c['Name']:
m=re.match(r'([0-9.]+)([A-Za-z]+)',c['MemUsage']);scale={'B':1,'KiB':1024,'MiB':1024**2,'GiB':1024**3}.get(m[2],1);mem.append(float(m[1])*scale)
out={'outage_seconds':p['end']-p['start'],'before':before,'after_drain':after,'counter_deltas':deltas,'collector_accepted_minus_sent_estimate':deltas['otelcol_receiver_accepted_spans_total']-deltas['otelcol_exporter_sent_spans_total'],'estimate_note':'Background health/observer spans remain active; balance is approximate by a few in-flight batches. SDK-side losses are not included; send_failed counts attempts, not unique spans.','peak_exporter_queue':peak,'queue_samples':samples,'collector_memory_peak_bytes':max(mem,default=0),'collector_memory_limit_bytes':256*1024**2,'drop_records':drops,'drop_records_items':sum(d.get('dropped_items',0) for d in drops),'message_persistence':x.get('message_persistence'),'attachment_states':x.get('attachment_states')}
(ROOT/'evidence/backend-outage.json').write_text(json.dumps(out,indent=2)+'\n')
print('Saved backend outage balance, queue, memory and business persistence evidence')
+17
View File
@@ -0,0 +1,17 @@
#!/usr/bin/env python3
"""Create synthetic local broker credentials, preserving existing settings."""
from pathlib import Path
import secrets
from urllib.parse import quote
root=Path(__file__).resolve().parents[1]
env=root/'.env.lab6'
values=dict(line.split('=',1) for line in env.read_text().splitlines() if '=' in line and not line.startswith('#'))
password=values.get('BROKER_PASSWORD')
if not password:
password=secrets.token_hex(24)
with env.open('a') as f: f.write('\nBROKER_PASSWORD='+password+'\n')
p=root/'secrets/lab6/broker_url'
if p.exists(): p.chmod(0o600)
p.write_text('amqp://messenger:'+quote(password,safe='')+'@broker:5672/\n')
p.chmod(0o444)
print('Local broker credentials ready.')
+25
View File
@@ -0,0 +1,25 @@
#!/usr/bin/env python3
"""Generate local synthetic storage credentials without rotating existing keys."""
import json
import os
from pathlib import Path
import secrets
root=Path(__file__).resolve().parents[1]/'secrets'/'lab6'
root.mkdir(parents=True,exist_ok=True)
for name in ('s3_admin_access','s3_admin_secret','s3_api_access','s3_api_secret'):
p=root/name
if not p.exists():
p.write_text(secrets.token_hex(24)+'\n')
p.chmod(0o444) # Non-root containers must read these local synthetic secrets.
bucket=os.environ.get('S3_BUCKET','messenger-attachments')
identities=[]
for role,actions in [('admin',['Admin','Read','Write','List','Tagging']),('api',['Read:'+bucket,'Write:'+bucket,'List:'+bucket])]:
identities.append({'name':role,'credentials':[{'accessKey':(root/('s3_'+role+'_access')).read_text().strip(),'secretKey':(root/('s3_'+role+'_secret')).read_text().strip()}],'actions':actions})
p=root/'s3.json'
content=json.dumps({'identities':identities})+'\n'
if not p.exists() or p.read_text()!=content:
if p.exists(): p.chmod(0o600)
p.write_text(content)
p.chmod(0o444)
print('Local storage credentials ready; existing keys preserved.')
+30
View File
@@ -0,0 +1,30 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
umask 077
mkdir -p secrets/lab6
if [ ! -f .env.lab6 ]; then
{ printf 'POSTGRES_PASSWORD='; openssl rand -hex 24
printf 'REDIS_PASSWORD='; openssl rand -hex 24
printf 'HOST_PORT=18086\nAPI_REPLICAS=2\nSESSION_TTL_SECONDS=14400\n'
} > .env.lab6
fi
if [ ! -f secrets/lab6/private_key.pem ]; then
openssl genrsa -out secrets/lab6/private_key.pem 2048 2>/dev/null
fi
if [ ! -f secrets/lab6/public_key.pem ]; then
openssl rsa -in secrets/lab6/private_key.pem -pubout -out secrets/lab6/public_key.pem 2>/dev/null
fi
# Only synthetic stand keys, readable by the non-root application inside containers.
chmod 755 secrets secrets/lab6
chmod 444 secrets/lab6/*.pem
printf '%s\n' 'Local lab6 settings and synthetic keys ready; existing files preserved.'
python3 scripts/bootstrap-storage.py
python3 scripts/bootstrap-broker.py
if [ ! -f secrets/lab6/grafana_password ]; then
openssl rand -hex 24 > secrets/lab6/grafana_password
fi
chmod 444 secrets/lab6/grafana_password
+19
View File
@@ -0,0 +1,19 @@
#!/usr/bin/env python3
"""Record reproducibility metadata, with an allowlist excluding all secrets."""
import hashlib,json,platform,subprocess,time
from pathlib import Path
root=Path(__file__).resolve().parents[1]
def cmd(*args):return subprocess.check_output(args,cwd=root,text=True).strip()
paths=[p for d in ['service','infra','scripts','tests','contracts','student-tests'] for p in (root/d).rglob('*') if p.is_file() and '__pycache__' not in p.parts]
paths += [root/x for x in ['Dockerfile','compose.yaml','compose.faults.yaml','Makefile']]
h=hashlib.sha256()
for p in sorted(paths):h.update(str(p.relative_to(root)).encode());h.update(p.read_bytes())
containers=[json.loads(x) for x in cmd('docker','compose','--env-file','.env.lab6','ps','--format','json').splitlines()]
ids=[c['ID'] for c in containers]
inspect=json.loads(cmd('docker','inspect',*ids))
rows=[{'service':c['Config']['Labels']['com.docker.compose.service'],'image':c['Config']['Image'],'image_id':c['Image'],'started_at':c['State']['StartedAt'],'memory_limit':c['HostConfig']['Memory'],'nano_cpus':c['HostConfig']['NanoCpus'],'mounts':[{'type':m['Type'],'name':m.get('Name'),'destination':m['Destination']} for m in c['Mounts'] if m['Type']=='volume']} for c in inspect]
info=json.loads(cmd('docker','info','--format','{{json .}}'))
out={'timestamp':time.time(),'revision':cmd('git','rev-parse','HEAD'),'working_tree_dirty':bool(cmd('git','status','--porcelain')),'source_sha256':h.hexdigest(),'platform':platform.platform(),'go':cmd('go','version'),'docker':{'server_version':info['ServerVersion'],'cpus':info['NCPU'],'memory':info['MemTotal'],'architecture':info['Architecture']},'containers':rows}
if platform.system()=='Darwin':out['host']={'cpu':cmd('sysctl','-n','machdep.cpu.brand_string'),'memory':cmd('sysctl','-n','hw.memsize')}
(root/'evidence/environment.json').write_text(json.dumps(out,indent=2)+'\n')
print('Saved sanitized environment and source fingerprint')
+43
View File
@@ -0,0 +1,43 @@
#!/usr/bin/env python3
"""Passively save stored log/trace evidence during the running SLO experiment."""
import json,sys,time
from pathlib import Path
ROOT=Path(__file__).resolve().parents[1];sys.path.insert(0,str(ROOT/'scripts'))
from observations import get
source=ROOT/'evidence/slo-experiment.json';target=ROOT/'evidence/incident-signals.json'
result={};errors=[]
def logs(query,start,end):
return get('loki','/loki/api/v1/query_range',{'query':query,'start':str(int(start*1e9)),'end':str(int(end*1e9)),'limit':100})['data']['result']
def summary(d):
rows=[]
for b in d.get('batches',[]):
resource={a['key']:a['value'].get('stringValue') for a in b.get('resource',{}).get('attributes',[])}
for scope in b.get('scopeSpans',[]):
for s in scope.get('spans',[]):
rows.append({'service':resource.get('service.name'),'name':s['name'],'kind':s.get('kind'),'span_id':s.get('spanId'),'parent_span_id':s.get('parentSpanId'),'duration_ms':(int(s['endTimeUnixNano'])-int(s['startTimeUnixNano']))/1e6,'status':s.get('status')})
return rows
end=time.time()+8000
while time.time()<end:
try:
x=json.loads(source.read_text());last=x['observations'][-1];phase=last['phase'];event=next(e for e in reversed(x['events']) if e['phase']==phase)
if phase.startswith('db_long_') and time.time()-event['at']>60 and phase not in result:
rows=logs('{service_name="messenger-api"} |= "http_error"',event['at'],time.time()-3)
ids=[r['stream'].get('trace_id') for r in rows if r['stream'].get('trace_id')]
if ids:
traceid=ids[-1];spans=summary(get('tempo','/api/traces/'+traceid))
if any(s['name'].startswith('postgresql ') and s['duration_ms']>1000 for s in spans):result[phase]={'captured':time.time(),'observed':last,'trace_id':traceid,'spans':spans,'error_log_events':sum(len(r['values']) for r in rows),'diagnosis':'DB wait dominates server span; HTTP 5xx, not image processing'}
if phase.startswith('worker_long_') and time.time()-event['at']>70 and phase not in result:
worker=logs('{service_name="messenger-worker"} |= "job_received"',event['at'],time.time()-3)
uploads=logs('{service_name="messenger-api"} |= "http_request" | route="/api/v1/chats/{chat_id}/attachments" | status="202"',event['at'],time.time()-3)
result[phase]={'captured':time.time(),'observed':last,'worker_received_events':sum(len(r['values']) for r in worker),'accepted_upload_events':sum(len(r['values']) for r in uploads),'diagnosis':'Queue age and ready burn increase while API accepts uploads and no worker consumption is observed'}
if phase.startswith('worker_recovery_') and phase not in result and time.time()-event['at']>20:
rows=logs('{service_name="messenger-worker"} |= "job_ready"',event['at'],time.time()-3)
ids=[r['stream'].get('trace_id') for r in rows if r['stream'].get('trace_id')]
if ids:
traceid=ids[0];spans=summary(get('tempo','/api/traces/'+traceid))
if spans:result[phase]={'captured':time.time(),'trace_id':traceid,'spans':spans,'diagnosis':'Worker resumes consumption and publishes S3/DB results'}
target.write_text(json.dumps({'evidence':result,'collection_errors':errors[-20:]},indent=2)+'\n')
if x.get('completed') is not None:break
except Exception as e:errors.append({'at':time.time(),'type':type(e).__name__})
time.sleep(5)
print('Saved incident observations:',len(result))
+4
View File
@@ -0,0 +1,4 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
exec docker compose --env-file .env.lab6 "$@"
+147
View File
@@ -0,0 +1,147 @@
#!/usr/bin/env python3
"""Bounded SLO experiment on the isolated lab6 stand, with raw observations.
No synthetic metrics or time acceleration. Restores every injected fault in finally.
"""
import argparse,concurrent.futures,hashlib,json,math,os,signal,statistics,subprocess,sys,threading,time,urllib.request,urllib.error,uuid
from pathlib import Path
ROOT=Path(__file__).resolve().parents[1]
sys.path[:0]=[str(ROOT/'tests'),str(ROOT/'scripts')]
from client import Client
from observations import get,query,scalar
C=[str(ROOT/'scripts/compose.sh')]
def compose(*a):return subprocess.run(C+list(a),cwd=ROOT,check=True,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL,timeout=90)
def start_service(name):
ids=subprocess.check_output(C+['ps','-a','-q',name],text=True).split()
if not ids:raise RuntimeError('Missing existing '+name+' container')
subprocess.run(['docker','start',*ids],check=True,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL,timeout=30)
def sql(q):return subprocess.check_output(C+['exec','-T','postgres','psql','-U','messenger','-d','messenger','-Atc',q],text=True,timeout=10).strip()
def percentile(xs,q):return sorted(xs)[max(0,math.ceil(len(xs)*q)-1)] if xs else None
class Traffic:
def __init__(self):
c=Client('http://127.0.0.1:18086');a,b,e,chat=c.bootstrap()
self.chat=chat;self.token=a['token'];self.actor=a;self.client=c
self.fixture=(ROOT/'tests/fixtures/sample.png').read_bytes()
self.lock=threading.Lock();self.rows=[];self.attachments=[];self.messages=[];self.dropped=0;self.stop=threading.Event();self.slots=threading.BoundedSemaphore(64)
def request(self,index,scheduled,upload=False):
try:
path='/api/v1/chats/'+self.chat['id']
headers={'Authorization':'Bearer '+self.token};body=None
if upload:method='POST';path+='/attachments?filename=sample.png';body=self.fixture;headers['Content-Type']='image/png'
elif index%10==0:method='POST';path+='/messages';body=b'{"text":"synthetic-slo"}';headers.update({'Content-Type':'application/json','Idempotency-Key':uuid.uuid4().hex})
else:method='GET';path+='/messages?limit=50'
r=urllib.request.Request('http://127.0.0.1:18086'+path,data=body,headers=headers,method=method)
status=0;att=None;message_id=None
try:
try:response=urllib.request.urlopen(r,timeout=6)
except urllib.error.HTTPError as e:response=e
with response:
status=response.status;data=response.read()
if upload and status==202:att=json.loads(data)['id']
if not upload and method=='POST' and status==201:message_id=json.loads(data)['id']
except (OSError,TimeoutError):pass
with self.lock:
self.rows.append((time.time(),upload,status,(time.monotonic()-scheduled)*1000))
if att:self.attachments.append(att)
if message_id:self.messages.append(message_id)
finally:self.slots.release()
def run(self):
start=time.monotonic();i=0
with concurrent.futures.ThreadPoolExecutor(max_workers=64) as pool:
while not self.stop.is_set():
scheduled=start+i*.1
if self.stop.wait(max(0,scheduled-time.monotonic())):break
if self.slots.acquire(False):pool.submit(self.request,i,scheduled)
else:
with self.lock:self.dropped+=1
if i%20==0:
if self.slots.acquire(False):pool.submit(self.request,i,scheduled,True)
else:
with self.lock:self.dropped+=1
i+=1
def summary(self):
with self.lock:rows=list(self.rows);dropped=self.dropped
xs=[r[3] for r in rows if 200<=r[2]<300]
return {'requests':len(rows),'success':sum(200<=r[2]<300 for r in rows),'errors':sum(not 200<=r[2]<300 for r in rows),'generator_dropped':dropped,'uploads_accepted':sum(r[1] and r[2]==202 for r in rows),'latency_success_ms':{str(q):percentile(xs,q) for q in [.5,.95,.99]},'statuses':{str(s):sum(r[2]==s for r in rows) for s in sorted(set(r[2] for r in rows))}}
def main():
def interrupted(*_):raise KeyboardInterrupt()
signal.signal(signal.SIGTERM,interrupted)
p=argparse.ArgumentParser();p.add_argument('--warmup',type=int,default=900);p.add_argument('--long',type=int,default=240);p.add_argument('--cooldown',type=int,default=660);p.add_argument('--repeats',type=int,default=3);p.add_argument('--backend',type=int,default=300);p.add_argument('--output',default='evidence/slo-experiment.json');a=p.parse_args()
out={'started':time.time(),'revision':subprocess.check_output(['git','rev-parse','HEAD'],text=True).strip(),'parameters':vars(a),'profile':{'message_rps':10,'read_fraction':.9,'image_interval_seconds':2,'max_inflight':64,'transparent_retries':False},'events':[],'observations':[],'resources':[],'phases':[]}
out['image_id']=subprocess.check_output(['docker','image','inspect','mtusi-messenger:lab6','--format','{{.Id}}'],text=True).strip()
target=ROOT/a.output
def save():target.write_text(json.dumps(out,indent=2)+'\n')
traffic=Traffic();t=threading.Thread(target=traffic.run);t.start()
def phase(name,seconds):
at=time.time();out['events'].append({'phase':name,'at':at});print(name,seconds,'seconds',flush=True);save()
before=traffic.summary();nextstats=0;nextprint=0
while time.time()<at+seconds:
now=time.time()
row={'at':now,'phase':name}
try:
row.update({'alerts':get('prometheus','/api/v1/alerts')['data']['alerts'],'pending':scalar('sum(messenger_jobs_pending)'), 'db_lock_waiting':scalar('sum(messenger_db_locks_waiting)'), 'queue_age':scalar('max(messenger_queue_oldest_age_seconds)'), 'api_burn':scalar('messenger:api_error_ratio:1m / 0.01'), 'ready_burn':scalar('messenger:ready_error_ratio:1m / 0.01')})
except Exception as e:row['collection_error']=type(e).__name__
out['observations'].append(row)
if now>=nextstats:
ids=subprocess.check_output(C+['ps','-q','api','worker','dispatcher','otel-collector'],text=True).split()
if ids:
raw=subprocess.check_output(['docker','stats','--no-stream','--format','{{json .}}',*ids],text=True,timeout=20)
out['resources'].append({'at':time.time(),'phase':name,'containers':[json.loads(x) for x in raw.splitlines()]})
nextstats=now+30
if now>=nextprint:print(name,round(time.time()-at),'s',traffic.summary()['requests'],'requests',flush=True);nextprint=now+60
save();time.sleep(min(5,max(0,at+seconds-time.time())))
with traffic.lock: rows=[r for r in traffic.rows if at<=r[0]<=time.time()]
lat=[r[3] for r in rows if 200<=r[2]<300]
elapsed=time.time()-at;success=sum(200<=r[2]<300 for r in rows)
out['phases'].append({'name':name,'start':at,'end':time.time(),'before':before,'after':traffic.summary(),'successful_rps':success/elapsed,'error_ratio':1-success/len(rows) if rows else None,'latency_success_ms':{str(q):percentile(lat,q) for q in [.5,.95,.99]},'statuses':{str(s):sum(r[2]==s for r in rows) for s in sorted(set(r[2] for r in rows))}});save()
def worker(duration,name):
compose('stop','worker')
try:phase(name,duration)
finally:start_service('worker')
def database(duration,name):
# A separate bounded transaction blocks only the local message table.
proc=subprocess.Popen(C+['exec','-T','postgres','psql','-U','messenger','-d','messenger','-v','ON_ERROR_STOP=1','-c',f"SET application_name='lab6_slo_injection'; BEGIN; LOCK TABLE messages IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep({duration}); ROLLBACK;"],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
try:
deadline=time.time()+10
while time.time()<deadline:
if sql("SELECT count(*) FROM pg_locks l JOIN pg_stat_activity a USING(pid) WHERE a.application_name='lab6_slo_injection' AND l.mode='AccessExclusiveLock' AND l.granted")!='0':break
time.sleep(.1)
else:raise RuntimeError('DB injection did not acquire lock')
phase(name,duration)
finally:
sql("SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE application_name='lab6_slo_injection'")
proc.wait(timeout=10)
try:
phase('baseline',a.warmup)
for i in range(1,a.repeats+1):
worker(5,f'worker_short_{i}');phase(f'worker_short_recovery_{i}',60)
database(5,f'db_short_{i}');phase(f'db_short_recovery_{i}',60)
worker(a.long,f'worker_long_{i}');phase(f'worker_recovery_{i}',a.cooldown)
database(a.long,f'db_long_{i}');phase(f'db_recovery_{i}',a.cooldown)
if a.backend:
out['backend_before']={n:query(n) for n in ['otelcol_exporter_sent_spans_total','otelcol_exporter_send_failed_spans_total','otelcol_exporter_queue_size','otelcol_receiver_accepted_spans_total']}
compose('stop','tempo')
try:phase('tempo_unavailable',a.backend)
finally:start_service('tempo')
phase('tempo_recovery',60)
out['backend_after']={n:query(n) for n in out['backend_before']}
traffic.stop.set();t.join(timeout=15)
phase('final_drain',30)
# Every accepted upload must have a published result after recovery.
with traffic.lock:ids=list(traffic.attachments)
out['accepted_ids_count']=len(ids)
counts={}
for id in ids:
state=traffic.client.call('GET','/api/v1/attachments/'+id,actor=traffic.actor,schema='Attachment')['json']['status'];counts[state]=counts.get(state,0)+1
out['attachment_states']=counts
stored=set(sql("SELECT id FROM messages WHERE chat_id='%s'"%str(uuid.UUID(traffic.chat['id']))).splitlines())
with traffic.lock:confirmed=set(traffic.messages)
out['message_persistence']={'confirmed':len(confirmed),'stored':len(stored),'confirmed_missing':len(confirmed-stored)}
if confirmed-stored or counts.get('ready',0)!=len(ids):raise RuntimeError('Confirmed business data not preserved')
out['completed']=True
except BaseException as e:
out['completed']=False;out['failure_type']=type(e).__name__;raise
finally:
traffic.stop.set();t.join(timeout=15)
out['finished']=time.time();out['traffic']=traffic.summary();save()
if __name__=='__main__':main()
+17
View File
@@ -0,0 +1,17 @@
#!/usr/bin/env python3
"""Convert a temporary generation view to the oapi-codegen 3.0 subset.
Runtime handlers enforce anyOf and required fields; normative contract is untouched.
"""
import json
from pathlib import Path
s=json.loads(Path('contracts/openapi.json').read_text())
for schema in s['components']['schemas'].values():
schema.pop('anyOf',None)
for prop in schema.get('properties',{}).values():
if prop.get('type')==['string','null']:
prop.update(type='string',nullable=True)
# Preserve pointers so handlers can distinguish missing fields from empty values.
for name,field in [('CreateUser','password'),('CreateMessage','text')]:
s['components']['schemas'][name]['properties'][field]['x-go-type-skip-optional-pointer']=False
s['components']['schemas'][name]['required'].remove(field)
print(json.dumps(s))
+27
View File
@@ -0,0 +1,27 @@
#!/usr/bin/env python3
"""Read local telemetry through authenticated Grafana proxies; never print credentials."""
import math,base64,json,urllib.request,urllib.parse,time
from pathlib import Path
ROOT=Path(__file__).resolve().parents[1]
def grafana(path,params=None):
password=(ROOT/'secrets/lab6/grafana_password').read_text().strip()
token=base64.b64encode(('admin:'+password).encode()).decode()
url='http://127.0.0.1:13006'+path
if params:url+='?'+urllib.parse.urlencode(params)
r=urllib.request.Request(url,headers={'Authorization':'Basic '+token,'Accept':'application/json'})
with urllib.request.urlopen(r,timeout=20) as response:return json.load(response)
def get(source,path,params=None):return grafana('/api/datasources/proxy/uid/'+source+path,params)
def query(expr):return get('prometheus','/api/v1/query',{'query':expr})['data']['result']
def scalar(expr,default=None):
x=query(expr)
v=float(x[0]['value'][1]) if x else default
return v if v is not None and math.isfinite(v) else default
def collect():
names=['messenger_http_requests_total','messenger_jobs_total','messenger_job_duration_seconds_count','messenger_job_ready_duration_seconds_count','messenger_db_pool_used','messenger_worker_busy','messenger_queue_depth','messenger_queue_oldest_age_seconds','messenger_ready_slo_total','messenger_observation_timestamp','ALERTS','otelcol_exporter_send_failed_spans_total','otelcol_exporter_sent_spans_total','otelcol_receiver_accepted_spans_total']
return {'timestamp':time.time(),'queries':{n:query(n) for n in names}}
if __name__=='__main__':
import argparse
p=argparse.ArgumentParser();p.add_argument('--output',default='evidence/signals.json');a=p.parse_args()
result=collect();Path(a.output).write_text(json.dumps(result,indent=2)+'\n')
print('Saved signal snapshot:',a.output)
+68
View File
@@ -0,0 +1,68 @@
#!/usr/bin/env python3
"""Reproduce ordinary and retried uploads and save real trace/log relationships."""
import json,subprocess,sys,time,uuid,os
from pathlib import Path
ROOT=Path(__file__).resolve().parents[1]
sys.path[:0]=[str(ROOT/'tests'),str(ROOT/'scripts')]
from client import Client,check
from observations import get
C=[str(ROOT/'scripts/compose.sh')]
def compose(*args):subprocess.run(C+list(args),cwd=ROOT,check=True,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL,timeout=120)
def sql(q):return subprocess.check_output(C+['exec','-T','postgres','psql','-U','messenger','-d','messenger','-Atc',q],text=True).strip()
def wait(fn,seconds=60):
end=time.monotonic()+seconds
while time.monotonic()<end:
try:
x=fn()
if x:return x
except Exception:pass
time.sleep(1)
raise RuntimeError('Timed out waiting for correlation evidence')
def trace_summary(d):
spans=[]
for b in d.get('batches',d.get('resourceSpans',[])):
attrs={a['key']:a['value'].get('stringValue') for a in b.get('resource',{}).get('attributes',[])}
for scope in b.get('scopeSpans',b.get('instrumentationLibrarySpans',[])):
for s in scope.get('spans',[]):
spans.append({k:s.get(k) for k in ['name','kind','traceId','spanId','parentSpanId','startTimeUnixNano','endTimeUnixNano','status'] }|{'service':attrs.get('service.name')})
return spans
c=Client('http://127.0.0.1:18086');a,b,e,chat=c.bootstrap()
results=[];faults=ROOT/'.lab6-faults';faults.mkdir(exist_ok=True);faults.chmod(0o777)
for retry in [False,True]:
traceid=uuid.uuid4().hex
marker=None
try:
if retry:compose('stop','worker')
data=(ROOT/'tests/fixtures/sample.png').read_bytes()
att=c.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=sample.png',202,actor=a,raw=data,headers={'Content-Type':'image/png','traceparent':'00-'+traceid+'-'+uuid.uuid4().hex[:16]+'-01'},schema='Attachment')['json']
jid=sql("SELECT id FROM attachment_jobs WHERE attachment_id='%s'"%str(uuid.UUID(att['id'])))
if retry:
marker=faults/('before_objects.'+jid);marker.write_text('temporary');marker.chmod(0o644)
compose('-f','compose.yaml','-f','compose.faults.yaml','up','-d','--no-deps','worker')
wait(lambda:int(sql("SELECT attempt FROM attachment_jobs WHERE id='%s'"%jid))>=2)
marker.unlink()
ready=c.wait_ready(a,att)
def complete():
spans=trace_summary(get('tempo','/api/traces/'+traceid))
return spans if any(s['service']=='messenger-api' and s['name'].startswith('POST ') for s in spans) and any(s['name']=='attachment.process' for s in spans) and any(s['name']=='s3.put' and s['service']=='messenger-worker' for s in spans) else None
spans=wait(complete)
def logs():
d=get('loki','/loki/api/v1/query_range',{'query':'{service_name=~"messenger-.+"} | trace_id="'+traceid+'"','start':str(int((time.time()-300)*1e9)),'end':str(int(time.time()*1e9)),'limit':'1000'})
rows=d['data']['result']
return rows if {'messenger-api','messenger-worker'}.issubset({r['stream'].get('service_name') for r in rows}) else None
rows=wait(logs)
events=[]
for r in rows:
stream=r['stream'];events.extend({'service':stream.get('service_name'),'trace_id':stream.get('trace_id'),'span_id':stream.get('span_id'),'event':line,'timestamp':t} for t,line,*_ in r['values'])
check(any(s['kind'] in ['SPAN_KIND_PRODUCER',4] for s in spans),'producer span missing')
check(any(s['kind'] in ['SPAN_KIND_CONSUMER',5] for s in spans),'consumer span missing')
producers={s['spanId'] for s in spans if s['kind'] in ['SPAN_KIND_PRODUCER',4]}
consumers=[s for s in spans if s['kind'] in ['SPAN_KIND_CONSUMER',5]]
check(all(s['parentSpanId'] in producers for s in consumers),'consumer parent is not a producer')
check(sql("SELECT split_part(traceparent,'-',2) FROM attachment_jobs WHERE id='%s'"%jid)==traceid,'durable context was lost')
results.append({'trace_id' :traceid,'retry':retry,'attachment_id':att['id'],'job_id':jid,'attempt':int(sql("SELECT attempt FROM attachment_jobs WHERE id='%s'"%jid)),'spans':spans,'logs':events})
print('PASS: correlated upload, retry='+str(retry),flush=True)
finally:
if marker:marker.unlink(missing_ok=True)
if retry:compose('up','-d','--no-deps','worker')
(ROOT/'evidence/correlation.json').write_text(json.dumps(results,indent=2)+'\n')
+31
View File
@@ -0,0 +1,31 @@
#!/usr/bin/env python3
"""Confirm that Prometheus sees the first 5xx spike after zero-series initialization."""
import concurrent.futures,json,subprocess,sys,time
from pathlib import Path
ROOT=Path(__file__).resolve().parents[1];sys.path[:0]=[str(ROOT/'tests'),str(ROOT/'scripts')]
from client import Client,check
from observations import query,scalar
C=[str(ROOT/'scripts/compose.sh')]
def sql(q):return subprocess.check_output(C+['exec','-T','postgres','psql','-U','messenger','-d','messenger','-Atc',q],text=True,timeout=10).strip()
c=Client('http://127.0.0.1:18086');a,b,e,chat=c.bootstrap()
before=query('messenger_http_requests_total{route="/api/v1/chats/{chat_id}/messages",status_class="5xx"}')
proc=subprocess.Popen(C+['exec','-T','postgres','psql','-U','messenger','-d','messenger','-c',"SET application_name='lab6_counter_probe'; BEGIN; LOCK TABLE messages IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep(5); ROLLBACK;"],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
def request():return c.call('GET','/api/v1/chats/'+chat['id']+'/messages',expected=[200,500,503],actor=a)['status']
try:
end=time.time()+5
while time.time()<end:
if sql("SELECT count(*) FROM pg_locks l JOIN pg_stat_activity a USING(pid) WHERE a.application_name='lab6_counter_probe' AND l.mode='AccessExclusiveLock' AND l.granted")!='0':break
time.sleep(.05)
futures=[]
with concurrent.futures.ThreadPoolExecutor(max_workers=24) as pool:
start=time.monotonic()
for i in range(50):
time.sleep(max(0,start+i*.1-time.monotonic()));futures.append(pool.submit(request))
statuses=[f.result() for f in futures]
finally:
sql("SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE application_name='lab6_counter_probe'");proc.wait(timeout=10)
time.sleep(15)
increase=scalar('sum(increase(messenger_http_requests_total{route="/api/v1/chats/{chat_id}/messages",status_class="5xx"}[1m]))')
check(increase is not None and increase>0,'first spike missing from rate/increase')
out={'at':time.time(),'before':before,'after':query('messenger_http_requests_total{route="/api/v1/chats/{chat_id}/messages",status_class="5xx"}'),'increase_1m':increase,'statuses':{str(s):statuses.count(s) for s in set(statuses)},'passed':True}
(ROOT/'evidence/first-spike.json').write_text(json.dumps(out,indent=2)+'\n');print('PASS: first 5xx spike appears in Prometheus increase:',round(increase,2))
+22
View File
@@ -0,0 +1,22 @@
#!/usr/bin/env python3
"""Attach container/volume evidence to the existing public persistence scenario."""
import json,subprocess,time
from pathlib import Path
root=Path(__file__).resolve().parents[1]
compose=[str(root/'scripts/compose.sh')]
def snapshot():
ids=subprocess.check_output(compose+['ps','-q'],text=True).split()
rows=json.loads(subprocess.check_output(['docker','inspect',*ids],text=True))
return {r['Name'].lstrip('/'):{'service':r['Config']['Labels']['com.docker.compose.service'],'started_at':r['State']['StartedAt'],'volumes':sorted(m['Name'] for m in r['Mounts'] if m['Type']=='volume')} for r in rows}
before=snapshot();start=time.time()
p=subprocess.run(['make','test-persistence','ACTION_SCRIPT=scripts/restart.sh'],cwd=root,text=True,stdout=subprocess.PIPE,stderr=subprocess.STDOUT)
(root/'evidence/persistence.txt').write_text(p.stdout)
after=snapshot();stores=['postgres','redis','s3','broker','prometheus','loki','tempo','grafana']
checks={}
for service in stores:
old=next(v for v in before.values() if v['service']==service);new=next(v for v in after.values() if v['service']==service)
checks[service]={'restarted':old['started_at']!=new['started_at'],'same_volumes':old['volumes']==new['volumes']}
out={'started':start,'finished':time.time(),'command':['make','test-persistence','ACTION_SCRIPT=scripts/restart.sh'],'returncode':p.returncode,'before':before,'after':after,'checks':checks}
(root/'evidence/persistence-processes.json').write_text(json.dumps(out,indent=2)+'\n')
if p.returncode or not all(v['restarted'] and v['same_volumes'] for v in checks.values()):raise SystemExit('FAIL: persistence or actual store restart evidence')
print('PASS: persistence, real restart of all stores, unchanged volume identities')
+37
View File
@@ -0,0 +1,37 @@
#!/usr/bin/env python3
"""Inspect real stored signals, label cardinality and synthetic privacy canaries."""
import json,sys,time,uuid
from pathlib import Path
root=Path(__file__).resolve().parents[1];sys.path[:0]=[str(root/'tests'),str(root/'scripts')]
from client import Client,check
from observations import get,grafana,query,collect
c=Client('http://127.0.0.1:18086');a,b,e,chat=c.bootstrap()
marker='private_canary_'+uuid.uuid4().hex
c.message(a,chat,marker)
c.call('POST','/api/v1/chats',400,actor=a,body={marker:marker})
att,_=c.attachment(a,chat);c.wait_ready(a,att)
time.sleep(12)
series=query('{__name__=~"messenger_.*"}')
for row in series:
labels=row['metric']
check(not set(labels).intersection({'user_id','chat_id','message_id','attachment_id','request_id','filename','url'}),'high-cardinality metric label')
check(not __import__('re').search(r'[0-9a-f]{8}-[0-9a-f]{4}-',labels.get('route','')),'raw route contains UUID')
required=['messenger_http_requests_total','messenger_http_duration_seconds_bucket','messenger_jobs_total','messenger_job_duration_seconds_bucket','messenger_job_ready_duration_seconds_bucket','messenger_db_pool_used','messenger_db_pool_waiting','messenger_worker_busy','messenger_worker_capacity','messenger_process_memory','messenger_process_cpu','messenger_queue_depth','messenger_queue_oldest_age_seconds','messenger_ready_slo_total']
found={r['metric']['__name__'] for r in series}
check(set(required)<=found,'missing required metrics: '+str(set(required)-found))
logs=get('loki','/loki/api/v1/query_range',{'query':'{service_name=~"messenger-.+"} |= "'+marker+'"','start':str(int((time.time()-180)*1e9)),'end':str(int(time.time()*1e9))})
check(not logs['data']['result'],'canary leaked into logs')
# Search trace attributes; HTTP bodies and headers are never recorded.
traces=get('tempo','/api/search',{'q':'{ resource.service.name = "messenger-api" }','limit':10})
trace_rows=[]
for tr in traces.get('traces',[]):
d=get('tempo','/api/traces/'+tr['traceID']);check(marker not in json.dumps(d),'canary leaked into traces');trace_rows.append(tr['traceID'])
check(bool(trace_rows),'trace search returned no observations')
# Verify that the imported dashboard references valid expressions.
dashboard=grafana('/api/dashboards/uid/messenger-lab6')['dashboard']
check(len(dashboard['panels'])>=18,'dashboard not provisioned')
provisioned=grafana('/api/datasources');check({'prometheus','loki','tempo'}<=set(d['uid'] for d in provisioned),'datasources not provisioned')
for panel in dashboard['panels']:
for target in panel.get('targets',[]):query(target['expr'])
out=collect();out.update({'checks':{'required_metrics_present':True,'forbidden_labels_absent':True,'raw_uuid_routes_absent':True,'synthetic_canary_absent_in_logs':True,'synthetic_canary_absent_in_traces':True,'dashboard_expressions_valid':True},'series_count':len(series),'trace_ids_checked':trace_rows})
(root/'evidence/signals.json').write_text(json.dumps(out,indent=2)+'\n');print('PASS: stored metrics, safe labels, privacy canaries, trace search and dashboard queries')
+7
View File
@@ -0,0 +1,7 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
./scripts/compose.sh stop api lb worker dispatcher
./scripts/compose.sh restart postgres redis s3 broker prometheus loki tempo otel-collector grafana
./scripts/compose.sh run --rm -T s3-init
./scripts/compose.sh up -d --wait --wait-timeout 90 api lb worker dispatcher
+45
View File
@@ -0,0 +1,45 @@
#!/usr/bin/env python3
"""Drain and replace API processes individually, retaining other container IDs."""
import json,os,subprocess,time
from pathlib import Path
ROOT=Path(__file__).resolve().parents[1]
def run(args,input=None):
p=subprocess.run(args,cwd=ROOT,input=input,text=True,capture_output=True,timeout=100)
if p.returncode:raise RuntimeError(p.stderr.strip()[:300])
return p.stdout.strip()
def runtime(lb,cmd):return run(['docker','exec','-i',lb,'nc','-w','1','127.0.0.1','9999'],cmd+'\n')
def ids():return run(['./scripts/compose.sh','ps','-q','api']).split()
def states(lb):
lines=runtime(lb,'show servers state').splitlines();head=lines[1].lstrip('# ').split()
return [dict(zip(head,l.split())) for l in lines[2:] if l.strip()]
def main():
initial=ids()
if len(initial)<2:raise SystemExit('rolling update requires at least two running API replicas')
lb=run(['./scripts/compose.sh','ps','-q','lb'])
for old in initial:
info=json.loads(run(['docker','inspect',old]))[0]
ip=next(iter(info['NetworkSettings']['Networks'].values()))['IPAddress']
backend=next(s['srv_name'] for s in states(lb) if s['srv_addr']==ip and s['be_name']=='api')
runtime(lb,f'set server api/{backend} state drain')
try:
deadline=time.monotonic()+10
while True:
# Existing transactions get a chance to finish before SIGTERM.
import csv,io
raw=run(['docker','exec',lb,'wget','-qO-','http://127.0.0.1:8404/stats;csv'])
row=next(r for r in csv.DictReader(io.StringIO(raw)) if r['# pxname']=='api' and r['svname']==backend)
if int(row['scur'])==0:break
if time.monotonic()>deadline:raise RuntimeError('backend did not drain')
time.sleep(.1)
before=set(ids());run(['docker','stop','--time','8',old]);run(['docker','rm',old])
run(['./scripts/compose.sh','up','-d','--no-deps','--no-recreate','--scale',f'api={len(initial)}','api'])
after=set(ids())
if not (before-{old}).issubset(after):raise RuntimeError('another replica was replaced unexpectedly')
new=(after-before).pop();deadline=time.monotonic()+60
while run(['docker','inspect','--format','{{.State.Health.Status}}',new])!='healthy':
if time.monotonic()>deadline:raise RuntimeError('replacement failed readiness')
time.sleep(.5)
print(json.dumps({'old_container':old,'new_container':new,'name':info['Name'],'drained_backend':backend}),flush=True)
finally:runtime(lb,f'set server api/{backend} state ready')
time.sleep(3) # allow DNS refresh and a successful active readiness check
if __name__=='__main__':main()
+4
View File
@@ -0,0 +1,4 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
exec python3 scripts/rolling-update.py
+19
View File
@@ -0,0 +1,19 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
: "${TARGET_INSTANCE_ID:?missing target alias}"
for id in $(./scripts/compose.sh ps -q api); do
current=$(docker exec "$id" wget -T 2 -qO- http://127.0.0.1:8080/internal/metrics | python3 -c 'import json,sys; print(json.load(sys.stdin)["instance_id"])')
if [ "$current" = "$TARGET_INSTANCE_ID" ]; then
docker stop --time 8 "$id" >/dev/null
python3 - "$id" "$TARGET_INSTANCE_ID" <<'PYLOG'
import json,sys,time
from pathlib import Path
with Path('evidence/failover-action.jsonl').open('a') as f:
f.write(json.dumps({'at':time.time(),'container':sys.argv[1],'instance_id':sys.argv[2],'action':'docker stop','project':'mtusi2026lab6'})+'\n')
PYLOG
exit 0
fi
done
printf '%s\n' 'Target does not match a running lab6 API process' >&2
exit 1
+27
View File
@@ -0,0 +1,27 @@
#!/usr/bin/env python3
"""Derive incident timing from recorded observations, without inventing missing data."""
import json
from pathlib import Path
root=Path(__file__).resolve().parents[1]
x=json.loads((root/'evidence/slo-experiment.json').read_text())
if not x.get('completed'):raise SystemExit('Experiment has not completed successfully')
rows=x['observations'];events=x['events'];results=[]
for i,event in enumerate(events):
name=event['phase']
if not any(name.startswith(s) for s in ['worker_short_','db_short_','worker_long_','db_long_']):continue
if 'recovery' in name:continue
prefix=name.rsplit('_',1)[0];start=event['at']
phase=next(p for p in x['phases'] if p['name']==name);recovered=phase['end']
nextfault=next((e['at'] for e in events[i+1:] if ('_short_' in e['phase'] or '_long_' in e['phase']) and 'recovery' not in e['phase']),x.get('finished',float('inf')))
observed=[r for r in rows if start<=r['at']<nextfault]
expected=['QueueLag','ReadyBurnRateFast','ReadyBurnRateSlow'] if name.startswith('worker') else ['ApiBurnRateFast','ApiBurnRateSlow']
alerts={}
for alert in expected:
def firing(r):return any(a['labels']['alertname']==alert and a['state']=='firing' for a in r.get('alerts',[]))
first=next((r['at'] for r in observed if firing(r)),None)
resolved=next((r['at'] for r in observed if first is not None and r['at']>max(first,recovered) and not firing(r)),None)
alerts[alert]={'detection_seconds':None if first is None else first-start,'resolved_after_restore_seconds':None if resolved is None else resolved-recovered,'fired':first is not None}
results.append({'phase':name,'duration':recovered-start,'alerts':alerts,'successful_rps':phase['successful_rps'],'error_ratio':phase['error_ratio'],'latency_success_ms':phase['latency_success_ms']})
summary={'completed':x.get('completed',False),'duration_seconds':x.get('finished',0)-x['started'],'parameters':x['parameters'],'incidents':results,'traffic':x.get('traffic'),'attachment_states':x.get('attachment_states'),'image_id':x.get('image_id')}
(root/'evidence/slo-summary.json').write_text(json.dumps(summary,indent=2)+'\n')
print(json.dumps(summary,indent=2))
+11
View File
@@ -0,0 +1,11 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
mkdir -p bin
(cd service && GOOS=linux GOARCH=$(docker info --format '{{.Architecture}}' | sed 's/aarch64/arm64/;s/x86_64/amd64/') CGO_ENABLED=0 go test -c -o ../bin/storage.test ./internal/adapter/repository/storage)
database="lab5_atomic_$(date +%s)_$$"
./scripts/compose.sh exec -T postgres psql -U messenger -d messenger -v ON_ERROR_STOP=1 -c "CREATE DATABASE $database" >/dev/null
trap './scripts/compose.sh exec -T postgres psql -U messenger -d messenger -c "DROP DATABASE IF EXISTS $database WITH (FORCE)" >/dev/null' EXIT
id=$(./scripts/compose.sh ps -q api | head -n 1)
docker cp bin/storage.test "$id:/tmp/storage.test" >/dev/null
docker exec -e "TEST_DATABASE_NAME=$database" "$id" /tmp/storage.test -test.v
+16
View File
@@ -0,0 +1,16 @@
#!/usr/bin/env python3
"""Wait for the business endpoint and each telemetry backend, not only running processes."""
import subprocess,time,urllib.request
from pathlib import Path
root=Path(__file__).resolve().parents[1]
checks=['http://otel-collector:13133/','http://prometheus:9090/-/ready','http://loki:3100/ready','http://tempo:3200/ready','http://grafana:3000/api/health']
end=time.monotonic()+120
while time.monotonic()<end:
try:
with urllib.request.urlopen('http://127.0.0.1:18086/health/ready',timeout=2) as r:
if r.status!=200:raise RuntimeError()
for url in checks:
subprocess.run([str(root/'scripts/compose.sh'),'exec','-T','--index','1','api','wget','-q','-O','/dev/null',url],check=True,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL,timeout=5)
print('PASS: business, Collector, Prometheus, Loki, Tempo and Grafana ready');break
except Exception:time.sleep(2)
else:raise SystemExit('FAIL: stand readiness timeout')
+169
View File
@@ -0,0 +1,169 @@
package main
import (
"context"
"fmt"
"log/slog"
"os"
"os/signal"
"syscall"
"time"
"encoding/json"
"gitea.privcloud.tech/mtusi2026/internal/adapter/controller/http"
"gitea.privcloud.tech/mtusi2026/internal/adapter/repository/cache/outside"
"gitea.privcloud.tech/mtusi2026/internal/adapter/repository/storage"
"gitea.privcloud.tech/mtusi2026/internal/config"
"gitea.privcloud.tech/mtusi2026/internal/pkg/auth/jwtprovider"
"gitea.privcloud.tech/mtusi2026/internal/pkg/cache"
"gitea.privcloud.tech/mtusi2026/internal/pkg/encrypting"
"gitea.privcloud.tech/mtusi2026/internal/pkg/migrator"
"gitea.privcloud.tech/mtusi2026/internal/pkg/objectstore"
"gitea.privcloud.tech/mtusi2026/internal/pkg/postgres"
"gitea.privcloud.tech/mtusi2026/internal/pkg/postgres/txmanager"
"gitea.privcloud.tech/mtusi2026/internal/pkg/rest/middleware"
"gitea.privcloud.tech/mtusi2026/internal/pkg/rest/router"
"gitea.privcloud.tech/mtusi2026/internal/pkg/telemetry"
"gitea.privcloud.tech/mtusi2026/internal/pkg/validator"
"gitea.privcloud.tech/mtusi2026/internal/service/attachment"
"gitea.privcloud.tech/mtusi2026/internal/service/auth"
"gitea.privcloud.tech/mtusi2026/internal/service/chat"
"gitea.privcloud.tech/mtusi2026/internal/service/outbox"
"gitea.privcloud.tech/mtusi2026/internal/service/user"
stdhttp "net/http"
)
func main() {
time.Local = time.UTC
if err := run(); err != nil {
slog.Error("application stopped", "error", err)
os.Exit(1)
}
}
func run() error {
cfg, err := config.New()
if err != nil {
return fmt.Errorf("load server config: %w", err)
}
if len(os.Args) > 1 && os.Args[1] == "migrate" {
cfg.Postgres.RunMigrations = true
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
return migrator.New(cfg).Start(ctx)
}
name := "messenger-api"
if len(os.Args) > 1 {
name = "messenger-" + os.Args[1]
}
shutdown, err := telemetry.Init(context.Background(), name)
if err != nil {
return err
}
defer shutdown()
attachmentConfig, err := config.LoadAttachment()
if err != nil {
return err
}
if len(os.Args) > 1 && (os.Args[1] == "worker" || os.Args[1] == "dispatcher" || os.Args[1] == "jobs" || os.Args[1] == "outbox") {
return runPipeline(cfg, attachmentConfig, os.Args[1:])
}
if len(os.Args) > 1 && os.Args[1] == "storage" {
return runStorage(cfg, attachmentConfig, os.Args[2:])
}
// API processes never run migrations; the one-shot migrator owns schema changes.
cfg.Postgres.RunMigrations = false
var level slog.Level
if err := level.UnmarshalText([]byte(cfg.Application.LogLevel)); err != nil {
return err
}
// Logging is initialized for every process by telemetry.Init.
instanceID := telemetry.InstanceID
slog.Info("instance_started", "instance_id", instanceID)
pgx := postgres.New(cfg.Postgres)
rx := cache.New(cfg)
postgresMigrator := migrator.New(cfg)
defer pgx.Close()
telemetry.ObservePool(pgx, 0)
defer rx.Close()
tm := txmanager.New(pgx)
provider := jwtprovider.New(cfg)
encryptor := encrypting.New()
chatsRepo := storage.NewChatsRepo(pgx)
messageRepo := storage.NewMessagesRepo(pgx)
refreshTokensRepo := storage.NewRefreshTokensRepo(pgx)
userRepo := storage.NewUsersRepo(pgx)
chatCache := outside.NewChatCache(rx)
authSvc := auth.New(provider, tm, encryptor, userRepo, refreshTokensRepo)
userSvc := user.New(encryptor, userRepo)
chatSvc := chat.New(tm, chatsRepo, messageRepo, chatCache)
objectStore := objectstore.New(attachmentConfig)
attachmentSvc := attachment.New(&storage.AttachmentsRepo{Pool: pgx}, objectStore, attachmentConfig, chatSvc)
pipelineConfig, err := config.LoadPipeline()
if err != nil {
return err
}
outboxSvc := outbox.New(&storage.OutboxRepo{Pool: pgx}, tm, nil)
attachmentSvc.Configure(tm, outboxSvc, pipelineConfig)
httpRouter := router.New(
middleware.NewAuthMiddleware(authSvc),
middleware.Instance(instanceID),
middleware.RequestIDMiddleware(),
telemetry.HTTP,
middleware.RequestTimeout(cfg.Postgres.QueryTimeout, attachmentConfig.Timeout),
)
httpRouter.Get("/internal/metrics", func(w stdhttp.ResponseWriter, r *stdhttp.Request) {
stat := pgx.Stat()
_ = json.NewEncoder(w).Encode(map[string]any{"instance_id": instanceID, "acquire_count": stat.AcquireCount(), "acquire_duration_ns": stat.AcquireDuration().Nanoseconds(), "empty_acquire_count": stat.EmptyAcquireCount(), "acquired_conns": stat.AcquiredConns(), "total_conns": stat.TotalConns(), "max_conns": stat.MaxConns()})
})
server := http.New(cfg, httpRouter)
requestValidator := validator.NewValidatorImpl()
registerRoutes(
http.NewAuthHandler(httpRouter, authSvc),
http.NewHealthHandler(httpRouter, server, pgx, objectStore),
http.NewChatHandler(httpRouter, provider, requestValidator, chatSvc),
http.NewUserHandler(httpRouter, provider, requestValidator, userSvc),
http.NewAttachmentHandler(httpRouter, provider, attachmentSvc),
)
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
if err = postgresMigrator.Start(ctx); err != nil {
return fmt.Errorf("start postgres migrator: %w", err)
}
if err = server.Start(ctx); err != nil {
return fmt.Errorf("start HTTP server: %w", err)
}
if err = rx.Start(ctx); err != nil {
slog.Warn("redis unavailable at startup; continuing without cache", "error", err)
}
slog.Info("HTTP server started", "address", cfg.Server.BindAddress())
<-ctx.Done()
shutdownCtx, cancel := context.WithTimeout(context.Background(), cfg.Server.ShutdownTimeout)
defer cancel()
if err = server.Stop(shutdownCtx); err != nil {
return fmt.Errorf("stop HTTP server: %w", err)
}
if err = postgresMigrator.Close(shutdownCtx); err != nil {
return fmt.Errorf("close postgres migrator: %w", err)
}
if err = rx.Close(); err != nil {
return fmt.Errorf("close redis: %w", err)
}
pgx.Close()
return nil
}
func registerRoutes(handlers ...http.Handler) {
for _, handler := range handlers {
handler.RegisterRoutes()
}
}
+156
View File
@@ -0,0 +1,156 @@
package main
import (
"context"
"encoding/json"
"errors"
"flag"
"gitea.privcloud.tech/mtusi2026/internal/adapter/broker"
"gitea.privcloud.tech/mtusi2026/internal/adapter/repository/storage"
"gitea.privcloud.tech/mtusi2026/internal/config"
"gitea.privcloud.tech/mtusi2026/internal/pkg/objectstore"
"gitea.privcloud.tech/mtusi2026/internal/pkg/postgres"
"gitea.privcloud.tech/mtusi2026/internal/pkg/postgres/txmanager"
"gitea.privcloud.tech/mtusi2026/internal/pkg/telemetry"
"gitea.privcloud.tech/mtusi2026/internal/service/attachment"
"gitea.privcloud.tech/mtusi2026/internal/service/observation"
"gitea.privcloud.tech/mtusi2026/internal/service/outbox"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
"log/slog"
"os"
"os/signal"
"syscall"
"time"
)
func runPipeline(cfg *config.Config, a *config.Attachment, args []string) error {
c, e := config.LoadPipeline()
if e != nil {
return e
}
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
pool := postgres.New(cfg.Postgres)
defer pool.Close()
telemetry.ObservePool(pool, c.Concurrency)
tm := txmanager.New(pool)
repo := &storage.AttachmentsRepo{Pool: pool}
rabbit := &broker.Rabbit{URL: c.BrokerURL}
ob := outbox.New(&storage.OutboxRepo{Pool: pool}, tm, rabbit)
svc := attachment.New(repo, objectstore.New(a), a, nil)
svc.Configure(tm, ob, c)
ob.OnDead = svc.PublishFailed
if args[0] == "jobs" || args[0] == "outbox" {
if len(args) < 2 {
return errors.New("operator action required")
}
fs := flag.NewFlagSet(args[1], flag.ContinueOnError)
idText := fs.String("id", "", "job ID (attachment ID for reprocess)")
version := fs.Int("pipeline-version", 2, "new pipeline version")
if e = fs.Parse(args[2:]); e != nil {
return e
}
if args[1] == "list" {
jobs, e := repo.ListJobs(ctx)
if e != nil {
return e
}
return json.NewEncoder(os.Stdout).Encode(jobs)
}
id, e := uuid.Parse(*idText)
if e != nil {
return e
}
switch args[1] {
case "redeliver":
if os.Getenv("LAB5_FAULTS") != "1" {
return errors.New("redeliver is restricted to the explicit fault profile")
}
var payload []byte
e = tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error {
j, e := repo.JobTx(ctx, tx, id)
if e != nil {
return e
}
payload, e = json.Marshal(j.Message)
return e
})
if e == nil {
e = rabbit.Publish(ctx, outbox.DeliveryMessage{ID: uuid.New(), AggregateID: id, Payload: payload})
}
case "replay":
e = svc.Replay(ctx, id)
case "reprocess":
e = svc.Reprocess(ctx, id, *version)
default:
return errors.New("unknown operator action")
}
if e != nil {
return e
}
return json.NewEncoder(os.Stdout).Encode(map[string]any{"accepted": true, "id": id})
}
if c.BrokerURL == "" {
return errors.New("BROKER_URL_FILE is required")
}
if args[0] == "worker" {
// Recovery continues while the consumer is disconnected or a delivery is paused.
recoveryDone := make(chan struct{})
defer func() { stop(); <-recoveryDone }()
go func() {
defer close(recoveryDone)
tick := time.NewTicker(time.Second)
defer tick.Stop()
for {
select {
case <-ctx.Done():
return
case <-tick.C:
if e := svc.Recover(ctx); e != nil {
slog.Warn("job_recovery_failed")
}
if e := svc.Cleanup(ctx); e != nil {
slog.Warn("source_cleanup_failed")
}
}
}
}()
for ctx.Err() == nil {
if e = rabbit.Consume(ctx, c.Concurrency, c.Prefetch, svc.Process); e != nil && ctx.Err() == nil {
slog.Warn("consumer_reconnecting")
}
select {
case <-ctx.Done():
case <-time.After(time.Second):
}
}
return nil
}
observer := &observation.Service{Repo: &storage.ObservationRepo{Pool: pool}, Publish: func(s observation.Snapshot) {
telemetry.PublishObservation(s.Queued, s.Outbox, s.Good, s.Bad, s.Oldest, s.Locked)
}}
telemetry.RegisterObservation()
nextObservation := time.Time{}
for ctx.Err() == nil {
if time.Now().After(nextObservation) {
if err := observer.Observe(ctx); err != nil {
slog.Warn("observation_unavailable")
}
depth, err := rabbit.Depth(ctx)
if err == nil {
telemetry.QueueDepth.Store(int64(depth))
}
nextObservation = time.Now().Add(time.Second)
}
_, e = ob.ProcessOnce(ctx)
if e != nil {
slog.Warn("dispatcher_retry")
}
select {
case <-ctx.Done():
case <-time.After(100 * time.Millisecond):
}
}
return nil
}
+183
View File
@@ -0,0 +1,183 @@
package main
import (
"context"
"encoding/json"
"errors"
"flag"
"fmt"
"log/slog"
"net/http"
"os"
"strings"
"time"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/smithy-go"
"github.com/google/uuid"
"gitea.privcloud.tech/mtusi2026/internal/adapter/repository/storage"
"gitea.privcloud.tech/mtusi2026/internal/config"
"gitea.privcloud.tech/mtusi2026/internal/pkg/objectstore"
"gitea.privcloud.tech/mtusi2026/internal/pkg/postgres"
"gitea.privcloud.tech/mtusi2026/internal/pkg/postgres/txmanager"
"gitea.privcloud.tech/mtusi2026/internal/service/attachment"
"gitea.privcloud.tech/mtusi2026/internal/service/outbox"
)
func runStorage(cfg *config.Config, c *config.Attachment, args []string) error {
slog.SetDefault(slog.New(slog.NewJSONHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelWarn})))
if len(args) == 0 {
return fmt.Errorf("storage command required")
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
defer cancel()
store := objectstore.New(c)
if args[0] == "bootstrap" {
var err error
for i := 0; i < 60; i++ {
_, err = store.Client.CreateBucket(ctx, &s3.CreateBucketInput{Bucket: &c.Bucket})
var api smithy.APIError
if err == nil || (errors.As(err, &api) && (api.ErrorCode() == "BucketAlreadyOwnedByYou" || api.ErrorCode() == "BucketAlreadyExists")) {
if store.Ping(ctx) == nil {
// HEAD alone can succeed before the restarted master/volume write path.
// This constant key is outside the application's immutable originals prefix.
key := "bootstrap/write-check"
if store.Put(ctx, key, "application/octet-stream", strings.NewReader("ready"), 5) == nil && store.Delete(ctx, key) == nil {
return json.NewEncoder(os.Stdout).Encode(map[string]any{"bucket_ready": true, "write_checked": true})
}
}
}
time.Sleep(time.Second)
}
return fmt.Errorf("storage bootstrap failed: %w", err)
}
if args[0] == "private-check" {
// Unsigned requests must reach the S3 server and receive an auth failure,
// rather than passing because a network error hid a public object.
fs := flag.NewFlagSet("private-check", flag.ContinueOnError)
key := fs.String("key", "", "known object key")
if e := fs.Parse(args[1:]); e != nil {
return e
}
client := http.Client{Timeout: 5 * time.Second, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }}
resp, e := client.Get(c.Endpoint + "/" + c.Bucket + "/" + *key)
if e != nil {
return e
}
defer resp.Body.Close()
if resp.StatusCode != 401 && resp.StatusCode != 403 {
return fmt.Errorf("unsigned GET returned %d", resp.StatusCode)
}
_, e = store.Client.CreateBucket(ctx, &s3.CreateBucketInput{Bucket: new("forbidden-" + uuid.NewString())})
var api smithy.APIError
if !errors.As(e, &api) || api.ErrorCode() != "AccessDenied" {
return fmt.Errorf("API key bucket creation was not denied")
}
return json.NewEncoder(os.Stdout).Encode(map[string]any{"anonymous_status": resp.StatusCode, "api_create_bucket_denied": true})
}
if args[0] == "object-check" {
fs := flag.NewFlagSet("object-check", flag.ContinueOnError)
key := fs.String("key", "", "synthetic object key")
if e := fs.Parse(args[1:]); e != nil {
return e
}
head, e := store.Head(ctx, *key)
if objectstore.Missing(e) {
return json.NewEncoder(os.Stdout).Encode(map[string]any{"exists": false})
}
if e != nil {
return e
}
return json.NewEncoder(os.Stdout).Encode(map[string]any{"exists": true, "last_modified": head.LastModified, "size_bytes": head.ContentLength})
}
pool := postgres.New(cfg.Postgres)
defer pool.Close()
svc := attachment.New(&storage.AttachmentsRepo{Pool: pool}, store, c, nil)
jobs, e := config.LoadPipeline()
if e != nil {
return e
}
tm := txmanager.New(pool)
svc.Configure(tm, outbox.New(&storage.OutboxRepo{Pool: pool}, tm, nil), jobs)
switch args[0] {
case "reconcile":
fs := flag.NewFlagSet("reconcile", flag.ContinueOnError)
apply := fs.Bool("apply", false, "apply safe changes")
if e := fs.Parse(args[1:]); e != nil {
return e
}
result, e := svc.Reconcile(ctx, *apply)
_ = json.NewEncoder(os.Stdout).Encode(result)
return e
case "restore":
fs := flag.NewFlagSet("restore", flag.ContinueOnError)
id := fs.String("id", "", "attachment id")
source := fs.String("source", "", "original file")
if e := fs.Parse(args[1:]); e != nil {
return e
}
uid, e := uuid.Parse(*id)
if e != nil {
return e
}
if e = svc.Restore(ctx, uid, *source); e != nil {
return e
}
return json.NewEncoder(os.Stdout).Encode(map[string]any{"restored": uid})
case "seed":
fs := flag.NewFlagSet("seed", flag.ContinueOnError)
count := fs.Int("count", 1, "synthetic orphan count")
keyOverride := fs.String("key", "", "explicit key for a single synthetic object")
if e := fs.Parse(args[1:]); e != nil {
return e
}
if *count < 1 || *count > 10000 {
return fmt.Errorf("count must be 1..10000")
}
keys := []string{}
for i := 0; i < *count; i++ {
key := attachment.Prefix + uuid.NewString()
if *keyOverride != "" {
if *count != 1 {
return fmt.Errorf("explicit key requires count=1")
}
key = *keyOverride
}
if e := store.Put(ctx, key, "image/png", strings.NewReader("synthetic-orphan"), 16); e != nil {
return e
}
if *count == 1 {
keys = append(keys, key)
}
}
return json.NewEncoder(os.Stdout).Encode(map[string]any{"seeded": *count, "keys": keys})
case "remove-object":
fs := flag.NewFlagSet("remove-object", flag.ContinueOnError)
id := fs.String("id", "", "synthetic attachment id")
if e := fs.Parse(args[1:]); e != nil {
return e
}
uid, e := uuid.Parse(*id)
if e != nil {
return e
}
a, e := svc.Repo.Get(ctx, uid)
if e != nil {
return e
}
conn, unlock, e := svc.Repo.Lock(ctx, a.Key)
_ = conn
if e != nil {
return e
}
defer unlock()
if e = store.Delete(ctx, a.Key); e != nil {
return e
}
return json.NewEncoder(os.Stdout).Encode(map[string]any{"removed_object": uid})
default:
return fmt.Errorf("unknown storage command")
}
}
+260
View File
@@ -0,0 +1,260 @@
// Command benchmark generates paced lab5 traffic through the public load balancer.
package main
import (
"bytes"
"encoding/json"
"flag"
"fmt"
"io"
"math"
"net/http"
"os"
"slices"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/google/uuid"
)
type client struct {
base, token string
http *http.Client
}
func (c *client) call(method, path string, body any, key string) (map[string]any, int, error) {
var data io.Reader
if body != nil {
b, e := json.Marshal(body)
if e != nil {
return nil, 0, e
}
data = bytes.NewReader(b)
}
req, e := http.NewRequest(method, c.base+path, data)
if e != nil {
return nil, 0, e
}
if body != nil {
req.Header.Set("Content-Type", "application/json")
}
if c.token != "" {
req.Header.Set("Authorization", "Bearer "+c.token)
}
if key != "" {
req.Header.Set("Idempotency-Key", key)
}
// net/http otherwise treats Idempotency-Key + GetBody as permission to retry POST.
if method == http.MethodPost {
req.GetBody = nil
}
resp, e := c.http.Do(req)
if e != nil {
return nil, 0, e
}
defer resp.Body.Close()
b, e := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
if e != nil {
return nil, resp.StatusCode, e
}
if resp.StatusCode != 200 && resp.StatusCode != 201 {
return nil, resp.StatusCode, fmt.Errorf("HTTP %d", resp.StatusCode)
}
var out map[string]any
e = json.Unmarshal(b, &out)
return out, resp.StatusCode, e
}
func (c *client) setup(seed int) (string, error) {
username := "bench_" + strings.ReplaceAll(uuid.NewString(), "-", "")[:20]
password := "Synthetic-benchmark-123"
user, _, e := c.call("POST", "/api/v1/users", map[string]string{"username": username, "display_name": "Benchmark", "password": password}, "")
if e != nil {
return "", e
}
req, _ := http.NewRequest("POST", c.base+"/api/v1/auth/sessions", nil)
req.SetBasicAuth(username, password)
resp, e := c.http.Do(req)
if e != nil {
return "", e
}
var session struct {
Token string `json:"token"`
}
e = json.NewDecoder(resp.Body).Decode(&session)
resp.Body.Close()
if e != nil || resp.StatusCode != 201 || session.Token == "" {
return "", fmt.Errorf("login status %d: %v", resp.StatusCode, e)
}
c.token = session.Token
chat, _, e := c.call("POST", "/api/v1/chats", map[string]any{"title": "Benchmark", "member_ids": []any{user["id"]}}, "")
if e != nil {
return "", e
}
path := "/api/v1/chats/" + chat["id"].(string) + "/messages"
// Bounded concurrent setup is excluded from the measured interval.
var group sync.WaitGroup
slots := make(chan struct{}, 8)
errs := make(chan error, seed)
for i := range seed {
slots <- struct{}{}
group.Go(func() {
defer func() { <-slots }()
_, _, e := c.call("POST", path, map[string]string{"text": fmt.Sprintf("seed-%d", i)}, uuid.NewString())
if e != nil {
errs <- e
}
})
}
group.Wait()
close(errs)
for e := range errs {
if e != nil {
return "", e
}
}
return path, nil
}
type result struct {
MeasurementUnix float64 `json:"measurement_started_unix"`
Label string `json:"label"`
TargetRate int `json:"target_rps"`
Duration float64 `json:"duration_seconds"`
Elapsed float64 `json:"elapsed_including_drain_seconds"`
Scheduled int `json:"scheduled"`
Dropped int `json:"generator_dropped"`
Success int `json:"successful_requests"`
Errors int `json:"failed_requests"`
ReadSuccess int `json:"read_success"`
WriteSuccess int `json:"write_success"`
SuccessfulRPS float64 `json:"successful_rps"`
ErrorRatio float64 `json:"error_ratio_including_drops"`
Latency map[string]float64 `json:"successful_latency_ms"`
Statuses map[int]int `json:"statuses"`
PeakInFlight int64 `json:"peak_in_flight"`
Note string `json:"note"`
}
func phase(c *client, path string, rate int, duration time.Duration, maxInFlight int, label string) result {
out := result{Label: label, TargetRate: rate, Duration: duration.Seconds(), Statuses: map[int]int{}, Latency: map[string]float64{}, Note: "Paced open-loop, 10ms dispatch batches; 90% first-page GET, 10% POST; no POST retries. Percentiles of successes include scheduler delay; failures and capacity drops reported separately. RPS denominator includes drain."}
start := time.Now()
out.MeasurementUnix = float64(start.UnixNano()) / 1e9
end := start.Add(duration)
slots := make(chan struct{}, maxInFlight)
var group sync.WaitGroup
var mu sync.Mutex
var active atomic.Int64
var peak atomic.Int64
latencies := make([]float64, 0, rate*int(duration.Seconds()))
index := 0
for tick := 0; ; tick++ {
now := time.Now()
if !now.Before(end) {
break
}
target := start.Add(time.Duration(tick) * 10 * time.Millisecond)
if target.After(now) {
time.Sleep(time.Until(target))
}
until := min(int(time.Since(start).Seconds()*float64(rate))+1, int(duration.Seconds()*float64(rate)))
for index < until {
i := index
index++
out.Scheduled++
scheduled := start.Add(time.Duration(float64(i) / float64(rate) * float64(time.Second)))
select {
case slots <- struct{}{}:
group.Go(func() {
n := active.Add(1)
for old := peak.Load(); n > old; old = peak.Load() {
if peak.CompareAndSwap(old, n) {
break
}
}
defer func() { active.Add(-1); <-slots }()
method, p := "GET", path+"?limit=50"
var body any
key := ""
if i%10 == 0 {
method = "POST"
p = path
key = uuid.NewString()
body = map[string]string{"text": key}
}
_, status, err := c.call(method, p, body, key)
latency := float64(time.Since(scheduled).Microseconds()) / 1000
mu.Lock()
defer mu.Unlock()
out.Statuses[status]++
if err != nil {
out.Errors++
} else {
out.Success++
if method == "GET" {
out.ReadSuccess++
} else {
out.WriteSuccess++
}
latencies = append(latencies, max(0, latency))
}
})
default:
out.Dropped++
}
}
}
group.Wait()
out.Elapsed = time.Since(start).Seconds()
out.PeakInFlight = peak.Load()
out.SuccessfulRPS = float64(out.Success) / out.Elapsed
if out.Scheduled > 0 {
out.ErrorRatio = float64(out.Errors+out.Dropped) / float64(out.Scheduled)
}
slices.Sort(latencies)
for name, p := range map[string]float64{"p50": .5, "p95": .95, "p99": .99} {
if len(latencies) > 0 {
out.Latency[name] = latencies[max(0, int(math.Ceil(p*float64(len(latencies))))-1)]
}
}
return out
}
func main() {
base := flag.String("base-url", "http://localhost:18086", "public LB URL")
rate := flag.Int("rate", 100, "scheduled requests per second")
duration := flag.Duration("duration", 30*time.Second, "measured time")
warmup := flag.Duration("warmup", 5*time.Second, "warmup excluded from results")
seed := flag.Int("seed", 1000, "initial messages")
maxFlight := flag.Int("max-in-flight", 256, "bounded in-flight limit")
label := flag.String("label", "manual", "case label")
output := flag.String("output", "", "JSON destination")
flag.Parse()
if *rate < 1 || *rate > 100000 || *duration <= 0 || *warmup < 0 || *maxFlight < 1 || *seed < 0 {
fmt.Fprintln(os.Stderr, "invalid benchmark settings")
os.Exit(2)
}
c := &client{base: *base, http: &http.Client{Timeout: 2 * time.Second, Transport: &http.Transport{MaxIdleConns: 512, MaxIdleConnsPerHost: 512, MaxConnsPerHost: *maxFlight, IdleConnTimeout: 60 * time.Second}}}
path, e := c.setup(*seed)
if e != nil {
fmt.Fprintln(os.Stderr, "setup:", e)
os.Exit(1)
}
if *warmup > 0 {
phase(c, path, *rate, *warmup, *maxFlight, "warmup")
}
row := phase(c, path, *rate, *duration, *maxFlight, *label)
b, e := json.MarshalIndent(row, "", " ")
if e != nil {
panic(e)
}
if *output != "" {
if e = os.WriteFile(*output, append(b, '\n'), 0644); e != nil {
panic(e)
}
}
fmt.Println(string(b))
if row.Success == 0 {
os.Exit(1)
}
}
+42
View File
@@ -0,0 +1,42 @@
package main
import (
"io"
"net/http"
"net/http/httptest"
"sync/atomic"
"testing"
"time"
)
func TestLostPostResponseIsNotRetried(t *testing.T) {
var posts atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method == http.MethodGet {
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{}`))
return
}
posts.Add(1)
_, _ = io.Copy(io.Discard, r.Body)
conn, _, err := w.(http.Hijacker).Hijack()
if err != nil {
t.Error(err)
return
}
_ = conn.Close()
}))
defer server.Close()
transport := &http.Transport{MaxIdleConnsPerHost: 1}
defer transport.CloseIdleConnections()
c := &client{base: server.URL, http: &http.Client{Timeout: time.Second, Transport: transport}}
if _, _, err := c.call(http.MethodGet, "/warm", nil, ""); err != nil {
t.Fatal(err)
}
if _, _, err := c.call(http.MethodPost, "/message", map[string]string{"text": "uncertain result"}, "same-key"); err == nil {
t.Fatal("lost response must be reported as an error")
}
if posts.Load() != 1 {
t.Fatalf("transport sent %d POSTs, want exactly 1", posts.Load())
}
}
+96
View File
@@ -0,0 +1,96 @@
module gitea.privcloud.tech/mtusi2026
go 1.27.0
require (
github.com/aws/aws-sdk-go-v2 v1.47.8
github.com/aws/aws-sdk-go-v2/credentials v1.20.16
github.com/aws/aws-sdk-go-v2/service/s3 v1.114.9
github.com/aws/smithy-go v1.28.8
github.com/go-chi/chi/v5 v5.5.0
github.com/go-playground/validator/v10 v10.30.5
github.com/golang-jwt/jwt/v5 v5.3.1
github.com/google/uuid v1.6.0
github.com/jackc/pgerrcode v0.0.0-20250907135507-afb5586c32a6
github.com/jackc/pgx/v5 v5.11.0
github.com/jackc/tern/v2 v2.4.3
github.com/oapi-codegen/runtime v1.6.0
github.com/rabbitmq/amqp091-go v1.10.0
github.com/redis/go-redis/v9 v9.23.0
github.com/shirou/gopsutil/v4 v4.26.10
github.com/spf13/viper v1.21.0
github.com/vgarvardt/pgx-google-uuid/v5 v5.6.0
go.opentelemetry.io/contrib/bridges/otelslog v0.21.0
go.opentelemetry.io/otel v1.47.0
go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.23.0
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.47.0
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.47.0
go.opentelemetry.io/otel/metric v1.47.0
go.opentelemetry.io/otel/sdk v1.47.0
go.opentelemetry.io/otel/sdk/log v1.47.0
go.opentelemetry.io/otel/sdk/metric v1.47.0
go.opentelemetry.io/otel/trace v1.47.0
go.uber.org/multierr v1.11.0
golang.org/x/crypto v0.57.0
golang.org/x/sync v0.23.0
google.golang.org/grpc v1.85.0
)
require (
dario.cat/mergo v1.0.2 // indirect
github.com/Masterminds/goutils v1.1.1 // indirect
github.com/Masterminds/semver/v3 v3.5.0 // indirect
github.com/Masterminds/sprig/v3 v3.3.0 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.26 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.11 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.11 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.11 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.24 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.12 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.11 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.11 // indirect
github.com/cenkalti/backoff/v5 v5.0.3 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/ebitengine/purego v0.11.2 // indirect
github.com/fsnotify/fsnotify v1.9.0 // indirect
github.com/gabriel-vasile/mimetype v1.4.15 // indirect
github.com/go-logr/logr v1.4.4 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-ole/go-ole v1.2.6 // indirect
github.com/go-playground/locales v0.14.1 // indirect
github.com/go-playground/universal-translator v0.18.1 // indirect
github.com/go-viper/mapstructure/v2 v2.5.0 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 // indirect
github.com/huandu/xstrings v1.5.0 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
github.com/jackc/puddle/v2 v2.2.2 // indirect
github.com/leodido/go-urn v1.5.0 // indirect
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
github.com/mitchellh/copystructure v1.2.0 // indirect
github.com/mitchellh/reflectwalk v1.0.2 // indirect
github.com/pelletier/go-toml/v2 v2.4.3-0.20260624044523-35f78d5908f1 // indirect
github.com/power-devops/perfstat v0.0.0-20260805114148-88456608a4f6 // indirect
github.com/sagikazarmark/locafero v0.11.0 // indirect
github.com/shopspring/decimal v1.4.0 // indirect
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect
github.com/spf13/afero v1.15.0 // indirect
github.com/spf13/cast v1.10.0 // indirect
github.com/spf13/pflag v1.0.10 // indirect
github.com/subosito/gotenv v1.6.0 // indirect
github.com/tklauser/go-sysconf v0.4.0 // indirect
github.com/tklauser/numcpus v0.12.0 // indirect
github.com/yusufpapurcu/wmi v1.2.4 // indirect
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.47.0 // indirect
go.opentelemetry.io/otel/log v1.47.0 // indirect
go.opentelemetry.io/proto/otlp v1.11.0 // indirect
go.uber.org/atomic v1.12.0 // indirect
go.yaml.in/yaml/v3 v3.0.5 // indirect
golang.org/x/net v0.60.0 // indirect
golang.org/x/sys v0.48.0 // indirect
golang.org/x/text v0.42.0 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20260928230214-8a89bd6388cc // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20260928230214-8a89bd6388cc // indirect
google.golang.org/protobuf v1.36.12 // indirect
)
+218
View File
@@ -0,0 +1,218 @@
dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8=
dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA=
github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI=
github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU=
github.com/Masterminds/semver/v3 v3.5.0 h1:kQceYJfbupGfZOKZQg0kou0DgAKhzDg2NZPAwZ/2OOE=
github.com/Masterminds/semver/v3 v3.5.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM=
github.com/Masterminds/sprig/v3 v3.3.0 h1:mQh0Yrg1XPo6vjYXgtf5OtijNAKJRNcTdOOGZe3tPhs=
github.com/Masterminds/sprig/v3 v3.3.0/go.mod h1:Zy1iXRYNqNLUolqCpL4uhk6SHUMAOSCzdgBfDb35Lz0=
github.com/aws/aws-sdk-go-v2 v1.47.8 h1:fuClo7Un+Bi0EDTnSIPWxVEoJcze+wkGzNA4Vo3jRcQ=
github.com/aws/aws-sdk-go-v2 v1.47.8/go.mod h1:I8lFSH4MGrZiw2m7miCrRYHsgpC1+HW8uiVGOiPYPcw=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.26 h1:SJNA4XaM8Z09adeTHf/vME93jzl44lJV70pSawIBXac=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.26/go.mod h1:l0ME5pd2QytH8UZzETcrt90x9Bm+3TyBVeWfVEcZ82c=
github.com/aws/aws-sdk-go-v2/credentials v1.20.16 h1:cqRWTY4fcctJGqZedMC1Khu4h2VsgRODUOIZwn39n3Y=
github.com/aws/aws-sdk-go-v2/credentials v1.20.16/go.mod h1:x8JgCOAnbnEVQOLVX6lOtsyTrG2EHgiRNqzku9jiVxA=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.11 h1:LTxHSJ/rwTFfaXqi5mlKKHJOW+shFB+HQ7NrCa4OQ48=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.11/go.mod h1:GF0bVKZVQX4UeUtPmTuWZj3gw0eRCdkVhW0xFnSRLmU=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.11 h1:DGUM0yztI3Zz5kePulT/mMP/gVZFGTTISUiCcqSQtT0=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.11/go.mod h1:kbcdJgN9cgSzZlUqsMveDl9IfexJqOzCihXjQXTNaPk=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.11 h1:kOHCwgWFyfrz9GqnJct8knQw7Apyl3xHH+OGnome77s=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.11/go.mod h1:854cZkT2fdkdrzQklD49uzxkZnajXoelpVNgT/yk4J0=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.24 h1:jGzHjc2jDx08Rji1JTBtMxuXUDG7muTEw+ExYXqFzPg=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.24/go.mod h1:uJt75pK2cEyY80F5C+DK9YkCkJWsMkJViHxLYZRHl8Y=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.12 h1:hfO8FblXX93ZKHXSCqCjehwul1EtNJa9G8iFSOiJ+6w=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.12/go.mod h1:r8Eypc7ZRNdS7BsaLBUUdcXIEuKE6cVHZJ29u+IGaIc=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.11 h1:PO7Tx443ecMI00nIxDrT99DVcbeMBQ6BUtJlDMRnY24=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.11/go.mod h1:n/9RixikO12YvNomZLP8k4TgUSoEkT1W23QIrpFAPvw=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.11 h1:jNP7wZylDeSJhjc1nmjZBuG5deotp3cEYIfMazmzp+Y=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.11/go.mod h1:1US+8JBnF0xNm6GE6wM5u5ugFwSZpkd2686gMuql5tI=
github.com/aws/aws-sdk-go-v2/service/s3 v1.114.9 h1:MLHGyNy4melTyk9e63s+9QgLKCHPDJrP+1NfJhtU1g4=
github.com/aws/aws-sdk-go-v2/service/s3 v1.114.9/go.mod h1:zfUkiNK2JiYlR1/SnU8aJ8Gj1nMFLkPZ69rVLv6jWPo=
github.com/aws/smithy-go v1.28.8 h1:aEmdAq7gVCoGCnvf1NnELlu3I6D/nhqrC321FyFiQkQ=
github.com/aws/smithy-go v1.28.8/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM=
github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/ebitengine/purego v0.11.2 h1:WWiTx8rD8zYdOwSj8O/NiNbsOvSAgJMTlyV1XlbUSeU=
github.com/ebitengine/purego v0.11.2/go.mod h1:DCHPP08djqhNSoTfImcnHYQRZmd0qhakvrozqaEYhGQ=
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
github.com/gabriel-vasile/mimetype v1.4.15 h1:05iP/CYtZ/w455R/KZM6rZ5ieAdh99UPtd+d3YzLmaI=
github.com/gabriel-vasile/mimetype v1.4.15/go.mod h1:azpTcoLcDZRNgFou5j+APrqQx9HqVPWa6ijYQIIVswQ=
github.com/go-chi/chi/v5 v5.5.0 h1:xdKQ1dJmVJsmgahrXSKhUx8NyVPzpwdlyrrQFpMPJgg=
github.com/go-chi/chi/v5 v5.5.0/go.mod h1:d1IG7GtKPSjPyBP365PH8sAGVHk3qfB/IhjFNRH1wWM=
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8=
github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA=
github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY=
github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY=
github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
github.com/go-playground/validator/v10 v10.30.5 h1:YyCXvVShZbs2Sm3Mb53eNOlhRXctSOzW5QJAouCTZL4=
github.com/go-playground/validator/v10 v10.30.5/go.mod h1:wEqiaov48pXX1kjhc3Da8y0M0Dtg/BK7gurFBLgwFrQ=
github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro=
github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY=
github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE=
github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 h1:/Tnpcb2E0Pz/tN9s3bfEY2Q8ePCEX9iuS+cneUwncnw=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0/go.mod h1:zOBXOsUaBSjKgmH4OGzV1esUpR3oUSCPYVd2cUBjKYY=
github.com/huandu/xstrings v1.5.0 h1:2ag3IFq9ZDANvthTwTiqSSZLjDc+BedvHPAp5tJy2TI=
github.com/huandu/xstrings v1.5.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
github.com/jackc/pgerrcode v0.0.0-20250907135507-afb5586c32a6 h1:D/V0gu4zQ3cL2WKeVNVM4r2gLxGGf6McLwgXzRTo2RQ=
github.com/jackc/pgerrcode v0.0.0-20250907135507-afb5586c32a6/go.mod h1:a/s9Lp5W7n/DD0VrVoyJ00FbP2ytTPDVOivvn2bMlds=
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
github.com/jackc/pgx/v5 v5.11.0 h1:IzBBtyK9AHqf98cctWFifYSci2hgQR/cd56wB4p+ogg=
github.com/jackc/pgx/v5 v5.11.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4=
github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
github.com/jackc/tern/v2 v2.4.3 h1:g293d3OZgW7OFEhsYXgEv0C21jea2boNr0VR4K5I7OY=
github.com/jackc/tern/v2 v2.4.3/go.mod h1:rMpMuRYcff5wWLptoTSO1qcDxJ4OodysvK17i2SVBys=
github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw=
github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/leodido/go-urn v1.5.0 h1:pLqT2kq1zpHW/1D18QMjMpdtX7cekxqtJJjg5ANyWw0=
github.com/leodido/go-urn v1.5.0/go.mod h1:9BORnCDhdPBJNDEX+w1bJisa8yOKYi116VeO96s4ifE=
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4=
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I=
github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw=
github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s=
github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ=
github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
github.com/oapi-codegen/runtime v1.6.0 h1:7Xx+GlueD6nRuyKoCPzL434Jfi3BetbiJOrzCHp/VPU=
github.com/oapi-codegen/runtime v1.6.0/go.mod h1:GwV7hC2hviaMzj+ITfHVRESK5J2W/GefVwIND/bMGvU=
github.com/pelletier/go-toml/v2 v2.4.3-0.20260624044523-35f78d5908f1 h1:gIZQQ2+wbsdQGWLPDSdJFptR25tBCjSC0KbSIQZ1rgE=
github.com/pelletier/go-toml/v2 v2.4.3-0.20260624044523-35f78d5908f1/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/power-devops/perfstat v0.0.0-20260805114148-88456608a4f6 h1:jL3a8soXdzuTCcRnKhOmtcsVOObdDTFf4O2B403HPRU=
github.com/power-devops/perfstat v0.0.0-20260805114148-88456608a4f6/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw=
github.com/rabbitmq/amqp091-go v1.10.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o=
github.com/redis/go-redis/v9 v9.23.0 h1:/TTVdJa8BRNsybwwGGnmiYktHNlUXtbOQxlWpHBPL3s=
github.com/redis/go-redis/v9 v9.23.0/go.mod h1:EWP2UUk+XDaI1s5nQB5mablZN6vccXlmBQTyfnF87D0=
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc=
github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik=
github.com/shirou/gopsutil/v4 v4.26.10 h1:MJN6euesJx53MVEjscYeyRSRcrdIcy3sz5k9nosjldg=
github.com/shirou/gopsutil/v4 v4.26.10/go.mod h1:XpkIrcUMQhwte/dGxu57KtnMMFsbKelI7PP23zweKBk=
github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k=
github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME=
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 h1:+jumHNA0Wrelhe64i8F6HNlS8pkoyMv5sreGx2Ry5Rw=
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8/go.mod h1:3n1Cwaq1E1/1lhQhtRK2ts/ZwZEhjcQeJQ1RuC6Q/8U=
github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I=
github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg=
github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY=
github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo=
github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk=
github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU=
github.com/spf13/viper v1.21.0/go.mod h1:P0lhsswPGWD/1lZJ9ny3fYnVqxiegrlNrEmgLjbTCAY=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE=
github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg=
github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
github.com/tklauser/go-sysconf v0.4.0 h1:7H0uAN+7RkwWRaxhYXDLqa5V3LPrJeV8wmD9dRUgPQU=
github.com/tklauser/go-sysconf v0.4.0/go.mod h1:8mTNWyog7H+MpKijp4VmKJAd2bbYQ2zuUwkYRbUArPI=
github.com/tklauser/numcpus v0.12.0 h1:NR85qdvHA9pFse3x3weVZ0r0ST8R6l5RHbZrlRaqob4=
github.com/tklauser/numcpus v0.12.0/go.mod h1:ABHeXzJnr/qqwguhClkZKT1/8VABcYrsyUiUGobwWJg=
github.com/vgarvardt/pgx-google-uuid/v5 v5.6.0 h1:EhPtK0mgrgaTMXpegE69hvoSOVC1Ahk8+QJ9B8b+OdU=
github.com/vgarvardt/pgx-google-uuid/v5 v5.6.0/go.mod h1:5LtFrNEkgzxHvXPO9eOvcXsSn9/KeKYgx9kjeI2oXQI=
github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=
github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs=
github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s=
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
go.opentelemetry.io/contrib/bridges/otelslog v0.21.0 h1:/CdgE52i1OEPHhsGUqrggYNRaZy4v4YgVenGd52dRBI=
go.opentelemetry.io/contrib/bridges/otelslog v0.21.0/go.mod h1:9pBdbVbuehIDzxN4accnzzxEUPT8Nq7grbZMo+J1VN4=
go.opentelemetry.io/otel v1.47.0 h1:j7ALJ/zgkS7Z6aeJW09p8VC9804bC+PpeTfCD4XPnOM=
go.opentelemetry.io/otel v1.47.0/go.mod h1:8wS9O2qfXrYrzp6hIF/HOYJJf/wIhFPhR2xLuP+iXQU=
go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.23.0 h1:5WVp40k2fJHuc5BtNTgKk+EugKFT8Eg3lywFfCPI/nE=
go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.23.0/go.mod h1:59nH0RYCGHRlwt15WyioeSYINBbuObeBpTowPhEKWHY=
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.47.0 h1:7kszRN4Ze5GeAkUnBjBSUIunVBMA+mZIG04auueqguM=
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.47.0/go.mod h1:CLzakYy8jOdkBfcveCWYkeKJxujcF4xPhh2Z7IfE98U=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.47.0 h1:julhjPeUH/q/7hinbSdDdqt5h7Zw9YWmRlWRhI0jd54=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.47.0/go.mod h1:Ao2mz688LH/tFf0yMAenidq6k2YNSx6SIY2q6jDACck=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.47.0 h1:aXZXsZ012wOgVkqaiQv+Z/d8V0xKjmg70F6m4CE94lc=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.47.0/go.mod h1:ziu1gUIJhAaxM3EV1e3Ralk5AAyL1UOYz3lJJ4VSI38=
go.opentelemetry.io/otel/log v1.47.0 h1:cOTS1CcLbSQeZKanGJ+0JpF/+t4PELi3O3bbl2lqCcI=
go.opentelemetry.io/otel/log v1.47.0/go.mod h1:9byitSQ5pLC6PpqwGXjqdMKya6ZTswHRZh2vvXT33nw=
go.opentelemetry.io/otel/metric v1.47.0 h1:4PptaldXx3Eat1XjMZ68pPJEs5wrhlemctZE9a3UdWY=
go.opentelemetry.io/otel/metric v1.47.0/go.mod h1:ADGSXxRrXM6bjbvLo535EstVFlPpPYZm4LBKixjDHwU=
go.opentelemetry.io/otel/metric/x v0.69.0 h1:DjRLr15H83v+hCW7JA9NoJvOkYTtmq5YoDRbe9deYpM=
go.opentelemetry.io/otel/metric/x v0.69.0/go.mod h1:uVvsMPMFFyj/HUQfrUnH3JjnOQ1dwFDorgFLRBasM0k=
go.opentelemetry.io/otel/sdk v1.47.0 h1:zWXEr4j2lFefG87TU6Yg8a7ngfohIKFZHKp0Hf5hC6I=
go.opentelemetry.io/otel/sdk v1.47.0/go.mod h1:VUc24kiOeoGsxG8G9ULx3fWKvB7jMhnGE8Oi607lgR0=
go.opentelemetry.io/otel/sdk/log v1.47.0 h1:W4N/ZfgpkRRnmARnqYoQ6BrWA0mrj8IzVDNLlpmUJZY=
go.opentelemetry.io/otel/sdk/log v1.47.0/go.mod h1:Wu74Z2EmOKXi2FleSNAb0Hyau2G+r5kubSEMJw5aAMs=
go.opentelemetry.io/otel/sdk/log/logtest v0.23.0 h1:j3pm42QCrKYYLLkI6OrV1QHFkUfDA6/4EP+Y1PzggLg=
go.opentelemetry.io/otel/sdk/log/logtest v0.23.0/go.mod h1:zt0gEnuTZgDvqvczHwYswz93eqMaIr6upTTsdV/HUlU=
go.opentelemetry.io/otel/sdk/metric v1.47.0 h1:lfISg2j93VT6yqdk9OfUaZmw/GfcZqCCV3jdXtsPnKw=
go.opentelemetry.io/otel/sdk/metric v1.47.0/go.mod h1:ypLp+mW1Nt2x+Szt3b5/i1syodyts49lMOwxpDI3VGw=
go.opentelemetry.io/otel/trace v1.47.0 h1:JOjX/Oci8K94QHddo+bbfya/Ai/nf6/dt9ZfrFNWSrM=
go.opentelemetry.io/otel/trace v1.47.0/go.mod h1:jNaSLa2PZEYFG6fRjJABAu+bw4FS08uDmPg28lTghu0=
go.opentelemetry.io/proto/otlp v1.11.0 h1:5rrYs0Ykyj50sdU/JU0x8etU+LubXWb+gED6TbEdMIk=
go.opentelemetry.io/proto/otlp v1.11.0/go.mod h1:SmVizdCOAm3XBtG1g1NnOdhW6jtddT72hLMhv8VwA8E=
go.uber.org/atomic v1.12.0 h1:BvcXdFKuviU4fTL/f+SxdQ5qJX/Jix8pAkgdUcb3XOE=
go.uber.org/atomic v1.12.0/go.mod h1:I6c4cg+6HCxRjfjSsYtApoFILnpc0CGUdGkXVqbYVNk=
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
golang.org/x/net v0.60.0 h1:79p50tfZlm0J9YfoDsSi639qSXNGVwEzOPLCxM2FsYU=
golang.org/x/net v0.60.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg=
golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk=
golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI=
golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=
gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E=
google.golang.org/genproto/googleapis/api v0.0.0-20260928230214-8a89bd6388cc h1:rZHRz0ogq4Pid1IKtA/ivHi8XONVF5tqUQXA2TtgpkE=
google.golang.org/genproto/googleapis/api v0.0.0-20260928230214-8a89bd6388cc/go.mod h1:K/n0XLazCJHjD73YuMyNJsN38BPPy1kfFnKxUXaThFU=
google.golang.org/genproto/googleapis/rpc v0.0.0-20260928230214-8a89bd6388cc h1:4bNTbnb44EqGVy9HaQxSY2jnafSUdgV3qHtedvNpDKg=
google.golang.org/genproto/googleapis/rpc v0.0.0-20260928230214-8a89bd6388cc/go.mod h1:OaIUM3+LpYcK2GXM4FTmhWoIq371Owdr+Cc7/BsYHHc=
google.golang.org/grpc v1.85.0 h1:BPBCfuVRQ1qvBhKDOq9uvQllMvO59rKsYhXpp7Dd8vw=
google.golang.org/grpc v1.85.0/go.mod h1:0rQrOD58yjal/0EaVbGBANindixsCZNQPDV3VKtaKuQ=
google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc=
google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

Some files were not shown because too many files have changed in this diff Show More