diff --git a/.dockerignore b/.dockerignore index bc16e55..a4397ed 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,20 +1,8 @@ -.git -.github -.env -.env.* -!.env.example -secrets -certs -volumes -data -backups -*.key -*.pem -*.p12 -*.pfx -*.jks -.venv -__pycache__ -node_modules -evidence -.DS_Store +* +!go.mod +!go.sum +!cmd/ +!cmd/** +!internal/ +!internal/** +!Dockerfile diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml new file mode 100644 index 0000000..53f7fd6 --- /dev/null +++ b/.github/workflows/go.yml @@ -0,0 +1,21 @@ +name: Go correctness and per-file coverage +on: [push, pull_request] +jobs: + verify: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: '1.26.1' + - run: sudo apt-get update && sudo apt-get install -y jq ruby webp + - run: go vet ./... + - run: test -z "$(gofmt -l cmd internal)" + - run: LOCAL_COVERAGE=true scripts/test-local.sh + - run: docker run --rm --entrypoint /bin/promtool -v "$PWD/charts/messenger/files/observability:/rules:ro" -v "$PWD/student-tests/alerts.test.yaml:/tests.yaml:ro" prom/prometheus:v3.5.0 test rules /tests.yaml + - uses: actions/upload-artifact@v4 + if: always() + with: + name: coverage + path: coverage/summary.json diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000..1a5f7df --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,21 @@ +version: "2" +run: + timeout: 3m +linters: + default: none + enable: + - errcheck + - govet + - ineffassign + - staticcheck + - unused + - revive + - gocritic +formatters: + enable: + - gofmt + - goimports + settings: + goimports: + local-prefixes: + - gitea.privcloud.tech/vlad_marlo/mtusi2026lab4 diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md new file mode 100644 index 0000000..21a57a4 --- /dev/null +++ b/ARCHITECTURE.md @@ -0,0 +1,9 @@ +# Наблюдаемость мессенджера + +Бизнес-архитектура lab5 сохраняется. Компонент telemetry создаётся в composition root и передаётся HTTP/worker/storage через явные границы. SDK провайдеры принадлежат Fx lifecycle. Отказ Collector не входит в readiness бизнес-API. + +HTTP принимает W3C carrier, создаёт server span и записывает только шаблон маршрута, метод и класс статуса. Durable job хранит trace context. Dispatch создаёт producer span, consumer извлекает его parent, SQL и S3 операции создают дочерние spans. Retry сохраняет исходную причинную связь, но получает отдельный span попытки. + +OTLP Collector принимает три сигнала и ограничивает память и очереди. Prometheus хранит метрики, Loki — структурные логи, Tempo — трассы; Grafana подключается через provisioning. Все компоненты имеют учебные имена messenger-* и находятся внутри существующего Helm release, отдельно от внешнего monitoring. + +Метрики не содержат ID пользователя/чата/вложения/запроса и raw URL. Histogram boundaries заданы в секундах. Error ratio исключает health и клиентские 4xx. SLO alerts используют одновременно короткое и длинное окна; проверка краткого всплеска нужна для оценки ложных срабатываний. diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..c106394 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,17 @@ +FROM golang:1.26.1-alpine3.23 AS build +WORKDIR /src +COPY go.mod go.sum ./ +RUN go mod download +COPY cmd/ ./cmd/ +COPY internal/ ./internal/ +RUN CGO_ENABLED=0 go build -trimpath -ldflags="-s -w" -o /out/server ./cmd/server + +FROM alpine:3.23 AS runtime +RUN apk add --no-cache ca-certificates libwebp-tools +COPY --from=build /out/server /server +USER 65532:65532 +ENV APP_HOST=0.0.0.0 APP_PORT=8080 GOMEMLIMIT=160MiB +EXPOSE 8080 +STOPSIGNAL SIGTERM +HEALTHCHECK --interval=5s --timeout=3s --start-period=3s --retries=3 CMD ["/server", "healthcheck"] +ENTRYPOINT ["/server"] diff --git a/Makefile b/Makefile index 5d86d17..e53bf38 100644 --- a/Makefile +++ b/Makefile @@ -28,3 +28,32 @@ test-persistence: test-failover: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2) $(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) + +.PHONY: test-student vet lint coverage chart-check +test-student: + go test -race ./... +vet: + go vet ./... +lint: + golangci-lint run ./... +coverage: + scripts/coverage.sh +chart-check: + helm lint charts/messenger + helm template lab2 charts/messenger >/dev/null + +KUBE_CONTEXT ?= dev +NAMESPACE ?= highload-applications +RELEASE ?= lab2 +IMAGE ?= asisti.registry.twcstorage.ru/mtusi2026lab6 +TAG ?= local +.PHONY: init-secrets deploy build-push test-local +init-secrets: + KUBE_CONTEXT=$(KUBE_CONTEXT) NAMESPACE=$(NAMESPACE) scripts/init-secrets.sh +build-push: + docker buildx build --platform linux/amd64 --target runtime --tag $(IMAGE):$(TAG) --metadata-file image-build.json --push . + @echo 'Copy containerimage.digest from image-build.json into charts/messenger/values.yaml before deploy.' +deploy: chart-check + helm upgrade --install $(RELEASE) --reset-values charts/messenger --kube-context $(KUBE_CONTEXT) --namespace $(NAMESPACE) --wait --wait-for-jobs --timeout 8m +test-local: + scripts/test-local.sh diff --git a/REPORT.md b/REPORT.md index b97ae42..536724d 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,118 @@ -# Отчёт — лабораторная 6 +# Лабораторная 6: наблюдаемость -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +Агапов, БПИ2303. Целевой уровень — 5, трек 2: SLO и обнаружение инцидентов. Оценка не выставлена. Бизнес-контракт lab5 сохранён. -## Запуск из чистого clone +## Запуск -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Нужны Go 1.26.1, Docker, Helm, kubectl, jq, openssl, Ruby и Python 3.9+ для неизменённых публичных тестов. Native worker использует cwebp; в контейнере установлен libwebp-tools. Все образы закреплены в chart по digest. -## Архитектура и решения +```sh +export GOTOOLCHAIN=go1.26.1 +make check chart-check +make test-local +LOCAL_COVERAGE=true scripts/test-local.sh +# Учебный dev: секрет registry берётся из ~/.secrets/twc-registry-token. +make init-secrets +make deploy +make test BASE_URL=https://hl.agapov.space +``` -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +Один Helm release `lab2`, namespace `highload-applications`, context `dev`. Ingress Traefik — https://hl.agapov.space, issuer letsencrypt-prod. Добавлены Collector, Prometheus, Loki, Tempo и Grafana; прежние API/worker/PostgreSQL/Valkey/S3/broker и PVC сохранены. Migration и S3 bootstrap выполняются идемпотентными Jobs. Registry image обновляется через `make build-push TAG=` и перенос digest в values. `--reset-values` исключает неявное использование образа предыдущей лабы. -## Доказательства критериев +[Настройки и соответствие .env.example](docs/CONFIGURATION.md). [Матрица контрактов](docs/CONTRACT-CHECK.md). Kubernetes заменяет требуемый заданием Compose/Swarm; подтверждение преподавателя на такую замену остаётся открытым вопросом. Наличие работающего dev не доказывает формальное принятие этой замены. -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +Grafana закрыта внутри кластера. Доступ: -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +```sh +kubectl --context dev -n highload-applications port-forward svc/messenger-grafana 3000:3000 +# В другом терминале: прочитать пароль только для своего локального входа. +kubectl --context dev -n highload-applications get secret messenger-grafana -o jsonpath='{.data.password}' | base64 --decode +``` -## Эксперименты и отказы +Открыть http://localhost:3000, username admin. Пароль генерируется один раз; не хранить вывод в отчёте/логах. Dashboard `Messenger: API, worker and SLO` и три datasource создаются provisioning. Остановка port-forward не останавливает приложения. `helm uninstall lab2` намеренно удаляет учебные workloads всего общего release, поэтому это не обычный способ остановки одной лабы. PVC остаются; для намеренного удаления данных выбирать конкретные учебные PVC после проверки владельца. Общий namespace удалять нельзя. -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +## Архитектура -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +API/worker → OTLP HTTP/protobuf → Collector → Prometheus (метрики), Loki (структурные логи), Tempo (трассы) → Grafana (запросы и отображение). Collector не является долговременной БД. Prometheus/Loki/Tempo сохраняют данные на PVC, retention — 24 часа; Grafana хранит настройки на своём PVC. -## Ограничения и следующие шаги +SDK trace/metric 1.38.0, log 0.14.0; Collector contrib 0.135.0, Prometheus 3.5.0, Loki 3.5.5, Tempo 2.8.2, Grafana 12.1.1. Точные версии зависимостей — go.mod; digest образов — values.yaml. Resource attributes: service.name=messenger-api/worker, service.version=lab6, service.instance.id=уникальный ID процесса, deployment.environment.name=lab. Метрики — собственные имена, единицы ниже; W3C TraceContext без baggage. -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +Контекст запроса сохраняется вместе с SQL outbox; producer inject помещает дочерний контекст в Streams. Worker извлекает его для consumer span, вложенные S3/SQL spans показывают реальные операции. Retry сохраняет исходную причинную связь. SQL и параметры, ключи S3, filename, тела, пароли и токены не экспортируются. Логи содержат timestamp/severity/event/outcome и trace/span IDs; HTTP-событие также request_id. Метрики не содержат пользовательские UUID или request_id. Число service.instance.id растёт при рестартах и ограничивается retention; это не бизнес-идентификатор. + +Конструкторы не проверяют доступность telemetry backend; readiness зависит от синхронных бизнес-зависимостей PostgreSQL/Valkey/S3. OTLP не участвует в успехе business commit. SDK очереди trace/log — 256 записей, batch 64, timeout 1 s, retry отключён. Метрики экспортируются раз в 5 s. Collector memory limiter 192 MiB/spike 48 MiB, batch 128/max256; exporter queues 256 batches, два consumers, retry максимум 30 s. Переполнение/отказ допускает потерю telemetry, а не бесконечное накопление. SDK/Collector shutdown ограничен lifecycle deadline. + +## Сигналы и запросы + +| Имя Prometheus | Значение/единица | +| --- | --- | +| messenger_http_requests_total | Количество запросов; method, route template, status_class | +| messenger_http_duration_seconds_* | Histogram длительности HTTP handler в секундах; streaming download может завершиться позже, его S3 span живёт до Close | +| messenger_jobs_total | Реальные попытки worker, job_type/outcome; duplicate без lease не считается обработкой | +| messenger_job_duration_seconds_* | Время попытки обработки, секунды | +| messenger_ready_duration_seconds_* | От durable SQL acceptance до подтверждённой публикации ready, секунды | +| messenger_queue_depth / messenger_queue_oldest_seconds | Текущие незавершённые SQL jobs / возраст старейшей; max по наблюдателям, не sum дублированных снимков | +| messenger_db_pool_used / messenger_db_pool_waiting | Соединения конкретного процесса / ожидающие acquire | +| messenger_worker_busy / messenger_worker_capacity | Занятость / одна job на процесс | +| messenger_process_cpu_seconds_total | Накопленное CPU time; rate даёт CPU cores | +| messenger_process_peak_rss_bytes | Максимальный RSS процесса, bytes; это не мгновенный RSS | + +p95: `histogram_quantile(0.95, sum by (le) (rate(messenger_http_duration_seconds_bucket[1m])))`. Аналогично p50/p99; percentiles реплик не усредняются. Для RPS — `sum(rate(messenger_http_requests_total[1m]))`. Dashboard JSON содержит точные запросы и единицы всех 14 panels. [Реальные имена dev](evidence/current/dev-metric-names.json), [цели scrape](evidence/current/dev-prometheus-targets.json). + +## SLO и правила + +Учебное целевое окно SLO — rolling 24 h. API availability ≥99%: `1 - 5xx / все business requests`, исключены health/metrics, ошибки клиента остаются в denominator. Ready ≥95% завершённых jobs за ≤60 s от durable acceptance. Это latency SLI завершений: зависшие и terminal failed не входят в него, поэтому обязательный дополнительный alert queue lag закрывает слепую зону незавершённой работы. Эти цели выбраны для небольших учебных PNG/JPEG при доступных зависимостях, не являются подтверждённым production SLA. + +Для демонстрации обнаружения используются два окна 1 m/5 m одновременно и `for=30s`. API error ratio >5% означает burn >5 относительно бюджета 1%; доля completed jobs >60 s больше25% — burn >5 относительно бюджета5%. Queue lag: oldest>60 s при depth>0, for15s. Полный день измерений не собирался; проверено поведение alerts в коротком контролируемом эксперименте. Условия и ограничения можно воспроизвести из [rules](charts/messenger/files/observability/alerts.yaml). + +## Эксперимент и диагностика + +29.10.2026, Apple Silicon, native API/worker + Docker Desktop. Две API, один worker, два клиента с паузой250ms; polling alerts раз5s. Нагрузка синтетическая, без Kubernetes CPU/memory limits; один проход и отдельный повтор исправленного ready alert. Гипотеза: multiwindow/for отсекают короткий всплеск, но выявляют устойчивый отказ и накопление jobs, а отказ Tempo не блокирует business API. + +[Raw HTTP, события и alert snapshots](evidence/incidents/), [агрегаты](evidence/incidents/summary.json), [график](evidence/incidents/incident-timeline.png). Длительности определяются по журналу событий; точность обнаружения порядка5s плюс export/scrape задержки. Фаза HTTP назначается перед запросом, поэтому переходные запросы могут пересекать границу воздействия. + +| Воздействие | Реальное наблюдение | +| --- | --- | +| PostgreSQL pause5s | 19/19 запросов фазы завершились ошибкой; API burn не firing ни во всплеске, ни в следующем91s восстановления | +| PostgreSQL pause122s | 552/552 ошибок; API burn firing через137s от начала, то есть через15s после снятия отказа. Resolved через51s после восстановления. Задержка показывает ограничение окон/export/for, а не мгновенное обнаружение | +| Worker STOP113s, 20 uploads | 734 запросов истории без ошибок; queue lag firing через87s от остановки, resolved через10s после CONT. Итоговый backlog0 | +| Tempo stop46s | 328 запросов истории,0 ошибок, p95=23.36ms; полноценный smoke с upload/message/thumbnail прошёл. Зафиксированы136 failed span deliveries, очередь после восстановления0 | + +Во время основной серии пик RSS трёх процессов:30736/45600/46240KiB; CPU от0:00.09/0:00.10/0:00.10 до0:01.95/0:04.72/0:04.38. Единственный snapshot Collector:66.68MiB, CPU0.93%; это snapshot, не доказанный глобальный максимум памяти. Ограничение очередей дополнительно проверено кодом/config и тестом2000spans против недоступного OTLP. Количество failed deliveries не равнозначно точному числу безвозвратно потерянных уникальных spans. + +Первый запуск выявил ошибку ready rule: selector `le="60"` не совпадал с экспортированным `60.0`. Исправление принимает обе формы. [Повтор с настоящим STOP/CONT](evidence/ready-slo/): очередь firing через86s от остановки и resolved через10s после CONT; ready burn firing через41s после CONT, resolved через66s после CONT. [Результат](evidence/ready-slo-result.txt). [promtool regression](evidence/current/alert-rule-tests.txt) дополнительно проверяет короткий/длительный error rate и canonical bucket. Первоначальная проверка полного сценария завершилась ненулевым кодом: assertion ожидал API firing до снятия отказа. Raw данные сохранены; текущий сценарий учитывает lagged detection и использует180s длительного отказа. Полный сценарий после этого изменения заново не запускался. + +Runbook: + +1. Открыть API errors/p95 и queue oldest/depth. Если выросли5xx, найти error trace и HTTP request_id в Loki. +2. В [DB outage trace](evidence/current/db-outage-trace.json) проверить postgres.acquire/query: около1s и error, безопасное сообщение operation failed. Это указывает на storage path, а не на decode изображения. Проверить доступность БД/pool, восстановить зависимость, наблюдать уменьшение error ratio; alert снимается с задержкой окна. +3. Если HTTP работает, но растёт queue age при нулевой занятости, проверить worker process/replicas и последние consumer logs. Возобновить worker; дождаться depth0 и resolved. После накопленной задержки ready burn некоторое время остаётся firing — это ожидаемая память окна. +4. Если Collector send_failed/queue растут, а business RPS стабилен, проверить Tempo/Loki и их PVC. Не включать их в readiness API. После восстановления проверить exporter queue0 и свежие traces; уже потерянные spans восстановить нельзя. + +## Доказательства и повторение + +```sh +export LOCAL_TELEMETRY_STATE=/tmp/messenger-telemetry.env +scripts/telemetry-local.sh +. "$LOCAL_TELEMETRY_STATE" +OBSERVABILITY_TESTS=true scripts/test-local.sh +RETRY_TRACE_TEST=true scripts/test-local.sh +scripts/telemetry-local.sh down +``` + +Скрипт down удаляет только контейнеры/данные, созданные указанным локальным state file. Не использовать state file от другого стенда. Сценарии не останавливают dev workloads. Native source snapshot — [хэши](evidence/current/source.sha256), образ — [build metadata](evidence/current/image.json). Первый длинный эксперимент выполнен до дополнительной проверки входной telemetry configuration; instrumentation/бизнес-путь не менялись. Повтор ready и проверки dev используют исправления. + +| Критерий | Доказательство | +| --- | --- | +| Бизнес API, failover, persistence, очередь переживает restart | evidence/local/smoke.txt, failover.txt, persistence.txt, durable.txt; реальные действия в actions.txt | +| API→producer→consumer→S3/SQL + logs | evidence/current/correlated-trace.json, correlated-logs.json; dev-correlation.txt | +| Retry после настоящего S3 pause | retry-trace.json: первый consumer error15s, следующий успешен; retry-result.json. Assertion enum исправлен и перепроверен по сохранённому trace | +| Метрики/provisioning/live datasource | dev-dashboard.json, dev-datasources.jsonl, dev-tempo-health.txt | +| Race и каждый Go-файл≥90% | coverage-per-file.json, coverage.txt; lint.txt, static.txt | +| Burn/queue alerts и восстановление | evidence/incidents и evidence/ready-slo; alert-rule-tests.txt | +| Dev upgrade и HTTPS smoke | dev-deploy-final.txt: release lab2 revision45; dev-smoke.txt; dev-pods.txt | + +Remote CI не проверялся; локальные результаты не заменяют зелёный pipeline Gitea. Нет HA telemetry backend/PostgreSQL/S3 и восстановления после потери узла/диска. Фиксированная частота sampling1 предназначена для учебной нагрузки; при большой нагрузке её следует отдельно исследовать. Рост RAM на бесконечном горизонте экспериментом не доказан; архитектурные очереди и Kubernetes memory limits конечны. + +## Источники + +[OTel Go](https://opentelemetry.io/docs/languages/go/instrumentation/), [Collector](https://opentelemetry.io/docs/collector/configuration/), [Loki OTLP](https://grafana.com/docs/loki/latest/send-data/otel/), обязательные контракты этого репозитория. При реализации использовалась помощь ИИ. Измерения и ограничения указаны отдельно от проектных гарантий. diff --git a/charts/messenger/Chart.yaml b/charts/messenger/Chart.yaml new file mode 100644 index 0000000..a8d5c19 --- /dev/null +++ b/charts/messenger/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: messenger +description: Учебный мессенджер с PostgreSQL и шардированным Valkey +version: 0.1.0 +appVersion: lab6 +type: application diff --git a/charts/messenger/files/observability/alerts.yaml b/charts/messenger/files/observability/alerts.yaml new file mode 100644 index 0000000..c432550 --- /dev/null +++ b/charts/messenger/files/observability/alerts.yaml @@ -0,0 +1,38 @@ +groups: + - name: messenger-slo + interval: 5s + rules: + - record: messenger:api_error_ratio:1m + expr: sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics",status_class="5xx"}[1m])) / clamp_min(sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics"}[1m])), 0.001) + - record: messenger:api_error_ratio:5m + expr: sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics",status_class="5xx"}[5m])) / clamp_min(sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics"}[5m])), 0.001) + - alert: MessengerAPIBurnRate + expr: (messenger:api_error_ratio:1m > 0.05) and (messenger:api_error_ratio:5m > 0.05) + for: 30s + labels: + severity: warning + annotations: + summary: API consumes the 99% availability budget at more than 5x in both windows + runbook: REPORT.md#диагностика + - alert: MessengerReadyBurnRate + expr: ((sum(rate(messenger_ready_duration_seconds_count[1m])) - sum(rate(messenger_ready_duration_seconds_bucket{le=~"60|60.0"}[1m]))) / clamp_min(sum(rate(messenger_ready_duration_seconds_count[1m])), 0.001) > 0.25) and ((sum(rate(messenger_ready_duration_seconds_count[5m])) - sum(rate(messenger_ready_duration_seconds_bucket{le=~"60|60.0"}[5m]))) / clamp_min(sum(rate(messenger_ready_duration_seconds_count[5m])), 0.001) > 0.25) + for: 30s + labels: + severity: warning + annotations: + summary: More than 25% of completed jobs missed 60s in both windows (5x burn of 95% SLO) + runbook: REPORT.md#диагностика + - alert: MessengerQueueLag + expr: max(messenger_queue_oldest_seconds) > 60 and max(messenger_queue_depth) > 0 + for: 15s + labels: + severity: warning + annotations: + summary: Accepted work has been waiting for over 60 seconds + runbook: REPORT.md#диагностика + - alert: MessengerTelemetryDrops + expr: sum(increase(otelcol_exporter_send_failed_spans_total[1m])) > 0 + labels: + severity: info + annotations: + summary: Trace backend delivery is failing; inspect bounded collector queues diff --git a/charts/messenger/files/observability/collector.yaml b/charts/messenger/files/observability/collector.yaml new file mode 100644 index 0000000..a59bc8a --- /dev/null +++ b/charts/messenger/files/observability/collector.yaml @@ -0,0 +1,71 @@ +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 +processors: + memory_limiter: + check_interval: 1s + limit_mib: 192 + spike_limit_mib: 48 + batch: + timeout: 1s + send_batch_size: 128 + send_batch_max_size: 256 +exporters: + prometheus: + endpoint: 0.0.0.0:8889 + resource_to_telemetry_conversion: + enabled: true + otlphttp/tempo: + endpoint: http://messenger-tempo:4318 + timeout: 2s + sending_queue: + enabled: true + queue_size: 256 + num_consumers: 2 + retry_on_failure: + enabled: true + initial_interval: 1s + max_interval: 5s + max_elapsed_time: 30s + otlphttp/loki: + endpoint: http://messenger-loki:3100/otlp + timeout: 2s + sending_queue: + enabled: true + queue_size: 256 + num_consumers: 2 + retry_on_failure: + enabled: true + initial_interval: 1s + max_interval: 5s + max_elapsed_time: 30s +extensions: + health_check: + endpoint: 0.0.0.0:13133 +service: + telemetry: + metrics: + readers: + - pull: + exporter: + prometheus: + host: 0.0.0.0 + port: 8888 + extensions: [health_check] + pipelines: + traces: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlphttp/tempo] + metrics: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [prometheus] + logs: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlphttp/loki] diff --git a/charts/messenger/files/observability/dashboards.yaml b/charts/messenger/files/observability/dashboards.yaml new file mode 100644 index 0000000..ea74f2d --- /dev/null +++ b/charts/messenger/files/observability/dashboards.yaml @@ -0,0 +1,10 @@ +apiVersion: 1 +providers: + - name: messenger + orgId: 1 + folder: Highload applications + type: file + disableDeletion: true + editable: false + options: + path: /dashboards diff --git a/charts/messenger/files/observability/datasources.yaml b/charts/messenger/files/observability/datasources.yaml new file mode 100644 index 0000000..2c74d27 --- /dev/null +++ b/charts/messenger/files/observability/datasources.yaml @@ -0,0 +1,35 @@ +apiVersion: 1 +datasources: + - name: Prometheus + uid: prometheus + type: prometheus + access: proxy + url: http://messenger-prometheus:9090 + isDefault: true + editable: false + - name: Loki + uid: loki + type: loki + access: proxy + url: http://messenger-loki:3100 + editable: false + jsonData: + derivedFields: + - name: TraceID + matcherType: label + matcherRegex: trace_id + datasourceUid: tempo + url: '$${__value.raw}' + - name: Tempo + uid: tempo + type: tempo + access: proxy + url: http://messenger-tempo:3200 + editable: false + jsonData: + tracesToLogsV2: + datasourceUid: loki + filterByTraceID: true + spanStartTimeShift: '-1m' + spanEndTimeShift: '1m' + tags: [{key: service.name, value: service_name}] diff --git a/charts/messenger/files/observability/loki.yaml b/charts/messenger/files/observability/loki.yaml new file mode 100644 index 0000000..f4180b4 --- /dev/null +++ b/charts/messenger/files/observability/loki.yaml @@ -0,0 +1,41 @@ +auth_enabled: false +server: + http_listen_port: 3100 +common: + path_prefix: /data + replication_factor: 1 + ring: + kvstore: + store: inmemory + storage: + filesystem: + chunks_directory: /data/chunks + rules_directory: /data/rules +schema_config: + configs: + - from: 2024-01-01 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h +limits_config: + allow_structured_metadata: true + retention_period: 24h + ingestion_rate_mb: 2 + ingestion_burst_size_mb: 4 + otlp_config: + resource_attributes: + ignore_defaults: true + attributes_config: + - action: index_label + attributes: [service.name, deployment.environment.name] + - action: structured_metadata + attributes: [service.instance.id, service.version] +compactor: + working_directory: /data/compactor + retention_enabled: true + delete_request_store: filesystem +analytics: + reporting_enabled: false diff --git a/charts/messenger/files/observability/messenger.json b/charts/messenger/files/observability/messenger.json new file mode 100644 index 0000000..6554293 --- /dev/null +++ b/charts/messenger/files/observability/messenger.json @@ -0,0 +1,519 @@ +{ + "uid": "messenger", + "title": "Messenger: API, worker and SLO", + "schemaVersion": 41, + "version": 1, + "refresh": "5s", + "time": { + "from": "now-15m", + "to": "now" + }, + "panels": [ + { + "id": 1, + "title": "HTTP RPS", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 0, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "sum by (route) (rate(messenger_http_requests_total{route!~\"/health/.*|/metrics\"}[1m]))" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps" + }, + "overrides": [ + + ] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 2, + "title": "API 5xx ratio", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 0, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "sum(rate(messenger_http_requests_total{status_class=\"5xx\",route!~\"/health/.*|/metrics\"}[1m])) / clamp_min(sum(rate(messenger_http_requests_total{route!~\"/health/.*|/metrics\"}[1m])),0.001)" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit" + }, + "overrides": [ + + ] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 3, + "title": "HTTP latency p50/p95/p99", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 8, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "histogram_quantile(0.5, sum by (le) (rate(messenger_http_duration_seconds_bucket{route!~\"/health/.*|/metrics\"}[1m])))" + }, + { + "refId": "B", + "expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_http_duration_seconds_bucket{route!~\"/health/.*|/metrics\"}[1m])))" + }, + { + "refId": "C", + "expr": "histogram_quantile(0.99, sum by (le) (rate(messenger_http_duration_seconds_bucket{route!~\"/health/.*|/metrics\"}[1m])))" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [ + + ] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 4, + "title": "Jobs by outcome", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 8, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "sum by (outcome) (rate(messenger_jobs_total[1m]))" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops" + }, + "overrides": [ + + ] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 5, + "title": "Job duration p95", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 16, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_job_duration_seconds_bucket[1m])))" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [ + + ] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 6, + "title": "Time to ready p95", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 16, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_ready_duration_seconds_bucket[1m])))" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [ + + ] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 7, + "title": "Queue depth", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 24, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "max(messenger_queue_depth)" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [ + + ] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 8, + "title": "Oldest accepted job", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 24, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "max(messenger_queue_oldest_seconds)" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [ + + ] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 9, + "title": "DB pool used/waiting", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 32, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "sum(messenger_db_pool_used)" + }, + { + "refId": "B", + "expr": "sum(messenger_db_pool_waiting)" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [ + + ] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 10, + "title": "Worker saturation", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 32, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "sum(messenger_worker_busy) / clamp_min(sum(messenger_worker_capacity),1)" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit" + }, + "overrides": [ + + ] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 11, + "title": "Process CPU", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 40, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "sum by (service_name) (rate(messenger_process_cpu_seconds_total[1m]))" + } + ], + "fieldConfig": { + "defaults": { + "unit": "cores" + }, + "overrides": [ + + ] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 12, + "title": "Process peak RSS", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 40, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "max by (service_name) (messenger_process_peak_rss_bytes)" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes" + }, + "overrides": [ + + ] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 13, + "title": "Collector queue and failed exports", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 48, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "sum(otelcol_exporter_queue_size)" + }, + { + "refId": "B", + "expr": "sum(rate(otelcol_exporter_send_failed_spans_total[1m]))" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [ + + ] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 14, + "title": "Active alerts", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 48, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "ALERTS{alertstate=\"firing\"}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [ + + ] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + } + ] +} \ No newline at end of file diff --git a/charts/messenger/files/observability/prometheus.yaml b/charts/messenger/files/observability/prometheus.yaml new file mode 100644 index 0000000..7ae66be --- /dev/null +++ b/charts/messenger/files/observability/prometheus.yaml @@ -0,0 +1,12 @@ +global: + scrape_interval: 5s + evaluation_interval: 5s +rule_files: + - /config/alerts.yaml +scrape_configs: + - job_name: messenger + static_configs: + - targets: ['messenger-collector:8889'] + - job_name: collector + static_configs: + - targets: ['messenger-collector:8888'] diff --git a/charts/messenger/files/observability/tempo.yaml b/charts/messenger/files/observability/tempo.yaml new file mode 100644 index 0000000..cef2804 --- /dev/null +++ b/charts/messenger/files/observability/tempo.yaml @@ -0,0 +1,24 @@ +server: + http_listen_port: 3200 +distributor: + receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 +storage: + trace: + backend: local + wal: + path: /data/wal + local: + path: /data/blocks +compactor: + compaction: + block_retention: 24h +ingester: + max_block_duration: 5m +usage_report: + reporting_enabled: false diff --git a/charts/messenger/templates/_helpers.tpl b/charts/messenger/templates/_helpers.tpl new file mode 100644 index 0000000..2fcf371 --- /dev/null +++ b/charts/messenger/templates/_helpers.tpl @@ -0,0 +1,7 @@ +{{- define "messenger.shards" -}} +{{- $nodes := list -}} +{{- range $i := until (int .Values.valkey.shards) -}} +{{- $nodes = append $nodes (printf "shard%d=valkey-%d.valkey:6379" $i $i) -}} +{{- end -}} +{{- join "," $nodes -}} +{{- end -}} diff --git a/charts/messenger/templates/api.yaml b/charts/messenger/templates/api.yaml new file mode 100644 index 0000000..0868bb9 --- /dev/null +++ b/charts/messenger/templates/api.yaml @@ -0,0 +1,140 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: messenger-config + namespace: {{ .Release.Namespace }} +data: + APP_PORT: '8080' +{{- if .Values.observability.enabled }} + OTEL_EXPORTER_OTLP_ENDPOINT: http://messenger-collector:4318 + OTEL_SAMPLE_RATIO: '1' + APP_VERSION: lab6 +{{- end }} + VALKEY_COORDINATOR: shard0 + VALKEY_SHARDS: {{ include "messenger.shards" . | quote }} +{{- range $key, $value := .Values.api.env }} + {{ $key }}: {{ $value | quote }} +{{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: api + name: api + namespace: {{ .Release.Namespace }} +spec: + selector: + app: api + ports: + - name: http + port: 8080 + targetPort: 8080 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: api + namespace: {{ .Release.Namespace }} +spec: +{{- if not .Values.autoscaling.enabled }} + replicas: {{ .Values.api.replicas }} +{{- end }} + strategy: + rollingUpdate: + maxUnavailable: 0 + maxSurge: 1 + selector: + matchLabels: + app: api + template: + metadata: + annotations: + checksum/config: {{ printf "%s%s" (include "messenger.shards" .) (toJson .Values.api.env) | sha256sum }} + labels: + app: api + spec: + {{- with .Values.api.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + imagePullSecrets: + - name: {{ .Values.imagePullSecret }} + terminationGracePeriodSeconds: 20 + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: ScheduleAnyway + labelSelector: + matchLabels: + app: api + securityContext: + runAsNonRoot: true + runAsUser: 65532 + seccompProfile: + type: RuntimeDefault + containers: + - name: api + image: "{{ .Values.image.repository }}@{{ .Values.image.digest }}" + imagePullPolicy: IfNotPresent + ports: + - containerPort: 8080 + envFrom: + - configMapRef: + name: messenger-config + env: + - name: S3_ENDPOINT + value: s3:9000 + - name: S3_BUCKET + value: {{ .Values.s3.bucket }} + - name: S3_TLS + value: "false" + - name: S3_ACCESS_KEY + valueFrom: + secretKeyRef: {name: {{ .Values.s3.secret }}, key: access-key} + - name: S3_SECRET_KEY + valueFrom: + secretKeyRef: {name: {{ .Values.s3.secret }}, key: secret-key} + - name: INSTANCE_ID + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: CURSOR_SIGNING_KEY + valueFrom: + secretKeyRef: + name: {{ .Values.databaseSecret }} + key: cursor-key + - name: DATABASE_URL + valueFrom: + secretKeyRef: + name: {{ .Values.databaseSecret }} + key: database-url + resources: + {{- toYaml .Values.api.resources | nindent 12 }} + lifecycle: + preStop: + sleep: + seconds: 3 + startupProbe: + httpGet: + path: /health/ready + port: 8080 + periodSeconds: 2 + failureThreshold: 30 + readinessProbe: + timeoutSeconds: {{ .Values.api.readiness.timeoutSeconds }} + failureThreshold: {{ .Values.api.readiness.failureThreshold }} + httpGet: + path: /health/ready + port: 8080 + periodSeconds: 2 + livenessProbe: + httpGet: + path: /health/live + port: 8080 + periodSeconds: 5 + securityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: + drop: [ALL] diff --git a/charts/messenger/templates/guard.yaml b/charts/messenger/templates/guard.yaml new file mode 100644 index 0000000..313b843 --- /dev/null +++ b/charts/messenger/templates/guard.yaml @@ -0,0 +1,7 @@ +{{- $live := lookup "v1" "ConfigMap" .Release.Namespace "messenger-config" -}} +{{- if and $live (ne (index $live.data "VALKEY_SHARDS") (include "messenger.shards" .)) -}} +{{- $api := lookup "apps/v1" "Deployment" .Release.Namespace "api" -}} +{{- if or (gt (int .Values.api.replicas) 0) (and $api (gt (int $api.spec.replicas) 0)) -}} +{{- fail "Changing shards requires drained API: use make reshard SHARDS=2 or SHARDS=3" -}} +{{- end -}} +{{- end -}} diff --git a/charts/messenger/templates/hpa.yaml b/charts/messenger/templates/hpa.yaml new file mode 100644 index 0000000..a6b8d72 --- /dev/null +++ b/charts/messenger/templates/hpa.yaml @@ -0,0 +1,33 @@ +{{- if .Values.autoscaling.enabled }} +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: api +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: api + minReplicas: {{ .Values.autoscaling.minReplicas }} + maxReplicas: {{ .Values.autoscaling.maxReplicas }} + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.cpuPercent }} + behavior: + scaleUp: + stabilizationWindowSeconds: 15 + policies: + - type: Pods + value: 1 + periodSeconds: 30 + scaleDown: + stabilizationWindowSeconds: 120 + policies: + - type: Pods + value: 1 + periodSeconds: 60 +{{- end }} diff --git a/charts/messenger/templates/ingress.yaml b/charts/messenger/templates/ingress.yaml new file mode 100644 index 0000000..3bc3d8a --- /dev/null +++ b/charts/messenger/templates/ingress.yaml @@ -0,0 +1,23 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: messenger + annotations: + cert-manager.io/cluster-issuer: {{ .Values.ingress.clusterIssuer | quote }} +spec: + ingressClassName: {{ .Values.ingress.className }} + tls: + - hosts: + - {{ .Values.ingress.host | quote }} + secretName: messenger-tls + rules: + - host: {{ .Values.ingress.host | quote }} + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: api + port: + number: 8080 diff --git a/charts/messenger/templates/load.yaml b/charts/messenger/templates/load.yaml new file mode 100644 index 0000000..b9a5f13 --- /dev/null +++ b/charts/messenger/templates/load.yaml @@ -0,0 +1,39 @@ +{{- if .Values.load.enabled }} +apiVersion: batch/v1 +kind: Job +metadata: + name: load-{{ .Values.load.run }} +spec: + backoffLimit: 0 + activeDeadlineSeconds: 300 + ttlSecondsAfterFinished: 86400 + template: + metadata: + labels: + app: load + spec: + restartPolicy: Never + containers: + - name: k6 + image: grafana/k6:1.3.0 + args: [run, --quiet, /scripts/{{ .Values.load.script }}] + env: + - name: BASE_URL + value: "https://{{ .Values.ingress.host }}" + - name: TARGET_INSTANCE_ID + value: {{ .Values.load.target | default "" | quote }} + - name: DURATION + value: {{ .Values.load.duration | quote }} + - name: VUS + value: {{ .Values.load.vus | quote }} + resources: + requests: {cpu: 250m, memory: 128Mi} + limits: {cpu: '2', memory: 256Mi} + volumeMounts: + - name: scripts + mountPath: /scripts + volumes: + - name: scripts + configMap: + name: load-scripts +{{- end }} diff --git a/charts/messenger/templates/migrate.yaml b/charts/messenger/templates/migrate.yaml new file mode 100644 index 0000000..db5f372 --- /dev/null +++ b/charts/messenger/templates/migrate.yaml @@ -0,0 +1,32 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: migrate-{{ .Release.Revision }} +spec: + activeDeadlineSeconds: 180 + backoffLimit: 3 + ttlSecondsAfterFinished: 86400 + template: + metadata: + labels: + app: migrate + spec: + restartPolicy: OnFailure + imagePullSecrets: + - name: {{ .Values.imagePullSecret }} + containers: + - name: migrate + image: "{{ .Values.image.repository }}@{{ .Values.image.digest }}" + args: [migrate] + envFrom: + - configMapRef: + name: messenger-config + env: + - name: DATABASE_URL + valueFrom: + secretKeyRef: + name: {{ .Values.databaseSecret }} + key: database-url + resources: + requests: {cpu: 50m, memory: 32Mi} + limits: {cpu: 500m, memory: 128Mi} diff --git a/charts/messenger/templates/monitoring.yaml b/charts/messenger/templates/monitoring.yaml new file mode 100644 index 0000000..57db5d5 --- /dev/null +++ b/charts/messenger/templates/monitoring.yaml @@ -0,0 +1,19 @@ +{{- if .Values.monitoring.serviceMonitor }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: api + labels: + app: api +spec: + namespaceSelector: + matchNames: [{{ .Release.Namespace }}] + selector: + matchLabels: + app: api + endpoints: + - port: http + path: /metrics + interval: 15s + scrapeTimeout: 3s +{{- end }} diff --git a/charts/messenger/templates/network-policy.yaml b/charts/messenger/templates/network-policy.yaml new file mode 100644 index 0000000..e2b8e4d --- /dev/null +++ b/charts/messenger/templates/network-policy.yaml @@ -0,0 +1,59 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: private-databases + namespace: {{ .Release.Namespace }} +spec: + podSelector: + matchExpressions: + - key: app + operator: In + values: [postgres, valkey] + policyTypes: [Ingress] + ingress: + - from: + - podSelector: + matchLabels: {app: worker} + ports: + - {protocol: TCP, port: 5432} + - from: + - podSelector: + matchLabels: + app: api + ports: + - protocol: TCP + port: 5432 + - protocol: TCP + port: 6379 + - from: + - podSelector: + matchLabels: + app: migrate + ports: + - protocol: TCP + port: 5432 +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: private-api +spec: + podSelector: + matchLabels: + app: api + policyTypes: [Ingress] + ingress: + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: traefik + ports: + - protocol: TCP + port: 8080 + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: monitoring + ports: + - protocol: TCP + port: 8080 diff --git a/charts/messenger/templates/observability.yaml b/charts/messenger/templates/observability.yaml new file mode 100644 index 0000000..6039267 --- /dev/null +++ b/charts/messenger/templates/observability.yaml @@ -0,0 +1,124 @@ +{{- if .Values.observability.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: messenger-observability + labels: {app: messenger-observability} +data: +{{- range $path, $_ := .Files.Glob "files/observability/*" }} + {{ base $path }}: | +{{ $.Files.Get $path | indent 4 }} +{{- end }} +{{- $components := list + (dict "name" "collector" "image" .Values.observability.collectorImage "port" 4318 "args" (list "--config=/config/collector.yaml") "memory" "256Mi" "uid" 10001) + (dict "name" "prometheus" "image" .Values.observability.prometheusImage "port" 9090 "args" (list "--config.file=/config/prometheus.yaml" "--storage.tsdb.path=/data" "--storage.tsdb.retention.time=24h" "--storage.tsdb.retention.size=1GB") "memory" "384Mi" "uid" 65534) + (dict "name" "loki" "image" .Values.observability.lokiImage "port" 3100 "args" (list "-config.file=/config/loki.yaml") "memory" "384Mi" "uid" 10001) + (dict "name" "tempo" "image" .Values.observability.tempoImage "port" 3200 "args" (list "-config.file=/config/tempo.yaml") "memory" "384Mi" "uid" 10001) + (dict "name" "grafana" "image" .Values.observability.grafanaImage "port" 3000 "args" list "memory" "256Mi" "uid" 472) +}} +{{- range $component := $components }} +--- +apiVersion: v1 +kind: Service +metadata: + name: messenger-{{ $component.name }} +spec: + selector: {app: 'messenger-{{ $component.name }}'} + ports: + - {name: http, port: {{ $component.port }}, targetPort: {{ $component.port }}} + {{- if eq $component.name "collector" }} + - {name: metrics, port: 8889, targetPort: 8889} + - {name: self-metrics, port: 8888, targetPort: 8888} + {{- end }} + {{- if eq $component.name "tempo" }} + - {name: otlp, port: 4318, targetPort: 4318} + {{- end }} +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: messenger-{{ $component.name }} +spec: + serviceName: messenger-{{ $component.name }} + replicas: 1 + selector: + matchLabels: {app: 'messenger-{{ $component.name }}'} + template: + metadata: + labels: {app: 'messenger-{{ $component.name }}'} + annotations: + checksum/config: {{ ($.Files.Glob "files/observability/*").AsConfig | sha256sum }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: {{ $component.uid }} + fsGroup: {{ $component.uid }} + containers: + - name: {{ $component.name }} + image: {{ $component.image }} + args: {{ $component.args | toJson }} + ports: + - {containerPort: {{ $component.port }}} + resources: + requests: {cpu: 50m, memory: 96Mi} + limits: {cpu: '1', memory: {{ $component.memory }}} + readinessProbe: + tcpSocket: {port: {{ $component.port }}} + {{- if eq $component.name "grafana" }} + env: + - {name: GF_PATHS_DATA, value: /data} + - {name: GF_AUTH_ANONYMOUS_ENABLED, value: 'false'} + - {name: GF_USERS_ALLOW_SIGN_UP, value: 'false'} + - {name: GF_ANALYTICS_REPORTING_ENABLED, value: 'false'} + - name: GF_SECURITY_ADMIN_USER + valueFrom: + secretKeyRef: {name: {{ $.Values.observability.secret }}, key: username} + - name: GF_SECURITY_ADMIN_PASSWORD + valueFrom: + secretKeyRef: {name: {{ $.Values.observability.secret }}, key: password} + {{- end }} + volumeMounts: + - {name: data, mountPath: /data} + - {name: config, mountPath: /config, readOnly: true} + {{- if eq $component.name "grafana" }} + - {name: config, mountPath: /etc/grafana/provisioning/datasources/datasources.yaml, subPath: datasources.yaml, readOnly: true} + - {name: config, mountPath: /etc/grafana/provisioning/dashboards/dashboards.yaml, subPath: dashboards.yaml, readOnly: true} + - {name: config, mountPath: /dashboards/messenger.json, subPath: messenger.json, readOnly: true} + {{- end }} + volumes: + - name: config + configMap: {name: messenger-observability} + volumeClaimTemplates: + - metadata: {name: data} + spec: + accessModes: [ReadWriteOnce] + {{- if $.Values.storageClass }} + storageClassName: {{ $.Values.storageClass | quote }} + {{- end }} + resources: + requests: {storage: 2Gi} +{{- end }} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: private-observability +spec: + podSelector: + matchExpressions: + - {key: app, operator: In, values: [messenger-collector, messenger-prometheus, messenger-loki, messenger-tempo, messenger-grafana]} + policyTypes: [Ingress] + ingress: + - from: + - podSelector: + matchExpressions: + - {key: app, operator: In, values: [api, worker, messenger-collector, messenger-prometheus, messenger-loki, messenger-tempo, messenger-grafana]} + ports: + - {protocol: TCP, port: 4318} + - {protocol: TCP, port: 8888} + - {protocol: TCP, port: 8889} + - {protocol: TCP, port: 9090} + - {protocol: TCP, port: 3100} + - {protocol: TCP, port: 3200} + - {protocol: TCP, port: 3000} +{{- end }} diff --git a/charts/messenger/templates/postgres.yaml b/charts/messenger/templates/postgres.yaml new file mode 100644 index 0000000..8d8704a --- /dev/null +++ b/charts/messenger/templates/postgres.yaml @@ -0,0 +1,66 @@ +apiVersion: v1 +kind: Service +metadata: + name: postgres + namespace: {{ .Release.Namespace }} +spec: + selector: + app: postgres + ports: + - port: 5432 + targetPort: 5432 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: postgres + namespace: {{ .Release.Namespace }} +spec: + serviceName: postgres + replicas: 1 + selector: + matchLabels: + app: postgres + template: + metadata: + labels: + app: postgres + spec: + terminationGracePeriodSeconds: 30 + containers: + - name: postgres + image: {{ .Values.postgres.image | quote }} + ports: + - containerPort: 5432 + env: + - name: POSTGRES_DB + value: messenger + - name: POSTGRES_USER + value: messenger + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.databaseSecret }} + key: postgres-password + - name: PGDATA + value: /var/lib/postgresql/data/pgdata + resources: + {{- toYaml .Values.postgres.resources | nindent 12 }} + readinessProbe: + exec: + command: [pg_isready, -U, messenger, -d, messenger] + initialDelaySeconds: 3 + volumeMounts: + - name: data + mountPath: /var/lib/postgresql/data + volumeClaimTemplates: + - metadata: + name: data + spec: +{{- if .Values.storageClass }} + storageClassName: {{ .Values.storageClass | quote }} +{{- end }} + accessModes: [ReadWriteOnce] + resources: + requests: + storage: {{ .Values.postgres.storage }} diff --git a/charts/messenger/templates/s3.yaml b/charts/messenger/templates/s3.yaml new file mode 100644 index 0000000..f015bab --- /dev/null +++ b/charts/messenger/templates/s3.yaml @@ -0,0 +1,97 @@ +apiVersion: v1 +kind: Service +metadata: + name: s3 +spec: + selector: {app: s3} + ports: + - {name: s3, port: 9000, targetPort: 9000} +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: s3 +spec: + serviceName: s3 + replicas: 1 + selector: + matchLabels: {app: s3} + template: + metadata: + labels: {app: s3} + spec: + securityContext: + runAsUser: 1000 + runAsNonRoot: true + fsGroup: 1000 + containers: + - name: s3 + image: {{ .Values.s3.image }} + args: [server, -dir=/data, -s3, -s3.port=9000, -s3.config=/identity/identities.json, -master.volumeSizeLimitMB=100, -volume.max=32] + ports: [{containerPort: 9000}] + readinessProbe: + tcpSocket: {port: 9000} + resources: + requests: {cpu: 100m, memory: 256Mi} + limits: {cpu: '1', memory: 512Mi} + securityContext: + allowPrivilegeEscalation: false + capabilities: {drop: [ALL]} + volumeMounts: + - {name: objects, mountPath: /data} + - {name: identity, mountPath: /identity, readOnly: true} + volumes: + - name: identity + secret: {secretName: {{ .Values.s3.secret }}, items: [{key: identities.json, path: identities.json}]} + volumeClaimTemplates: + - metadata: {name: objects} + spec: + accessModes: [ReadWriteOnce] + resources: + requests: {storage: {{ .Values.s3.storage }}} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: s3-bootstrap-{{ .Release.Revision }} +spec: + ttlSecondsAfterFinished: 3600 + backoffLimit: 4 + template: + metadata: + labels: {app: s3-bootstrap} + spec: + restartPolicy: OnFailure + imagePullSecrets: [{name: {{ .Values.imagePullSecret }}}] + containers: + - name: bootstrap + image: "{{ .Values.image.repository }}@{{ .Values.image.digest }}" + args: [s3-bootstrap] + env: + - {name: S3_ENDPOINT, value: 's3:9000'} + - {name: S3_TLS, value: 'false'} + - {name: S3_BUCKET, value: {{ .Values.s3.bucket }}} + - name: S3_ACCESS_KEY + valueFrom: + secretKeyRef: {name: {{ .Values.s3.secret }}, key: root-user} + - name: S3_SECRET_KEY + valueFrom: + secretKeyRef: {name: {{ .Values.s3.secret }}, key: root-password} + resources: + requests: {cpu: 50m, memory: 64Mi} + limits: {cpu: 200m, memory: 128Mi} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: private-s3 +spec: + podSelector: + matchLabels: {app: s3} + policyTypes: [Ingress] + ingress: + - from: + - podSelector: + matchExpressions: + - {key: app, operator: In, values: [api, worker, s3-bootstrap, reconcile]} + ports: [{protocol: TCP, port: 9000}] diff --git a/charts/messenger/templates/valkey.yaml b/charts/messenger/templates/valkey.yaml new file mode 100644 index 0000000..5c996cc --- /dev/null +++ b/charts/messenger/templates/valkey.yaml @@ -0,0 +1,57 @@ +apiVersion: v1 +kind: Service +metadata: + name: valkey + namespace: {{ .Release.Namespace }} +spec: + clusterIP: None + selector: + app: valkey + ports: + - port: 6379 + targetPort: 6379 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: valkey + namespace: {{ .Release.Namespace }} +spec: + serviceName: valkey + replicas: {{ .Values.valkey.shards }} + podManagementPolicy: Parallel + selector: + matchLabels: + app: valkey + template: + metadata: + labels: + app: valkey + spec: + terminationGracePeriodSeconds: 30 + containers: + - name: valkey + image: {{ .Values.valkey.image | quote }} + args: [valkey-server, --appendonly, 'yes', --appendfsync, always, --maxmemory, {{ .Values.valkey.maxmemory | quote }}, --maxmemory-policy, noeviction] + ports: + - containerPort: 6379 + resources: + {{- toYaml .Values.valkey.resources | nindent 12 }} + readinessProbe: + exec: + command: [valkey-cli, ping] + initialDelaySeconds: 2 + volumeMounts: + - name: data + mountPath: /data + volumeClaimTemplates: + - metadata: + name: data + spec: +{{- if .Values.storageClass }} + storageClassName: {{ .Values.storageClass | quote }} +{{- end }} + accessModes: [ReadWriteOnce] + resources: + requests: + storage: {{ .Values.valkey.storage }} diff --git a/charts/messenger/templates/worker.yaml b/charts/messenger/templates/worker.yaml new file mode 100644 index 0000000..273bd5e --- /dev/null +++ b/charts/messenger/templates/worker.yaml @@ -0,0 +1,111 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: worker +spec: + replicas: {{ .Values.worker.replicas }} + selector: + matchLabels: {app: worker} + template: + metadata: + labels: {app: worker} + spec: + imagePullSecrets: + - name: {{ .Values.imagePullSecret }} + terminationGracePeriodSeconds: 60 + securityContext: + runAsNonRoot: true + runAsUser: 65532 + seccompProfile: {type: RuntimeDefault} + containers: + - name: worker + image: "{{ .Values.image.repository }}@{{ .Values.image.digest }}" + args: [worker] + resources: + requests: {cpu: 250m, memory: 256Mi} + limits: {cpu: '1', memory: 1Gi} + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: {drop: [ALL]} + envFrom: + - configMapRef: {name: messenger-config} + env: + - {name: GOMEMLIMIT, value: 768MiB} + - {name: BROKER_ADDRESS, value: 'broker:6379'} + - {name: S3_ENDPOINT, value: 's3:9000'} + - {name: S3_TLS, value: 'false'} + - name: S3_BUCKET + value: {{ .Values.s3.bucket }} + - name: DATABASE_URL + valueFrom: + secretKeyRef: {name: {{ .Values.databaseSecret }}, key: database-url} + - name: CURSOR_SIGNING_KEY + valueFrom: + secretKeyRef: {name: {{ .Values.databaseSecret }}, key: cursor-key} + - name: S3_ACCESS_KEY + valueFrom: + secretKeyRef: {name: {{ .Values.s3.secret }}, key: access-key} + - name: S3_SECRET_KEY + valueFrom: + secretKeyRef: {name: {{ .Values.s3.secret }}, key: secret-key} +--- +apiVersion: v1 +kind: Service +metadata: + name: broker +spec: + selector: {app: broker} + ports: + - {name: streams, port: 6379} +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: broker +spec: + serviceName: broker + replicas: 1 + selector: + matchLabels: {app: broker} + template: + metadata: + labels: {app: broker} + spec: + containers: + - name: broker + image: {{ .Values.valkey.image }} + args: [valkey-server, --appendonly, 'yes', --appendfsync, always, --maxmemory, 128mb, --maxmemory-policy, noeviction] + resources: + requests: {cpu: 100m, memory: 160Mi} + limits: {cpu: '1', memory: 256Mi} + ports: + - {containerPort: 6379} + readinessProbe: + exec: {command: [valkey-cli, ping]} + volumeMounts: + - {name: data, mountPath: /data} + volumeClaimTemplates: + - metadata: {name: data} + spec: + accessModes: [ReadWriteOnce] + {{- if .Values.storageClass }} + storageClassName: {{ .Values.storageClass | quote }} + {{- end }} + resources: + requests: {storage: 1Gi} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: private-broker +spec: + podSelector: + matchLabels: {app: broker} + policyTypes: [Ingress] + ingress: + - from: + - podSelector: + matchLabels: {app: worker} + ports: + - {protocol: TCP, port: 6379} diff --git a/charts/messenger/values.schema.json b/charts/messenger/values.schema.json new file mode 100644 index 0000000..3640ffe --- /dev/null +++ b/charts/messenger/values.schema.json @@ -0,0 +1,9 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "valkey": {"type": "object", "properties": {"shards": {"type": "integer", "minimum": 2, "maximum": 3}}}, + "api": {"type": "object", "properties": {"replicas": {"type": "integer", "minimum": 0}}}, + "image": {"type": "object", "properties": {"digest": {"type": "string", "pattern": "^sha256:[a-f0-9]{64}$"}}} + } +} diff --git a/charts/messenger/values.yaml b/charts/messenger/values.yaml new file mode 100644 index 0000000..8e0189c --- /dev/null +++ b/charts/messenger/values.yaml @@ -0,0 +1,76 @@ +# Existing lab2 release; shared persistent stores and credentials. +image: + repository: asisti.registry.twcstorage.ru/mtusi2026lab6 + digest: sha256:04a28492ad5645ed56c58d51a95214a758426adc2b0271fc184e8f204c3fa7cc +imagePullSecret: registry-pull +databaseSecret: messenger-secrets +api: + readiness: + timeoutSeconds: 2 + failureThreshold: 2 + replicas: 2 + resources: + requests: {cpu: 250m, memory: 128Mi} + limits: {cpu: '1', memory: 768Mi} + env: + SESSION_TTL_SECONDS: '3600' + DEPENDENCY_TIMEOUT_MS: '2000' + DEPENDENCY_POOL_SIZE: '8' + REQUEST_TIMEOUT_SECONDS: '4' + SHUTDOWN_TIMEOUT_SECONDS: '5' + GOMEMLIMIT: 640MiB + MAX_CONNECTIONS: '16' + MAX_BODY_BYTES: '10485760' +postgres: + image: postgres:17.6-alpine + storage: 2Gi + resources: + requests: {cpu: 250m, memory: 256Mi} + limits: {cpu: '1', memory: 512Mi} +valkey: + image: valkey/valkey:8.1.3-alpine + shards: 3 + storage: 1Gi + maxmemory: 64mb + resources: + requests: {cpu: 100m, memory: 96Mi} + limits: {cpu: 500m, memory: 128Mi} +# Пустое значение использует default StorageClass. Задать до первой установки. +storageClass: '' + +autoscaling: + enabled: true + minReplicas: 2 + maxReplicas: 4 + cpuPercent: 60 +monitoring: + serviceMonitor: false +load: + enabled: false + run: manual + script: scaling.js + duration: 30 + vus: 32 + +ingress: + host: hl.agapov.space + className: traefik + clusterIssuer: letsencrypt-prod + +s3: + image: chrislusf/seaweedfs:3.85@sha256:49312939c00c01e5ee6afbd7d728b18027821d3764c35a797a72acd4fdf3296a + secret: messenger-s3 + bucket: attachments + storage: 5Gi + +worker: + replicas: 1 + +observability: + enabled: true + collectorImage: otel/opentelemetry-collector-contrib:0.135.0@sha256:89107a3a8f4636a396927edf7025bb9614b8da2d92f4cc3f43109e8d115736e2 + prometheusImage: prom/prometheus:v3.5.0@sha256:63805ebb8d2b3920190daf1cb14a60871b16fd38bed42b857a3182bc621f4996 + lokiImage: grafana/loki:3.5.5@sha256:31628519045e7f28692a7ae73b4a3fd293dccb425585ed5d16ceea3b5c9592e6 + tempoImage: grafana/tempo:2.8.2@sha256:0ef775495967cd5d7a6b2e146b6ea695d624803c8db8349fb8ce4164f719f9b7 + grafanaImage: grafana/grafana:12.1.1@sha256:a1701c2180249361737a99a01bc770db39381640e4d631825d38ff4535efa47d + secret: messenger-grafana diff --git a/cmd/server/app.go b/cmd/server/app.go new file mode 100644 index 0000000..9ed63d5 --- /dev/null +++ b/cmd/server/app.go @@ -0,0 +1,106 @@ +package main + +import ( + "time" + + "go.uber.org/fx" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/admission" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/config" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/controller" + fiberctrl "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/controller/http/fiber" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/cursor" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/password" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/service" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/service/production" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/store" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/store/postgres" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/valkey" +) + +const applicationStopTimeout = 9 * time.Second + +// CreateApp defines the dependency graph and Fx lifecycle for the lab2 server. +// Constructors do not open sockets; controller.RunControllerFx registers startup +// and shutdown hooks. The outer stop deadline bounds all component cleanup. +func CreateApp() fx.Option { + return fx.Options( + fx.StopTimeout(applicationStopTimeout), + fx.Supply(fx.Annotate(password.Hasher{}, fx.As(new(service.Passwords)))), + fx.Provide( + config.New, + config.NewPersistence, + config.NewObjectStore, + apiTelemetry, + observedObjects, + newMedia, + observedDatabase, + newValkey, + newAuthentication, + fx.Annotate(newMediaReadiness, fx.As(new(fiberctrl.Readiness))), + httpConfig, + + loggerConfig, + newLogger, + fx.Annotate(newAdmissionLimiter, fx.As(new(admission.Distributed))), + fiberctrl.NewErrorRegistry, + fx.Annotate(databaseStore, fx.As(new(store.Interface))), + fx.Annotate(databaseAccounts, fx.As(new(service.Accounts))), + fx.Annotate(profileCache, fx.As(new(service.ProfileCache))), + fx.Annotate(newCursor, fx.As(new(service.CursorCodec))), + fx.Annotate(production.NewCached, fx.As(new(service.Service))), + fx.Annotate(observedHTTP, fx.As(new(controller.Controller))), + ), + fx.WithLogger(newFxLogger), + fx.Invoke(controller.RunControllerFx, observeAPI), + ) +} + +// httpConfig provides the HTTP configuration section to Fx. +func httpConfig(cfg *config.Config) *config.Controller { + return cfg.HTTP +} + +// loggerConfig provides the logging configuration section to Fx. +func loggerConfig(cfg *config.Config) *config.Logger { + return cfg.Logger +} + +// newAdmissionLimiter translates configured bounds into separate shared policy budgets. +func newAdmissionLimiter(cfg *config.Controller, client *valkey.Client) (*valkey.Limiter, error) { + global := valkey.Budget{Rate: cfg.Admission.RequestsPerSecond, Slots: cfg.Admission.MaxInFlight} + return valkey.NewLimiter(client, valkey.Limits{ + Global: global, + Groups: map[string]valkey.Budget{ + "auth": {Rate: cfg.Admission.AuthRate, Slots: cfg.Admission.AuthSlots}, + "read": global, + "write": global, + }, + User: valkey.Budget{Rate: cfg.Admission.UserRate, Slots: cfg.Admission.UserSlots}, + UserGroups: map[string]valkey.Budget{ + "auth": {Rate: cfg.Admission.UserAuthRate, Slots: cfg.Admission.UserAuthSlots}, + "read": {Rate: cfg.Admission.UserRate, Slots: cfg.Admission.UserSlots}, + "write": {Rate: cfg.Admission.UserWriteRate, Slots: cfg.Admission.UserWriteSlots}, + }, + }) +} + +// databaseStore exposes business persistence through its consumer interface. +func databaseStore(database *postgres.Store) store.Interface { + return database +} + +// databaseAccounts exposes account and session persistence through its consumer interface. +func databaseAccounts(database *postgres.Store) service.Accounts { + return database +} + +// profileCache exposes immutable profiles without leaking the Valkey client. +func profileCache(client *valkey.Client) service.ProfileCache { + return client +} + +// newCursor binds the shared signing secret at the composition root. +func newCursor(cfg *config.Persistence) (*cursor.Codec, error) { + return cursor.NewShared(cfg.CursorKey) +} diff --git a/cmd/server/app_test.go b/cmd/server/app_test.go new file mode 100644 index 0000000..7b10bf0 --- /dev/null +++ b/cmd/server/app_test.go @@ -0,0 +1,108 @@ +package main + +import ( + "context" + "net" + "os" + "testing" + "time" + + "go.uber.org/fx" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/config" +) + +// TestDependencyGraph ensures constructor signatures and interface bindings agree. +func TestDependencyGraph(t *testing.T) { + if err := fx.ValidateApp(CreateApp(), fx.NopLogger); err != nil { + t.Fatal(err) + } +} + +// TestInvalidConfigurationStopsStartup checks failure before a listener is acquired. +func TestInvalidConfigurationStopsStartup(t *testing.T) { + t.Setenv("APP_PORT", "0") + app := fx.New(CreateApp(), fx.NopLogger) + if err := app.Start(context.Background()); err == nil { + t.Fatal("invalid listener configuration accepted") + } +} + +// TestCommandValidation ensures unsupported arguments cannot accidentally start a server. +func TestCommandValidation(t *testing.T) { + if err := run([]string{"unknown"}); err == nil { + t.Fatal("unknown command accepted") + } +} + +// TestLoggerConfiguration checks that process logging uses its own validated section. +func TestLoggerConfiguration(t *testing.T) { + if _, err := newLogger(&config.Logger{Level: "invalid"}); err == nil { + t.Fatal("invalid logger configuration accepted") + } +} + +// TestApplicationLifecycle starts the complete Fx graph and drains its real HTTP listener. +func TestApplicationLifecycle(t *testing.T) { + requirePersistence(t) + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + cfg := config.Default() + cfg.HTTP.Host = "127.0.0.1" + cfg.HTTP.Port = listener.Addr().(*net.TCPAddr).Port + cfg.Logger.Level = "error" + if err := listener.Close(); err != nil { + t.Fatal(err) + } + app := fx.New(CreateApp(), fx.Replace(cfg)) + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + if err := app.Start(ctx); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + stopCtx, stopCancel := context.WithTimeout(context.Background(), 5*time.Second) + defer stopCancel() + if err := app.Stop(stopCtx); err != nil { + t.Error(err) + } + }) + if err := checkReadiness(cfg.HTTP); err != nil { + t.Fatal(err) + } +} + +// TestOccupiedListenerFailsStartup ensures bind failures propagate through the Fx lifecycle. +func TestOccupiedListenerFailsStartup(t *testing.T) { + requirePersistence(t) + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer func() { + _ = listener.Close() + }() + cfg := config.Default() + cfg.HTTP.Host = "127.0.0.1" + cfg.HTTP.Port = listener.Addr().(*net.TCPAddr).Port + cfg.Logger.Level = "error" + app := fx.New(CreateApp(), fx.Replace(cfg), fx.NopLogger) + if err := app.Start(t.Context()); err == nil { + _ = app.Stop(t.Context()) + t.Fatal("startup succeeded on an occupied listener") + } +} + +// requirePersistence prevents infrastructure scenarios from silently passing on configuration failure. +func requirePersistence(t *testing.T) { + t.Helper() + if os.Getenv("DATABASE_URL") == "" || os.Getenv("VALKEY_SHARDS") == "" { + t.Skip("DATABASE_URL and VALKEY_SHARDS are required for real application lifecycle tests") + } + t.Setenv("CURSOR_SIGNING_KEY", "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=") + if err := migrate(); err != nil { + t.Fatal(err) + } +} diff --git a/cmd/server/commands.go b/cmd/server/commands.go new file mode 100644 index 0000000..bb3e742 --- /dev/null +++ b/cmd/server/commands.go @@ -0,0 +1,57 @@ +package main + +import ( + "context" + "fmt" + "time" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/config" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/store/postgres" + + "go.uber.org/fx" +) + +// run dispatches the server command or the container healthcheck. +func run(args []string) error { + if len(args) == 1 && args[0] == "worker" { + fx.New(workerApp()).Run() + return nil + } + if len(args) == 3 && args[0] == "replay" { + return replayJob(args[1], args[2]) + } + if len(args) == 1 && args[0] == "s3-bootstrap" { + return bootstrapS3() + } + if len(args) == 0 { + fx.New(CreateApp()).Run() + return nil + } + if len(args) == 1 && args[0] == "reconcile" { + return reconcile(false) + } + if len(args) == 2 && args[0] == "reconcile" && args[1] == "--apply" { + return reconcile(true) + } + if len(args) == 1 && args[0] == "migrate" { + return migrate() + } + if len(args) == 1 && args[0] == "healthcheck" { + return healthcheck() + } + return fmt.Errorf("usage: server [healthcheck|migrate]") +} + +// migrate applies versioned DDL once from an operator job, independently of the API. +func migrate() error { + cfg, err := config.NewPersistence() + if err != nil { + return err + } + database, err := postgres.New(postgres.Options{URL: cfg.DatabaseURL, MaxConnections: 1, Timeout: 30 * time.Second}) + if err != nil { + return err + } + defer database.Close() + return database.Start(context.Background()) +} diff --git a/cmd/server/healthcheck.go b/cmd/server/healthcheck.go new file mode 100644 index 0000000..def115e --- /dev/null +++ b/cmd/server/healthcheck.go @@ -0,0 +1,46 @@ +package main + +import ( + "fmt" + "net" + "net/http" + "strconv" + "time" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/config" +) + +const readinessProbeTimeout = 2 * time.Second + +// healthcheck loads configuration and runs the container readiness probe. +func healthcheck() error { + cfg, err := config.New() + if err != nil { + return err + } + return checkReadiness(cfg.HTTP) +} + +// checkReadiness checks the local readiness endpoint within the probe timeout. +func checkReadiness(cfg *config.Controller) error { + host := cfg.Host + if host == "0.0.0.0" { + host = "127.0.0.1" + } + if host == "::" { + host = "::1" + } + address := "http://" + net.JoinHostPort(host, strconv.Itoa(cfg.Port)) + "/health/ready" + client := http.Client{Timeout: readinessProbeTimeout} + response, err := client.Get(address) + if err != nil { + return fmt.Errorf("readiness probe: %w", err) + } + defer func() { + _ = response.Body.Close() + }() + if response.StatusCode != http.StatusOK { + return fmt.Errorf("not ready: %d", response.StatusCode) + } + return nil +} diff --git a/cmd/server/healthcheck_test.go b/cmd/server/healthcheck_test.go new file mode 100644 index 0000000..b49d091 --- /dev/null +++ b/cmd/server/healthcheck_test.go @@ -0,0 +1,111 @@ +package main + +import ( + "errors" + "net" + "net/http" + "net/http/httptest" + "net/url" + "strconv" + "testing" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/config" +) + +// TestRun_Healthcheck verifies command routing to the readiness endpoint. +func TestRun_Healthcheck(t *testing.T) { + // Arrange. + requests := make(chan string, 1) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests <- r.Method + " " + r.URL.Path + w.WriteHeader(http.StatusOK) + })) + t.Cleanup(server.Close) + address := server.Listener.Addr().(*net.TCPAddr) + t.Setenv("APP_HOST", "0.0.0.0") + t.Setenv("APP_PORT", strconv.Itoa(address.Port)) + + // Act. + err := run([]string{"healthcheck"}) + + // Assert. + if err != nil { + t.Fatal(err) + } + select { + case got := <-requests: + if got != "GET /health/ready" { + t.Fatalf("probe=%q", got) + } + default: + t.Fatal("command returned success without probing readiness") + } +} + +// TestHealthcheck_InvalidConfig rejects invalid startup settings before attempting HTTP. +func TestHealthcheck_InvalidConfig(t *testing.T) { + // Arrange. + t.Setenv("APP_PORT", "invalid") + + // Act. + err := healthcheck() + + // Assert. + if err == nil { + t.Fatal("invalid port accepted") + } +} + +// TestCheckReadiness distinguishes unhealthy responses from connection failures. +func TestCheckReadiness(t *testing.T) { + t.Run("Negative: unhealthy response", func(t *testing.T) { + // Arrange. + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusServiceUnavailable) + })) + t.Cleanup(server.Close) + address := server.Listener.Addr().(*net.TCPAddr) + cfg := config.Default().HTTP + cfg.Host = "127.0.0.1" + cfg.Port = address.Port + + // Act. + err := checkReadiness(cfg) + + // Assert. + if err == nil { + t.Fatalf("probe accepted %s", "Negative: unhealthy response") + } + if err.Error() != "not ready: 503" { + t.Fatalf("unexpected health failure: %v", err) + } + + }) + + t.Run("Negative: server unavailable", func(t *testing.T) { + // Arrange. + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusServiceUnavailable) + })) + t.Cleanup(server.Close) + address := server.Listener.Addr().(*net.TCPAddr) + cfg := config.Default().HTTP + cfg.Host = "127.0.0.1" + cfg.Port = address.Port + + server.Close() + + // Act. + err := checkReadiness(cfg) + + // Assert. + if err == nil { + t.Fatalf("probe accepted %s", "Negative: server unavailable") + } + + if _, ok := errors.AsType[*url.Error](err); !ok { + t.Fatalf("connection failure was not preserved: %v", err) + } + + }) +} diff --git a/cmd/server/logger.go b/cmd/server/logger.go new file mode 100644 index 0000000..7673a20 --- /dev/null +++ b/cmd/server/logger.go @@ -0,0 +1,29 @@ +package main + +import ( + "go.uber.org/fx/fxevent" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/config" +) + +// newLogger creates a structured logger with the configured severity threshold. +func newLogger(cfg *config.Logger) (*zap.Logger, error) { + if err := cfg.Validate(); err != nil { + return nil, err + } + level, err := zapcore.ParseLevel(cfg.Level) + if err != nil { + return nil, err + } + settings := zap.NewProductionConfig() + settings.Level = zap.NewAtomicLevelAt(level) + settings.DisableStacktrace = true + return settings.Build() +} + +// newFxLogger adapts the application logger for Fx lifecycle events. +func newFxLogger(log *zap.Logger) fxevent.Logger { + return &fxevent.ZapLogger{Logger: log.Named("fx")} +} diff --git a/cmd/server/main.go b/cmd/server/main.go new file mode 100644 index 0000000..1d1fbd5 --- /dev/null +++ b/cmd/server/main.go @@ -0,0 +1,15 @@ +// Command server runs the messenger or its container readiness probe. +package main + +import ( + "fmt" + "os" +) + +// main runs the selected command and reports failures to stderr. +func main() { + if err := run(os.Args[1:]); err != nil { + _, _ = fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} diff --git a/cmd/server/media.go b/cmd/server/media.go new file mode 100644 index 0000000..0cc3835 --- /dev/null +++ b/cmd/server/media.go @@ -0,0 +1,39 @@ +package main + +import ( + "go.lab260.ru/fielderr" + "go.uber.org/zap" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/admission" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/config" + fiberctrl "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/controller/http/fiber" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/media" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/objectstore" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/service" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/store/postgres" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/valkey" +) + +// newObjects binds the S3 adapter at the composition root. +func newObjects(cfg *config.ObjectStore) (*objectstore.Client, error) { + return objectstore.New(cfg) +} + +// newMedia injects infrastructure through the media service's consumer interfaces. +func newMedia(database *postgres.Store, objects *objectstore.Client, chats service.Service) (*media.Service, error) { + return media.New(database, objects, chats) +} + +// newHTTPController connects the independently tested transport and attachment service. +func newHTTPController(log *zap.Logger, cfg *config.Controller, svc service.Service, registry *fielderr.Registry, limiter admission.Distributed, auth service.Authentication, readiness fiberctrl.Readiness, attachments *media.Service) (*fiberctrl.Controller, error) { + ctrl, err := fiberctrl.New(log, cfg, svc, registry, limiter, auth, readiness) + if err != nil { + return nil, err + } + return ctrl.WithAttachments(attachments), nil +} + +// newMediaReadiness includes private originals in the mandatory synchronous dependency path. +func newMediaReadiness(database *postgres.Store, cache *valkey.Client, objects *objectstore.Client) *dependencyReadiness { + return &dependencyReadiness{database: database, cache: cache, objects: objects} +} diff --git a/cmd/server/media_test.go b/cmd/server/media_test.go new file mode 100644 index 0000000..ab8e9f7 --- /dev/null +++ b/cmd/server/media_test.go @@ -0,0 +1,88 @@ +package main + +import ( + "os" + "testing" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/config" +) + +// TestMediaHTTPConfigurationFailure rejects invalid listener settings before serving files. +func TestMediaHTTPConfigurationFailure(t *testing.T) { + if _, err := newHTTPController(nil, &config.Controller{}, nil, nil, nil, nil, nil, nil); err == nil { + t.Fatal("invalid HTTP composition accepted") + } +} + +// TestReconcileConfigurationFailures checks operator commands fail closed before side effects. +func TestReconcileConfigurationFailures(t *testing.T) { + cases := []struct { + name string + key string + value string + }{ + {"database missing", "DATABASE_URL", ""}, + {"s3 missing", "S3_ENDPOINT", ""}, + {"database malformed", "DATABASE_URL", "://invalid"}, + {"s3 malformed", "S3_ENDPOINT", "http://invalid/path"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Setenv("DATABASE_URL", "postgres://localhost/test") + t.Setenv("VALKEY_SHARDS", "shard0=localhost:6379") + t.Setenv("S3_ENDPOINT", "localhost:9000") + t.Setenv("S3_BUCKET", "attachments") + t.Setenv("S3_ACCESS_KEY", "test") + t.Setenv("S3_SECRET_KEY", "synthetic") + t.Setenv(tc.key, tc.value) + if err := run([]string{"reconcile"}); err == nil { + t.Fatal("invalid reconciliation configuration accepted") + } + }) + } +} + +// TestReconcileRealInventory verifies the operator command can scan both real dependencies. +func TestReconcileRealInventory(t *testing.T) { + requirePersistence(t) + if os.Getenv("S3_ENDPOINT") == "" { + t.Skip("isolated S3 required") + } + if err := run([]string{"reconcile"}); err != nil { + t.Fatal(err) + } + if err := run([]string{"reconcile", "--apply"}); err != nil { + t.Fatal(err) + } +} + +// TestBootstrapConfigurationFailures rejects missing credentials and malformed endpoints. +func TestBootstrapConfigurationFailures(t *testing.T) { + t.Run("missing", func(t *testing.T) { + t.Setenv("S3_ENDPOINT", "") + if err := run([]string{"s3-bootstrap"}); err == nil { + t.Fatal("missing config accepted") + } + }) + t.Run("malformed", func(t *testing.T) { + t.Setenv("S3_ENDPOINT", "http://invalid/path") + t.Setenv("S3_BUCKET", "attachments") + t.Setenv("S3_ACCESS_KEY", "test") + t.Setenv("S3_SECRET_KEY", "synthetic") + if err := run([]string{"s3-bootstrap"}); err == nil { + t.Fatal("malformed endpoint accepted") + } + }) +} + +// TestBootstrapExistingBucket verifies repeatable operator execution against the isolated bucket. +func TestBootstrapExistingBucket(t *testing.T) { + if os.Getenv("S3_ENDPOINT") == "" || os.Getenv("S3_ADMIN_SECRET") == "" { + t.Skip("isolated S3 operator credentials required") + } + t.Setenv("S3_ACCESS_KEY", "operator") + t.Setenv("S3_SECRET_KEY", os.Getenv("S3_ADMIN_SECRET")) + if err := run([]string{"s3-bootstrap"}); err != nil { + t.Fatal(err) + } +} diff --git a/cmd/server/persistence.go b/cmd/server/persistence.go new file mode 100644 index 0000000..7b3ac68 --- /dev/null +++ b/cmd/server/persistence.go @@ -0,0 +1,98 @@ +package main + +import ( + "context" + "time" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/telemetry" + "github.com/jackc/pgx/v5" + "go.uber.org/fx" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/config" + fiberctrl "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/controller/http/fiber" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/service" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/service/authentication" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/store/postgres" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/valkey" +) + +type dependencyReadiness struct { + database fiberctrl.Readiness + cache fiberctrl.Readiness + objects fiberctrl.Readiness +} + +// Ping checks persistence and all limiter shards under a shared deadline. +func (d *dependencyReadiness) Ping(ctx context.Context) error { + if err := d.database.Ping(ctx); err != nil { + return err + } + if err := d.cache.Ping(ctx); err != nil { + return err + } + if d.objects != nil { + return d.objects.Ping(ctx) + } + return nil +} + +// newDatabase creates the pool and registers its migration/start and shutdown hooks. +func newDatabase(lifecycle fx.Lifecycle, cfg *config.Persistence) (*postgres.Store, error) { + return databaseWithTracer(lifecycle, cfg, nil) +} + +// observedDatabase binds the SDK tracer to the driver at composition time. +func observedDatabase(lifecycle fx.Lifecycle, cfg *config.Persistence, runtime *telemetry.Runtime) (*postgres.Store, error) { + return databaseWithTracer(lifecycle, cfg, runtime) +} + +// databaseWithTracer creates and owns the optionally instrumented SQL pool. +func databaseWithTracer(lifecycle fx.Lifecycle, cfg *config.Persistence, tracer pgx.QueryTracer) (*postgres.Store, error) { + database, err := postgres.New(postgres.Options{ + Tracer: tracer, + URL: cfg.DatabaseURL, + MaxConnections: int32(cfg.PoolSize), + Timeout: time.Duration(cfg.TimeoutMilliseconds) * time.Millisecond, + }) + if err != nil { + return nil, err + } + lifecycle.Append(fx.Hook{ + OnStart: database.CheckSchema, + OnStop: func(context.Context) error { + database.Close() + return nil + }, + }) + return database, nil +} + +// newValkey creates bounded clients and registers connectivity and cleanup hooks. +func newValkey(lifecycle fx.Lifecycle, cfg *config.Persistence) (*valkey.Client, error) { + client, err := valkey.New(valkey.Options{ + Nodes: cfg.Shards, + Coordinator: cfg.Coordinator, + PoolSize: cfg.PoolSize, + Timeout: time.Duration(cfg.TimeoutMilliseconds) * time.Millisecond, + }) + if err != nil { + return nil, err + } + lifecycle.Append(fx.Hook{ + OnStart: client.Ping, + OnStop: func(context.Context) error { + return client.Close() + }, + }) + return client, nil +} + +// newAuthentication binds credentials and KDF implementations to authentication policy. +func newAuthentication(accounts service.Accounts, passwords service.Passwords, cfg *config.Persistence) (service.Authentication, error) { + return authentication.New(accounts, passwords, time.Duration(cfg.SessionTTLSeconds)*time.Second) +} + +// newReadiness assembles infrastructure checks at the composition boundary. +func newReadiness(database *postgres.Store, cache *valkey.Client) *dependencyReadiness { + return &dependencyReadiness{database: database, cache: cache} +} diff --git a/cmd/server/persistence_test.go b/cmd/server/persistence_test.go new file mode 100644 index 0000000..1369b81 --- /dev/null +++ b/cmd/server/persistence_test.go @@ -0,0 +1,66 @@ +package main + +import ( + "testing" + "time" + + "go.uber.org/fx" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/config" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/store/postgres" +) + +// TestDatabaseProviderRejectsInvalidConfiguration verifies dependency errors propagate before startup. +func TestDatabaseProviderRejectsInvalidConfiguration(t *testing.T) { + var lifecycle fx.Lifecycle + app := fx.New(fx.NopLogger, fx.Populate(&lifecycle)) + if err := app.Err(); err != nil { + t.Fatal(err) + } + if _, err := newDatabase(lifecycle, &config.Persistence{}); err == nil { + t.Fatal("invalid database settings accepted") + } +} + +// TestValkeyProviderRejectsInvalidConfiguration verifies the cache constructor is a real boundary. +func TestValkeyProviderRejectsInvalidConfiguration(t *testing.T) { + var lifecycle fx.Lifecycle + app := fx.New(fx.NopLogger, fx.Populate(&lifecycle)) + if err := app.Err(); err != nil { + t.Fatal(err) + } + if _, err := newValkey(lifecycle, &config.Persistence{}); err == nil { + t.Fatal("invalid cache settings accepted") + } +} + +// TestReadinessStopsAtUnavailableDatabase checks that mandatory persistence failure is visible. +func TestReadinessStopsAtUnavailableDatabase(t *testing.T) { + database, err := postgres.New(postgres.Options{URL: "postgres://localhost/test", MaxConnections: 1, Timeout: time.Second}) + if err != nil { + t.Fatal(err) + } + database.Close() + readiness := newReadiness(database, nil) + if err := readiness.Ping(t.Context()); err == nil { + t.Fatal("closed database reported ready") + } +} + +// TestMigrationCommandRejectsMissingConfiguration fails before opening a database. +func TestMigrationCommandRejectsMissingConfiguration(t *testing.T) { + t.Setenv("DATABASE_URL", "") + t.Setenv("VALKEY_SHARDS", "shard0=localhost:6379") + if err := run([]string{"migrate"}); err == nil { + t.Fatal("missing database configuration accepted") + } +} + +// TestMigrationCommandRejectsMalformedDatabaseURL fails without logging credentials. +func TestMigrationCommandRejectsMalformedDatabaseURL(t *testing.T) { + t.Setenv("DATABASE_URL", "://invalid") + t.Setenv("VALKEY_SHARDS", "shard0=localhost:6379") + if err := run([]string{"migrate"}); err == nil { + t.Fatal("malformed database configuration accepted") + } +} diff --git a/cmd/server/reconcile.go b/cmd/server/reconcile.go new file mode 100644 index 0000000..2b14f4a --- /dev/null +++ b/cmd/server/reconcile.go @@ -0,0 +1,43 @@ +package main + +import ( + "context" + + "os" + "time" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/config" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/media" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/objectstore" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/pipeline" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/store/postgres" +) + +// reconcile runs a bounded dry-run scan unless the operator explicitly supplies apply. +func reconcile(apply bool) error { + cfg, err := config.NewPersistence() + if err != nil { + return err + } + s3, err := config.NewObjectStore() + if err != nil { + return err + } + database, err := postgres.New(postgres.Options{URL: cfg.DatabaseURL, MaxConnections: 1, Timeout: 5 * time.Second}) + if err != nil { + return err + } + defer database.Close() + objects, err := objectstore.New(s3) + if err != nil { + return err + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + if apply { + if _, err := pipeline.Cleanup(ctx, database, objects); err != nil { + return err + } + } + return media.RunReconciliation(ctx, database, objects, os.Stdout, apply) +} diff --git a/cmd/server/s3_bootstrap.go b/cmd/server/s3_bootstrap.go new file mode 100644 index 0000000..d88b022 --- /dev/null +++ b/cmd/server/s3_bootstrap.go @@ -0,0 +1,24 @@ +package main + +import ( + "context" + "time" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/config" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/objectstore" +) + +// bootstrapS3 creates the private bucket using separate operator credentials. +func bootstrapS3() error { + cfg, err := config.NewObjectStore() + if err != nil { + return err + } + objects, err := objectstore.New(cfg) + if err != nil { + return err + } + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + return objects.Bootstrap(ctx) +} diff --git a/cmd/server/telemetry.go b/cmd/server/telemetry.go new file mode 100644 index 0000000..51aed7e --- /dev/null +++ b/cmd/server/telemetry.go @@ -0,0 +1,93 @@ +package main + +import ( + "context" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/admission" + fiberctrl "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/controller/http/fiber" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/media" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/service" + "go.lab260.ru/fielderr" + "go.uber.org/fx" + "go.uber.org/zap" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/config" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/objectstore" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/store/postgres" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/telemetry" +) + +// telemetryFor constructs one process-owned SDK runtime with bounded lifecycle shutdown. +func telemetryFor(lc fx.Lifecycle, service string) (*telemetry.Runtime, error) { + cfg, err := config.NewTelemetry(service) + if err != nil { + return nil, err + } + runtime, err := telemetry.New(context.Background(), cfg) + if err != nil { + return nil, err + } + lc.Append(fx.Hook{OnStop: runtime.Shutdown}) + return runtime, nil +} + +// apiTelemetry assigns the API's resource identity. +func apiTelemetry(lc fx.Lifecycle) (*telemetry.Runtime, error) { + return telemetryFor(lc, "messenger-api") +} + +// workerTelemetry assigns the worker's resource identity. +func workerTelemetry(lc fx.Lifecycle) (*telemetry.Runtime, error) { + return telemetryFor(lc, "messenger-worker") +} + +// observedObjects adds S3 operation spans at the infrastructure composition boundary. +func observedObjects(cfg *config.ObjectStore, runtime *telemetry.Runtime) (*objectstore.Client, error) { + client, err := newObjects(cfg) + if err != nil { + return nil, err + } + return client.WithTracer(runtime), nil +} + +// observeAPI registers pool and backlog metrics outside the HTTP request path. +func observeAPI(runtime *telemetry.Runtime, database *postgres.Store) error { + return runtime.ObserveSaturation(database, false) +} + +// observeWorker registers worker occupancy and database observations. +func observeWorker(runtime *telemetry.Runtime, database *postgres.Store) error { + return runtime.ObserveSaturation(database, true) +} + +// observedHTTP binds the transport observer after constructing its business dependencies. +func observedHTTP(params observedHTTPParams) (*fiberctrl.Controller, error) { + ctrl, err := newHTTPController(params.Log, params.Config, params.Service, params.Errors, params.Limiter, params.Auth, params.Readiness, params.Attachments) + if err != nil { + return nil, err + } + return ctrl.WithObserver(params.Telemetry), nil +} + +// observedHTTPParams declares the complete HTTP composition dependencies. +type observedHTTPParams struct { + fx.In + // Log emits process diagnostics. + Log *zap.Logger + // Config provides listener policy. + Config *config.Controller + // Service owns messaging operations. + Service service.Service + // Errors maps domain failures. + Errors *fielderr.Registry + // Limiter bounds shared admission. + Limiter admission.Distributed + // Auth resolves bearer sessions. + Auth service.Authentication + // Readiness checks mandatory dependencies. + Readiness fiberctrl.Readiness + // Attachments owns file authorization and intake. + Attachments *media.Service + // Telemetry observes safe transport metadata. + Telemetry *telemetry.Runtime +} diff --git a/cmd/server/telemetry_test.go b/cmd/server/telemetry_test.go new file mode 100644 index 0000000..af31ec0 --- /dev/null +++ b/cmd/server/telemetry_test.go @@ -0,0 +1,29 @@ +package main + +import ( + "testing" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/config" +) + +// TestTelemetryInvalidConfiguration rejects invalid sampling before lifecycle registration. +func TestTelemetryInvalidConfiguration(t *testing.T) { + t.Setenv("OTEL_SAMPLE_RATIO", "invalid") + if _, err := apiTelemetry(nil); err == nil { + t.Fatal("invalid sampling accepted") + } +} + +// TestObservedObjectsInvalidEndpoint propagates S3 construction errors before instrumentation. +func TestObservedObjectsInvalidEndpoint(t *testing.T) { + if _, err := observedObjects(&config.ObjectStore{Endpoint: "http://invalid/path"}, nil); err == nil { + t.Fatal("invalid object endpoint accepted") + } +} + +// TestObservedHTTPInvalidConfiguration rejects invalid listener settings before adding an observer. +func TestObservedHTTPInvalidConfiguration(t *testing.T) { + if _, err := observedHTTP(observedHTTPParams{Config: &config.Controller{}}); err == nil { + t.Fatal("invalid listener accepted") + } +} diff --git a/cmd/server/worker.go b/cmd/server/worker.go new file mode 100644 index 0000000..df884fb --- /dev/null +++ b/cmd/server/worker.go @@ -0,0 +1,89 @@ +package main + +import ( + "context" + "fmt" + "strconv" + "time" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/telemetry" + "github.com/google/uuid" + "go.uber.org/fx" + "go.uber.org/zap" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/config" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/imageprocessor" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/objectstore" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/pipeline" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/store/postgres" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/streamqueue" +) + +// workerApp composes an independently deployable worker with owned Fx lifecycle. +func workerApp() fx.Option { + return fx.Options(fx.StopTimeout(55*time.Second), fx.Provide(config.New, config.NewPersistence, config.NewObjectStore, config.NewWorker, loggerConfig, newLogger, workerTelemetry, observedDatabase, observedObjects, newQueue, imageprocessor.New, newWorker), fx.WithLogger(newFxLogger), fx.Invoke(startWorker, observeWorker)) +} + +// newQueue closes the queue after the worker has relinquished its processing loop. +func newQueue(lc fx.Lifecycle, cfg *config.Worker) (*streamqueue.Queue, error) { + queue, err := streamqueue.New(cfg.BrokerAddress, cfg.Consumer) + if err != nil { + return nil, err + } + lc.Append(fx.Hook{OnStop: func(context.Context) error { + return queue.Close() + }}) + return queue, nil +} + +// newWorker binds concrete adapters only at the composition root. +func newWorker(database *postgres.Store, queue *streamqueue.Queue, objects *objectstore.Client, processor *imageprocessor.Processor, log *zap.Logger, runtime *telemetry.Runtime) (*pipeline.Worker, error) { + worker, err := pipeline.New(database, queue, objects, processor, log) + if err != nil { + return nil, err + } + return worker.WithObserver(runtime), nil +} + +// startWorker owns the background context and joins its goroutine during shutdown. +func startWorker(lc fx.Lifecycle, worker *pipeline.Worker) { + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + lc.Append(fx.Hook{OnStart: func(context.Context) error { + go func() { + defer close(done) + worker.Run(ctx) + }() + return nil + }, OnStop: func(stop context.Context) error { + cancel() + select { + case <-done: + return nil + case <-stop.Done(): + return stop.Err() + } + }}) +} + +// replayJob explicitly retries a failed version or advances a pipeline version. +func replayJob(rawID, rawVersion string) error { + id, err := uuid.Parse(rawID) + if err != nil { + return fmt.Errorf("invalid attachment ID") + } + version, err := strconv.Atoi(rawVersion) + if err != nil || version < 1 { + return fmt.Errorf("positive pipeline version required") + } + cfg, err := config.NewPersistence() + if err != nil { + return err + } + database, err := postgres.New(postgres.Options{URL: cfg.DatabaseURL, MaxConnections: 1, Timeout: 5 * time.Second}) + if err != nil { + return err + } + defer database.Close() + return database.Replay(context.Background(), id, version) +} diff --git a/cmd/server/worker_test.go b/cmd/server/worker_test.go new file mode 100644 index 0000000..3c3dcd1 --- /dev/null +++ b/cmd/server/worker_test.go @@ -0,0 +1,80 @@ +package main + +import ( + "os" + "testing" + "time" + + "github.com/google/uuid" + "go.uber.org/fx" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/config" +) + +// TestWorkerLifecycle starts and joins the real worker without exposing an HTTP listener. +func TestWorkerLifecycle(t *testing.T) { + requirePersistence(t) + if os.Getenv("BROKER_ADDRESS") == "" || os.Getenv("S3_ENDPOINT") == "" { + t.Skip("isolated broker and S3 required") + } + app := fx.New(workerApp()) + if err := app.Start(t.Context()); err != nil { + t.Fatal(err) + } + time.Sleep(30 * time.Millisecond) + if err := app.Stop(t.Context()); err != nil { + t.Fatal(err) + } +} + +// TestQueueProviderRejectsMissingAddress propagates configuration failure before startup. +func TestQueueProviderRejectsMissingAddress(t *testing.T) { + var lc fx.Lifecycle + app := fx.New(fx.NopLogger, fx.Populate(&lc)) + if err := app.Err(); err != nil { + t.Fatal(err) + } + if _, err := newQueue(lc, &config.Worker{}); err == nil { + t.Fatal("missing broker accepted") + } +} + +// TestReplayCommandRejectsInvalidInput prevents unintended operator transitions. +func TestReplayCommandRejectsInvalidInput(t *testing.T) { + for _, tc := range []struct { + name string + id string + version string + }{{"invalid ID", "invalid", "1"}, {"invalid version", uuid.NewString(), "x"}, {"negative version", uuid.NewString(), "-1"}} { + t.Run(tc.name, func(t *testing.T) { + if err := run([]string{"replay", tc.id, tc.version}); err == nil { + t.Fatal("invalid operator input accepted") + } + }) + } +} + +// TestReplayCommandConfiguration checks failure before database access. +func TestReplayCommandConfiguration(t *testing.T) { + t.Run("missing", func(t *testing.T) { + t.Setenv("DATABASE_URL", "") + if err := replayJob(uuid.NewString(), "1"); err == nil { + t.Fatal("missing database accepted") + } + }) + t.Run("malformed", func(t *testing.T) { + t.Setenv("DATABASE_URL", "://invalid") + t.Setenv("VALKEY_SHARDS", "shard0=localhost:6379") + if err := replayJob(uuid.NewString(), "1"); err == nil { + t.Fatal("malformed database accepted") + } + }) +} + +// TestReplayUnknownAttachment verifies safe operator failure against the actual schema. +func TestReplayUnknownAttachment(t *testing.T) { + requirePersistence(t) + if err := replayJob(uuid.NewString(), "1"); err == nil { + t.Fatal("unknown attachment replayed") + } +} diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md new file mode 100644 index 0000000..2fb428e --- /dev/null +++ b/docs/CONFIGURATION.md @@ -0,0 +1,27 @@ +# Фактическая конфигурация + +`.env.example` задания содержит логические предложения. Программа не загружает его автоматически и не читает переменные с суффиксом `_FILE`. Helm передаёт значения из Kubernetes Secrets в env; локальный test-local создаёт отдельные синтетические credentials и удаляет их после запуска. + +| Настройка задания | Реальная настройка | +| --- | --- | +| DATABASE_URL_FILE | DATABASE_URL из messenger-secrets | +| SESSION_STORE | PostgreSQL, сессии не находятся только в Valkey | +| CACHE_URL_FILE | VALKEY_SHARDS: именованные host:port, consistent hashing | +| API_REPLICAS | Helm replicaCount / autoscaling minReplicas | +| APP_HOST/PORT | APP_HOST и APP_PORT | +| SESSION_TTL_SECONDS | Одноимённая переменная, по умолчанию 3600 | +| IDEMPOTENCY_TTL_SECONDS | Фиксированные контрактные 24 часа | +| S3_ENDPOINT | host:port без схемы; S3_TLS=false для приватного учебного HTTP | +| S3_ACCESS_KEY_ID_FILE / S3_SECRET_ACCESS_KEY_FILE | S3_ACCESS_KEY / S3_SECRET_KEY из messenger-s3 | +| S3_BUCKET | attachments; bootstrap отдельной административной identity | +| MAX_UPLOAD_BYTES / MAX_IMAGE_PIXELS | Именованные константы 10 MiB / 40 миллионов | +| BROKER_URL_FILE | BROKER_ADDRESS, отдельный Valkey Streams | +| WORKER_CONCURRENCY | Один обработчик в процессе; Helm worker.replicas масштабирует процессы | +| JOB_MAX_ATTEMPTS / JOB_TIMEOUT_SECONDS | Пять попыток / 45 s операции; lease 120 s | +| STAGING_RETENTION_SECONDS | Семь дней после terminal transition; orphan grace час | +| OTEL_SERVICE_NAME / RESOURCE_ATTRIBUTES | Явные SDK resource attributes messenger-api/worker, версия APP_VERSION, уникальный instance, environment=lab | +| OTEL_EXPORTER_OTLP_ENDPOINT | HTTP(S) base URL Collector; пусто отключает export | +| OTEL_EXPORTER_OTLP_PROTOCOL | Реализован OTLP HTTP/protobuf | +| Sampling | OTEL_SAMPLE_RATIO в [0,1], по умолчанию 1 | + +Значения политик, не читаемые из env, нельзя менять вымышленными переменными. Фактические limits/resources/image digests находятся в chart values и Go-конфигурации. CURSOR_SIGNING_KEY берётся из Secret и сохраняется между обновлениями. diff --git a/docs/CONTRACT-CHECK.md b/docs/CONTRACT-CHECK.md new file mode 100644 index 0000000..856ef61 --- /dev/null +++ b/docs/CONTRACT-CHECK.md @@ -0,0 +1,24 @@ +# Проверка обязательных контрактов + +Источники: неизменённые README, COURSE, contracts/README.md, OpenAPI и observability.md. Проверка не сводится к публичному smoke. Унаследованные требования проверяются на текущем коде; результаты предыдущих репозиториев не выдаются за результаты lab6. + +| Контракт | Реализация и проверка | +| --- | --- | +| Строгий JSON, UUID, Unicode, pagination, общая ошибка | strictjson, service/production, controller/http/fiber; Go regression tests и публичный smoke | +| Basic только при login, Bearer, expiry/revoke, игнорирование X-User-Id | service/authentication, postgres/sessions; session tests, smoke и persistence | +| X-Request-Id и X-Instance-Id, включая 413 | controller/http/fiber/errors.go; tests ранних transport errors | +| Две API, видимость записи, атомарная идемпотентность 24h | PostgreSQL unique key + транзакция; TestMessageRetryRace/Conflict/Expiry/Rollback, failover | +| 10 MiB / 40M pixels, MIME и filename, приватные bytes | media/validation, ограниченное чтение, objectstore; validation/ACL/private bucket tests | +| 202 только после S3 staging + durable SQL intent | QueueAttachment; fault matrix и реальный restart при остановленном worker | +| queued/processing можно прикреплять; failed запрещён; content до ready — 409 | media и PostgreSQL message transaction; attachments/pipeline tests, smoke | +| Неизменный original, JPEG fit256, alpha/EXIF, crop128, WebP fit1280 | imageprocessor tests + checksum/размеры в smoke и durable | +| At-least-once, bounded retry, fencing, replay, retention | pipeline, streamqueue, jobs/retention tests с настоящими SQL/S3 | +| OTLP API/worker, три backend, provisioning и PVC | charts/messenger; сохранённые live metric names, trace, logs, dashboard API | +| Parent chain и retry | trace-probe.sh, trace-retry.sh, OTLP tests; JSON traces в evidence/current | +| Histogram percentiles, bounded labels, queue/pool/CPU/RAM | telemetry + dashboard; histogram границы проверены по живому Prometheus | +| Multiwindow SLO, DB/worker/backend incidents | observability-incidents.sh, ready-slo.sh; raw evidence и разбор REPORT | +| Graceful lifecycle и ≥90% каждого Go-файла | coverage.sh: race tests + инструментированный настоящий процесс | + +## Формальная граница + +В infrastructure.json требуется Compose/Swarm. Реализован Helm/Kubernetes по принятому направлению проекта; эквивалентные роли и bootstrap доступны, но разрешение преподавателя на замену не подтверждено. Это открытое расхождение задания, а не доказанный пункт соответствия. Обязательные контракты и публичные tests не изменялись. diff --git a/docs/PLAN.md b/docs/PLAN.md new file mode 100644 index 0000000..11eaef3 --- /dev/null +++ b/docs/PLAN.md @@ -0,0 +1,10 @@ +# План lab6 + +1. L6-01/02 — выполнено: перенос lab5, owned SDK providers, durable W3C propagation, реальная цепочка upload и retry. +2. L6-03 — выполнено: histogram/counters/saturation, три backend, provisioning14panels; проверены реальные dev имена/источники. +3. L6-04 — выполнено в заявленных границах: конечные очереди/timeouts,2000spans failure test, реальная остановка Tempo с работающим business smoke. Длительный soak не проводился. +4. L6-05/06 — выполнено: raw инциденты, API lagged firing/resolved, исправленный ready alert отдельно проверен STOP/CONT, promtool regression; REPORT сохраняет первоначальные ошибки сценария. +5. Проверки — race, per-file≥90%, lint, Helm/static, HTTPS dev smoke прошли. Общий release lab2 revision45. +6. Описание PR подготовлено в корне рабочего каталога. Публикация фиксируется отдельным Git commit; remote SHA проверяется после push. + +Открытый внешний вопрос: принятие Kubernetes вместо Compose/Swarm. Remote CI и оценка преподавателя не подтверждены. diff --git a/docs/SPEC.md b/docs/SPEC.md new file mode 100644 index 0000000..073eff7 --- /dev/null +++ b/docs/SPEC.md @@ -0,0 +1,12 @@ +# Спецификация lab6 + +Источники: README, COURSE, contracts/README.md и contracts/observability.md. Бизнес-контракт lab5 сохраняется. Цель — уровни 3/4 и трек 5.2. + +- L6-01: API/worker отправляют OTLP traces, metrics, logs в Collector; Prometheus/Loki/Tempo/Grafana разворачиваются тем же Helm release с PVC и provisioning. +- L6-02: W3C context проходит HTTP → SQL outbox → producer → broker → consumer → S3/DB. Логи содержат trace/span/request IDs, не содержат credentials, body и filename. +- L6-03: bounded labels, histogram HTTP/job/e2e в секундах, queue depth/oldest, DB used/waiting, worker busy/capacity, CPU/RAM. Dashboard с реальными именами и p50/p95/p99 из bucket aggregation. +- L6-04: bounded SDK/export queues и Collector memory limiter/batch. Отказ telemetry backend не меняет бизнес-готовность API и не приводит к неограниченному накоплению. +- L6-05: проверяемые SLO API и ready; multiwindow burn alerts и queue lag, runbook. Подтверждены firing/resolved, краткий всплеск и длительная деградация, измерено время реакции. +- L6-06: реальные инциденты остановки worker и отказа DB, связь логов/трасс; smoke с telemetry, race, coverage ≥90% каждого Go-файла, raw evidence и ограничения. + +Границы: namespace highload-applications, release lab2, context dev, hl.agapov.space. Чужой monitoring не изменяется. Kubernetes substitution требует подтверждения преподавателя. Внешняя отправка уведомлений не настраивается. diff --git a/evidence/current/alert-rule-tests.txt b/evidence/current/alert-rule-tests.txt new file mode 100644 index 0000000..1e5777a --- /dev/null +++ b/evidence/current/alert-rule-tests.txt @@ -0,0 +1,2 @@ + SUCCESS + diff --git a/evidence/current/collector-metric-names.json b/evidence/current/collector-metric-names.json new file mode 100644 index 0000000..cb7efa0 --- /dev/null +++ b/evidence/current/collector-metric-names.json @@ -0,0 +1 @@ +{"status":"success","data":{"resultType":"vector","result":[{"metric":{"__name__":"up"},"value":[1793283940.497,"1"]},{"metric":{"__name__":"scrape_duration_seconds"},"value":[1793283940.497,"1"]},{"metric":{"__name__":"scrape_samples_scraped"},"value":[1793283940.497,"1"]},{"metric":{"__name__":"scrape_samples_post_metric_relabeling"},"value":[1793283940.497,"1"]},{"metric":{"__name__":"scrape_series_added"},"value":[1793283940.497,"1"]}]}} \ No newline at end of file diff --git a/evidence/current/correlated-logs.json b/evidence/current/correlated-logs.json new file mode 100644 index 0000000..a85ee72 --- /dev/null +++ b/evidence/current/correlated-logs.json @@ -0,0 +1 @@ +{"status":"success","data":{"resultType":"streams","result":[{"stream":{"deployment_environment_name":"lab","detected_level":"info","event":"s3.put","flags":"1","observed_timestamp":"1793285484543478948","outcome":"ok","scope_name":"messenger","service_instance_id":"4c56cc55-c3a5-49a2-a434-d452ae6d30cb","service_name":"messenger-worker","service_version":"lab6","severity_number":"9","span_id":"648957516300f9ef","trace_id":"4486ae744b284857a148568406c8ff4c"},"values":[["1793285484543469889","s3.put"]]},{"stream":{"deployment_environment_name":"lab","detected_level":"info","event":"s3.put","flags":"1","observed_timestamp":"1793285484539329392","outcome":"ok","scope_name":"messenger","service_instance_id":"4c56cc55-c3a5-49a2-a434-d452ae6d30cb","service_name":"messenger-worker","service_version":"lab6","severity_number":"9","span_id":"f6582245403d72c8","trace_id":"4486ae744b284857a148568406c8ff4c"},"values":[["1793285484539321281","s3.put"]]},{"stream":{"deployment_environment_name":"lab","detected_level":"info","event":"s3.put","flags":"1","observed_timestamp":"1793285484535447360","outcome":"ok","scope_name":"messenger","service_instance_id":"4c56cc55-c3a5-49a2-a434-d452ae6d30cb","service_name":"messenger-worker","service_version":"lab6","severity_number":"9","span_id":"e1cc8b15c569c119","trace_id":"4486ae744b284857a148568406c8ff4c"},"values":[["1793285484535434656","s3.put"]]},{"stream":{"deployment_environment_name":"lab","detected_level":"info","event":"s3.put","flags":"1","observed_timestamp":"1793285484529061503","outcome":"ok","scope_name":"messenger","service_instance_id":"4c56cc55-c3a5-49a2-a434-d452ae6d30cb","service_name":"messenger-worker","service_version":"lab6","severity_number":"9","span_id":"7a01332473e9c681","trace_id":"4486ae744b284857a148568406c8ff4c"},"values":[["1793285484529050768","s3.put"]]},{"stream":{"deployment_environment_name":"lab","detected_level":"info","event":"s3.put","flags":"1","observed_timestamp":"1793285483815873066","outcome":"ok","scope_name":"messenger","service_instance_id":"e106424b-537d-45f3-bf84-f2dcb383e1c2","service_name":"messenger-api","service_version":"lab6","severity_number":"9","span_id":"2244e09d6e0fe868","trace_id":"4486ae744b284857a148568406c8ff4c"},"values":[["1793285483815867220","s3.put"]]},{"stream":{"deployment_environment_name":"lab","detected_level":"info","event":"s3.get","flags":"1","observed_timestamp":"1793285484469221465","outcome":"ok","scope_name":"messenger","service_instance_id":"4c56cc55-c3a5-49a2-a434-d452ae6d30cb","service_name":"messenger-worker","service_version":"lab6","severity_number":"9","span_id":"9e7ba0ac47c73e1b","trace_id":"4486ae744b284857a148568406c8ff4c"},"values":[["1793285484469210296","s3.get"]]},{"stream":{"deployment_environment_name":"lab","detected_level":"info","event":"postgres.query","flags":"1","observed_timestamp":"1793285484552054462","outcome":"ok","scope_name":"messenger","service_instance_id":"4c56cc55-c3a5-49a2-a434-d452ae6d30cb","service_name":"messenger-worker","service_version":"lab6","severity_number":"9","span_id":"be5bab75b83e555b","trace_id":"4486ae744b284857a148568406c8ff4c"},"values":[["1793285484552039626","postgres.query"]]},{"stream":{"deployment_environment_name":"lab","detected_level":"info","event":"postgres.query","flags":"1","observed_timestamp":"1793285484549932639","outcome":"ok","scope_name":"messenger","service_instance_id":"4c56cc55-c3a5-49a2-a434-d452ae6d30cb","service_name":"messenger-worker","service_version":"lab6","severity_number":"9","span_id":"5e4df9c472e6729c","trace_id":"4486ae744b284857a148568406c8ff4c"},"values":[["1793285484549921394","postgres.query"]]},{"stream":{"deployment_environment_name":"lab","detected_level":"info","event":"postgres.query","flags":"1","observed_timestamp":"1793285484545889441","outcome":"ok","scope_name":"messenger","service_instance_id":"4c56cc55-c3a5-49a2-a434-d452ae6d30cb","service_name":"messenger-worker","service_version":"lab6","severity_number":"9","span_id":"a07c164e94ace2fd","trace_id":"4486ae744b284857a148568406c8ff4c"},"values":[["1793285484545881441","postgres.query"]]},{"stream":{"deployment_environment_name":"lab","detected_level":"info","event":"postgres.query","flags":"1","observed_timestamp":"1793285484544993745","outcome":"ok","scope_name":"messenger","service_instance_id":"4c56cc55-c3a5-49a2-a434-d452ae6d30cb","service_name":"messenger-worker","service_version":"lab6","severity_number":"9","span_id":"fcdbf40b504840d8","trace_id":"4486ae744b284857a148568406c8ff4c"},"values":[["1793285484544988762","postgres.query"]]},{"stream":{"deployment_environment_name":"lab","detected_level":"info","event":"postgres.query","flags":"1","observed_timestamp":"1793285484544091934","outcome":"ok","scope_name":"messenger","service_instance_id":"4c56cc55-c3a5-49a2-a434-d452ae6d30cb","service_name":"messenger-worker","service_version":"lab6","severity_number":"9","span_id":"c2d4e0558f6b6a12","trace_id":"4486ae744b284857a148568406c8ff4c"},"values":[["1793285484544087117","postgres.query"]]},{"stream":{"deployment_environment_name":"lab","detected_level":"info","event":"postgres.query","flags":"1","observed_timestamp":"1793285484452698109","outcome":"ok","scope_name":"messenger","service_instance_id":"4c56cc55-c3a5-49a2-a434-d452ae6d30cb","service_name":"messenger-worker","service_version":"lab6","severity_number":"9","span_id":"3a12a48421d3c820","trace_id":"4486ae744b284857a148568406c8ff4c"},"values":[["1793285484452677291","postgres.query"]]},{"stream":{"deployment_environment_name":"lab","detected_level":"info","event":"postgres.query","flags":"1","observed_timestamp":"1793285484450319733","outcome":"ok","scope_name":"messenger","service_instance_id":"4c56cc55-c3a5-49a2-a434-d452ae6d30cb","service_name":"messenger-worker","service_version":"lab6","severity_number":"9","span_id":"d3b6f7ff702d749b","trace_id":"4486ae744b284857a148568406c8ff4c"},"values":[["1793285484450316092","postgres.query"]]},{"stream":{"deployment_environment_name":"lab","detected_level":"info","event":"postgres.query","flags":"1","observed_timestamp":"1793285484449712220","outcome":"ok","scope_name":"messenger","service_instance_id":"4c56cc55-c3a5-49a2-a434-d452ae6d30cb","service_name":"messenger-worker","service_version":"lab6","severity_number":"9","span_id":"d28280dd1ed026fa","trace_id":"4486ae744b284857a148568406c8ff4c"},"values":[["1793285484449707899","postgres.query"]]},{"stream":{"deployment_environment_name":"lab","detected_level":"info","event":"postgres.query","flags":"1","observed_timestamp":"1793285484448496776","outcome":"ok","scope_name":"messenger","service_instance_id":"4c56cc55-c3a5-49a2-a434-d452ae6d30cb","service_name":"messenger-worker","service_version":"lab6","severity_number":"9","span_id":"53f41e805b7ebb23","trace_id":"4486ae744b284857a148568406c8ff4c"},"values":[["1793285484448492841","postgres.query"]]},{"stream":{"deployment_environment_name":"lab","detected_level":"info","event":"postgres.query","flags":"1","observed_timestamp":"1793285484447402754","outcome":"ok","scope_name":"messenger","service_instance_id":"4c56cc55-c3a5-49a2-a434-d452ae6d30cb","service_name":"messenger-worker","service_version":"lab6","severity_number":"9","span_id":"adac5cdcbb78e916","trace_id":"4486ae744b284857a148568406c8ff4c"},"values":[["1793285484447400225","postgres.query"]]},{"stream":{"deployment_environment_name":"lab","detected_level":"info","event":"postgres.query","flags":"1","observed_timestamp":"1793285483910858759","outcome":"ok","scope_name":"messenger","service_instance_id":"e106424b-537d-45f3-bf84-f2dcb383e1c2","service_name":"messenger-api","service_version":"lab6","severity_number":"9","span_id":"a55f98939a7b8c2e","trace_id":"4486ae744b284857a148568406c8ff4c"},"values":[["1793285483910846452","postgres.query"]]},{"stream":{"deployment_environment_name":"lab","detected_level":"info","event":"postgres.query","flags":"1","observed_timestamp":"1793285483886833878","outcome":"ok","scope_name":"messenger","service_instance_id":"e106424b-537d-45f3-bf84-f2dcb383e1c2","service_name":"messenger-api","service_version":"lab6","severity_number":"9","span_id":"e943906f44a41106","trace_id":"4486ae744b284857a148568406c8ff4c"},"values":[["1793285483886829727","postgres.query"]]},{"stream":{"deployment_environment_name":"lab","detected_level":"info","event":"postgres.query","flags":"1","observed_timestamp":"1793285483860467830","outcome":"ok","scope_name":"messenger","service_instance_id":"e106424b-537d-45f3-bf84-f2dcb383e1c2","service_name":"messenger-api","service_version":"lab6","severity_number":"9","span_id":"df94cf8b2ff6ae6c","trace_id":"4486ae744b284857a148568406c8ff4c"},"values":[["1793285483860462424","postgres.query"]]},{"stream":{"deployment_environment_name":"lab","detected_level":"info","event":"postgres.query","flags":"1","observed_timestamp":"1793285483830376195","outcome":"ok","scope_name":"messenger","service_instance_id":"e106424b-537d-45f3-bf84-f2dcb383e1c2","service_name":"messenger-api","service_version":"lab6","severity_number":"9","span_id":"7e7e583802b6a48c","trace_id":"4486ae744b284857a148568406c8ff4c"},"values":[["1793285483830367584","postgres.query"]]},{"stream":{"deployment_environment_name":"lab","detected_level":"info","event":"postgres.query","flags":"1","observed_timestamp":"1793285483726443887","outcome":"ok","scope_name":"messenger","service_instance_id":"e106424b-537d-45f3-bf84-f2dcb383e1c2","service_name":"messenger-api","service_version":"lab6","severity_number":"9","span_id":"ee0a402e2db1612d","trace_id":"4486ae744b284857a148568406c8ff4c"},"values":[["1793285483726437533","postgres.query"]]},{"stream":{"deployment_environment_name":"lab","detected_level":"info","event":"postgres.query","flags":"1","observed_timestamp":"1793285483658682105","outcome":"ok","scope_name":"messenger","service_instance_id":"e106424b-537d-45f3-bf84-f2dcb383e1c2","service_name":"messenger-api","service_version":"lab6","severity_number":"9","span_id":"134364cd8e150e5e","trace_id":"4486ae744b284857a148568406c8ff4c"},"values":[["1793285483658676657","postgres.query"]]},{"stream":{"deployment_environment_name":"lab","detected_level":"info","event":"postgres.query","flags":"1","observed_timestamp":"1793285483599994505","outcome":"ok","scope_name":"messenger","service_instance_id":"e106424b-537d-45f3-bf84-f2dcb383e1c2","service_name":"messenger-api","service_version":"lab6","severity_number":"9","span_id":"344fc31e74c67908","trace_id":"4486ae744b284857a148568406c8ff4c"},"values":[["1793285483599989043","postgres.query"]]},{"stream":{"deployment_environment_name":"lab","detected_level":"info","event":"postgres.query","flags":"1","observed_timestamp":"1793285483533390750","outcome":"ok","scope_name":"messenger","service_instance_id":"e106424b-537d-45f3-bf84-f2dcb383e1c2","service_name":"messenger-api","service_version":"lab6","severity_number":"9","span_id":"be41f2ea10a3a0c1","trace_id":"4486ae744b284857a148568406c8ff4c"},"values":[["1793285483533384976","postgres.query"]]},{"stream":{"deployment_environment_name":"lab","detected_level":"info","event":"postgres.acquire","flags":"1","observed_timestamp":"1793285484543733962","outcome":"ok","scope_name":"messenger","service_instance_id":"4c56cc55-c3a5-49a2-a434-d452ae6d30cb","service_name":"messenger-worker","service_version":"lab6","severity_number":"9","span_id":"2b31dc923f6946c3","trace_id":"4486ae744b284857a148568406c8ff4c"},"values":[["1793285484543732318","postgres.acquire"]]},{"stream":{"deployment_environment_name":"lab","detected_level":"info","event":"postgres.acquire","flags":"1","observed_timestamp":"1793285484447041031","outcome":"ok","scope_name":"messenger","service_instance_id":"4c56cc55-c3a5-49a2-a434-d452ae6d30cb","service_name":"messenger-worker","service_version":"lab6","severity_number":"9","span_id":"5b3337cbbbd0f07d","trace_id":"4486ae744b284857a148568406c8ff4c"},"values":[["1793285484447035677","postgres.acquire"]]},{"stream":{"deployment_environment_name":"lab","detected_level":"info","event":"postgres.acquire","flags":"1","observed_timestamp":"1793285483815927500","outcome":"ok","scope_name":"messenger","service_instance_id":"e106424b-537d-45f3-bf84-f2dcb383e1c2","service_name":"messenger-api","service_version":"lab6","severity_number":"9","span_id":"cecb68424b6c11fa","trace_id":"4486ae744b284857a148568406c8ff4c"},"values":[["1793285483815926510","postgres.acquire"]]},{"stream":{"deployment_environment_name":"lab","detected_level":"info","event":"postgres.acquire","flags":"1","observed_timestamp":"1793285483668593755","outcome":"ok","scope_name":"messenger","service_instance_id":"e106424b-537d-45f3-bf84-f2dcb383e1c2","service_name":"messenger-api","service_version":"lab6","severity_number":"9","span_id":"ce94cb59a135b68a","trace_id":"4486ae744b284857a148568406c8ff4c"},"values":[["1793285483668591988","postgres.acquire"]]},{"stream":{"deployment_environment_name":"lab","detected_level":"info","event":"postgres.acquire","flags":"1","observed_timestamp":"1793285483600062891","outcome":"ok","scope_name":"messenger","service_instance_id":"e106424b-537d-45f3-bf84-f2dcb383e1c2","service_name":"messenger-api","service_version":"lab6","severity_number":"9","span_id":"4820ddcac3a7a4b8","trace_id":"4486ae744b284857a148568406c8ff4c"},"values":[["1793285483600061602","postgres.acquire"]]},{"stream":{"deployment_environment_name":"lab","detected_level":"info","event":"postgres.acquire","flags":"1","observed_timestamp":"1793285483539951557","outcome":"ok","scope_name":"messenger","service_instance_id":"e106424b-537d-45f3-bf84-f2dcb383e1c2","service_name":"messenger-api","service_version":"lab6","severity_number":"9","span_id":"c126b481c82ee033","trace_id":"4486ae744b284857a148568406c8ff4c"},"values":[["1793285483539948530","postgres.acquire"]]},{"stream":{"deployment_environment_name":"lab","detected_level":"info","event":"postgres.acquire","flags":"1","observed_timestamp":"1793285483470970521","outcome":"ok","scope_name":"messenger","service_instance_id":"e106424b-537d-45f3-bf84-f2dcb383e1c2","service_name":"messenger-api","service_version":"lab6","severity_number":"9","span_id":"d172e304aa2fce81","trace_id":"4486ae744b284857a148568406c8ff4c"},"values":[["1793285483470966816","postgres.acquire"]]},{"stream":{"deployment_environment_name":"lab","detected_level":"info","event":"http.request","flags":"1","observed_timestamp":"1793285483929997464","outcome":"ok","scope_name":"messenger","service_instance_id":"e106424b-537d-45f3-bf84-f2dcb383e1c2","service_name":"messenger-api","service_version":"lab6","severity_number":"9","span_id":"7ace86813b22de3e","trace_id":"4486ae744b284857a148568406c8ff4c"},"values":[["1793285483929996270","http.request"]]},{"stream":{"deployment_environment_name":"lab","detected_level":"info","event":"http.request","flags":"1","observed_timestamp":"1793285483929984930","outcome":"2xx","request_id":"6d30cdaf-6fb8-40d0-ad06-fc74d7dc656a","scope_name":"messenger","service_instance_id":"e106424b-537d-45f3-bf84-f2dcb383e1c2","service_name":"messenger-api","service_version":"lab6","severity_number":"9","span_id":"7ace86813b22de3e","trace_id":"4486ae744b284857a148568406c8ff4c"},"values":[["1793285483929981139","http.request"]]},{"stream":{"deployment_environment_name":"lab","detected_level":"info","event":"attachment.publish","flags":"1","observed_timestamp":"1793285484439285677","outcome":"ok","scope_name":"messenger","service_instance_id":"4c56cc55-c3a5-49a2-a434-d452ae6d30cb","service_name":"messenger-worker","service_version":"lab6","severity_number":"9","span_id":"0a8e6d8ae09e1935","trace_id":"4486ae744b284857a148568406c8ff4c"},"values":[["1793285484439273267","attachment.publish"]]},{"stream":{"deployment_environment_name":"lab","detected_level":"info","event":"attachment.process","flags":"1","observed_timestamp":"1793285484560022445","outcome":"ok","scope_name":"messenger","service_instance_id":"4c56cc55-c3a5-49a2-a434-d452ae6d30cb","service_name":"messenger-worker","service_version":"lab6","severity_number":"9","span_id":"52c48a9b63b0034d","trace_id":"4486ae744b284857a148568406c8ff4c"},"values":[["1793285484560009017","attachment.process"]]}],"stats":{"summary":{"bytesProcessedPerSecond":5234934,"linesProcessedPerSecond":50600,"totalBytesProcessed":373370,"totalLinesProcessed":3609,"execTime":0.071323,"queueTime":0.000605,"subqueries":0,"totalEntriesReturned":35,"splits":2,"shards":2,"totalPostFilterLines":35,"totalStructuredMetadataBytesProcessed":323016},"querier":{"store":{"totalChunksRef":0,"totalChunksDownloaded":0,"chunksDownloadTime":0,"queryReferencedStructuredMetadata":false,"chunk":{"headChunkBytes":0,"headChunkLines":0,"decompressedBytes":0,"decompressedLines":0,"compressedBytes":0,"totalDuplicates":0,"postFilterLines":0,"headChunkStructuredMetadataBytes":0,"decompressedStructuredMetadataBytes":0},"chunkRefsFetchTime":0,"congestionControlLatency":0,"pipelineWrapperFilteredLines":0}},"ingester":{"totalReached":2,"totalChunksMatched":2,"totalBatches":3,"totalLinesSent":35,"store":{"totalChunksRef":0,"totalChunksDownloaded":0,"chunksDownloadTime":0,"queryReferencedStructuredMetadata":true,"chunk":{"headChunkBytes":373370,"headChunkLines":3609,"decompressedBytes":0,"decompressedLines":0,"compressedBytes":0,"totalDuplicates":0,"postFilterLines":35,"headChunkStructuredMetadataBytes":323016,"decompressedStructuredMetadataBytes":0},"chunkRefsFetchTime":3299029,"congestionControlLatency":0,"pipelineWrapperFilteredLines":0}},"cache":{"chunk":{"entriesFound":0,"entriesRequested":0,"entriesStored":0,"bytesReceived":0,"bytesSent":0,"requests":0,"downloadTime":0,"queryLengthServed":0},"index":{"entriesFound":0,"entriesRequested":0,"entriesStored":0,"bytesReceived":0,"bytesSent":0,"requests":0,"downloadTime":0,"queryLengthServed":0},"result":{"entriesFound":0,"entriesRequested":0,"entriesStored":0,"bytesReceived":0,"bytesSent":0,"requests":0,"downloadTime":0,"queryLengthServed":0},"statsResult":{"entriesFound":0,"entriesRequested":1,"entriesStored":1,"bytesReceived":0,"bytesSent":0,"requests":2,"downloadTime":77638,"queryLengthServed":0},"volumeResult":{"entriesFound":0,"entriesRequested":0,"entriesStored":0,"bytesReceived":0,"bytesSent":0,"requests":0,"downloadTime":0,"queryLengthServed":0},"seriesResult":{"entriesFound":0,"entriesRequested":0,"entriesStored":0,"bytesReceived":0,"bytesSent":0,"requests":0,"downloadTime":0,"queryLengthServed":0},"labelResult":{"entriesFound":0,"entriesRequested":0,"entriesStored":0,"bytesReceived":0,"bytesSent":0,"requests":0,"downloadTime":0,"queryLengthServed":0},"instantMetricResult":{"entriesFound":0,"entriesRequested":0,"entriesStored":0,"bytesReceived":0,"bytesSent":0,"requests":0,"downloadTime":0,"queryLengthServed":0}},"index":{"totalChunks":0,"postFilterChunks":0,"shardsDuration":0,"usedBloomFilters":false}}}} diff --git a/evidence/current/correlated-trace.json b/evidence/current/correlated-trace.json new file mode 100644 index 0000000..343fbe6 --- /dev/null +++ b/evidence/current/correlated-trace.json @@ -0,0 +1 @@ +{"batches":[{"resource":{"attributes":[{"key":"deployment.environment.name","value":{"stringValue":"lab"}},{"key":"service.instance.id","value":{"stringValue":"e106424b-537d-45f3-bf84-f2dcb383e1c2"}},{"key":"service.name","value":{"stringValue":"messenger-api"}},{"key":"service.version","value":{"stringValue":"lab6"}}]},"scopeSpans":[{"scope":{"name":"messenger"},"spans":[{"traceId":"RIaudEsoSFehSFaEBsj/TA==","spanId":"es6GgTsi3j4=","parentSpanId":"AlmMCiVDsO4=","flags":768,"name":"POST /api/v1/chats/:chat_id/attachments","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1793285483437747691","endTimeUnixNano":"1793285483929999544","attributes":[{"key":"http.route","value":{"stringValue":"/api/v1/chats/:chat_id/attachments"}},{"key":"http.request.method","value":{"stringValue":"POST"}},{"key":"http.response.status_code","value":{"intValue":"202"}}],"status":{}},{"traceId":"RIaudEsoSFehSFaEBsj/TA==","spanId":"0XLjBKovzoE=","parentSpanId":"es6GgTsi3j4=","flags":256,"name":"postgres.acquire","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793285483470956266","endTimeUnixNano":"1793285483470982534","status":{}},{"traceId":"RIaudEsoSFehSFaEBsj/TA==","spanId":"vkHy6hCjoME=","parentSpanId":"es6GgTsi3j4=","flags":256,"name":"postgres.query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793285483471014239","endTimeUnixNano":"1793285483533402891","status":{}},{"traceId":"RIaudEsoSFehSFaEBsj/TA==","spanId":"wSa0gcgu4DM=","parentSpanId":"es6GgTsi3j4=","flags":256,"name":"postgres.acquire","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793285483539937823","endTimeUnixNano":"1793285483539960159","status":{}},{"traceId":"RIaudEsoSFehSFaEBsj/TA==","spanId":"NE/DHnTGeQg=","parentSpanId":"es6GgTsi3j4=","flags":256,"name":"postgres.query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793285483539988495","endTimeUnixNano":"1793285483600006929","status":{}},{"traceId":"RIaudEsoSFehSFaEBsj/TA==","spanId":"SCDdysOnpLg=","parentSpanId":"es6GgTsi3j4=","flags":256,"name":"postgres.acquire","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793285483600053034","endTimeUnixNano":"1793285483600065416","status":{}},{"traceId":"RIaudEsoSFehSFaEBsj/TA==","spanId":"E0NkzY4VDl4=","parentSpanId":"es6GgTsi3j4=","flags":256,"name":"postgres.query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793285483600069275","endTimeUnixNano":"1793285483658694833","status":{}},{"traceId":"RIaudEsoSFehSFaEBsj/TA==","spanId":"zpTLWaE1too=","parentSpanId":"es6GgTsi3j4=","flags":256,"name":"postgres.acquire","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793285483668582427","endTimeUnixNano":"1793285483668599787","status":{}},{"traceId":"RIaudEsoSFehSFaEBsj/TA==","spanId":"7gpALi2xYS0=","parentSpanId":"es6GgTsi3j4=","flags":256,"name":"postgres.query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793285483668614058","endTimeUnixNano":"1793285483726456411","status":{}},{"traceId":"RIaudEsoSFehSFaEBsj/TA==","spanId":"IkTgnW4P6Gg=","parentSpanId":"es6GgTsi3j4=","flags":256,"name":"s3.put","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793285483726491831","endTimeUnixNano":"1793285483815884615","status":{}},{"traceId":"RIaudEsoSFehSFaEBsj/TA==","spanId":"zstoQktsEfo=","parentSpanId":"es6GgTsi3j4=","flags":256,"name":"postgres.acquire","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793285483815915119","endTimeUnixNano":"1793285483815929632","status":{}},{"traceId":"RIaudEsoSFehSFaEBsj/TA==","spanId":"fn5YOAK2pIw=","parentSpanId":"es6GgTsi3j4=","flags":256,"name":"postgres.query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793285483815935387","endTimeUnixNano":"1793285483830389443","status":{}},{"traceId":"RIaudEsoSFehSFaEBsj/TA==","spanId":"35TPiy/2rmw=","parentSpanId":"es6GgTsi3j4=","flags":256,"name":"postgres.query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793285483830424632","endTimeUnixNano":"1793285483860479118","status":{}},{"traceId":"RIaudEsoSFehSFaEBsj/TA==","spanId":"6UOQb0SkEQY=","parentSpanId":"es6GgTsi3j4=","flags":256,"name":"postgres.query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793285483860522602","endTimeUnixNano":"1793285483886842926","status":{}},{"traceId":"RIaudEsoSFehSFaEBsj/TA==","spanId":"pV+Yk5p7jC4=","parentSpanId":"es6GgTsi3j4=","flags":256,"name":"postgres.query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793285483886862357","endTimeUnixNano":"1793285483910867500","status":{}}]}]},{"resource":{"attributes":[{"key":"deployment.environment.name","value":{"stringValue":"lab"}},{"key":"service.instance.id","value":{"stringValue":"4c56cc55-c3a5-49a2-a434-d452ae6d30cb"}},{"key":"service.name","value":{"stringValue":"messenger-worker"}},{"key":"service.version","value":{"stringValue":"lab6"}}]},"scopeSpans":[{"scope":{"name":"messenger"},"spans":[{"traceId":"RIaudEsoSFehSFaEBsj/TA==","spanId":"Co5tiuCeGTU=","parentSpanId":"es6GgTsi3j4=","flags":768,"name":"attachment.publish","kind":"SPAN_KIND_PRODUCER","startTimeUnixNano":"1793285484435356196","endTimeUnixNano":"1793285484439303866","status":{}},{"traceId":"RIaudEsoSFehSFaEBsj/TA==","spanId":"UsSKm2OwA00=","parentSpanId":"Co5tiuCeGTU=","flags":768,"name":"attachment.process","kind":"SPAN_KIND_CONSUMER","startTimeUnixNano":"1793285484447002116","endTimeUnixNano":"1793285484560039712","status":{}},{"traceId":"RIaudEsoSFehSFaEBsj/TA==","spanId":"WzM3y7vQ8H0=","parentSpanId":"UsSKm2OwA00=","flags":256,"name":"postgres.acquire","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793285484447027775","endTimeUnixNano":"1793285484447052707","status":{}},{"traceId":"RIaudEsoSFehSFaEBsj/TA==","spanId":"raxc3Lt46RY=","parentSpanId":"UsSKm2OwA00=","flags":256,"name":"postgres.query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793285484447075178","endTimeUnixNano":"1793285484447410074","status":{}},{"traceId":"RIaudEsoSFehSFaEBsj/TA==","spanId":"U/QegFt+uyM=","parentSpanId":"UsSKm2OwA00=","flags":256,"name":"postgres.query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793285484447422399","endTimeUnixNano":"1793285484448598892","status":{}},{"traceId":"RIaudEsoSFehSFaEBsj/TA==","spanId":"0oKA3R7QJvo=","parentSpanId":"UsSKm2OwA00=","flags":256,"name":"postgres.query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793285484448636585","endTimeUnixNano":"1793285484449721358","status":{}},{"traceId":"RIaudEsoSFehSFaEBsj/TA==","spanId":"07b3/3AtdJs=","parentSpanId":"UsSKm2OwA00=","flags":256,"name":"postgres.query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793285484449743755","endTimeUnixNano":"1793285484450326115","status":{}},{"traceId":"RIaudEsoSFehSFaEBsj/TA==","spanId":"OhKkhCHTyCA=","parentSpanId":"UsSKm2OwA00=","flags":256,"name":"postgres.query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793285484450343774","endTimeUnixNano":"1793285484452712321","status":{}},{"traceId":"RIaudEsoSFehSFaEBsj/TA==","spanId":"nnugrEfHPhs=","parentSpanId":"UsSKm2OwA00=","flags":256,"name":"s3.get","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793285484452764244","endTimeUnixNano":"1793285484469240472","status":{}},{"traceId":"RIaudEsoSFehSFaEBsj/TA==","spanId":"egEzJHPpxoE=","parentSpanId":"UsSKm2OwA00=","flags":256,"name":"s3.put","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793285484518809573","endTimeUnixNano":"1793285484529082010","status":{}},{"traceId":"RIaudEsoSFehSFaEBsj/TA==","spanId":"4cyLFcVpwRk=","parentSpanId":"UsSKm2OwA00=","flags":256,"name":"s3.put","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793285484529114615","endTimeUnixNano":"1793285484535466150","status":{}},{"traceId":"RIaudEsoSFehSFaEBsj/TA==","spanId":"9lgiRUA9csg=","parentSpanId":"UsSKm2OwA00=","flags":256,"name":"s3.put","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793285484535497690","endTimeUnixNano":"1793285484539343039","status":{}},{"traceId":"RIaudEsoSFehSFaEBsj/TA==","spanId":"ZIlXUWMA+e8=","parentSpanId":"UsSKm2OwA00=","flags":256,"name":"s3.put","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793285484539373826","endTimeUnixNano":"1793285484543492514","status":{}},{"traceId":"RIaudEsoSFehSFaEBsj/TA==","spanId":"KzHckj9pRsM=","parentSpanId":"UsSKm2OwA00=","flags":256,"name":"postgres.acquire","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793285484543719450","endTimeUnixNano":"1793285484543737689","status":{}},{"traceId":"RIaudEsoSFehSFaEBsj/TA==","spanId":"wtTgVY9rahI=","parentSpanId":"UsSKm2OwA00=","flags":256,"name":"postgres.query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793285484543742977","endTimeUnixNano":"1793285484544099659","status":{}},{"traceId":"RIaudEsoSFehSFaEBsj/TA==","spanId":"/Nv0C1BIQNg=","parentSpanId":"UsSKm2OwA00=","flags":256,"name":"postgres.query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793285484544126120","endTimeUnixNano":"1793285484545002306","status":{}},{"traceId":"RIaudEsoSFehSFaEBsj/TA==","spanId":"oHwWTpSs4v0=","parentSpanId":"UsSKm2OwA00=","flags":256,"name":"postgres.query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793285484545020431","endTimeUnixNano":"1793285484545903462","status":{}},{"traceId":"RIaudEsoSFehSFaEBsj/TA==","spanId":"Xk35xHLmcpw=","parentSpanId":"UsSKm2OwA00=","flags":256,"name":"postgres.query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793285484545936975","endTimeUnixNano":"1793285484549948385","status":{}},{"traceId":"RIaudEsoSFehSFaEBsj/TA==","spanId":"vlurdbg+VVs=","parentSpanId":"UsSKm2OwA00=","flags":256,"name":"postgres.query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793285484549991254","endTimeUnixNano":"1793285484552071169","status":{}}]}]}]} \ No newline at end of file diff --git a/evidence/current/correlation.json b/evidence/current/correlation.json new file mode 100644 index 0000000..89d8717 --- /dev/null +++ b/evidence/current/correlation.json @@ -0,0 +1,5 @@ +{ + "trace_id": "4486ae744b284857a148568406c8ff4c", + "attachment_id": "b726dbfb-5ddc-497b-80d7-59efd8debae3", + "ready": true +} diff --git a/evidence/current/correlation.txt b/evidence/current/correlation.txt new file mode 100644 index 0000000..e14848a --- /dev/null +++ b/evidence/current/correlation.txt @@ -0,0 +1,2 @@ +curl: (22) The requested URL returned error: 404 +PASS: upload and worker share trace bfeacf9a52db37463ddefff2ed23b0cb; correlated structured logs found diff --git a/evidence/current/coverage-per-file.json b/evidence/current/coverage-per-file.json new file mode 100644 index 0000000..bfc60af --- /dev/null +++ b/evidence/current/coverage-per-file.json @@ -0,0 +1,113 @@ +{"threshold_per_file":90,"files":[ +{"file":"internal/service/production/validation.go","statements":13,"covered":13,"percent":100.00000000,"sha256":"19442e5931252a040559e34af83e8e22d47fa127677ed5dd273e0ba5bbf99ab0"}, +{"file":"internal/model/http/models.go","statements":0,"covered":0,"percent":null,"sha256":"d68827b0bd3e6df4037621b6941298d3923dc348bda5bf314a34861e7c34f04e"}, +{"file":"internal/service/production/members.go","statements":19,"covered":19,"percent":100.00000000,"sha256":"861263597b62be27cfbc98a5dec11fd348994cba95773405086ab39e4d18e223"}, +{"file":"internal/telemetry/runtime.go","statements":32,"covered":29,"percent":90.62500000,"sha256":"7b973eadb1610369f4cc8d3dc2a4fa78d0732fb8677091c3c2db12d628868657"}, +{"file":"internal/controller/http/fiber/public_errors.go","statements":15,"covered":15,"percent":100.00000000,"sha256":"c2af04ba4a1a9a1b5e6596ac5a924113bf37954b27d54037016dc1ca9ee48e77"}, +{"file":"internal/store/memory/capacity.go","statements":7,"covered":7,"percent":100.00000000,"sha256":"8d8c50b868c725c0303a3cbc9578f6ea2b27d1a229700d2f80fe91f0de8622f3"}, +{"file":"internal/controller/http/fiber/attachments.go","statements":28,"covered":28,"percent":100.00000000,"sha256":"7b4f03ae4414de081409ff443813e7c854a132dea9bbbce5dc9354a6bf7f989c"}, +{"file":"internal/controller/http/fiber/health.go","statements":7,"covered":7,"percent":100.00000000,"sha256":"29ca7d1f1207b2fbbd272062beadcfe869d007350b8d50cc2a909b0dbe5c08d1"}, +{"file":"internal/model/dto/session.go","statements":0,"covered":0,"percent":null,"sha256":"896a694b0c217240343077d4287f4a6e447338c38571abbebcb3d16a519137fb"}, +{"file":"internal/service/production/service.go","statements":10,"covered":10,"percent":100.00000000,"sha256":"5f9009ad92a7e5ea5b7ed4aae7146a1805bdbb4ded66b57661dc1950a33b32e4"}, +{"file":"internal/valkey/admission.go","statements":42,"covered":42,"percent":100.00000000,"sha256":"8cdb6f5e2e83341b624e9b14a52b73e17ec03b41dad620f6e3d39a3244930b4a"}, +{"file":"internal/controller/http/fiber/observation.go","statements":2,"covered":2,"percent":100.00000000,"sha256":"3b36952bd2c08dfdbe04a3c1e0eeeb0b550edefee44b7816bfdb6a094787bda4"}, +{"file":"internal/controller/http/fiber/error_registry.go","statements":2,"covered":2,"percent":100.00000000,"sha256":"9d5da8d98ece2706e324cb2deca915e0811b6ad98f7aaac5805ca0f3cc0a1f9a"}, +{"file":"internal/service/interface.go","statements":0,"covered":0,"percent":null,"sha256":"58b49648c1b6e73971255cd9c5ae95853abd8bb44653ee2a912122c599ef23cb"}, +{"file":"internal/store/memory/store.go","statements":3,"covered":3,"percent":100.00000000,"sha256":"e00a97d2b6d408bd8964fefd62f5a187a1de81c96f93637c0905a346b4beaa5b"}, +{"file":"internal/config/memory.go","statements":9,"covered":9,"percent":100.00000000,"sha256":"86cdf9e90b576f4484946bfd9605333fc52077f38d3a39c5d71ddca4cdeecda0"}, +{"file":"internal/service/cursor.go","statements":0,"covered":0,"percent":null,"sha256":"1815bf20f8ebfa8f8f394f8aef5e3e7954ff7652a5c4f4d6b14ace4d57b99ee2"}, +{"file":"internal/store/memory/messages.go","statements":19,"covered":18,"percent":94.73684211,"sha256":"03d054b385ae51d160b6ce0441e86c2f6c4763724aa9e91228224cc88a12fdd2"}, +{"file":"internal/streamqueue/queue.go","statements":33,"covered":30,"percent":90.90909091,"sha256":"0b6655c6c7d7f60758b05b3d5fc6abfea4a7de3466f7fd56d1cac0b537cc76d5"}, +{"file":"internal/pipeline/types.go","statements":0,"covered":0,"percent":null,"sha256":"559c98b3fa4d4ea4aa2a07480ccdea52b37cc0dbf0910aacef7589757effaa8c"}, +{"file":"internal/store/postgres/idempotency.go","statements":21,"covered":19,"percent":90.47619048,"sha256":"0f1d4b91a9deb26f276235f3a069570b3962f279aeb5a8491d6384ad2fa942ca"}, +{"file":"internal/model/dto/models.go","statements":0,"covered":0,"percent":null,"sha256":"5e2536b8eba6bb92b751a7b817b6338755008857e3b53920d8da6fb67adeebdb"}, +{"file":"internal/service/errors.go","statements":0,"covered":0,"percent":null,"sha256":"a1ac40c39260c9babeaf8259985de5f84abe1ce80dc8ed9f26ad1b66b1ef35af"}, +{"file":"internal/config/telemetry.go","statements":13,"covered":13,"percent":100.00000000,"sha256":"a5289a9ee5c8d6b092c9719f7ccece0abbbc409ecedd187352718ec3a083bf90"}, +{"file":"internal/service/cache.go","statements":0,"covered":0,"percent":null,"sha256":"d3ef483a9fcc47c0a056c0ed9e06907f51688cbd0f87c5ce1b657b2f12fc714c"}, +{"file":"internal/controller/http/fiber/stream.go","statements":14,"covered":14,"percent":100.00000000,"sha256":"6bc72fdc23332e08e4dee8a5ed3f9110952208af86b529d5d4847ebcf00dd4c3"}, +{"file":"internal/store/postgres/reconcile.go","statements":32,"covered":31,"percent":96.87500000,"sha256":"a770985780d475f03ca9990522f02d9bcffaa69b7701128df11e0939b55848ba"}, +{"file":"cmd/server/worker.go","statements":36,"covered":34,"percent":94.44444444,"sha256":"aae78ee2a9c844684410dd6d727427d3437ec7b19f417b6e75dbafa203a86b51"}, +{"file":"internal/controller/http/fiber/middleware.go","statements":53,"covered":52,"percent":98.11320755,"sha256":"18d69e4e21f9b2231d75ce9ac6dfe0243d9aa2a5ebf51932380adcd0dfec2a3a"}, +{"file":"internal/objectstore/observation.go","statements":5,"covered":5,"percent":100.00000000,"sha256":"d11d878aa4b1e0e093ae39028e1cc77fbf1841036145d9ce44392b8465ebddf5"}, +{"file":"internal/store/postgres/store.go","statements":58,"covered":54,"percent":93.10344828,"sha256":"ac5a64c966681f8baf5dbbc1232c9f6a2a54ad137869acfe88052e7cb943cfd5"}, +{"file":"internal/config/logger.go","statements":7,"covered":7,"percent":100.00000000,"sha256":"3a69e20a00fa2ed816f344a990e9e0ef0077e1607d4207858b4b00e4ff5d1988"}, +{"file":"cmd/server/logger.go","statements":10,"covered":9,"percent":90.00000000,"sha256":"d4c33ca5a93583caffd63e78bef00a2af906c477e29f1ce89495ee1737219579"}, +{"file":"internal/service/auth.go","statements":0,"covered":0,"percent":null,"sha256":"f584e498d5f902f2a8eb22bb91b0dfd7a86547797995e750356415002240a50b"}, +{"file":"internal/service/production/creation.go","statements":24,"covered":23,"percent":95.83333333,"sha256":"f7767482299f26cc44cb8d2c2a6fd024f5caec4a2818fbde617fe99ba687a097"}, +{"file":"internal/media/service.go","statements":55,"covered":54,"percent":98.18181818,"sha256":"e0e55fcaad09af39caf5d481014de408552d28b7925de8e9492aa208938e637d"}, +{"file":"internal/service/production/messages.go","statements":18,"covered":18,"percent":100.00000000,"sha256":"5ae4aae2622bad3673a5b0b4a460954a1f8ff38e337145dcfec678a401b490d4"}, +{"file":"internal/controller/http/fiber/routes.go","statements":18,"covered":18,"percent":100.00000000,"sha256":"e248223517102e1eecdeba9a5e2d844c0afc14116a56e9a1d76c6afb755e63d1"}, +{"file":"internal/store/postgres/jobs.go","statements":116,"covered":110,"percent":94.82758621,"sha256":"9b4445f552ad08923a59be3853fad2091ce62a4ab0f6b4875711505a4e6e5fe1"}, +{"file":"internal/config/persistence.go","statements":13,"covered":13,"percent":100.00000000,"sha256":"28c61d23db4ca2ef7ed952451f465318865076d2393eaa94ff5d7bb339c4c06b"}, +{"file":"internal/controller/http/fiber/error_codes.go","statements":0,"covered":0,"percent":null,"sha256":"3351a019c43d0360ee0fd68afabb5200422d065ae7bd908b1c0a5c02a0c1ad41"}, +{"file":"internal/media/run_reconcile.go","statements":21,"covered":21,"percent":100.00000000,"sha256":"3255403afcb9486d33c5bb45b5afa9c3c9b85596ce97f86ac2f224011cd8a860"}, +{"file":"internal/controller/http/fiber/handlers.go","statements":37,"covered":36,"percent":97.29729730,"sha256":"f514263df465c6c1e3df76590553b37f7e32330dee48add9fdfb194d9bcde28e"}, +{"file":"internal/store/postgres/retention.go","statements":11,"covered":11,"percent":100.00000000,"sha256":"6ca456767aec350fe81af74deb82a04fcb8689896d11ed547d3855a01f1b57e7"}, +{"file":"internal/store/postgres/chats.go","statements":26,"covered":24,"percent":92.30769231,"sha256":"1a7f671d6c781a5f3b6a03ce80e38f87e8a11e7e4cd51fca57829be9a01811ff"}, +{"file":"cmd/server/media.go","statements":7,"covered":7,"percent":100.00000000,"sha256":"acc63500e1475752bc8f82472e6714d52bd30af62d82d07e44f0de3f117c22ac"}, +{"file":"internal/store/memory/users.go","statements":22,"covered":21,"percent":95.45454545,"sha256":"7ef5f5000ed30aac7e9e058d33ff4a75caa4b83040a59269ae1223081cab68af"}, +{"file":"internal/service/production/chats.go","statements":22,"covered":22,"percent":100.00000000,"sha256":"e917208ece7e11d442c3c3ca7e33ac303a9e50b11288f2476390d601a5d68e41"}, +{"file":"internal/valkey/client.go","statements":35,"covered":34,"percent":97.14285714,"sha256":"b30fccec3b12374e76b93a2a76ffa7d9fc6a9d6a123793b8effb6b0da679a7d0"}, +{"file":"internal/store/memory/ordering.go","statements":6,"covered":6,"percent":100.00000000,"sha256":"0eb10952ceb798a5e6bacfd115eeaaed9cd37e8232a56c20d2c120cf457d3c5e"}, +{"file":"internal/controller/http/fiber/decode.go","statements":12,"covered":11,"percent":91.66666667,"sha256":"f5e9c3f8a8a22f51639a570e2ab2e626bde0bf29434dd6527d1da8f32f021cf0"}, +{"file":"internal/controller/http/fiber/errors.go","statements":29,"covered":29,"percent":100.00000000,"sha256":"a490411aca9d3a6ddc730981cdc4e0f785172db6773db23357a05c51f7889ea3"}, +{"file":"internal/controller/http/fiber/controller.go","statements":10,"covered":10,"percent":100.00000000,"sha256":"cc0fd1f1735385a5b09c836c4fbbf7da22e8343489cc9737f926bc38dcc1add9"}, +{"file":"internal/observation/snapshot.go","statements":0,"covered":0,"percent":null,"sha256":"124a8a6393c3c23040b2fd7ec0dbc77b05b4a317e9b1ecddd6df366250db56a6"}, +{"file":"cmd/server/telemetry.go","statements":20,"covered":19,"percent":95.00000000,"sha256":"c5c54cf34133c297c88a401f8183c8b5a269864323e24100ce0a3994e3a4f12f"}, +{"file":"internal/pipeline/worker.go","statements":87,"covered":85,"percent":97.70114943,"sha256":"357355bc4420f9b4c8306967cdc468de6989ccb96d90d3a6ff5afa80f07c3259"}, +{"file":"internal/service/production/access.go","statements":27,"covered":27,"percent":100.00000000,"sha256":"878415267a409244b5fa490ed09da3ed446ab5d45963f4b1c02f626fd5248614"}, +{"file":"internal/valkey/cache.go","statements":17,"covered":16,"percent":94.11764706,"sha256":"6aa3f9dbf0e5f5e60a06245fefe4d5d139ab181a47ec0e2d2e88346158fd4359"}, +{"file":"internal/service/production/errors.go","statements":5,"covered":5,"percent":100.00000000,"sha256":"d002ded303f0aaf8acef41f5e5846e548c52e5b4e683d7cb986be38db4227bbd"}, +{"file":"internal/store/postgres/observability.go","statements":5,"covered":5,"percent":100.00000000,"sha256":"7eebf916fdc3ee58b2d19098a835405aa95cb985d2c8c7f5934e1e64d1f2b9f9"}, +{"file":"internal/observation/tracer.go","statements":0,"covered":0,"percent":null,"sha256":"3642e33cdb0ae1ca5ed0a88cc0b6d0748f836d45ba0712ae1368533310aa5a7a"}, +{"file":"internal/service/authentication/service.go","statements":48,"covered":47,"percent":97.91666667,"sha256":"87a752eb08988aa0b76fa3c6e37e6180508d9bf774f3d9cec0cd825162094d34"}, +{"file":"internal/service/production/pagination.go","statements":21,"covered":21,"percent":100.00000000,"sha256":"2ab7be10ded3332e4bbfc0c825f5d355a68b2ef79385fc3af490ae3a11077859"}, +{"file":"internal/media/orphans.go","statements":23,"covered":23,"percent":100.00000000,"sha256":"327ee894c8c9afa5bcf8236ed4c91befd66699c0ca33178e9690705764faf002"}, +{"file":"internal/pipeline/cleanup.go","statements":11,"covered":11,"percent":100.00000000,"sha256":"eb4a71e1d2cbc93d539770b425305b4826b98dd7770741c6677a50c0ec43b4f8"}, +{"file":"cmd/server/persistence.go","statements":24,"covered":22,"percent":91.66666667,"sha256":"8332e76c999388078bc176da84fb38535b1105e32d8d4044cf6b701188476065"}, +{"file":"internal/admission/limiter.go","statements":16,"covered":16,"percent":100.00000000,"sha256":"a373c9539167a4b7c35d45334d35ffd4e9d124c1740c88c12094fededd9643b4"}, +{"file":"internal/media/validation.go","statements":17,"covered":17,"percent":100.00000000,"sha256":"3662f14499e4106d3148a79a001ab92cb0d31279516ccba9a524e6bcec2a16b4"}, +{"file":"cmd/server/reconcile.go","statements":19,"covered":18,"percent":94.73684211,"sha256":"42848ffa729c73773b301ab27da96633ef5fe36fd4da2ea059bb622f49a5766f"}, +{"file":"internal/controller/http/fiber/lifecycle.go","statements":24,"covered":22,"percent":91.66666667,"sha256":"f39cf7a0049a2ed39d6cc1fcc66b44656d9d6120914d0e973b032e12f3d0de08"}, +{"file":"internal/config/controller.go","statements":23,"covered":22,"percent":95.65217391,"sha256":"1c853c0f771e2c75958db8158797f2287e06c8d9efe8856365b7b9c71b1747b0"}, +{"file":"internal/config/config.go","statements":17,"covered":17,"percent":100.00000000,"sha256":"8cb675a0bef8b1645071e4cb85b79a467620e99b375db5d8d3e16a54b4c5b5fa"}, +{"file":"internal/model/dto/attachment.go","statements":0,"covered":0,"percent":null,"sha256":"801a6290037459a1f9d28669b4f3cdce5191317da8a48189027b7a507139fdc7"}, +{"file":"internal/controller/http/fiber/auth.go","statements":29,"covered":29,"percent":100.00000000,"sha256":"a8fb7892f713529a9359b1c77202ef5fb9a2dd4d921ee9f1ec104da67ee1ad87"}, +{"file":"internal/strictjson/decode.go","statements":47,"covered":47,"percent":100.00000000,"sha256":"d99595e1f74406c5aa617c0ec062afb7f12f116bd86776e87cc9c9a436ff7eea"}, +{"file":"internal/config/objectstore.go","statements":4,"covered":4,"percent":100.00000000,"sha256":"c6283fe310747e8297850b48c0a119c1d62db4903dd2c779c6296678563df188"}, +{"file":"internal/config/worker.go","statements":4,"covered":4,"percent":100.00000000,"sha256":"eb502257c4badca1e8a2d4eb17701937edcc2c45dd6d697d70abb1624f457e14"}, +{"file":"internal/objectstore/list.go","statements":10,"covered":9,"percent":90.00000000,"sha256":"5eb695bb582e545a3b43ba15e7a88b6bb70bdfd43155f2169defff2ffd41fd7c"}, +{"file":"cmd/server/app.go","statements":9,"covered":9,"percent":100.00000000,"sha256":"21a514acf6287bcc9c39b96e6ce22b5c58f7e0425f7cf796744f4547af11ff31"}, +{"file":"cmd/server/main.go","statements":3,"covered":3,"percent":100.00000000,"sha256":"c22d8cdf4ce31fde29945e967d3f025e280a7dc4d2022329d51be972d2a0f495"}, +{"file":"cmd/server/s3_bootstrap.go","statements":9,"covered":9,"percent":100.00000000,"sha256":"e8e1d54f869025360853a5a73eabb3146914d4caf6b8d4f20e1a250e79058ea7"}, +{"file":"internal/config/environment.go","statements":14,"covered":14,"percent":100.00000000,"sha256":"027761b059b76898a84318ec1a8b43158d8bafd6e0b15b52efbe902b4d8c5ea3"}, +{"file":"internal/telemetry/spans.go","statements":42,"covered":42,"percent":100.00000000,"sha256":"4f9ef25f02df303db401d99780888bac94ebd7601f8227be710acefd80f09987"}, +{"file":"internal/store/memory/chats.go","statements":31,"covered":31,"percent":100.00000000,"sha256":"618eaf4e1c042d29abaa7df456d83a36f6b4dcc6e7a401071ec8489805dfb640"}, +{"file":"internal/imageprocessor/processor.go","statements":43,"covered":39,"percent":90.69767442,"sha256":"7f1fed5f8276e19ed13e0576c633c11d213d8ad1af7e27179aea893948a795ad"}, +{"file":"internal/controller/fx.go","statements":1,"covered":1,"percent":100.00000000,"sha256":"af72da25fea9811c13a50c8133e9e8b35e070c5295069c40c0ae89cdc0aff19a"}, +{"file":"internal/controller/interface.go","statements":0,"covered":0,"percent":null,"sha256":"24b6b06d6fab919cd2130697b44fd0d358265184dca0ca8362e0b4fcc2ae5a68"}, +{"file":"internal/media/reconcile.go","statements":38,"covered":38,"percent":100.00000000,"sha256":"90c319dc2ed9cc8193474ba84793094839d5c7a6ef19a103f4170da5558348fb"}, +{"file":"internal/hashring/ring.go","statements":21,"covered":20,"percent":95.23809524,"sha256":"e205901cec2e66011ec764b70879b0eb0d28a78c4dea8913a9fb30c2d778fa8c"}, +{"file":"internal/telemetry/postgres.go","statements":10,"covered":10,"percent":100.00000000,"sha256":"ddba713ce3941cef9fddd4968d63359105e4f32366a87b469268a57d0da6364a"}, +{"file":"internal/pipeline/observation.go","statements":10,"covered":10,"percent":100.00000000,"sha256":"d844cf88188841f6e5ba393c48f79d393a70531eea9fedfce07da419c49bd366"}, +{"file":"cmd/server/commands.go","statements":27,"covered":27,"percent":100.00000000,"sha256":"ebc7c7d4bb1a0ca63cf949ca349f4819b4e5672a3bb1d714fee90ab7ec48ceff"}, +{"file":"internal/controller/http/fiber/dependencies.go","statements":0,"covered":0,"percent":null,"sha256":"6ee07f110bdb29d8d2191ab8889ba60474b33a5b225739875c8a57d8434be15d"}, +{"file":"internal/service/production/pages.go","statements":17,"covered":16,"percent":94.11764706,"sha256":"a19420fa45469f36a4a624b8f8a696b81af56cbfb94f28dcc26fc5940fc0aec7"}, +{"file":"internal/store/postgres/sessions.go","statements":13,"covered":13,"percent":100.00000000,"sha256":"889a63b51b29012f4ae5499d1c0577fe29f2f2959b05629f8e123715cfc1f75f"}, +{"file":"internal/controller/http/fiber/configure.go","statements":7,"covered":7,"percent":100.00000000,"sha256":"be356cf26c228958d51f5bafa4305d7325e81ab01065575acbc9b2d7a6ddd4bd"}, +{"file":"internal/admission/distributed.go","statements":0,"covered":0,"percent":null,"sha256":"41ef40897886c0878cbfe52f6716bdcb8433fb7eaddcfff298e1d54c55db423d"}, +{"file":"internal/store/postgres/messages.go","statements":31,"covered":29,"percent":93.54838710,"sha256":"041454c30d1e1cc980567b6e11c8de0bd57cf9e0e8703d132a5172e7bc0ea901"}, +{"file":"internal/cursor/codec.go","statements":36,"covered":34,"percent":94.44444444,"sha256":"678e197744fb2fc4b51715f51593b98761125bbaddbe4cb1b7032bd6b173ef04"}, +{"file":"internal/store/interface.go","statements":0,"covered":0,"percent":null,"sha256":"f683306c33020d2403e91e93b0fc260ff1d3b3f1911425a7550384dfa60d9937"}, +{"file":"cmd/server/healthcheck.go","statements":19,"covered":18,"percent":94.73684211,"sha256":"2308066b0bc2cb117adced1181a64d8b9998aa404d2198143000729a3b9138a1"}, +{"file":"internal/controller/http/fiber/mapping.go","statements":16,"covered":16,"percent":100.00000000,"sha256":"7e67fc7c7d3076dc0927a9601871d38002d31d988c9e4c7abcac9c7a65eba327"}, +{"file":"internal/password/password.go","statements":18,"covered":17,"percent":94.44444444,"sha256":"fe2df6990331573bbf0202ebc81c58263a7c16a29b713eefa0f0e852e4ee4a76"}, +{"file":"internal/objectstore/client.go","statements":48,"covered":44,"percent":91.66666667,"sha256":"e2a8f39cf5ecff860bca6bceeda78779117ea122b7619621ef9a80c84eeb62cb"}, +{"file":"internal/telemetry/metrics.go","statements":18,"covered":18,"percent":100.00000000,"sha256":"009d4d61832b93f1e014e1d567d9e1828efc023ec2c40204d96c5d8097c0551f"}, +{"file":"internal/controller/http/fiber/headers.go","statements":0,"covered":0,"percent":null,"sha256":"d32a9a01a8b391f4d986e14947a25a0504f4363335ea1524df05e0dd8e9e61bd"}, +{"file":"internal/telemetry/saturation.go","statements":28,"covered":27,"percent":96.42857143,"sha256":"4f92838ad08685b5470f9a35ec9c48a136de333ba8d2287b1384f0a35ddb1cb6"}, +{"file":"internal/store/postgres/users.go","statements":21,"covered":20,"percent":95.23809524,"sha256":"15de54e4bea80c8523429e3ef5b22c49a2b36f48c9fa58c2b76bc0653a6eae23"}, +{"file":"internal/controller/http/fiber/metrics.go","statements":17,"covered":17,"percent":100.00000000,"sha256":"54db82201697bdfcc65712afe61b650f8f0885fc65222dc4cc6d1c3bb07d82dd"}, +{"file":"internal/service/production/users.go","statements":19,"covered":19,"percent":100.00000000,"sha256":"6e72b534a0ac2b9bfc7867826b5596da614eb1ef883b86b27fde5098bf818ca2"}, +{"file":"internal/store/postgres/attachments.go","statements":21,"covered":20,"percent":95.23809524,"sha256":"5de665ff2f33918a736a97859ff9df766087e8b5deb2e01dff5d35e115a1af19"} +],"passed":true} diff --git a/evidence/current/coverage.txt b/evidence/current/coverage.txt new file mode 100644 index 0000000..1608764 --- /dev/null +++ b/evidence/current/coverage.txt @@ -0,0 +1,139 @@ +warning: GOCOVERDIR not set, no coverage data emitted +ok gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/cmd/server 3.478s coverage: 33.3% of statements in ./... +ok gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/admission 1.354s coverage: 0.8% of statements in ./... +ok gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/config 1.356s coverage: 4.8% of statements in ./... + gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/controller coverage: 0.0% of statements +ok gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/controller/http/fiber 1.880s coverage: 31.7% of statements in ./... +ok gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/cursor 1.386s coverage: 1.7% of statements in ./... +ok gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/hashring 1.850s coverage: 1.0% of statements in ./... +ok gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/imageprocessor 3.381s coverage: 1.9% of statements in ./... +ok gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/media 1.354s coverage: 7.1% of statements in ./... +? gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/model/dto [no test files] +? gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/model/http [no test files] +ok gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/objectstore 1.642s coverage: 3.0% of statements in ./... +? gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/observation [no test files] +ok gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/password 2.373s coverage: 0.8% of statements in ./... +ok gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/pipeline 1.326s coverage: 5.1% of statements in ./... +? gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/service [no test files] +ok gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/service/authentication 1.338s coverage: 2.6% of statements in ./... +ok gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/service/production 1.339s coverage: 10.5% of statements in ./... +? gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/store [no test files] +ok gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/store/memory 1.329s coverage: 4.4% of statements in ./... +ok gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/store/postgres 5.321s coverage: 23.6% of statements in ./... +ok gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/streamqueue 2.510s coverage: 1.5% of statements in ./... +ok gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/strictjson 1.375s coverage: 2.3% of statements in ./... +ok gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/telemetry 1.493s coverage: 6.1% of statements in ./... +ok gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/valkey 1.393s coverage: 5.4% of statements in ./... +ok gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/student-tests 1.618s coverage: 0.0% of statements in ./... +PASS: lab6 public API scenario; infrastructure and grading evidence remain separate. + 100.00% internal/service/production/validation.go + N/A internal/model/http/models.go + 100.00% internal/service/production/members.go + 90.62% internal/telemetry/runtime.go + 100.00% internal/controller/http/fiber/public_errors.go + 100.00% internal/store/memory/capacity.go + 100.00% internal/controller/http/fiber/attachments.go + 100.00% internal/controller/http/fiber/health.go + N/A internal/model/dto/session.go + 100.00% internal/service/production/service.go + 100.00% internal/valkey/admission.go + 100.00% internal/controller/http/fiber/observation.go + 100.00% internal/controller/http/fiber/error_registry.go + N/A internal/service/interface.go + 100.00% internal/store/memory/store.go + 100.00% internal/config/memory.go + N/A internal/service/cursor.go + 94.74% internal/store/memory/messages.go + 90.91% internal/streamqueue/queue.go + N/A internal/pipeline/types.go + 90.48% internal/store/postgres/idempotency.go + N/A internal/model/dto/models.go + N/A internal/service/errors.go + 100.00% internal/config/telemetry.go + N/A internal/service/cache.go + 100.00% internal/controller/http/fiber/stream.go + 96.88% internal/store/postgres/reconcile.go + 94.44% cmd/server/worker.go + 98.11% internal/controller/http/fiber/middleware.go + 100.00% internal/objectstore/observation.go + 93.10% internal/store/postgres/store.go + 100.00% internal/config/logger.go + 90.00% cmd/server/logger.go + N/A internal/service/auth.go + 95.83% internal/service/production/creation.go + 98.18% internal/media/service.go + 100.00% internal/service/production/messages.go + 100.00% internal/controller/http/fiber/routes.go + 94.83% internal/store/postgres/jobs.go + 100.00% internal/config/persistence.go + N/A internal/controller/http/fiber/error_codes.go + 100.00% internal/media/run_reconcile.go + 97.30% internal/controller/http/fiber/handlers.go + 100.00% internal/store/postgres/retention.go + 92.31% internal/store/postgres/chats.go + 100.00% cmd/server/media.go + 95.45% internal/store/memory/users.go + 100.00% internal/service/production/chats.go + 97.14% internal/valkey/client.go + 100.00% internal/store/memory/ordering.go + 91.67% internal/controller/http/fiber/decode.go + 100.00% internal/controller/http/fiber/errors.go + 100.00% internal/controller/http/fiber/controller.go + N/A internal/observation/snapshot.go + 95.00% cmd/server/telemetry.go + 97.70% internal/pipeline/worker.go + 100.00% internal/service/production/access.go + 94.12% internal/valkey/cache.go + 100.00% internal/service/production/errors.go + 100.00% internal/store/postgres/observability.go + N/A internal/observation/tracer.go + 97.92% internal/service/authentication/service.go + 100.00% internal/service/production/pagination.go + 100.00% internal/media/orphans.go + 100.00% internal/pipeline/cleanup.go + 91.67% cmd/server/persistence.go + 100.00% internal/admission/limiter.go + 100.00% internal/media/validation.go + 94.74% cmd/server/reconcile.go + 91.67% internal/controller/http/fiber/lifecycle.go + 95.65% internal/config/controller.go + 100.00% internal/config/config.go + N/A internal/model/dto/attachment.go + 100.00% internal/controller/http/fiber/auth.go + 100.00% internal/strictjson/decode.go + 100.00% internal/config/objectstore.go + 100.00% internal/config/worker.go + 90.00% internal/objectstore/list.go + 100.00% cmd/server/app.go + 100.00% cmd/server/main.go + 100.00% cmd/server/s3_bootstrap.go + 100.00% internal/config/environment.go + 100.00% internal/telemetry/spans.go + 100.00% internal/store/memory/chats.go + 90.70% internal/imageprocessor/processor.go + 100.00% internal/controller/fx.go + N/A internal/controller/interface.go + 100.00% internal/media/reconcile.go + 95.24% internal/hashring/ring.go + 100.00% internal/telemetry/postgres.go + 100.00% internal/pipeline/observation.go + 100.00% cmd/server/commands.go + N/A internal/controller/http/fiber/dependencies.go + 94.12% internal/service/production/pages.go + 100.00% internal/store/postgres/sessions.go + 100.00% internal/controller/http/fiber/configure.go + N/A internal/admission/distributed.go + 93.55% internal/store/postgres/messages.go + 94.44% internal/cursor/codec.go + N/A internal/store/interface.go + 94.74% cmd/server/healthcheck.go + 100.00% internal/controller/http/fiber/mapping.go + 94.44% internal/password/password.go + 91.67% internal/objectstore/client.go + 100.00% internal/telemetry/metrics.go + N/A internal/controller/http/fiber/headers.go + 96.43% internal/telemetry/saturation.go + 95.24% internal/store/postgres/users.go + 100.00% internal/controller/http/fiber/metrics.go + 100.00% internal/service/production/users.go + 95.24% internal/store/postgres/attachments.go diff --git a/evidence/current/db-outage-trace.json b/evidence/current/db-outage-trace.json new file mode 100644 index 0000000..3d3def9 --- /dev/null +++ b/evidence/current/db-outage-trace.json @@ -0,0 +1 @@ +{"batches":[{"resource":{"attributes":[{"key":"deployment.environment.name","value":{"stringValue":"lab"}},{"key":"service.instance.id","value":{"stringValue":"cd2ca7bb-0626-4ccb-bdb4-6d4b50af696e"}},{"key":"service.version","value":{"stringValue":"lab6"}},{"key":"service.name","value":{"stringValue":"messenger-api"}}]},"scopeSpans":[{"scope":{"name":"messenger"},"spans":[{"traceId":"Lo9ArJnbv1/0A4F99ifvEw==","spanId":"mue1B0sSIGc=","parentSpanId":"V49QQjAf4B0=","name":"postgres.acquire","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793284547292826000","endTimeUnixNano":"1793284548293359458","status":{"message":"operation failed","code":"STATUS_CODE_ERROR"}},{"traceId":"Lo9ArJnbv1/0A4F99ifvEw==","spanId":"V49QQjAf4B0=","name":"GET /api/v1","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1793284547290694000","endTimeUnixNano":"1793284548296871166","attributes":[{"key":"http.route","value":{"stringValue":"/api/v1"}},{"key":"http.request.method","value":{"stringValue":"GET"}},{"key":"http.response.status_code","value":{"intValue":"503"}}],"status":{"message":"server error","code":"STATUS_CODE_ERROR"}}]}]}]} \ No newline at end of file diff --git a/evidence/current/dev-correlation.txt b/evidence/current/dev-correlation.txt new file mode 100644 index 0000000..26d9c55 --- /dev/null +++ b/evidence/current/dev-correlation.txt @@ -0,0 +1 @@ +PASS: upload and worker share trace 4486ae744b284857a148568406c8ff4c; correlated structured logs found diff --git a/evidence/current/dev-dashboard.json b/evidence/current/dev-dashboard.json new file mode 100644 index 0000000..847eb3b --- /dev/null +++ b/evidence/current/dev-dashboard.json @@ -0,0 +1,4 @@ +{ + "title": "Messenger: API, worker and SLO", + "panels": 14 +} diff --git a/evidence/current/dev-datasources.jsonl b/evidence/current/dev-datasources.jsonl new file mode 100644 index 0000000..4b003d2 --- /dev/null +++ b/evidence/current/dev-datasources.jsonl @@ -0,0 +1,2 @@ +{"uid":"prometheus","result":{"details":{"application":"Prometheus","features":{"rulerApiEnabled":false}},"message":"Successfully queried the Prometheus API.","status":"OK"}} +{"uid":"loki","result":{"message":"Data source successfully connected.","status":"OK"}} diff --git a/evidence/current/dev-deploy-final.txt b/evidence/current/dev-deploy-final.txt new file mode 100644 index 0000000..4089e9a --- /dev/null +++ b/evidence/current/dev-deploy-final.txt @@ -0,0 +1,15 @@ +helm lint charts/messenger +==> Linting charts/messenger +[INFO] Chart.yaml: icon is recommended + +1 chart(s) linted, 0 chart(s) failed +helm template lab2 charts/messenger >/dev/null +helm upgrade --install lab2 --reset-values charts/messenger --kube-context dev --namespace highload-applications --wait --wait-for-jobs --timeout 8m +Release "lab2" has been upgraded. Happy Helming! +NAME: lab2 +LAST DEPLOYED: Thu Oct 29 17:48:29 2026 +NAMESPACE: highload-applications +STATUS: deployed +REVISION: 45 +DESCRIPTION: Upgrade complete +TEST SUITE: None diff --git a/evidence/current/dev-deploy.txt b/evidence/current/dev-deploy.txt new file mode 100644 index 0000000..c3d72c9 --- /dev/null +++ b/evidence/current/dev-deploy.txt @@ -0,0 +1,15 @@ +helm lint charts/messenger +==> Linting charts/messenger +[INFO] Chart.yaml: icon is recommended + +1 chart(s) linted, 0 chart(s) failed +helm template lab2 charts/messenger >/dev/null +helm upgrade --install lab2 --reset-values charts/messenger --kube-context dev --namespace highload-applications --wait --wait-for-jobs --timeout 8m +Release "lab2" has been upgraded. Happy Helming! +NAME: lab2 +LAST DEPLOYED: Thu Oct 29 17:42:04 2026 +NAMESPACE: highload-applications +STATUS: deployed +REVISION: 44 +DESCRIPTION: Upgrade complete +TEST SUITE: None diff --git a/evidence/current/dev-metric-names.json b/evidence/current/dev-metric-names.json new file mode 100644 index 0000000..231b584 --- /dev/null +++ b/evidence/current/dev-metric-names.json @@ -0,0 +1 @@ +{"status":"success","data":["messenger_db_pool_used","messenger_db_pool_waiting","messenger_http_duration_seconds_bucket","messenger_http_duration_seconds_count","messenger_http_duration_seconds_sum","messenger_http_requests_total","messenger_job_duration_seconds_bucket","messenger_job_duration_seconds_count","messenger_job_duration_seconds_sum","messenger_jobs_total","messenger_process_cpu_seconds_total","messenger_process_peak_rss_bytes","messenger_queue_depth","messenger_queue_oldest_seconds","messenger_ready_duration_seconds_bucket","messenger_ready_duration_seconds_count","messenger_ready_duration_seconds_sum","messenger_worker_busy","messenger_worker_capacity","otel_scope_info","otelcol_exporter_queue_batch_send_size_bucket","otelcol_exporter_queue_batch_send_size_count","otelcol_exporter_queue_batch_send_size_sum","otelcol_exporter_queue_capacity","otelcol_exporter_queue_size","otelcol_exporter_send_failed_log_records_total","otelcol_exporter_send_failed_metric_points_total","otelcol_exporter_send_failed_spans_total","otelcol_exporter_sent_log_records_total","otelcol_exporter_sent_metric_points_total","otelcol_exporter_sent_spans_total","otelcol_process_cpu_seconds_total","otelcol_process_memory_rss_bytes","otelcol_process_runtime_heap_alloc_bytes","otelcol_process_runtime_total_alloc_bytes_total","otelcol_process_runtime_total_sys_memory_bytes","otelcol_process_uptime_seconds_total","otelcol_processor_accepted_log_records_total","otelcol_processor_accepted_metric_points_total","otelcol_processor_accepted_spans_total","otelcol_processor_batch_batch_send_size_bucket","otelcol_processor_batch_batch_send_size_count","otelcol_processor_batch_batch_send_size_sum","otelcol_processor_batch_metadata_cardinality","otelcol_processor_batch_timeout_trigger_send_total","otelcol_processor_incoming_items_total","otelcol_processor_outgoing_items_total","otelcol_receiver_accepted_log_records_total","otelcol_receiver_accepted_metric_points_total","otelcol_receiver_accepted_spans_total","otelcol_receiver_failed_log_records_total","otelcol_receiver_failed_metric_points_total","otelcol_receiver_failed_spans_total","otelcol_receiver_refused_log_records_total","otelcol_receiver_refused_metric_points_total","otelcol_receiver_refused_spans_total","promhttp_metric_handler_errors_total","scrape_duration_seconds","scrape_samples_post_metric_relabeling","scrape_samples_scraped","scrape_series_added","target_info","up"]} \ No newline at end of file diff --git a/evidence/current/dev-pods.txt b/evidence/current/dev-pods.txt new file mode 100644 index 0000000..ddcb784 --- /dev/null +++ b/evidence/current/dev-pods.txt @@ -0,0 +1,15 @@ +NAME READY RESTARTS +api-c99bb859b-hnv8w true 0 +api-c99bb859b-jnfg7 true 1 +broker-0 true 0 +messenger-collector-0 true 0 +messenger-grafana-0 true 1 +messenger-loki-0 true 0 +messenger-prometheus-0 true 0 +messenger-tempo-0 true 0 +postgres-0 true 0 +s3-0 true 0 +valkey-0 true 0 +valkey-1 true 1 +valkey-2 true 0 +worker-55bb489c77-74xsw true 0 diff --git a/evidence/current/dev-prometheus-targets.json b/evidence/current/dev-prometheus-targets.json new file mode 100644 index 0000000..82e8094 --- /dev/null +++ b/evidence/current/dev-prometheus-targets.json @@ -0,0 +1,12 @@ +[ + { + "job": "collector", + "health": "up", + "lastError": "" + }, + { + "job": "messenger", + "health": "up", + "lastError": "" + } +] diff --git a/evidence/current/dev-secrets.txt b/evidence/current/dev-secrets.txt new file mode 100644 index 0000000..d5a0ce0 --- /dev/null +++ b/evidence/current/dev-secrets.txt @@ -0,0 +1,4 @@ +KUBE_CONTEXT=dev NAMESPACE=highload-applications scripts/init-secrets.sh +namespace/highload-applications unchanged +secret/registry-pull unchanged +secret/messenger-grafana created diff --git a/evidence/current/dev-smoke.txt b/evidence/current/dev-smoke.txt new file mode 100644 index 0000000..88029fe --- /dev/null +++ b/evidence/current/dev-smoke.txt @@ -0,0 +1,2 @@ +python3 tests/smoke.py --base-url "https://hl.agapov.space" +PASS: lab6 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/current/dev-tempo-health.txt b/evidence/current/dev-tempo-health.txt new file mode 100644 index 0000000..395527d --- /dev/null +++ b/evidence/current/dev-tempo-health.txt @@ -0,0 +1 @@ +ready diff --git a/evidence/current/error-traces.json b/evidence/current/error-traces.json new file mode 100644 index 0000000..e8a87cc --- /dev/null +++ b/evidence/current/error-traces.json @@ -0,0 +1 @@ +{"traces":[{"traceID":"2e8f40ac99dbbf5ff403817df627ef13","rootServiceName":"messenger-api","rootTraceName":"GET /api/v1","startTimeUnixNano":"1793284547290694000","durationMs":1006,"spanSet":{"spans":[{"spanID":"9ae7b5074b122067","startTimeUnixNano":"1793284547292826000","durationNanos":"1000533458","attributes":[{"key":"status","value":{"stringValue":"error"}}]},{"spanID":"578f5042301fe01d","startTimeUnixNano":"1793284547290694000","durationNanos":"1006177166","attributes":[{"key":"status","value":{"stringValue":"error"}}]}],"matched":2},"spanSets":[{"spans":[{"spanID":"9ae7b5074b122067","startTimeUnixNano":"1793284547292826000","durationNanos":"1000533458","attributes":[{"key":"status","value":{"stringValue":"error"}}]},{"spanID":"578f5042301fe01d","startTimeUnixNano":"1793284547290694000","durationNanos":"1006177166","attributes":[{"key":"status","value":{"stringValue":"error"}}]}],"matched":2}],"serviceStats":{"messenger-api":{"spanCount":2,"errorCount":2}}},{"traceID":"7d6e8b7a786d5e7de16ea79029e243e1","rootServiceName":"messenger-api","rootTraceName":"postgres.acquire","startTimeUnixNano":"1793284544649754000","durationMs":251,"spanSet":{"spans":[{"spanID":"a59b7a04b5dfee44","startTimeUnixNano":"1793284544649754000","durationNanos":"251038167","attributes":[{"key":"status","value":{"stringValue":"error"}}]}],"matched":1},"spanSets":[{"spans":[{"spanID":"a59b7a04b5dfee44","startTimeUnixNano":"1793284544649754000","durationNanos":"251038167","attributes":[{"key":"status","value":{"stringValue":"error"}}]}],"matched":1}],"serviceStats":{"messenger-api":{"spanCount":1,"errorCount":1}}},{"traceID":"40199dbe36c3f62d071365f655735562","rootServiceName":"messenger-worker","rootTraceName":"postgres.acquire","startTimeUnixNano":"1793284544649567000","durationMs":251,"spanSet":{"spans":[{"spanID":"2f227dfd1b877abb","startTimeUnixNano":"1793284544649567000","durationNanos":"251167250","attributes":[{"key":"status","value":{"stringValue":"error"}}]}],"matched":1},"spanSets":[{"spans":[{"spanID":"2f227dfd1b877abb","startTimeUnixNano":"1793284544649567000","durationNanos":"251167250","attributes":[{"key":"status","value":{"stringValue":"error"}}]}],"matched":1}],"serviceStats":{"messenger-worker":{"spanCount":1,"errorCount":1}}},{"traceID":"e7201df61dbee6fb4ab1681a97d7cc7","rootServiceName":"messenger-api","rootTraceName":"GET /api/v1","startTimeUnixNano":"1793284544160672000","durationMs":1002,"spanSet":{"spans":[{"spanID":"98914b3724af3daf","startTimeUnixNano":"1793284544161691000","durationNanos":"1000283292","attributes":[{"key":"status","value":{"stringValue":"error"}}]},{"spanID":"dbf75536baba25d6","startTimeUnixNano":"1793284544160672000","durationNanos":"1002449708","attributes":[{"key":"status","value":{"stringValue":"error"}}]}],"matched":2},"spanSets":[{"spans":[{"spanID":"98914b3724af3daf","startTimeUnixNano":"1793284544161691000","durationNanos":"1000283292","attributes":[{"key":"status","value":{"stringValue":"error"}}]},{"spanID":"dbf75536baba25d6","startTimeUnixNano":"1793284544160672000","durationNanos":"1002449708","attributes":[{"key":"status","value":{"stringValue":"error"}}]}],"matched":2}],"serviceStats":{"messenger-api":{"spanCount":2,"errorCount":2}}},{"traceID":"8e7889cbde8f15e08636055ed8ab655b","rootServiceName":"messenger-api","rootTraceName":"GET /api/v1","startTimeUnixNano":"1793284544160324000","durationMs":1002,"spanSet":{"spans":[{"spanID":"e3391c46671af98a","startTimeUnixNano":"1793284544161690000","durationNanos":"1000239833","attributes":[{"key":"status","value":{"stringValue":"error"}}]},{"spanID":"60e5464f3f9924ba","startTimeUnixNano":"1793284544160324000","durationNanos":"1002764375","attributes":[{"key":"status","value":{"stringValue":"error"}}]}],"matched":2},"spanSets":[{"spans":[{"spanID":"e3391c46671af98a","startTimeUnixNano":"1793284544161690000","durationNanos":"1000239833","attributes":[{"key":"status","value":{"stringValue":"error"}}]},{"spanID":"60e5464f3f9924ba","startTimeUnixNano":"1793284544160324000","durationNanos":"1002764375","attributes":[{"key":"status","value":{"stringValue":"error"}}]}],"matched":2}],"serviceStats":{"messenger-api":{"spanCount":2,"errorCount":2}}},{"traceID":"94303c7ee72e64ab5901eb8f442e1a60","rootServiceName":"messenger-worker","rootTraceName":"postgres.acquire","startTimeUnixNano":"1793284543466799000","durationMs":1000,"spanSet":{"spans":[{"spanID":"4869414df9b6bddc","startTimeUnixNano":"1793284543466799000","durationNanos":"1000498625","attributes":[{"key":"status","value":{"stringValue":"error"}}]}],"matched":1},"spanSets":[{"spans":[{"spanID":"4869414df9b6bddc","startTimeUnixNano":"1793284543466799000","durationNanos":"1000498625","attributes":[{"key":"status","value":{"stringValue":"error"}}]}],"matched":1}],"serviceStats":{"messenger-worker":{"spanCount":1,"errorCount":1}}},{"traceID":"306fd9d0865c2fe3f2efc74ddd39e374","rootServiceName":"messenger-worker","rootTraceName":"postgres.acquire","startTimeUnixNano":"1793284541436519000","durationMs":1001,"spanSet":{"spans":[{"spanID":"de79e4d3c07a09a5","startTimeUnixNano":"1793284541436519000","durationNanos":"1001161375","attributes":[{"key":"status","value":{"stringValue":"error"}}]}],"matched":1},"spanSets":[{"spans":[{"spanID":"de79e4d3c07a09a5","startTimeUnixNano":"1793284541436519000","durationNanos":"1001161375","attributes":[{"key":"status","value":{"stringValue":"error"}}]}],"matched":1}],"serviceStats":{"messenger-worker":{"spanCount":1,"errorCount":1}}},{"traceID":"70b95554890600a2046393544fcdb4aa","rootServiceName":"messenger-api","rootTraceName":"GET /api/v1","startTimeUnixNano":"1793284541089626000","durationMs":1002,"spanSet":{"spans":[{"spanID":"d147bfff4376a59b","startTimeUnixNano":"1793284541090529000","durationNanos":"1000184750","attributes":[{"key":"status","value":{"stringValue":"error"}}]},{"spanID":"a9c9741f44ee22eb","startTimeUnixNano":"1793284541089626000","durationNanos":"1002670916","attributes":[{"key":"status","value":{"stringValue":"error"}}]}],"matched":2},"spanSets":[{"spans":[{"spanID":"d147bfff4376a59b","startTimeUnixNano":"1793284541090529000","durationNanos":"1000184750","attributes":[{"key":"status","value":{"stringValue":"error"}}]},{"spanID":"a9c9741f44ee22eb","startTimeUnixNano":"1793284541089626000","durationNanos":"1002670916","attributes":[{"key":"status","value":{"stringValue":"error"}}]}],"matched":2}],"serviceStats":{"messenger-api":{"spanCount":2,"errorCount":2}}},{"traceID":"40abca6f464d84eeb787becc23521e08","rootServiceName":"messenger-worker","rootTraceName":"postgres.acquire","startTimeUnixNano":"1793284539412725000","durationMs":1001,"spanSet":{"spans":[{"spanID":"78cb675c04daab7a","startTimeUnixNano":"1793284539412725000","durationNanos":"1001161625","attributes":[{"key":"status","value":{"stringValue":"error"}}]}],"matched":1},"spanSets":[{"spans":[{"spanID":"78cb675c04daab7a","startTimeUnixNano":"1793284539412725000","durationNanos":"1001161625","attributes":[{"key":"status","value":{"stringValue":"error"}}]}],"matched":1}],"serviceStats":{"messenger-worker":{"spanCount":1,"errorCount":1}}},{"traceID":"794ae9cd3bab9de6b2c6dbb6192e08ae","rootServiceName":"messenger-api","rootTraceName":"GET /api/v1","startTimeUnixNano":"1793284538039184000","durationMs":1002,"spanSet":{"spans":[{"spanID":"32e9518dd106e69d","startTimeUnixNano":"1793284538039909000","durationNanos":"1000154000","attributes":[{"key":"status","value":{"stringValue":"error"}}]},{"spanID":"142bd754a907131d","startTimeUnixNano":"1793284538039184000","durationNanos":"1002136500","attributes":[{"key":"status","value":{"stringValue":"error"}}]}],"matched":2},"spanSets":[{"spans":[{"spanID":"32e9518dd106e69d","startTimeUnixNano":"1793284538039909000","durationNanos":"1000154000","attributes":[{"key":"status","value":{"stringValue":"error"}}]},{"spanID":"142bd754a907131d","startTimeUnixNano":"1793284538039184000","durationNanos":"1002136500","attributes":[{"key":"status","value":{"stringValue":"error"}}]}],"matched":2}],"serviceStats":{"messenger-api":{"spanCount":2,"errorCount":2}}}],"metrics":{"inspectedBytes":"320391","completedJobs":1,"totalJobs":1}} \ No newline at end of file diff --git a/evidence/current/grafana-dashboard.json b/evidence/current/grafana-dashboard.json new file mode 100644 index 0000000..847eb3b --- /dev/null +++ b/evidence/current/grafana-dashboard.json @@ -0,0 +1,4 @@ +{ + "title": "Messenger: API, worker and SLO", + "panels": 14 +} diff --git a/evidence/current/grafana-datasources.jsonl b/evidence/current/grafana-datasources.jsonl new file mode 100644 index 0000000..4b003d2 --- /dev/null +++ b/evidence/current/grafana-datasources.jsonl @@ -0,0 +1,2 @@ +{"uid":"prometheus","result":{"details":{"application":"Prometheus","features":{"rulerApiEnabled":false}},"message":"Successfully queried the Prometheus API.","status":"OK"}} +{"uid":"loki","result":{"message":"Data source successfully connected.","status":"OK"}} diff --git a/evidence/current/grafana-tempo-health.txt b/evidence/current/grafana-tempo-health.txt new file mode 100644 index 0000000..e69de29 diff --git a/evidence/current/grafana-tempo.json b/evidence/current/grafana-tempo.json new file mode 100644 index 0000000..09f4a26 --- /dev/null +++ b/evidence/current/grafana-tempo.json @@ -0,0 +1,5 @@ +{ + "uid": "tempo", + "type": "tempo", + "url": "http://messenger-tempo:3200" +} diff --git a/evidence/current/image-build.txt b/evidence/current/image-build.txt new file mode 100644 index 0000000..94b9040 --- /dev/null +++ b/evidence/current/image-build.txt @@ -0,0 +1,78 @@ +#0 building with "asisti-builder" instance using docker-container driver + +#1 [internal] load build definition from Dockerfile +#1 transferring dockerfile: 640B done +#1 DONE 0.0s + +#2 [internal] load metadata for docker.io/library/golang:1.26.1-alpine3.23 +#2 ... + +#3 [internal] load metadata for docker.io/library/alpine:3.23 +#3 DONE 1.9s + +#2 [internal] load metadata for docker.io/library/golang:1.26.1-alpine3.23 +#2 DONE 2.0s + +#4 [internal] load .dockerignore +#4 transferring context: 145B done +#4 DONE 0.0s + +#5 [build 1/7] FROM docker.io/library/golang:1.26.1-alpine3.23@sha256:2389ebfa5b7f43eeafbd6be0c3700cc46690ef842ad962f6c5bd6be49ed82039 +#5 resolve docker.io/library/golang:1.26.1-alpine3.23@sha256:2389ebfa5b7f43eeafbd6be0c3700cc46690ef842ad962f6c5bd6be49ed82039 done +#5 DONE 0.0s + +#6 [runtime 1/3] FROM docker.io/library/alpine:3.23@sha256:ff7f0c727dd796aa36787bfcd9a64bf35a6ce72a73451b3359fde849b6703d63 +#6 resolve docker.io/library/alpine:3.23@sha256:ff7f0c727dd796aa36787bfcd9a64bf35a6ce72a73451b3359fde849b6703d63 done +#6 DONE 0.0s + +#7 [internal] load build context +#7 transferring context: 32.58kB 0.0s done +#7 DONE 0.0s + +#8 [build 2/7] WORKDIR /src +#8 CACHED + +#9 [build 3/7] COPY go.mod go.sum ./ +#9 CACHED + +#10 [build 4/7] RUN go mod download +#10 CACHED + +#11 [build 5/7] COPY cmd/ ./cmd/ +#11 DONE 0.0s + +#12 [build 6/7] COPY internal/ ./internal/ +#12 DONE 0.0s + +#13 [build 7/7] RUN CGO_ENABLED=0 go build -trimpath -ldflags="-s -w" -o /out/server ./cmd/server +#13 DONE 82.9s + +#14 [runtime 2/3] RUN apk add --no-cache ca-certificates libwebp-tools +#14 CACHED + +#15 [runtime 3/3] COPY --from=build /out/server /server +#15 DONE 0.1s + +#16 exporting to image +#16 exporting layers +#16 exporting layers 0.8s done +#16 exporting manifest sha256:2a623e438729aea6730100d852a47325f3cc8eb4bb776f8928db1229efced109 done +#16 exporting config sha256:8af675a4c84386022a49c32acdf57ef883c375af4b85ef28491e7826a4a38e78 done +#16 exporting attestation manifest sha256:f0d4b2173b833f4863f1302da3d0743df949f0d96468778f5c4ae5df8fc07f05 done +#16 exporting manifest list sha256:04a28492ad5645ed56c58d51a95214a758426adc2b0271fc184e8f204c3fa7cc done +#16 pushing layers +#16 ... + +#17 [auth] mtusi2026lab6:pull,push token for asisti.registry.twcstorage.ru +#17 DONE 0.0s + +#16 exporting to image +#16 pushing layers 15.4s done +#16 pushing manifest for asisti.registry.twcstorage.ru/mtusi2026lab6:solution-20261029@sha256:04a28492ad5645ed56c58d51a95214a758426adc2b0271fc184e8f204c3fa7cc +#16 pushing manifest for asisti.registry.twcstorage.ru/mtusi2026lab6:solution-20261029@sha256:04a28492ad5645ed56c58d51a95214a758426adc2b0271fc184e8f204c3fa7cc 2.1s done +#16 DONE 18.3s + +#18 resolving provenance for metadata file +#18 DONE 0.0s + +View build details: docker-desktop://dashboard/build/asisti-builder/asisti-builder0/qgdek5lit26j8suzndebmuxl0 diff --git a/evidence/current/image.json b/evidence/current/image.json new file mode 100644 index 0000000..d2b430d --- /dev/null +++ b/evidence/current/image.json @@ -0,0 +1,20 @@ +{ + "buildx.build.provenance": { + "buildType": "", + "invocation": { + "configSource": {}, + "parameters": {}, + "environment": { + "platform": "" + } + } + }, + "buildx.build.ref": "asisti-builder/asisti-builder0/qgdek5lit26j8suzndebmuxl0", + "containerimage.descriptor": { + "mediaType": "application/vnd.oci.image.index.v1+json", + "digest": "sha256:04a28492ad5645ed56c58d51a95214a758426adc2b0271fc184e8f204c3fa7cc", + "size": 855 + }, + "containerimage.digest": "sha256:04a28492ad5645ed56c58d51a95214a758426adc2b0271fc184e8f204c3fa7cc", + "image.name": "asisti.registry.twcstorage.ru/mtusi2026lab6:solution-20261029" +} \ No newline at end of file diff --git a/evidence/current/lint.txt b/evidence/current/lint.txt new file mode 100644 index 0000000..6a3ebaa --- /dev/null +++ b/evidence/current/lint.txt @@ -0,0 +1 @@ +0 issues. diff --git a/evidence/current/logs.json b/evidence/current/logs.json new file mode 100644 index 0000000..0624be6 --- /dev/null +++ b/evidence/current/logs.json @@ -0,0 +1 @@ +{"status":"success","data":{"resultType":"streams","result":[{"stream":{"deployment_environment_name":"lab","detected_level":"info","event":"postgres.query","flags":"1","observed_timestamp":"1793283657265646000","outcome":"ok","scope_name":"messenger","service_instance_id":"0166055c-46c4-44e5-b3e4-48cdbeb37dc7","service_name":"messenger-api","service_version":"lab6","severity_number":"9","span_id":"9b221cde2b667472","trace_id":"106b26219026f5ea027093816cabcd74"},"values":[["1793283657265645000","postgres.query"]]},{"stream":{"deployment_environment_name":"lab","detected_level":"info","event":"postgres.query","flags":"1","observed_timestamp":"1793283657265145000","outcome":"ok","scope_name":"messenger","service_instance_id":"0166055c-46c4-44e5-b3e4-48cdbeb37dc7","service_name":"messenger-api","service_version":"lab6","severity_number":"9","span_id":"95d9ce336ed73f99","trace_id":"106b26219026f5ea027093816cabcd74"},"values":[["1793283657265144000","postgres.query"]]},{"stream":{"deployment_environment_name":"lab","detected_level":"info","event":"postgres.query","flags":"1","observed_timestamp":"1793283657264584000","outcome":"ok","scope_name":"messenger","service_instance_id":"0166055c-46c4-44e5-b3e4-48cdbeb37dc7","service_name":"messenger-api","service_version":"lab6","severity_number":"9","span_id":"373fc1d833fff487","trace_id":"106b26219026f5ea027093816cabcd74"},"values":[["1793283657264584000","postgres.query"]]},{"stream":{"deployment_environment_name":"lab","detected_level":"info","event":"postgres.query","flags":"1","observed_timestamp":"1793283657263428000","outcome":"ok","scope_name":"messenger","service_instance_id":"0166055c-46c4-44e5-b3e4-48cdbeb37dc7","service_name":"messenger-api","service_version":"lab6","severity_number":"9","span_id":"2b050a5aa4950580","trace_id":"106b26219026f5ea027093816cabcd74"},"values":[["1793283657263428000","postgres.query"]]},{"stream":{"deployment_environment_name":"lab","detected_level":"info","event":"postgres.query","flags":"1","observed_timestamp":"1793283657263038000","outcome":"ok","scope_name":"messenger","service_instance_id":"0166055c-46c4-44e5-b3e4-48cdbeb37dc7","service_name":"messenger-api","service_version":"lab6","severity_number":"9","span_id":"182ac7074b53ebdf","trace_id":"106b26219026f5ea027093816cabcd74"},"values":[["1793283657263038000","postgres.query"]]}],"stats":{"summary":{"bytesProcessedPerSecond":651053,"linesProcessedPerSecond":6346,"totalBytesProcessed":18362,"totalLinesProcessed":179,"execTime":0.028204,"queueTime":0.000286,"subqueries":0,"totalEntriesReturned":5,"splits":1,"shards":1,"totalPostFilterLines":179,"totalStructuredMetadataBytesProcessed":15960},"querier":{"store":{"totalChunksRef":0,"totalChunksDownloaded":0,"chunksDownloadTime":0,"queryReferencedStructuredMetadata":false,"chunk":{"headChunkBytes":0,"headChunkLines":0,"decompressedBytes":0,"decompressedLines":0,"compressedBytes":0,"totalDuplicates":0,"postFilterLines":0,"headChunkStructuredMetadataBytes":0,"decompressedStructuredMetadataBytes":0},"chunkRefsFetchTime":0,"congestionControlLatency":0,"pipelineWrapperFilteredLines":0}},"ingester":{"totalReached":1,"totalChunksMatched":2,"totalBatches":1,"totalLinesSent":5,"store":{"totalChunksRef":0,"totalChunksDownloaded":0,"chunksDownloadTime":0,"queryReferencedStructuredMetadata":false,"chunk":{"headChunkBytes":18362,"headChunkLines":179,"decompressedBytes":0,"decompressedLines":0,"compressedBytes":0,"totalDuplicates":0,"postFilterLines":179,"headChunkStructuredMetadataBytes":15960,"decompressedStructuredMetadataBytes":0},"chunkRefsFetchTime":120458,"congestionControlLatency":0,"pipelineWrapperFilteredLines":0}},"cache":{"chunk":{"entriesFound":0,"entriesRequested":0,"entriesStored":0,"bytesReceived":0,"bytesSent":0,"requests":0,"downloadTime":0,"queryLengthServed":0},"index":{"entriesFound":0,"entriesRequested":0,"entriesStored":0,"bytesReceived":0,"bytesSent":0,"requests":0,"downloadTime":0,"queryLengthServed":0},"result":{"entriesFound":0,"entriesRequested":0,"entriesStored":0,"bytesReceived":0,"bytesSent":0,"requests":0,"downloadTime":0,"queryLengthServed":0},"statsResult":{"entriesFound":0,"entriesRequested":0,"entriesStored":0,"bytesReceived":0,"bytesSent":0,"requests":0,"downloadTime":0,"queryLengthServed":0},"volumeResult":{"entriesFound":0,"entriesRequested":0,"entriesStored":0,"bytesReceived":0,"bytesSent":0,"requests":0,"downloadTime":0,"queryLengthServed":0},"seriesResult":{"entriesFound":0,"entriesRequested":0,"entriesStored":0,"bytesReceived":0,"bytesSent":0,"requests":0,"downloadTime":0,"queryLengthServed":0},"labelResult":{"entriesFound":0,"entriesRequested":0,"entriesStored":0,"bytesReceived":0,"bytesSent":0,"requests":0,"downloadTime":0,"queryLengthServed":0},"instantMetricResult":{"entriesFound":0,"entriesRequested":0,"entriesStored":0,"bytesReceived":0,"bytesSent":0,"requests":0,"downloadTime":0,"queryLengthServed":0}},"index":{"totalChunks":0,"postFilterChunks":0,"shardsDuration":0,"usedBloomFilters":false}}}} diff --git a/evidence/current/metric-names.json b/evidence/current/metric-names.json new file mode 100644 index 0000000..ce1c744 --- /dev/null +++ b/evidence/current/metric-names.json @@ -0,0 +1 @@ +{"status":"success","data":["ALERTS","ALERTS_FOR_STATE","messenger:api_error_ratio:1m","messenger:api_error_ratio:5m","messenger_db_pool_used","messenger_db_pool_waiting","messenger_http_duration_seconds_bucket","messenger_http_duration_seconds_count","messenger_http_duration_seconds_sum","messenger_http_requests_total","messenger_job_duration_seconds_bucket","messenger_job_duration_seconds_count","messenger_job_duration_seconds_sum","messenger_jobs_total","messenger_process_cpu_seconds_total","messenger_process_peak_rss_bytes","messenger_queue_depth","messenger_queue_oldest_seconds","messenger_ready_duration_seconds_bucket","messenger_ready_duration_seconds_count","messenger_ready_duration_seconds_sum","messenger_worker_busy","messenger_worker_capacity","otel_scope_info","otelcol_exporter_queue_batch_send_size_bucket","otelcol_exporter_queue_batch_send_size_count","otelcol_exporter_queue_batch_send_size_sum","otelcol_exporter_queue_capacity","otelcol_exporter_queue_size","otelcol_exporter_send_failed_log_records_total","otelcol_exporter_send_failed_metric_points_total","otelcol_exporter_send_failed_spans_total","otelcol_exporter_sent_log_records_total","otelcol_exporter_sent_metric_points_total","otelcol_exporter_sent_spans_total","otelcol_process_cpu_seconds_total","otelcol_process_memory_rss_bytes","otelcol_process_runtime_heap_alloc_bytes","otelcol_process_runtime_total_alloc_bytes_total","otelcol_process_runtime_total_sys_memory_bytes","otelcol_process_uptime_seconds_total","otelcol_processor_accepted_log_records_total","otelcol_processor_accepted_metric_points_total","otelcol_processor_accepted_spans_total","otelcol_processor_batch_batch_send_size_bucket","otelcol_processor_batch_batch_send_size_count","otelcol_processor_batch_batch_send_size_sum","otelcol_processor_batch_batch_size_trigger_send_total","otelcol_processor_batch_metadata_cardinality","otelcol_processor_batch_timeout_trigger_send_total","otelcol_processor_incoming_items_total","otelcol_processor_outgoing_items_total","otelcol_receiver_accepted_log_records_total","otelcol_receiver_accepted_metric_points_total","otelcol_receiver_accepted_spans_total","otelcol_receiver_failed_log_records_total","otelcol_receiver_failed_metric_points_total","otelcol_receiver_failed_spans_total","otelcol_receiver_refused_log_records_total","otelcol_receiver_refused_metric_points_total","otelcol_receiver_refused_spans_total","promhttp_metric_handler_errors_total","scrape_duration_seconds","scrape_samples_post_metric_relabeling","scrape_samples_scraped","scrape_series_added","target_info","up"]} \ No newline at end of file diff --git a/evidence/current/native-binary.sha256 b/evidence/current/native-binary.sha256 new file mode 100644 index 0000000..3581dba --- /dev/null +++ b/evidence/current/native-binary.sha256 @@ -0,0 +1 @@ +f7c5f8c6bf2665596f7fbac265d988feb32689081ad0aa584e3114c302f1cf3a /private/tmp/lab6-server diff --git a/evidence/current/openapi-validation.txt b/evidence/current/openapi-validation.txt new file mode 100644 index 0000000..405e9cf --- /dev/null +++ b/evidence/current/openapi-validation.txt @@ -0,0 +1,2 @@ +/private/tmp/lab-contract-validation/bin/python scripts/validate_openapi.py +PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested. diff --git a/evidence/current/prometheus-targets.json b/evidence/current/prometheus-targets.json new file mode 100644 index 0000000..82e8094 --- /dev/null +++ b/evidence/current/prometheus-targets.json @@ -0,0 +1,12 @@ +[ + { + "job": "collector", + "health": "up", + "lastError": "" + }, + { + "job": "messenger", + "health": "up", + "lastError": "" + } +] diff --git a/evidence/current/retry-result.json b/evidence/current/retry-result.json new file mode 100644 index 0000000..709f21f --- /dev/null +++ b/evidence/current/retry-result.json @@ -0,0 +1,6 @@ +{ + "ready_observed_before_trace_query": true, + "failed_attempt_and_retry": true, + "trace_file": "retry-trace.json", + "verification_note": "Original JSON assertion expected numeric status; Tempo returned STATUS_CODE_ERROR. Corrected assertion verified saved real trace." +} diff --git a/evidence/current/retry-result.txt b/evidence/current/retry-result.txt new file mode 100644 index 0000000..a3b4812 --- /dev/null +++ b/evidence/current/retry-result.txt @@ -0,0 +1 @@ +PASS: ready polling completed; saved real trace contains failed consumer and successful retry (JSON enum assertion corrected) diff --git a/evidence/current/retry-trace.json b/evidence/current/retry-trace.json new file mode 100644 index 0000000..1bd80fd --- /dev/null +++ b/evidence/current/retry-trace.json @@ -0,0 +1 @@ +{"batches":[{"resource":{"attributes":[{"key":"deployment.environment.name","value":{"stringValue":"lab"}},{"key":"service.instance.id","value":{"stringValue":"99decb38-f999-4102-a2eb-53eaada58a9b"}},{"key":"service.version","value":{"stringValue":"lab6"}},{"key":"service.name","value":{"stringValue":"messenger-api"}}]},"scopeSpans":[{"scope":{"name":"messenger"},"spans":[{"traceId":"3JP/ew6FDBub24+5gQhU+g==","spanId":"x+8o7DtSlgw=","parentSpanId":"CTcF/A5bDAc=","name":"POST /api/v1/chats/:chat_id/attachments","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1793285097106191000","endTimeUnixNano":"1793285097119613750","attributes":[{"key":"http.route","value":{"stringValue":"/api/v1/chats/:chat_id/attachments"}},{"key":"http.request.method","value":{"stringValue":"POST"}},{"key":"http.response.status_code","value":{"intValue":"202"}}],"status":{}},{"traceId":"3JP/ew6FDBub24+5gQhU+g==","spanId":"rLXKai2Y9fk=","parentSpanId":"x+8o7DtSlgw=","name":"postgres.acquire","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793285097107047000","endTimeUnixNano":"1793285097107056041","status":{}},{"traceId":"3JP/ew6FDBub24+5gQhU+g==","spanId":"+RMytVGquKs=","parentSpanId":"x+8o7DtSlgw=","name":"postgres.query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793285097107061000","endTimeUnixNano":"1793285097107776958","status":{}},{"traceId":"3JP/ew6FDBub24+5gQhU+g==","spanId":"gQ2DWI02Wrk=","parentSpanId":"x+8o7DtSlgw=","name":"postgres.acquire","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793285097108145000","endTimeUnixNano":"1793285097108148125","status":{}},{"traceId":"3JP/ew6FDBub24+5gQhU+g==","spanId":"1gn8JTjCE6o=","parentSpanId":"x+8o7DtSlgw=","name":"postgres.query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793285097108151000","endTimeUnixNano":"1793285097108672750","status":{}},{"traceId":"3JP/ew6FDBub24+5gQhU+g==","spanId":"goiBblws3eE=","parentSpanId":"x+8o7DtSlgw=","name":"postgres.acquire","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793285097108684000","endTimeUnixNano":"1793285097108686958","status":{}},{"traceId":"3JP/ew6FDBub24+5gQhU+g==","spanId":"n72FO8Cfuek=","parentSpanId":"x+8o7DtSlgw=","name":"postgres.query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793285097108688000","endTimeUnixNano":"1793285097109136292","status":{}},{"traceId":"3JP/ew6FDBub24+5gQhU+g==","spanId":"T6d+fd8RGNc=","parentSpanId":"x+8o7DtSlgw=","name":"postgres.acquire","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793285097110428000","endTimeUnixNano":"1793285097110430542","status":{}},{"traceId":"3JP/ew6FDBub24+5gQhU+g==","spanId":"qV2/WUjher8=","parentSpanId":"x+8o7DtSlgw=","name":"postgres.query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793285097110433000","endTimeUnixNano":"1793285097111692667","status":{}},{"traceId":"3JP/ew6FDBub24+5gQhU+g==","spanId":"A4EsAHuIAVg=","parentSpanId":"x+8o7DtSlgw=","name":"s3.put","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793285097111698000","endTimeUnixNano":"1793285097116350625","status":{}},{"traceId":"3JP/ew6FDBub24+5gQhU+g==","spanId":"C5lMG9Wj11k=","parentSpanId":"x+8o7DtSlgw=","name":"postgres.acquire","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793285097116358000","endTimeUnixNano":"1793285097116361125","status":{}},{"traceId":"3JP/ew6FDBub24+5gQhU+g==","spanId":"ZRmw59Xzdxk=","parentSpanId":"x+8o7DtSlgw=","name":"postgres.query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793285097116362000","endTimeUnixNano":"1793285097116714458","status":{}},{"traceId":"3JP/ew6FDBub24+5gQhU+g==","spanId":"AUjouWS+zvs=","parentSpanId":"x+8o7DtSlgw=","name":"postgres.query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793285097116718000","endTimeUnixNano":"1793285097117457500","status":{}},{"traceId":"3JP/ew6FDBub24+5gQhU+g==","spanId":"uloYWpsOAwQ=","parentSpanId":"x+8o7DtSlgw=","name":"postgres.query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793285097117466000","endTimeUnixNano":"1793285097118316667","status":{}},{"traceId":"3JP/ew6FDBub24+5gQhU+g==","spanId":"njelhX/NcJc=","parentSpanId":"x+8o7DtSlgw=","name":"postgres.query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793285097118320000","endTimeUnixNano":"1793285097118868584","status":{}}]}]},{"resource":{"attributes":[{"key":"deployment.environment.name","value":{"stringValue":"lab"}},{"key":"service.instance.id","value":{"stringValue":"dae1c270-1026-4781-a7ca-fa88fd6d8ed1"}},{"key":"service.version","value":{"stringValue":"lab6"}},{"key":"service.name","value":{"stringValue":"messenger-worker"}}]},"scopeSpans":[{"scope":{"name":"messenger"},"spans":[{"traceId":"3JP/ew6FDBub24+5gQhU+g==","spanId":"r0iuUOoBKQU=","parentSpanId":"x+8o7DtSlgw=","name":"attachment.publish","kind":"SPAN_KIND_PRODUCER","startTimeUnixNano":"1793285097995727000","endTimeUnixNano":"1793285097998052000","status":{}},{"traceId":"3JP/ew6FDBub24+5gQhU+g==","spanId":"hINLqZLvp34=","parentSpanId":"cbADgKGCKeA=","name":"postgres.acquire","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793285098002201000","endTimeUnixNano":"1793285098002219167","status":{}},{"traceId":"3JP/ew6FDBub24+5gQhU+g==","spanId":"IfG6ey9XPXI=","parentSpanId":"cbADgKGCKeA=","name":"postgres.query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793285098002237000","endTimeUnixNano":"1793285098002750375","status":{}},{"traceId":"3JP/ew6FDBub24+5gQhU+g==","spanId":"WtQL/8kNDf8=","parentSpanId":"cbADgKGCKeA=","name":"postgres.query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793285098002769000","endTimeUnixNano":"1793285098003462792","status":{}},{"traceId":"3JP/ew6FDBub24+5gQhU+g==","spanId":"nFondGbMbJo=","parentSpanId":"cbADgKGCKeA=","name":"postgres.query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793285098003536000","endTimeUnixNano":"1793285098004354125","status":{}},{"traceId":"3JP/ew6FDBub24+5gQhU+g==","spanId":"aeC4npUWod4=","parentSpanId":"cbADgKGCKeA=","name":"postgres.query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793285098004372000","endTimeUnixNano":"1793285098004965667","status":{}},{"traceId":"3JP/ew6FDBub24+5gQhU+g==","spanId":"5kzRUhJOFbA=","parentSpanId":"cbADgKGCKeA=","name":"postgres.query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793285098004974000","endTimeUnixNano":"1793285098005624083","status":{}}]}]},{"resource":{"attributes":[{"key":"deployment.environment.name","value":{"stringValue":"lab"}},{"key":"service.instance.id","value":{"stringValue":"dae1c270-1026-4781-a7ca-fa88fd6d8ed1"}},{"key":"service.name","value":{"stringValue":"messenger-worker"}},{"key":"service.version","value":{"stringValue":"lab6"}}]},"scopeSpans":[{"scope":{"name":"messenger"},"spans":[{"traceId":"3JP/ew6FDBub24+5gQhU+g==","spanId":"cbADgKGCKeA=","parentSpanId":"r0iuUOoBKQU=","flags":768,"name":"attachment.process","kind":"SPAN_KIND_CONSUMER","startTimeUnixNano":"1793285098002182000","endTimeUnixNano":"1793285113014585542","status":{"message":"operation failed","code":"STATUS_CODE_ERROR"}},{"traceId":"3JP/ew6FDBub24+5gQhU+g==","spanId":"8DwifHbQowA=","parentSpanId":"cbADgKGCKeA=","flags":256,"name":"s3.get","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793285098005647000","endTimeUnixNano":"1793285113007060417","status":{"message":"operation failed","code":"STATUS_CODE_ERROR"}},{"traceId":"3JP/ew6FDBub24+5gQhU+g==","spanId":"GzQb95xUSqQ=","parentSpanId":"cbADgKGCKeA=","flags":256,"name":"postgres.acquire","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793285113007130000","endTimeUnixNano":"1793285113007784417","status":{}},{"traceId":"3JP/ew6FDBub24+5gQhU+g==","spanId":"SLXMYIKC6Yk=","parentSpanId":"cbADgKGCKeA=","flags":256,"name":"postgres.query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793285113007803000","endTimeUnixNano":"1793285113008340042","status":{}},{"traceId":"3JP/ew6FDBub24+5gQhU+g==","spanId":"wDyhfC7zBIk=","parentSpanId":"cbADgKGCKeA=","flags":256,"name":"postgres.query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793285113008357000","endTimeUnixNano":"1793285113009157042","status":{}},{"traceId":"3JP/ew6FDBub24+5gQhU+g==","spanId":"ouwzt4AdbxE=","parentSpanId":"cbADgKGCKeA=","flags":256,"name":"postgres.query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793285113009171000","endTimeUnixNano":"1793285113010449209","status":{}},{"traceId":"3JP/ew6FDBub24+5gQhU+g==","spanId":"Be/Qzso2Y4s=","parentSpanId":"cbADgKGCKeA=","flags":256,"name":"postgres.query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793285113010467000","endTimeUnixNano":"1793285113011937792","status":{}},{"traceId":"3JP/ew6FDBub24+5gQhU+g==","spanId":"UlcmPfIp4jo=","parentSpanId":"cbADgKGCKeA=","flags":256,"name":"postgres.query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793285113011953000","endTimeUnixNano":"1793285113012668541","status":{}}]}]},{"resource":{"attributes":[{"key":"deployment.environment.name","value":{"stringValue":"lab"}},{"key":"service.instance.id","value":{"stringValue":"dae1c270-1026-4781-a7ca-fa88fd6d8ed1"}},{"key":"service.name","value":{"stringValue":"messenger-worker"}},{"key":"service.version","value":{"stringValue":"lab6"}}]},"scopeSpans":[{"scope":{"name":"messenger"},"spans":[{"traceId":"3JP/ew6FDBub24+5gQhU+g==","spanId":"cCd94epOiOU=","parentSpanId":"x+8o7DtSlgw=","flags":768,"name":"attachment.publish","kind":"SPAN_KIND_PRODUCER","startTimeUnixNano":"1793285116061844000","endTimeUnixNano":"1793285116065377375","status":{}},{"traceId":"3JP/ew6FDBub24+5gQhU+g==","spanId":"fyhNouSPGY0=","parentSpanId":"jU9tgclxswU=","flags":256,"name":"postgres.acquire","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793285116071366000","endTimeUnixNano":"1793285116071380875","status":{}},{"traceId":"3JP/ew6FDBub24+5gQhU+g==","spanId":"3iIANDbphWI=","parentSpanId":"jU9tgclxswU=","flags":256,"name":"postgres.query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793285116071393000","endTimeUnixNano":"1793285116072279208","status":{}},{"traceId":"3JP/ew6FDBub24+5gQhU+g==","spanId":"FykKi8zXg/0=","parentSpanId":"jU9tgclxswU=","flags":256,"name":"postgres.query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793285116072295000","endTimeUnixNano":"1793285116073625250","status":{}},{"traceId":"3JP/ew6FDBub24+5gQhU+g==","spanId":"uPz4gD+0X4Y=","parentSpanId":"jU9tgclxswU=","flags":256,"name":"postgres.query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793285116073682000","endTimeUnixNano":"1793285116074948000","status":{}},{"traceId":"3JP/ew6FDBub24+5gQhU+g==","spanId":"Ryo7Bfg4h9s=","parentSpanId":"jU9tgclxswU=","flags":256,"name":"postgres.query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793285116074962000","endTimeUnixNano":"1793285116075981958","status":{}},{"traceId":"3JP/ew6FDBub24+5gQhU+g==","spanId":"pMdSM3LAVOE=","parentSpanId":"jU9tgclxswU=","flags":256,"name":"postgres.query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793285116075993000","endTimeUnixNano":"1793285116077408291","status":{}}]}]},{"resource":{"attributes":[{"key":"deployment.environment.name","value":{"stringValue":"lab"}},{"key":"service.instance.id","value":{"stringValue":"dae1c270-1026-4781-a7ca-fa88fd6d8ed1"}},{"key":"service.name","value":{"stringValue":"messenger-worker"}},{"key":"service.version","value":{"stringValue":"lab6"}}]},"scopeSpans":[{"scope":{"name":"messenger"},"spans":[{"traceId":"3JP/ew6FDBub24+5gQhU+g==","spanId":"jU9tgclxswU=","parentSpanId":"cCd94epOiOU=","flags":768,"name":"attachment.process","kind":"SPAN_KIND_CONSUMER","startTimeUnixNano":"1793285116071350000","endTimeUnixNano":"1793285117404425000","status":{}},{"traceId":"3JP/ew6FDBub24+5gQhU+g==","spanId":"MfGs+GJuldw=","parentSpanId":"jU9tgclxswU=","flags":256,"name":"s3.get","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793285116077435000","endTimeUnixNano":"1793285117365832750","status":{}},{"traceId":"3JP/ew6FDBub24+5gQhU+g==","spanId":"qjY7tnRxPGs=","parentSpanId":"jU9tgclxswU=","flags":256,"name":"s3.put","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793285117387690000","endTimeUnixNano":"1793285117394299542","status":{}},{"traceId":"3JP/ew6FDBub24+5gQhU+g==","spanId":"zxVY1CUozJU=","parentSpanId":"jU9tgclxswU=","flags":256,"name":"s3.put","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793285117394313000","endTimeUnixNano":"1793285117396047375","status":{}},{"traceId":"3JP/ew6FDBub24+5gQhU+g==","spanId":"zaQgS1Ou4L4=","parentSpanId":"jU9tgclxswU=","flags":256,"name":"s3.put","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793285117396056000","endTimeUnixNano":"1793285117397315666","status":{}},{"traceId":"3JP/ew6FDBub24+5gQhU+g==","spanId":"Yopx03G4AjA=","parentSpanId":"jU9tgclxswU=","flags":256,"name":"s3.put","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793285117397321000","endTimeUnixNano":"1793285117398901958","status":{}},{"traceId":"3JP/ew6FDBub24+5gQhU+g==","spanId":"tpoepgr3KKA=","parentSpanId":"jU9tgclxswU=","flags":256,"name":"postgres.acquire","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793285117398910000","endTimeUnixNano":"1793285117399373750","status":{}},{"traceId":"3JP/ew6FDBub24+5gQhU+g==","spanId":"M3otmalPebk=","parentSpanId":"jU9tgclxswU=","flags":256,"name":"postgres.query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793285117399378000","endTimeUnixNano":"1793285117399662500","status":{}},{"traceId":"3JP/ew6FDBub24+5gQhU+g==","spanId":"PPP0623qqD4=","parentSpanId":"jU9tgclxswU=","flags":256,"name":"postgres.query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793285117399667000","endTimeUnixNano":"1793285117400373834","status":{}},{"traceId":"3JP/ew6FDBub24+5gQhU+g==","spanId":"XrtemphirBM=","parentSpanId":"jU9tgclxswU=","flags":256,"name":"postgres.query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793285117400379000","endTimeUnixNano":"1793285117400986666","status":{}},{"traceId":"3JP/ew6FDBub24+5gQhU+g==","spanId":"q6WB8X5phls=","parentSpanId":"jU9tgclxswU=","flags":256,"name":"postgres.query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793285117400990000","endTimeUnixNano":"1793285117401459583","status":{}},{"traceId":"3JP/ew6FDBub24+5gQhU+g==","spanId":"yquRBB7eyOY=","parentSpanId":"jU9tgclxswU=","flags":256,"name":"postgres.query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793285117401462000","endTimeUnixNano":"1793285117402168542","status":{}}]}]}]} \ No newline at end of file diff --git a/evidence/current/smoke.txt b/evidence/current/smoke.txt new file mode 100644 index 0000000..92297e3 --- /dev/null +++ b/evidence/current/smoke.txt @@ -0,0 +1,2 @@ +python3 tests/smoke.py --base-url "http://127.0.0.1:18086" +PASS: lab6 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/current/source.sha256 b/evidence/current/source.sha256 new file mode 100644 index 0000000..3486ce3 --- /dev/null +++ b/evidence/current/source.sha256 @@ -0,0 +1,184 @@ +21a514acf6287bcc9c39b96e6ce22b5c58f7e0425f7cf796744f4547af11ff31 cmd/server/app.go +fcbe015008f384eafeaf017286e53085f02d8e762e5791032fb15a4f6f5f339c cmd/server/app_test.go +ebc7c7d4bb1a0ca63cf949ca349f4819b4e5672a3bb1d714fee90ab7ec48ceff cmd/server/commands.go +2308066b0bc2cb117adced1181a64d8b9998aa404d2198143000729a3b9138a1 cmd/server/healthcheck.go +3b91d936af082e60f0160d053ea79d8821e131fceb8133ae04342f6996d578a7 cmd/server/healthcheck_test.go +d4c33ca5a93583caffd63e78bef00a2af906c477e29f1ce89495ee1737219579 cmd/server/logger.go +c22d8cdf4ce31fde29945e967d3f025e280a7dc4d2022329d51be972d2a0f495 cmd/server/main.go +acc63500e1475752bc8f82472e6714d52bd30af62d82d07e44f0de3f117c22ac cmd/server/media.go +308a01f9e933419f245de0cf22262fd11ea535fc277d9add5fabceb1150e938f cmd/server/media_test.go +8332e76c999388078bc176da84fb38535b1105e32d8d4044cf6b701188476065 cmd/server/persistence.go +426bf36d68c37566a1fc998f8f682466b594aa60a253152ce0216c8df0b135b0 cmd/server/persistence_test.go +42848ffa729c73773b301ab27da96633ef5fe36fd4da2ea059bb622f49a5766f cmd/server/reconcile.go +e8e1d54f869025360853a5a73eabb3146914d4caf6b8d4f20e1a250e79058ea7 cmd/server/s3_bootstrap.go +c5c54cf34133c297c88a401f8183c8b5a269864323e24100ce0a3994e3a4f12f cmd/server/telemetry.go +9509f8dd616ad59db6c3808caf052edb81e5234df097906dc5082be206919c38 cmd/server/telemetry_test.go +aae78ee2a9c844684410dd6d727427d3437ec7b19f417b6e75dbafa203a86b51 cmd/server/worker.go +1ee98eda5b0ea42c96ef0f11e6762c1f228da7c681e63fa2765bc2d19069e5fd cmd/server/worker_test.go +41ef40897886c0878cbfe52f6716bdcb8433fb7eaddcfff298e1d54c55db423d internal/admission/distributed.go +a373c9539167a4b7c35d45334d35ffd4e9d124c1740c88c12094fededd9643b4 internal/admission/limiter.go +d52fb501e76ddb23c645c1aaa599e32da9d01e50064c9ca97ac0e320ffc7f0c8 internal/admission/limiter_test.go +8cb675a0bef8b1645071e4cb85b79a467620e99b375db5d8d3e16a54b4c5b5fa internal/config/config.go +0181673dcc4db2557783941aa8c5b3f61bc0ca6b5196d87e69df000346c40391 internal/config/config_test.go +1c853c0f771e2c75958db8158797f2287e06c8d9efe8856365b7b9c71b1747b0 internal/config/controller.go +027761b059b76898a84318ec1a8b43158d8bafd6e0b15b52efbe902b4d8c5ea3 internal/config/environment.go +3a69e20a00fa2ed816f344a990e9e0ef0077e1607d4207858b4b00e4ff5d1988 internal/config/logger.go +86cdf9e90b576f4484946bfd9605333fc52077f38d3a39c5d71ddca4cdeecda0 internal/config/memory.go +c6283fe310747e8297850b48c0a119c1d62db4903dd2c779c6296678563df188 internal/config/objectstore.go +4e57110e53843506d1dd7336feaa603538b425a6127988590fdfc9fe3e878323 internal/config/objectstore_test.go +28c61d23db4ca2ef7ed952451f465318865076d2393eaa94ff5d7bb339c4c06b internal/config/persistence.go +7377c7fc286e04292b7c82e4e286cadf409ce833079f794a501456024e73434a internal/config/persistence_test.go +a5289a9ee5c8d6b092c9719f7ccece0abbbc409ecedd187352718ec3a083bf90 internal/config/telemetry.go +58a9af253f4de710326bb3d2310459ae40568bfaa14378423a1ca883030b28e7 internal/config/telemetry_test.go +eb502257c4badca1e8a2d4eb17701937edcc2c45dd6d697d70abb1624f457e14 internal/config/worker.go +6c68b5aa8cd0bce1f56b968555dff10f53e452f9d977bd8f21e29155b2cf8442 internal/config/worker_test.go +af72da25fea9811c13a50c8133e9e8b35e070c5295069c40c0ae89cdc0aff19a internal/controller/fx.go +7b4f03ae4414de081409ff443813e7c854a132dea9bbbce5dc9354a6bf7f989c internal/controller/http/fiber/attachments.go +ecc9a69b9f5cb2f7e9be140ff2056fddef2d2094872fd3d09c166f739a3c96db internal/controller/http/fiber/attachments_test.go +a8fb7892f713529a9359b1c77202ef5fb9a2dd4d921ee9f1ec104da67ee1ad87 internal/controller/http/fiber/auth.go +2c314670f635fd5382a2599b1415636d55c524a3440bf4354a7d7304a2e3c20d internal/controller/http/fiber/auth_test.go +efdad000a8a0773983ead41c69bd94c6f020007505761f3d8ddddfa0e4ef6af7 internal/controller/http/fiber/body_limit_test.go +9c092ba7bca191f25c3ba9c9a5d1ca844126bee740a4c0262b776e947263ac68 internal/controller/http/fiber/chats_test.go +be356cf26c228958d51f5bafa4305d7325e81ab01065575acbc9b2d7a6ddd4bd internal/controller/http/fiber/configure.go +cc0fd1f1735385a5b09c836c4fbbf7da22e8343489cc9737f926bc38dcc1add9 internal/controller/http/fiber/controller.go +ad444092926bed08371ad730d41fee060879dcdc097bd2dafed1d203437c9fad internal/controller/http/fiber/controller_test.go +f5e9c3f8a8a22f51639a570e2ab2e626bde0bf29434dd6527d1da8f32f021cf0 internal/controller/http/fiber/decode.go +6ee07f110bdb29d8d2191ab8889ba60474b33a5b225739875c8a57d8434be15d internal/controller/http/fiber/dependencies.go +3351a019c43d0360ee0fd68afabb5200422d065ae7bd908b1c0a5c02a0c1ad41 internal/controller/http/fiber/error_codes.go +9d5da8d98ece2706e324cb2deca915e0811b6ad98f7aaac5805ca0f3cc0a1f9a internal/controller/http/fiber/error_registry.go +a490411aca9d3a6ddc730981cdc4e0f785172db6773db23357a05c51f7889ea3 internal/controller/http/fiber/errors.go +7040d44ac15888b0e8498a02bddf649c1789e109201f37455bffcd41ed90783c internal/controller/http/fiber/errors_test.go +8241d83c3a9a09107cab83537a6529a14d0f39cfbeab9a8bed24af005a8ba37c internal/controller/http/fiber/fixtures_auth_test.go +f514263df465c6c1e3df76590553b37f7e32330dee48add9fdfb194d9bcde28e internal/controller/http/fiber/handlers.go +d32a9a01a8b391f4d986e14947a25a0504f4363335ea1524df05e0dd8e9e61bd internal/controller/http/fiber/headers.go +29ca7d1f1207b2fbbd272062beadcfe869d007350b8d50cc2a909b0dbe5c08d1 internal/controller/http/fiber/health.go +f39cf7a0049a2ed39d6cc1fcc66b44656d9d6120914d0e973b032e12f3d0de08 internal/controller/http/fiber/lifecycle.go +496f50e5693456574614b8a47086a550018884adb37919ce82abf8c4e7e5254d internal/controller/http/fiber/lifecycle_test.go +7e67fc7c7d3076dc0927a9601871d38002d31d988c9e4c7abcac9c7a65eba327 internal/controller/http/fiber/mapping.go +e45c78489236e58f724b1e5b9a360934851ed3587345a0242f19d14d89d3b670 internal/controller/http/fiber/messages_test.go +54db82201697bdfcc65712afe61b650f8f0885fc65222dc4cc6d1c3bb07d82dd internal/controller/http/fiber/metrics.go +fe59de68ad07ae56c061b01a7946e848b3ee41a3005a3ae9fee1843d31aba5ab internal/controller/http/fiber/metrics_labels_test.go +4b2316357f3bdcb74281fd63a11b88aaebeac89a048a152fed6ddd9069793fa9 internal/controller/http/fiber/metrics_test.go +18d69e4e21f9b2231d75ce9ac6dfe0243d9aa2a5ebf51932380adcd0dfec2a3a internal/controller/http/fiber/middleware.go +b949cb7690fa561462fa254413b48e8d88d6346d1231fa88d1249b22f394deb4 internal/controller/http/fiber/middleware_test.go +3b36952bd2c08dfdbe04a3c1e0eeeb0b550edefee44b7816bfdb6a094787bda4 internal/controller/http/fiber/observation.go +c2af04ba4a1a9a1b5e6596ac5a924113bf37954b27d54037016dc1ca9ee48e77 internal/controller/http/fiber/public_errors.go +19fd03ee526e0551bbeaa61513286b267b2b252ad3eca5c86a3a5575e4200a5a internal/controller/http/fiber/public_errors_test.go +e248223517102e1eecdeba9a5e2d844c0afc14116a56e9a1d76c6afb755e63d1 internal/controller/http/fiber/routes.go +6bc72fdc23332e08e4dee8a5ed3f9110952208af86b529d5d4847ebcf00dd4c3 internal/controller/http/fiber/stream.go +0b3fad8fc87f561573bf4ee906a48a8fb317385ecc7600bdca3cc61992fa9253 internal/controller/http/fiber/users_test.go +24b6b06d6fab919cd2130697b44fd0d358265184dca0ca8362e0b4fcc2ae5a68 internal/controller/interface.go +678e197744fb2fc4b51715f51593b98761125bbaddbe4cb1b7032bd6b173ef04 internal/cursor/codec.go +6041f376a0f69a66ebf0ecacbd20d03d5d57590ae67585613256b0713e106fdf internal/cursor/codec_test.go +e205901cec2e66011ec764b70879b0eb0d28a78c4dea8913a9fb30c2d778fa8c internal/hashring/ring.go +606cdf84342b97a63eedfb9cd37aa7c99a489eee54641d8645be930a9ce6be0d internal/hashring/ring_test.go +7f1fed5f8276e19ed13e0576c633c11d213d8ad1af7e27179aea893948a795ad internal/imageprocessor/processor.go +143fb96568f494efba46ec658e7b0a88b874938bf3f1322c090e88dabd75bd99 internal/imageprocessor/processor_test.go +327ee894c8c9afa5bcf8236ed4c91befd66699c0ca33178e9690705764faf002 internal/media/orphans.go +90c319dc2ed9cc8193474ba84793094839d5c7a6ef19a103f4170da5558348fb internal/media/reconcile.go +0713380557ed6e83e4d21d94c738a239b4bf81597e543d4ff3a5277c2ca6bf53 internal/media/reconcile_test.go +3255403afcb9486d33c5bb45b5afa9c3c9b85596ce97f86ac2f224011cd8a860 internal/media/run_reconcile.go +2c27b800f8848425efa2acbe4134d9b70b4e10728a22136f4e4046806972a1ee internal/media/run_reconcile_test.go +e0e55fcaad09af39caf5d481014de408552d28b7925de8e9492aa208938e637d internal/media/service.go +c2994186863134502a3837fa3284029b7b0a51d50d4a5fce45b708177826f459 internal/media/service_test.go +3662f14499e4106d3148a79a001ab92cb0d31279516ccba9a524e6bcec2a16b4 internal/media/validation.go +b5671dfff93be81899e729d0bef63b928ab6a1bd5a84c06117fc2ed536d654bf internal/media/validation_test.go +801a6290037459a1f9d28669b4f3cdce5191317da8a48189027b7a507139fdc7 internal/model/dto/attachment.go +5e2536b8eba6bb92b751a7b817b6338755008857e3b53920d8da6fb67adeebdb internal/model/dto/models.go +896a694b0c217240343077d4287f4a6e447338c38571abbebcb3d16a519137fb internal/model/dto/session.go +d68827b0bd3e6df4037621b6941298d3923dc348bda5bf314a34861e7c34f04e internal/model/http/models.go +e2a8f39cf5ecff860bca6bceeda78779117ea122b7619621ef9a80c84eeb62cb internal/objectstore/client.go +fc0c8d446809dc4f57bddf9699a57413f5751184a3ec38c7b9ff598aa945faa7 internal/objectstore/client_test.go +5eb695bb582e545a3b43ba15e7a88b6bb70bdfd43155f2169defff2ffd41fd7c internal/objectstore/list.go +d11d878aa4b1e0e093ae39028e1cc77fbf1841036145d9ce44392b8465ebddf5 internal/objectstore/observation.go +c3f949d4056c9c3354ffb54a4fce7d95d49c1ab65780c9798a79042987ab1e43 internal/objectstore/observation_test.go +124a8a6393c3c23040b2fd7ec0dbc77b05b4a317e9b1ecddd6df366250db56a6 internal/observation/snapshot.go +3642e33cdb0ae1ca5ed0a88cc0b6d0748f836d45ba0712ae1368533310aa5a7a internal/observation/tracer.go +fe2df6990331573bbf0202ebc81c58263a7c16a29b713eefa0f0e852e4ee4a76 internal/password/password.go +1ad8dcb7d055fcb8da56d10e1089b47fe1b9cbf848381ded1a4048f8fd41278a internal/password/password_test.go +eb4a71e1d2cbc93d539770b425305b4826b98dd7770741c6677a50c0ec43b4f8 internal/pipeline/cleanup.go +e3e65b686b8ed366b548e0515535f1a3bde1ceadd70fcff22917bb3dd06cb6c5 internal/pipeline/cleanup_test.go +d844cf88188841f6e5ba393c48f79d393a70531eea9fedfce07da419c49bd366 internal/pipeline/observation.go +a0536beb0d1dec237f907e01b6c646fbcf269dcbe4b88ec09058eb6b41e04fa6 internal/pipeline/observation_test.go +559c98b3fa4d4ea4aa2a07480ccdea52b37cc0dbf0910aacef7589757effaa8c internal/pipeline/types.go +357355bc4420f9b4c8306967cdc468de6989ccb96d90d3a6ff5afa80f07c3259 internal/pipeline/worker.go +8817f62116976afcb0fae3b12fe0849325301280db9f9f89a26b1704804e760d internal/pipeline/worker_test.go +f584e498d5f902f2a8eb22bb91b0dfd7a86547797995e750356415002240a50b internal/service/auth.go +87a752eb08988aa0b76fa3c6e37e6180508d9bf774f3d9cec0cd825162094d34 internal/service/authentication/service.go +18c050293d21f5439327b2e887c46f8be15508737cddeefa6c862d9d9781586f internal/service/authentication/service_test.go +d3ef483a9fcc47c0a056c0ed9e06907f51688cbd0f87c5ce1b657b2f12fc714c internal/service/cache.go +1815bf20f8ebfa8f8f394f8aef5e3e7954ff7652a5c4f4d6b14ace4d57b99ee2 internal/service/cursor.go +a1ac40c39260c9babeaf8259985de5f84abe1ce80dc8ed9f26ad1b66b1ef35af internal/service/errors.go +58b49648c1b6e73971255cd9c5ae95853abd8bb44653ee2a912122c599ef23cb internal/service/interface.go +878415267a409244b5fa490ed09da3ed446ab5d45963f4b1c02f626fd5248614 internal/service/production/access.go +35be54339836d885f0acc60cee572a22d15802cd88ada5e52875cce625537975 internal/service/production/cache_test.go +e917208ece7e11d442c3c3ca7e33ac303a9e50b11288f2476390d601a5d68e41 internal/service/production/chats.go +cdb0653e4b842f4e07fca041c05b037dcd5832e8641da9ce66fd98138e6b7c7e internal/service/production/chats_page_test.go +b309cd0ea7c01f994dffb4bb311472f5afb09c21b2c68f87cb7267d9d5f2da65 internal/service/production/chats_test.go +f7767482299f26cc44cb8d2c2a6fd024f5caec4a2818fbde617fe99ba687a097 internal/service/production/creation.go +d002ded303f0aaf8acef41f5e5846e548c52e5b4e683d7cb986be38db4227bbd internal/service/production/errors.go +279da30df9ceaead23d0fc961d37570330e7e80d32f8300e74cb56d51d220fad internal/service/production/fixture_test.go +861263597b62be27cfbc98a5dec11fd348994cba95773405086ab39e4d18e223 internal/service/production/members.go +5ae4aae2622bad3673a5b0b4a460954a1f8ff38e337145dcfec678a401b490d4 internal/service/production/messages.go +a8fd43ed0499db68950fdc0acef7670a0b81517fdf6ca7f5a3cc2aaeff970d6d internal/service/production/messages_page_test.go +9ad9871d7d17ffaba8f3563d6495ac3e115f4d5901c253c9db0fd66ce19aa6a0 internal/service/production/messages_test.go +a19420fa45469f36a4a624b8f8a696b81af56cbfb94f28dcc26fc5940fc0aec7 internal/service/production/pages.go +f08e44678853b768efa5b307cd93d87c5d7559a9040b588c83c8a39bada51b41 internal/service/production/pages_test.go +2ab7be10ded3332e4bbfc0c825f5d355a68b2ef79385fc3af490ae3a11077859 internal/service/production/pagination.go +d57a1e5a75ba7c3d77406ad5c316b42e7907e48f7b99785a7e75af1ea13b97d9 internal/service/production/pagination_test.go +5f9009ad92a7e5ea5b7ed4aae7146a1805bdbb4ded66b57661dc1950a33b32e4 internal/service/production/service.go +ed4900e5120a58ceb4c792057f9e6fedcc0998f94c7fe1a9de12eb81198720aa internal/service/production/service_test.go +6e72b534a0ac2b9bfc7867826b5596da614eb1ef883b86b27fde5098bf818ca2 internal/service/production/users.go +9e579bf971ca9b321c354b52761a83f4e50a1f517261b5655b4c7fc47dfe8744 internal/service/production/users_test.go +19442e5931252a040559e34af83e8e22d47fa127677ed5dd273e0ba5bbf99ab0 internal/service/production/validation.go +f683306c33020d2403e91e93b0fc260ff1d3b3f1911425a7550384dfa60d9937 internal/store/interface.go +8d8c50b868c725c0303a3cbc9578f6ea2b27d1a229700d2f80fe91f0de8622f3 internal/store/memory/capacity.go +618eaf4e1c042d29abaa7df456d83a36f6b4dcc6e7a401071ec8489805dfb640 internal/store/memory/chats.go +63ae1aab707daf6e305c847b87f920d8c39d755188e3ac1cb2aad7fdce64dff8 internal/store/memory/chats_test.go +03d054b385ae51d160b6ce0441e86c2f6c4763724aa9e91228224cc88a12fdd2 internal/store/memory/messages.go +de31d43d34d4dfa8a4aeb5f057de623e52111a01680eb3b685d43ab07cd4f175 internal/store/memory/messages_test.go +0eb10952ceb798a5e6bacfd115eeaaed9cd37e8232a56c20d2c120cf457d3c5e internal/store/memory/ordering.go +e00a97d2b6d408bd8964fefd62f5a187a1de81c96f93637c0905a346b4beaa5b internal/store/memory/store.go +7dca28303c8c2d1847e5333d48b739fa6d4d1201d9577aff32fef522014b78c7 internal/store/memory/store_test.go +7ef5f5000ed30aac7e9e058d33ff4a75caa4b83040a59269ae1223081cab68af internal/store/memory/users.go +225c05e14e0c98908edaa3d3eea413d6d347b1a2a91f1732b30d2bf02a1c6b21 internal/store/memory/users_test.go +5de665ff2f33918a736a97859ff9df766087e8b5deb2e01dff5d35e115a1af19 internal/store/postgres/attachments.go +38d0e3a31c116d098a20ad871ff4f0fb046746bd9d94a8f6b166cb7236935617 internal/store/postgres/attachments_test.go +1a7f671d6c781a5f3b6a03ce80e38f87e8a11e7e4cd51fca57829be9a01811ff internal/store/postgres/chats.go +14af394f88b01701efa0b89dd079c60793eed1811d114a880dd8e5e283de6b04 internal/store/postgres/errors_test.go +0f1d4b91a9deb26f276235f3a069570b3962f279aeb5a8491d6384ad2fa942ca internal/store/postgres/idempotency.go +a88d27065e86501d47986f048bbd34fba8f2205872df2e08ef6a04ae01f98218 internal/store/postgres/idempotency_test.go +9b4445f552ad08923a59be3853fad2091ce62a4ab0f6b4875711505a4e6e5fe1 internal/store/postgres/jobs.go +40f78e3fcd8137af0e6da0416bb587678adc789ae9a9e68f6f4a18b7bd70381b internal/store/postgres/jobs_test.go +041454c30d1e1cc980567b6e11c8de0bd57cf9e0e8703d132a5172e7bc0ea901 internal/store/postgres/messages.go +8b8bf6a255cc48cee4b3e4adc94c23cf6955ecf5dfba689bba973dfccbd1b1cb internal/store/postgres/migrations/001_initial.sql +3678bb935ed8006d4bdc3e928231cfc5e8ba29e397d6b70122675933acd2a6f4 internal/store/postgres/migrations/002_idempotency.sql +448f0287ddd1e68914edcda4b39cc90daaac952a8e19a9e938ad78d27601b590 internal/store/postgres/migrations/003_attachments.sql +8d5cdffcfadd99564821fa6f7adad8d8a2d70bfaa21b630ded6dd6d06fbf2b71 internal/store/postgres/migrations/004_pipeline.sql +1336ef711833e50342534b7186c37748fae187faccd94b8c5d06035a37925b16 internal/store/postgres/migrations/005_retention.sql +7eebf916fdc3ee58b2d19098a835405aa95cb985d2c8c7f5934e1e64d1f2b9f9 internal/store/postgres/observability.go +f7e8f54ee60cc44e12760a09c8aa7f660f69c1fd8284548b6791630c49e895f8 internal/store/postgres/observability_test.go +9f8e3e77cf1f65b17496ec65041e1546211b402e670a601db49c6bf9edfab09a internal/store/postgres/pipeline_integration_test.go +a770985780d475f03ca9990522f02d9bcffaa69b7701128df11e0939b55848ba internal/store/postgres/reconcile.go +375914d765b4ae8e1ba25767bb6e96d6de8ce29a8787055f4242971cb0b5900d internal/store/postgres/reconcile_fixture_test.go +5b300bfc628e8b8d84094b97fdebde9d021f626321323961e87801af8766f950 internal/store/postgres/research_test.go +6ca456767aec350fe81af74deb82a04fcb8689896d11ed547d3855a01f1b57e7 internal/store/postgres/retention.go +889a63b51b29012f4ae5499d1c0577fe29f2f2959b05629f8e123715cfc1f75f internal/store/postgres/sessions.go +ac5a64c966681f8baf5dbbc1232c9f6a2a54ad137869acfe88052e7cb943cfd5 internal/store/postgres/store.go +aed9f7ae7b2145f14ca4199dc98d0c3aedd454462abf2e981afce5ca2b02e74a internal/store/postgres/store_test.go +15de54e4bea80c8523429e3ef5b22c49a2b36f48c9fa58c2b76bc0653a6eae23 internal/store/postgres/users.go +0b6655c6c7d7f60758b05b3d5fc6abfea4a7de3466f7fd56d1cac0b537cc76d5 internal/streamqueue/queue.go +95bbe1f4808c3f07264d6f05a83e2cc89e187417c769095dfb13978e86f62287 internal/streamqueue/queue_test.go +d99595e1f74406c5aa617c0ec062afb7f12f116bd86776e87cc9c9a436ff7eea internal/strictjson/decode.go +ea6273526572b4bfcf681ef2659ea2c1ec74bfedeb5e9bdbd83ebfabe0647bf9 internal/strictjson/decode_test.go +3830b93225ff8056993643fd404ab990ddb752fd80fb8ece6410e834b6023f5f internal/strictjson/optional_test.go +009d4d61832b93f1e014e1d567d9e1828efc023ec2c40204d96c5d8097c0551f internal/telemetry/metrics.go +ddba713ce3941cef9fddd4968d63359105e4f32366a87b469268a57d0da6364a internal/telemetry/postgres.go +7b973eadb1610369f4cc8d3dc2a4fa78d0732fb8677091c3c2db12d628868657 internal/telemetry/runtime.go +f09654814b133000304dac635603addbc4ea6fb8dea3a3546dc7263149ab6ddb internal/telemetry/runtime_test.go +4f92838ad08685b5470f9a35ec9c48a136de333ba8d2287b1384f0a35ddb1cb6 internal/telemetry/saturation.go +4f9ef25f02df303db401d99780888bac94ebd7601f8227be710acefd80f09987 internal/telemetry/spans.go +8cdb6f5e2e83341b624e9b14a52b73e17ec03b41dad620f6e3d39a3244930b4a internal/valkey/admission.go +6aa3f9dbf0e5f5e60a06245fefe4d5d139ab181a47ec0e2d2e88346158fd4359 internal/valkey/cache.go +b30fccec3b12374e76b93a2a76ffa7d9fc6a9d6a123793b8effb6b0da679a7d0 internal/valkey/client.go +461201233ffb38201633ee6fc75d4ecd963e3ef7cf18ca6e2fa0eb88ab69e395 internal/valkey/client_test.go diff --git a/evidence/current/static.txt b/evidence/current/static.txt new file mode 100644 index 0000000..e61e4f6 --- /dev/null +++ b/evidence/current/static.txt @@ -0,0 +1,8 @@ +python3 scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +helm lint charts/messenger +==> Linting charts/messenger +[INFO] Chart.yaml: icon is recommended + +1 chart(s) linted, 0 chart(s) failed +helm template lab2 charts/messenger >/dev/null diff --git a/evidence/current/telemetry-images.txt b/evidence/current/telemetry-images.txt new file mode 100644 index 0000000..e7e8fe9 --- /dev/null +++ b/evidence/current/telemetry-images.txt @@ -0,0 +1,78 @@ +0.135.0: Pulling from otel/opentelemetry-collector-contrib +eae93a3313d3: Pulling fs layer +e431bff99ece: Pulling fs layer +df85c9d0ca2e: Pulling fs layer +eae93a3313d3: Download complete +e431bff99ece: Download complete +df85c9d0ca2e: Download complete +Digest: sha256:89107a3a8f4636a396927edf7025bb9614b8da2d92f4cc3f43109e8d115736e2 +Status: Downloaded newer image for otel/opentelemetry-collector-contrib:0.135.0 +docker.io/otel/opentelemetry-collector-contrib:0.135.0 +v3.5.0: Pulling from prom/prometheus +Digest: sha256:63805ebb8d2b3920190daf1cb14a60871b16fd38bed42b857a3182bc621f4996 +Status: Image is up to date for prom/prometheus:v3.5.0 +docker.io/prom/prometheus:v3.5.0 +3.5.5: Pulling from grafana/loki +56ce5a7a0a8c: Pulling fs layer +eb8f5749650b: Pulling fs layer +d557676654e5: Pulling fs layer +f04daeaed6c9: Pulling fs layer +d82bc7a76a83: Pulling fs layer +29852e282777: Pulling fs layer +2e4cf50eeb92: Pulling fs layer +b40161cd83fc: Pulling fs layer +d858cbc252ad: Pulling fs layer +e1089d61b200: Pulling fs layer +38066e8d70f6: Pulling fs layer +3f4e2c586348: Pulling fs layer +1069fc2daed1: Pulling fs layer +49e5f77b20e6: Pulling fs layer +0f8b424aa0b9: Pulling fs layer +ffeccacd1a73: Pulling fs layer +17f3dd0d85e8: Pulling fs layer +eb8f5749650b: Download complete +d858cbc252ad: Download complete +1069fc2daed1: Download complete +2e4cf50eeb92: Download complete +b40161cd83fc: Download complete +f04daeaed6c9: Download complete +3f4e2c586348: Download complete +0f8b424aa0b9: Download complete +17f3dd0d85e8: Download complete +d557676654e5: Download complete +d82bc7a76a83: Download complete +e1089d61b200: Download complete +38066e8d70f6: Download complete +49e5f77b20e6: Download complete +56ce5a7a0a8c: Download complete +ffeccacd1a73: Download complete +29852e282777: Download complete +Digest: sha256:31628519045e7f28692a7ae73b4a3fd293dccb425585ed5d16ceea3b5c9592e6 +Status: Downloaded newer image for grafana/loki:3.5.5 +docker.io/grafana/loki:3.5.5 +2.8.2: Pulling from grafana/tempo +Digest: sha256:0ef775495967cd5d7a6b2e146b6ea695d624803c8db8349fb8ce4164f719f9b7 +Status: Image is up to date for grafana/tempo:2.8.2 +docker.io/grafana/tempo:2.8.2 +12.1.1: Pulling from grafana/grafana +464d91115c2e: Pulling fs layer +9f1d07f40523: Pulling fs layer +7f46d17c4b0f: Pulling fs layer +476716c10da1: Pulling fs layer +6b1ef42fa3bc: Pulling fs layer +c049b4b5b81d: Pulling fs layer +ded2bfdd8adf: Pulling fs layer +3540d00d3a8d: Pulling fs layer +4f4fb700ef54: Pulling fs layer +4f4fb700ef54: Already exists +9f1d07f40523: Download complete +7f46d17c4b0f: Download complete +476716c10da1: Download complete +ded2bfdd8adf: Download complete +3540d00d3a8d: Download complete +464d91115c2e: Download complete +6b1ef42fa3bc: Download complete +c049b4b5b81d: Download complete +Digest: sha256:a1701c2180249361737a99a01bc770db39381640e4d631825d38ff4535efa47d +Status: Downloaded newer image for grafana/grafana:12.1.1 +docker.io/grafana/grafana:12.1.1 diff --git a/evidence/current/telemetry-local.txt b/evidence/current/telemetry-local.txt new file mode 100644 index 0000000..97d4bb4 --- /dev/null +++ b/evidence/current/telemetry-local.txt @@ -0,0 +1,77 @@ +Unable to find image 'grafana/tempo:2.8.2' locally +2.8.2: Pulling from grafana/tempo +8ab2969f3376: Pulling fs layer +2e4cf50eeb92: Pulling fs layer +7b8a369ef91e: Pulling fs layer +d557676654e5: Pulling fs layer +eb8f5749650b: Pulling fs layer +9f16f7a29f29: Pulling fs layer +e644e3619489: Pulling fs layer +4e9f20d26c87: Pulling fs layer +ffeccacd1a73: Pulling fs layer +f7a549fe62c7: Pulling fs layer +e1089d61b200: Pulling fs layer +f80d0db27f51: Pulling fs layer +f1f6c2e50dbe: Pulling fs layer +3f4e2c586348: Pulling fs layer +0f8b424aa0b9: Pulling fs layer +d858cbc252ad: Pulling fs layer +d82bc7a76a83: Pulling fs layer +1069fc2daed1: Pulling fs layer +b40161cd83fc: Pulling fs layer +eb8f5749650b: Already exists +d82bc7a76a83: Download complete +2e4cf50eeb92: Download complete +0f8b424aa0b9: Download complete +1069fc2daed1: Download complete +3f4e2c586348: Download complete +e1089d61b200: Download complete +d858cbc252ad: Download complete +b40161cd83fc: Download complete +d557676654e5: Download complete +ffeccacd1a73: Download complete +4e9f20d26c87: Download complete +f7a549fe62c7: Download complete +f80d0db27f51: Download complete +9f16f7a29f29: Download complete +e644e3619489: Download complete +7b8a369ef91e: Download complete +8ab2969f3376: Download complete +f1f6c2e50dbe: Download complete +Digest: sha256:0ef775495967cd5d7a6b2e146b6ea695d624803c8db8349fb8ce4164f719f9b7 +Status: Downloaded newer image for grafana/tempo:2.8.2 +Unable to find image 'grafana/loki:3.5.5' locally +3.5.5: Pulling from grafana/loki +f04daeaed6c9: Pulling fs layer +29852e282777: Pulling fs layer +17f3dd0d85e8: Pulling fs layer +38066e8d70f6: Pulling fs layer +17f3dd0d85e8: Already exists +38066e8d70f6: Already exists +f04daeaed6c9: Already exists +29852e282777: Download complete +Digest: sha256:31628519045e7f28692a7ae73b4a3fd293dccb425585ed5d16ceea3b5c9592e6 +Status: Downloaded newer image for grafana/loki:3.5.5 +Unable to find image 'grafana/grafana:12.1.1' locally +12.1.1: Pulling from grafana/grafana +476716c10da1: Pulling fs layer +7f46d17c4b0f: Pulling fs layer +ded2bfdd8adf: Pulling fs layer +464d91115c2e: Pulling fs layer +4f4fb700ef54: Pulling fs layer +3540d00d3a8d: Pulling fs layer +c049b4b5b81d: Pulling fs layer +6b1ef42fa3bc: Pulling fs layer +9f1d07f40523: Pulling fs layer +4f4fb700ef54: Already exists +9f1d07f40523: Download complete +476716c10da1: Download complete +ded2bfdd8adf: Download complete +7f46d17c4b0f: Download complete +3540d00d3a8d: Download complete +464d91115c2e: Download complete +6b1ef42fa3bc: Download complete +c049b4b5b81d: Download complete +Digest: sha256:a1701c2180249361737a99a01bc770db39381640e4d631825d38ff4535efa47d +Status: Downloaded newer image for grafana/grafana:12.1.1 +Telemetry stack created; state: /private/tmp/lab6-observability.env diff --git a/evidence/current/telemetry-tests.txt b/evidence/current/telemetry-tests.txt new file mode 100644 index 0000000..811dde8 --- /dev/null +++ b/evidence/current/telemetry-tests.txt @@ -0,0 +1,4 @@ +ok gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/telemetry (cached) +ok gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/pipeline 1.787s +ok gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/objectstore 2.021s +ok gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/config (cached) diff --git a/evidence/current/tempo-readiness.txt b/evidence/current/tempo-readiness.txt new file mode 100644 index 0000000..395527d --- /dev/null +++ b/evidence/current/tempo-readiness.txt @@ -0,0 +1 @@ +ready diff --git a/evidence/current/traces.json b/evidence/current/traces.json new file mode 100644 index 0000000..9acdc43 --- /dev/null +++ b/evidence/current/traces.json @@ -0,0 +1 @@ +{"traces":[{"traceID":"251b904fc2df1ae84715482aee19d976","rootServiceName":"messenger-worker","rootTraceName":"postgres.query","startTimeUnixNano":"1793283625863699000","durationMs":1},{"traceID":"41b3bf1c6d071c25fc47398e4295fbc8","rootServiceName":"messenger-worker","rootTraceName":"postgres.query","startTimeUnixNano":"1793283623836906000","durationMs":1},{"traceID":"55e55bb35edc0fb3bb0bcb8ba27d4362","rootServiceName":"messenger-worker","rootTraceName":"postgres.query","startTimeUnixNano":"1793283622818854000","durationMs":3},{"traceID":"5017ac3b7f0591dd098e6ffea9a3ec4e","rootServiceName":"messenger-worker","rootTraceName":"postgres.query","startTimeUnixNano":"1793283620792636000","durationMs":2},{"traceID":"13832d220b5e286749725102cd23307f","rootServiceName":"messenger-worker","rootTraceName":"postgres.query","startTimeUnixNano":"1793283617693074000","durationMs":3}],"metrics":{"inspectedTraces":14,"inspectedBytes":"18220","completedJobs":1,"totalJobs":1}} \ No newline at end of file diff --git a/evidence/current/vet.txt b/evidence/current/vet.txt new file mode 100644 index 0000000..e69de29 diff --git a/evidence/incidents-summary.txt b/evidence/incidents-summary.txt new file mode 100644 index 0000000..a5b4354 --- /dev/null +++ b/evidence/incidents-summary.txt @@ -0,0 +1 @@ +Initial run exited 1: assertion required API firing during outage, but first firing occurred in db-recovery. Raw snapshots show firing and resolution. Ready selector was corrected and verified separately in evidence/ready-slo. diff --git a/evidence/incidents/alerts.jsonl b/evidence/incidents/alerts.jsonl new file mode 100644 index 0000000..d561214 --- /dev/null +++ b/evidence/incidents/alerts.jsonl @@ -0,0 +1,134 @@ +{"at":1793284209,"phase":"baseline","alerts":[]} +{"at":1793284214,"phase":"baseline","alerts":[]} +{"at":1793284219,"phase":"baseline","alerts":[]} +{"at":1793284224,"phase":"baseline","alerts":[]} +{"at":1793284229,"phase":"baseline","alerts":[]} +{"at":1793284234,"phase":"baseline","alerts":[]} +{"at":1793284239,"phase":"baseline","alerts":[]} +{"at":1793284244,"phase":"baseline","alerts":[]} +{"at":1793284250,"phase":"baseline","alerts":[]} +{"at":1793284255,"phase":"baseline","alerts":[]} +{"at":1793284260,"phase":"baseline","alerts":[]} +{"at":1793284265,"phase":"baseline","alerts":[]} +{"at":1793284270,"phase":"baseline","alerts":[]} +{"at":1793284275,"phase":"baseline","alerts":[]} +{"at":1793284280,"phase":"baseline","alerts":[]} +{"at":1793284285,"phase":"baseline","alerts":[]} +{"at":1793284290,"phase":"baseline","alerts":[]} +{"at":1793284295,"phase":"baseline","alerts":[]} +{"at":1793284300,"phase":"baseline","alerts":[]} +{"at":1793284305,"phase":"baseline","alerts":[]} +{"at":1793284310,"phase":"baseline","alerts":[]} +{"at":1793284315,"phase":"baseline","alerts":[]} +{"at":1793284320,"phase":"baseline","alerts":[]} +{"at":1793284326,"phase":"baseline","alerts":[]} +{"at":1793284331,"phase":"short-db-spike","alerts":[]} +{"at":1793284336,"phase":"spike-recovery","alerts":[]} +{"at":1793284341,"phase":"spike-recovery","alerts":[]} +{"at":1793284346,"phase":"spike-recovery","alerts":[]} +{"at":1793284351,"phase":"spike-recovery","alerts":[]} +{"at":1793284356,"phase":"spike-recovery","alerts":[]} +{"at":1793284361,"phase":"spike-recovery","alerts":[]} +{"at":1793284366,"phase":"spike-recovery","alerts":[]} +{"at":1793284371,"phase":"spike-recovery","alerts":[]} +{"at":1793284376,"phase":"spike-recovery","alerts":[]} +{"at":1793284382,"phase":"spike-recovery","alerts":[]} +{"at":1793284387,"phase":"spike-recovery","alerts":[]} +{"at":1793284392,"phase":"spike-recovery","alerts":[]} +{"at":1793284397,"phase":"spike-recovery","alerts":[]} +{"at":1793284402,"phase":"spike-recovery","alerts":[]} +{"at":1793284407,"phase":"spike-recovery","alerts":[]} +{"at":1793284412,"phase":"spike-recovery","alerts":[]} +{"at":1793284417,"phase":"spike-recovery","alerts":[]} +{"at":1793284422,"phase":"spike-recovery","alerts":[]} +{"at":1793284427,"phase":"sustained-db-outage","alerts":[]} +{"at":1793284432,"phase":"sustained-db-outage","alerts":[]} +{"at":1793284437,"phase":"sustained-db-outage","alerts":[]} +{"at":1793284442,"phase":"sustained-db-outage","alerts":[]} +{"at":1793284448,"phase":"sustained-db-outage","alerts":[]} +{"at":1793284453,"phase":"sustained-db-outage","alerts":[]} +{"at":1793284458,"phase":"sustained-db-outage","alerts":[]} +{"at":1793284463,"phase":"sustained-db-outage","alerts":[]} +{"at":1793284468,"phase":"sustained-db-outage","alerts":[]} +{"at":1793284473,"phase":"sustained-db-outage","alerts":[]} +{"at":1793284478,"phase":"sustained-db-outage","alerts":[]} +{"at":1793284483,"phase":"sustained-db-outage","alerts":[]} +{"at":1793284488,"phase":"sustained-db-outage","alerts":[]} +{"at":1793284493,"phase":"sustained-db-outage","alerts":[]} +{"at":1793284498,"phase":"sustained-db-outage","alerts":[]} +{"at":1793284503,"phase":"sustained-db-outage","alerts":[]} +{"at":1793284508,"phase":"sustained-db-outage","alerts":[]} +{"at":1793284513,"phase":"sustained-db-outage","alerts":[]} +{"at":1793284518,"phase":"sustained-db-outage","alerts":[]} +{"at":1793284523,"phase":"sustained-db-outage","alerts":[]} +{"at":1793284528,"phase":"sustained-db-outage","alerts":[]} +{"at":1793284533,"phase":"sustained-db-outage","alerts":[{"labels":{"alertname":"MessengerAPIBurnRate","severity":"warning"},"annotations":{"runbook":"REPORT.md#диагностика","summary":"API consumes the 99% availability budget at more than 5x in both windows"},"state":"pending","activeAt":"2026-10-29T14:35:31.842599208Z","value":"1e+00"}]} +{"at":1793284539,"phase":"sustained-db-outage","alerts":[{"labels":{"alertname":"MessengerAPIBurnRate","severity":"warning"},"annotations":{"runbook":"REPORT.md#диагностика","summary":"API consumes the 99% availability budget at more than 5x in both windows"},"state":"pending","activeAt":"2026-10-29T14:35:31.842599208Z","value":"1e+00"}]} +{"at":1793284544,"phase":"sustained-db-outage","alerts":[{"labels":{"alertname":"MessengerAPIBurnRate","severity":"warning"},"annotations":{"runbook":"REPORT.md#диагностика","summary":"API consumes the 99% availability budget at more than 5x in both windows"},"state":"pending","activeAt":"2026-10-29T14:35:31.842599208Z","value":"1e+00"}]} +{"at":1793284549,"phase":"db-recovery","alerts":[{"labels":{"alertname":"MessengerAPIBurnRate","severity":"warning"},"annotations":{"runbook":"REPORT.md#диагностика","summary":"API consumes the 99% availability budget at more than 5x in both windows"},"state":"pending","activeAt":"2026-10-29T14:35:31.842599208Z","value":"1e+00"}]} +{"at":1793284554,"phase":"db-recovery","alerts":[{"labels":{"alertname":"MessengerAPIBurnRate","severity":"warning"},"annotations":{"runbook":"REPORT.md#диагностика","summary":"API consumes the 99% availability budget at more than 5x in both windows"},"state":"pending","activeAt":"2026-10-29T14:35:31.842599208Z","value":"1e+00"}]} +{"at":1793284559,"phase":"db-recovery","alerts":[{"labels":{"alertname":"MessengerAPIBurnRate","severity":"warning"},"annotations":{"runbook":"REPORT.md#диагностика","summary":"API consumes the 99% availability budget at more than 5x in both windows"},"state":"pending","activeAt":"2026-10-29T14:35:31.842599208Z","value":"1e+00"}]} +{"at":1793284564,"phase":"db-recovery","alerts":[{"labels":{"alertname":"MessengerAPIBurnRate","severity":"warning"},"annotations":{"runbook":"REPORT.md#диагностика","summary":"API consumes the 99% availability budget at more than 5x in both windows"},"state":"firing","activeAt":"2026-10-29T14:35:31.842599208Z","value":"5.3125e-01"}]} +{"at":1793284569,"phase":"db-recovery","alerts":[{"labels":{"alertname":"MessengerAPIBurnRate","severity":"warning"},"annotations":{"runbook":"REPORT.md#диагностика","summary":"API consumes the 99% availability budget at more than 5x in both windows"},"state":"firing","activeAt":"2026-10-29T14:35:31.842599208Z","value":"3.333333333333333e-01"}]} +{"at":1793284574,"phase":"db-recovery","alerts":[{"labels":{"alertname":"MessengerAPIBurnRate","severity":"warning"},"annotations":{"runbook":"REPORT.md#диагностика","summary":"API consumes the 99% availability budget at more than 5x in both windows"},"state":"firing","activeAt":"2026-10-29T14:35:31.842599208Z","value":"2.2033898305084748e-01"}]} +{"at":1793284579,"phase":"db-recovery","alerts":[{"labels":{"alertname":"MessengerAPIBurnRate","severity":"warning"},"annotations":{"runbook":"REPORT.md#диагностика","summary":"API consumes the 99% availability budget at more than 5x in both windows"},"state":"firing","activeAt":"2026-10-29T14:35:31.842599208Z","value":"1.6216216216216214e-01"}]} +{"at":1793284585,"phase":"db-recovery","alerts":[{"labels":{"alertname":"MessengerAPIBurnRate","severity":"warning"},"annotations":{"runbook":"REPORT.md#диагностика","summary":"API consumes the 99% availability budget at more than 5x in both windows"},"state":"firing","activeAt":"2026-10-29T14:35:31.842599208Z","value":"1.1363636363636365e-01"}]} +{"at":1793284590,"phase":"db-recovery","alerts":[{"labels":{"alertname":"MessengerAPIBurnRate","severity":"warning"},"annotations":{"runbook":"REPORT.md#диагностика","summary":"API consumes the 99% availability budget at more than 5x in both windows"},"state":"firing","activeAt":"2026-10-29T14:35:31.842599208Z","value":"7.84313725490196e-02"}]} +{"at":1793284595,"phase":"db-recovery","alerts":[{"labels":{"alertname":"MessengerAPIBurnRate","severity":"warning"},"annotations":{"runbook":"REPORT.md#диагностика","summary":"API consumes the 99% availability budget at more than 5x in both windows"},"state":"firing","activeAt":"2026-10-29T14:35:31.842599208Z","value":"5.982905982905984e-02"}]} +{"at":1793284600,"phase":"db-recovery","alerts":[]} +{"at":1793284605,"phase":"db-recovery","alerts":[]} +{"at":1793284610,"phase":"db-recovery","alerts":[]} +{"at":1793284615,"phase":"db-recovery","alerts":[]} +{"at":1793284620,"phase":"db-recovery","alerts":[]} +{"at":1793284625,"phase":"db-recovery","alerts":[]} +{"at":1793284630,"phase":"db-recovery","alerts":[]} +{"at":1793284635,"phase":"db-recovery","alerts":[]} +{"at":1793284661,"phase":"worker-stopped","alerts":[]} +{"at":1793284667,"phase":"worker-stopped","alerts":[]} +{"at":1793284672,"phase":"worker-stopped","alerts":[]} +{"at":1793284677,"phase":"worker-stopped","alerts":[]} +{"at":1793284682,"phase":"worker-stopped","alerts":[]} +{"at":1793284687,"phase":"worker-stopped","alerts":[]} +{"at":1793284692,"phase":"worker-stopped","alerts":[]} +{"at":1793284697,"phase":"worker-stopped","alerts":[]} +{"at":1793284702,"phase":"worker-stopped","alerts":[]} +{"at":1793284707,"phase":"worker-stopped","alerts":[]} +{"at":1793284712,"phase":"worker-stopped","alerts":[{"labels":{"alertname":"MessengerQueueLag","severity":"warning"},"annotations":{"runbook":"REPORT.md#диагностика","summary":"Accepted work has been waiting for over 60 seconds"},"state":"pending","activeAt":"2026-10-29T14:38:31.842599208Z","value":"6.3833531e+01"}]} +{"at":1793284717,"phase":"worker-stopped","alerts":[{"labels":{"alertname":"MessengerQueueLag","severity":"warning"},"annotations":{"runbook":"REPORT.md#диагностика","summary":"Accepted work has been waiting for over 60 seconds"},"state":"pending","activeAt":"2026-10-29T14:38:31.842599208Z","value":"6.8832463e+01"}]} +{"at":1793284722,"phase":"worker-stopped","alerts":[{"labels":{"alertname":"MessengerQueueLag","severity":"warning"},"annotations":{"runbook":"REPORT.md#диагностика","summary":"Accepted work has been waiting for over 60 seconds"},"state":"pending","activeAt":"2026-10-29T14:38:31.842599208Z","value":"7.3832915e+01"}]} +{"at":1793284727,"phase":"worker-stopped","alerts":[{"labels":{"alertname":"MessengerQueueLag","severity":"warning"},"annotations":{"runbook":"REPORT.md#диагностика","summary":"Accepted work has been waiting for over 60 seconds"},"state":"firing","activeAt":"2026-10-29T14:38:31.842599208Z","value":"7.8833019e+01"}]} +{"at":1793284732,"phase":"worker-stopped","alerts":[{"labels":{"alertname":"MessengerQueueLag","severity":"warning"},"annotations":{"runbook":"REPORT.md#диагностика","summary":"Accepted work has been waiting for over 60 seconds"},"state":"firing","activeAt":"2026-10-29T14:38:31.842599208Z","value":"8.3833952e+01"}]} +{"at":1793284737,"phase":"worker-stopped","alerts":[{"labels":{"alertname":"MessengerQueueLag","severity":"warning"},"annotations":{"runbook":"REPORT.md#диагностика","summary":"Accepted work has been waiting for over 60 seconds"},"state":"firing","activeAt":"2026-10-29T14:38:31.842599208Z","value":"8.883363e+01"}]} +{"at":1793284742,"phase":"worker-stopped","alerts":[{"labels":{"alertname":"MessengerQueueLag","severity":"warning"},"annotations":{"runbook":"REPORT.md#диагностика","summary":"Accepted work has been waiting for over 60 seconds"},"state":"firing","activeAt":"2026-10-29T14:38:31.842599208Z","value":"9.3834666e+01"}]} +{"at":1793284747,"phase":"worker-stopped","alerts":[{"labels":{"alertname":"MessengerQueueLag","severity":"warning"},"annotations":{"runbook":"REPORT.md#диагностика","summary":"Accepted work has been waiting for over 60 seconds"},"state":"firing","activeAt":"2026-10-29T14:38:31.842599208Z","value":"9.8832359e+01"}]} +{"at":1793284753,"phase":"worker-recovery","alerts":[{"labels":{"alertname":"MessengerQueueLag","severity":"warning"},"annotations":{"runbook":"REPORT.md#диагностика","summary":"Accepted work has been waiting for over 60 seconds"},"state":"firing","activeAt":"2026-10-29T14:38:31.842599208Z","value":"1.03832518e+02"}]} +{"at":1793284758,"phase":"worker-recovery","alerts":[{"labels":{"alertname":"MessengerQueueLag","severity":"warning"},"annotations":{"runbook":"REPORT.md#диагностика","summary":"Accepted work has been waiting for over 60 seconds"},"state":"firing","activeAt":"2026-10-29T14:38:31.842599208Z","value":"1.12255325e+02"}]} +{"at":1793284763,"phase":"worker-recovery","alerts":[]} +{"at":1793284768,"phase":"worker-recovery","alerts":[]} +{"at":1793284773,"phase":"worker-recovery","alerts":[]} +{"at":1793284778,"phase":"worker-recovery","alerts":[]} +{"at":1793284783,"phase":"worker-recovery","alerts":[]} +{"at":1793284788,"phase":"worker-recovery","alerts":[]} +{"at":1793284793,"phase":"worker-recovery","alerts":[]} +{"at":1793284798,"phase":"worker-recovery","alerts":[]} +{"at":1793284803,"phase":"worker-recovery","alerts":[]} +{"at":1793284808,"phase":"worker-recovery","alerts":[]} +{"at":1793284813,"phase":"worker-recovery","alerts":[]} +{"at":1793284818,"phase":"worker-recovery","alerts":[]} +{"at":1793284823,"phase":"worker-recovery","alerts":[]} +{"at":1793284828,"phase":"worker-recovery","alerts":[]} +{"at":1793284833,"phase":"worker-recovery","alerts":[]} +{"at":1793284838,"phase":"worker-recovery","alerts":[]} +{"at":1793284844,"phase":"trace-backend-outage","alerts":[]} +{"at":1793284849,"phase":"trace-backend-outage","alerts":[]} +{"at":1793284854,"phase":"trace-backend-outage","alerts":[]} +{"at":1793284859,"phase":"trace-backend-outage","alerts":[]} +{"at":1793284864,"phase":"trace-backend-outage","alerts":[]} +{"at":1793284869,"phase":"trace-backend-outage","alerts":[]} +{"at":1793284874,"phase":"trace-backend-outage","alerts":[]} +{"at":1793284879,"phase":"trace-backend-outage","alerts":[]} +{"at":1793284885,"phase":"trace-backend-outage","alerts":[{"labels":{"alertname":"MessengerTelemetryDrops","severity":"info"},"annotations":{"summary":"Trace backend delivery is failing; inspect bounded collector queues"},"state":"firing","activeAt":"2026-10-29T14:41:21.842599208Z","value":"1.456090181818182e+02"}]} +{"at":1793284893,"phase":"telemetry-recovery","alerts":[{"labels":{"alertname":"MessengerTelemetryDrops","severity":"info"},"annotations":{"summary":"Trace backend delivery is failing; inspect bounded collector queues"},"state":"firing","activeAt":"2026-10-29T14:41:21.842599208Z","value":"1.456118404305611e+02"}]} +{"at":1793284898,"phase":"telemetry-recovery","alerts":[{"labels":{"alertname":"MessengerTelemetryDrops","severity":"info"},"annotations":{"summary":"Trace backend delivery is failing; inspect bounded collector queues"},"state":"firing","activeAt":"2026-10-29T14:41:21.842599208Z","value":"1.4560619613832222e+02"}]} +{"at":1793284903,"phase":"telemetry-recovery","alerts":[{"labels":{"alertname":"MessengerTelemetryDrops","severity":"info"},"annotations":{"summary":"Trace backend delivery is failing; inspect bounded collector queues"},"state":"firing","activeAt":"2026-10-29T14:41:21.842599208Z","value":"1.4560884347557317e+02"}]} +{"at":1793284908,"phase":"telemetry-recovery","alerts":[{"labels":{"alertname":"MessengerTelemetryDrops","severity":"info"},"annotations":{"summary":"Trace backend delivery is failing; inspect bounded collector queues"},"state":"firing","activeAt":"2026-10-29T14:41:21.842599208Z","value":"1.4560884347557317e+02"}]} diff --git a/evidence/incidents/backend-resources.jsonl b/evidence/incidents/backend-resources.jsonl new file mode 100644 index 0000000..f81bc6e --- /dev/null +++ b/evidence/incidents/backend-resources.jsonl @@ -0,0 +1,3 @@ +{"BlockIO":"4.32MB / 0B","CPUPerc":"0.93%","Container":"e926c8bc38503a9892408725493cf0fd51f3cfbd2299061cd42dfa674fd61fcf","ID":"e926c8bc3850","MemPerc":"0.75%","MemUsage":"66.68MiB / 8.722GiB","Name":"agitated_zhukovsky","NetIO":"13.7MB / 5.67MB","PIDs":"10"} +{"BlockIO":"69.1MB / 0B","CPUPerc":"0.00%","Container":"35a42bc619d8c05701a2b17130474246fc9e91f17e89359f2eb1bc1b7d112706","ID":"35a42bc619d8","MemPerc":"0.47%","MemUsage":"42.39MiB / 8.722GiB","Name":"zen_torvalds","NetIO":"1.8MB / 365kB","PIDs":"10"} +{"BlockIO":"123kB / 0B","CPUPerc":"1.12%","Container":"0efe7da49496d94cdac84540fdaf76ae43bbd5ead087b0e0f7afa3ad90348601","ID":"0efe7da49496","MemPerc":"1.25%","MemUsage":"111.9MiB / 8.722GiB","Name":"determined_rubin","NetIO":"1.95MB / 228kB","PIDs":"10"} diff --git a/evidence/incidents/events.csv b/evidence/incidents/events.csv new file mode 100644 index 0000000..085541c --- /dev/null +++ b/evidence/incidents/events.csv @@ -0,0 +1,19 @@ +1793284209,baseline,start +1793284331,baseline,end +1793284331,short-db-spike,start +1793284336,short-db-spike,end +1793284336,spike-recovery,start +1793284427,spike-recovery,end +1793284427,sustained-db-outage,start +1793284549,sustained-db-outage,end +1793284549,db-recovery,start +1793284640,db-recovery,end +1793284640,worker-stopped,start +1793284661,worker-stopped,start +1793284753,worker-stopped,end +1793284753,worker-recovery,start +1793284844,worker-recovery,end +1793284844,trace-backend-outage,start +1793284890,trace-backend-outage,end +1793284893,telemetry-recovery,start +1793284913,telemetry-recovery,end diff --git a/evidence/incidents/failed-spans.json b/evidence/incidents/failed-spans.json new file mode 100644 index 0000000..652e264 --- /dev/null +++ b/evidence/incidents/failed-spans.json @@ -0,0 +1 @@ +{"status":"success","data":{"resultType":"vector","result":[{"metric":{},"value":[1793284947.183,"136"]}]}} \ No newline at end of file diff --git a/evidence/incidents/final-metrics.jsonl b/evidence/incidents/final-metrics.jsonl new file mode 100644 index 0000000..48d12ea --- /dev/null +++ b/evidence/incidents/final-metrics.jsonl @@ -0,0 +1,5 @@ +{"query":"max(messenger_queue_oldest_seconds)","response":{"status":"success","data":{"resultType":"vector","result":[{"metric":{},"value":[1793284913.594,"0"]}]}}} +{"query":"max(messenger_queue_depth)","response":{"status":"success","data":{"resultType":"vector","result":[{"metric":{},"value":[1793284913.612,"0"]}]}}} +{"query":"sum(rate(messenger_http_requests_total{status_class=\"5xx\"}[1m]))","response":{"status":"success","data":{"resultType":"vector","result":[{"metric":{},"value":[1793284913.629,"0"]}]}}} +{"query":"sum(otelcol_exporter_send_failed_spans_total)","response":{"status":"success","data":{"resultType":"vector","result":[{"metric":{},"value":[1793284913.645,"136"]}]}}} +{"query":"sum(otelcol_exporter_queue_size)","response":{"status":"success","data":{"resultType":"vector","result":[{"metric":{},"value":[1793284913.659,"0"]}]}}} diff --git a/evidence/incidents/http-1.csv b/evidence/incidents/http-1.csv new file mode 100644 index 0000000..34a896a --- /dev/null +++ b/evidence/incidents/http-1.csv @@ -0,0 +1,2164 @@ +1793284209,baseline,1,200,0.017845 +1793284209,baseline,1,200,0.010871 +1793284210,baseline,1,200,0.018580 +1793284210,baseline,1,200,0.023096 +1793284210,baseline,1,200,0.013653 +1793284211,baseline,1,200,0.012778 +1793284211,baseline,1,200,0.017679 +1793284211,baseline,1,200,0.008067 +1793284211,baseline,1,200,0.025911 +1793284212,baseline,1,200,0.010281 +1793284212,baseline,1,200,0.022408 +1793284212,baseline,1,200,0.009879 +1793284213,baseline,1,200,0.021690 +1793284213,baseline,1,200,0.013421 +1793284213,baseline,1,200,0.009614 +1793284214,baseline,1,200,0.019664 +1793284214,baseline,1,200,0.009794 +1793284214,baseline,1,200,0.011518 +1793284214,baseline,1,200,0.015347 +1793284215,baseline,1,200,0.017569 +1793284215,baseline,1,200,0.008668 +1793284215,baseline,1,200,0.011795 +1793284216,baseline,1,200,0.010287 +1793284216,baseline,1,200,0.009354 +1793284216,baseline,1,200,0.008371 +1793284217,baseline,1,200,0.007658 +1793284217,baseline,1,200,0.015026 +1793284217,baseline,1,200,0.025302 +1793284217,baseline,1,200,0.010719 +1793284218,baseline,1,200,0.018335 +1793284218,baseline,1,200,0.019581 +1793284218,baseline,1,200,0.016237 +1793284219,baseline,1,200,0.023007 +1793284219,baseline,1,200,0.022008 +1793284219,baseline,1,200,0.010530 +1793284220,baseline,1,200,0.018984 +1793284220,baseline,1,200,0.028997 +1793284220,baseline,1,200,0.015293 +1793284220,baseline,1,200,0.021042 +1793284221,baseline,1,200,0.009503 +1793284221,baseline,1,200,0.011717 +1793284221,baseline,1,200,0.015693 +1793284222,baseline,1,200,0.017038 +1793284222,baseline,1,200,0.010393 +1793284222,baseline,1,200,0.017894 +1793284223,baseline,1,200,0.019616 +1793284223,baseline,1,200,0.015710 +1793284223,baseline,1,200,0.018554 +1793284224,baseline,1,200,0.010377 +1793284224,baseline,1,200,0.014761 +1793284224,baseline,1,200,0.009896 +1793284224,baseline,1,200,0.012398 +1793284225,baseline,1,200,0.015228 +1793284225,baseline,1,200,0.011114 +1793284225,baseline,1,200,0.012905 +1793284226,baseline,1,200,0.011294 +1793284226,baseline,1,200,0.021949 +1793284226,baseline,1,200,0.013724 +1793284226,baseline,1,200,0.020676 +1793284227,baseline,1,200,0.018953 +1793284227,baseline,1,200,0.016288 +1793284227,baseline,1,200,0.015818 +1793284228,baseline,1,200,0.010293 +1793284228,baseline,1,200,0.014740 +1793284228,baseline,1,200,0.012793 +1793284229,baseline,1,200,0.018799 +1793284229,baseline,1,200,0.017965 +1793284229,baseline,1,200,0.015364 +1793284229,baseline,1,200,0.015234 +1793284230,baseline,1,200,0.013844 +1793284230,baseline,1,200,0.015040 +1793284230,baseline,1,200,0.014783 +1793284231,baseline,1,200,0.014139 +1793284231,baseline,1,200,0.013207 +1793284231,baseline,1,200,0.017004 +1793284232,baseline,1,200,0.015827 +1793284232,baseline,1,200,0.016584 +1793284232,baseline,1,200,0.006970 +1793284233,baseline,1,200,0.014843 +1793284233,baseline,1,200,0.016069 +1793284233,baseline,1,200,0.014388 +1793284233,baseline,1,200,0.013695 +1793284234,baseline,1,200,0.012466 +1793284234,baseline,1,200,0.018660 +1793284234,baseline,1,200,0.024050 +1793284235,baseline,1,200,0.005731 +1793284235,baseline,1,200,0.010335 +1793284235,baseline,1,200,0.011248 +1793284235,baseline,1,200,0.011525 +1793284236,baseline,1,200,0.025444 +1793284236,baseline,1,200,0.010841 +1793284236,baseline,1,200,0.011827 +1793284237,baseline,1,200,0.012878 +1793284237,baseline,1,200,0.011781 +1793284237,baseline,1,200,0.008074 +1793284238,baseline,1,200,0.006572 +1793284238,baseline,1,200,0.008415 +1793284238,baseline,1,200,0.012384 +1793284238,baseline,1,200,0.008845 +1793284239,baseline,1,200,0.013444 +1793284239,baseline,1,200,0.014685 +1793284239,baseline,1,200,0.008564 +1793284240,baseline,1,200,0.014092 +1793284240,baseline,1,200,0.007576 +1793284240,baseline,1,200,0.014403 +1793284241,baseline,1,200,0.014347 +1793284241,baseline,1,200,0.012970 +1793284241,baseline,1,200,0.029721 +1793284241,baseline,1,200,0.018323 +1793284242,baseline,1,200,0.014533 +1793284242,baseline,1,200,0.015516 +1793284242,baseline,1,200,0.015949 +1793284243,baseline,1,200,0.014705 +1793284243,baseline,1,200,0.011068 +1793284243,baseline,1,200,0.010371 +1793284244,baseline,1,200,0.016093 +1793284244,baseline,1,200,0.010986 +1793284244,baseline,1,200,0.016907 +1793284244,baseline,1,200,0.012351 +1793284245,baseline,1,200,0.010952 +1793284245,baseline,1,200,0.013925 +1793284245,baseline,1,200,0.011377 +1793284246,baseline,1,200,0.012222 +1793284246,baseline,1,200,0.017201 +1793284246,baseline,1,200,0.013842 +1793284247,baseline,1,200,0.010311 +1793284247,baseline,1,200,0.015300 +1793284247,baseline,1,200,0.012875 +1793284247,baseline,1,200,0.008377 +1793284248,baseline,1,200,0.016563 +1793284248,baseline,1,200,0.009717 +1793284248,baseline,1,200,0.018521 +1793284249,baseline,1,200,0.015967 +1793284249,baseline,1,200,0.013401 +1793284249,baseline,1,200,0.015283 +1793284250,baseline,1,200,0.015404 +1793284250,baseline,1,200,0.020482 +1793284250,baseline,1,200,0.017132 +1793284251,baseline,1,200,0.015697 +1793284251,baseline,1,200,0.019926 +1793284251,baseline,1,200,0.031494 +1793284251,baseline,1,200,0.016427 +1793284252,baseline,1,200,0.015987 +1793284252,baseline,1,200,0.011825 +1793284252,baseline,1,200,0.018978 +1793284253,baseline,1,200,0.006102 +1793284253,baseline,1,200,0.006362 +1793284253,baseline,1,200,0.005759 +1793284253,baseline,1,200,0.011293 +1793284254,baseline,1,200,0.006599 +1793284254,baseline,1,200,0.010690 +1793284254,baseline,1,200,0.007332 +1793284255,baseline,1,200,0.019288 +1793284255,baseline,1,200,0.021130 +1793284255,baseline,1,200,0.009785 +1793284256,baseline,1,200,0.009436 +1793284256,baseline,1,200,0.014313 +1793284256,baseline,1,200,0.011765 +1793284256,baseline,1,200,0.012215 +1793284257,baseline,1,200,0.006774 +1793284257,baseline,1,200,0.013518 +1793284257,baseline,1,200,0.013291 +1793284258,baseline,1,200,0.012559 +1793284258,baseline,1,200,0.009866 +1793284258,baseline,1,200,0.019453 +1793284259,baseline,1,200,0.016585 +1793284259,baseline,1,200,0.017740 +1793284259,baseline,1,200,0.017662 +1793284260,baseline,1,200,0.009789 +1793284260,baseline,1,200,0.015300 +1793284260,baseline,1,200,1.014168 +1793284262,baseline,1,200,0.012007 +1793284262,baseline,1,200,0.013215 +1793284262,baseline,1,200,0.011104 +1793284262,baseline,1,200,0.019696 +1793284263,baseline,1,200,0.015108 +1793284263,baseline,1,200,0.013829 +1793284263,baseline,1,200,0.012026 +1793284264,baseline,1,200,0.014369 +1793284264,baseline,1,200,0.010230 +1793284264,baseline,1,200,0.017852 +1793284265,baseline,1,200,0.006658 +1793284265,baseline,1,200,0.012054 +1793284265,baseline,1,200,0.009595 +1793284265,baseline,1,200,0.010295 +1793284266,baseline,1,200,0.010426 +1793284266,baseline,1,200,0.011341 +1793284266,baseline,1,200,0.010641 +1793284266,baseline,1,200,0.006863 +1793284267,baseline,1,200,0.011019 +1793284267,baseline,1,200,0.010997 +1793284267,baseline,1,200,0.013806 +1793284268,baseline,1,200,0.009475 +1793284268,baseline,1,200,0.010187 +1793284268,baseline,1,200,0.004864 +1793284269,baseline,1,200,0.009380 +1793284269,baseline,1,200,0.015188 +1793284269,baseline,1,200,0.010554 +1793284269,baseline,1,200,0.004109 +1793284270,baseline,1,200,0.004333 +1793284270,baseline,1,200,0.003996 +1793284270,baseline,1,200,0.007044 +1793284271,baseline,1,200,0.003566 +1793284271,baseline,1,200,0.005513 +1793284271,baseline,1,200,0.033575 +1793284271,baseline,1,200,0.009937 +1793284272,baseline,1,200,0.004092 +1793284272,baseline,1,200,0.005056 +1793284272,baseline,1,200,0.003995 +1793284273,baseline,1,200,0.013840 +1793284273,baseline,1,200,0.010062 +1793284273,baseline,1,200,0.012659 +1793284273,baseline,1,200,0.016654 +1793284274,baseline,1,200,0.005594 +1793284274,baseline,1,200,0.004978 +1793284274,baseline,1,200,0.012188 +1793284275,baseline,1,200,0.006536 +1793284275,baseline,1,200,0.016234 +1793284275,baseline,1,200,0.011273 +1793284276,baseline,1,200,0.013170 +1793284276,baseline,1,200,0.015747 +1793284276,baseline,1,200,0.009828 +1793284277,baseline,1,200,0.015114 +1793284277,baseline,1,200,0.013614 +1793284277,baseline,1,200,0.009527 +1793284278,baseline,1,200,0.010675 +1793284278,baseline,1,200,0.012305 +1793284278,baseline,1,200,0.013474 +1793284278,baseline,1,200,0.016652 +1793284279,baseline,1,200,0.013320 +1793284279,baseline,1,200,0.011912 +1793284279,baseline,1,200,0.007648 +1793284280,baseline,1,200,0.007810 +1793284280,baseline,1,200,0.035144 +1793284280,baseline,1,200,0.010718 +1793284281,baseline,1,200,0.009852 +1793284281,baseline,1,200,0.009601 +1793284281,baseline,1,200,0.013235 +1793284281,baseline,1,200,0.009932 +1793284282,baseline,1,200,0.014036 +1793284282,baseline,1,200,0.011099 +1793284282,baseline,1,200,0.012814 +1793284283,baseline,1,200,0.011572 +1793284283,baseline,1,200,0.008094 +1793284283,baseline,1,200,0.013423 +1793284284,baseline,1,200,0.014998 +1793284284,baseline,1,200,0.013356 +1793284284,baseline,1,200,0.008694 +1793284284,baseline,1,200,0.009613 +1793284285,baseline,1,200,0.016462 +1793284285,baseline,1,200,0.005385 +1793284285,baseline,1,200,0.007546 +1793284286,baseline,1,200,0.009136 +1793284286,baseline,1,200,0.022810 +1793284286,baseline,1,200,0.009287 +1793284286,baseline,1,200,0.006814 +1793284287,baseline,1,200,0.012733 +1793284287,baseline,1,200,0.007928 +1793284287,baseline,1,200,0.012977 +1793284288,baseline,1,200,0.008925 +1793284288,baseline,1,200,0.014430 +1793284288,baseline,1,200,0.014478 +1793284289,baseline,1,200,0.010780 +1793284289,baseline,1,200,0.007349 +1793284289,baseline,1,200,0.009224 +1793284289,baseline,1,200,0.011084 +1793284290,baseline,1,200,0.010568 +1793284290,baseline,1,200,0.007834 +1793284290,baseline,1,200,0.016867 +1793284291,baseline,1,200,0.011425 +1793284291,baseline,1,200,0.015651 +1793284291,baseline,1,200,0.013649 +1793284292,baseline,1,200,0.007562 +1793284292,baseline,1,200,0.008075 +1793284292,baseline,1,200,0.009552 +1793284292,baseline,1,200,0.013842 +1793284293,baseline,1,200,0.015427 +1793284293,baseline,1,200,0.012574 +1793284293,baseline,1,200,0.014648 +1793284294,baseline,1,200,0.014910 +1793284294,baseline,1,200,0.006171 +1793284294,baseline,1,200,0.007379 +1793284295,baseline,1,200,0.009574 +1793284295,baseline,1,200,0.013489 +1793284295,baseline,1,200,0.004400 +1793284295,baseline,1,200,0.013340 +1793284296,baseline,1,200,0.015746 +1793284296,baseline,1,200,0.011416 +1793284296,baseline,1,200,0.014976 +1793284297,baseline,1,200,0.017248 +1793284297,baseline,1,200,0.013267 +1793284297,baseline,1,200,0.007672 +1793284298,baseline,1,200,0.008880 +1793284298,baseline,1,200,0.013629 +1793284298,baseline,1,200,0.010483 +1793284299,baseline,1,200,0.011007 +1793284299,baseline,1,200,0.012853 +1793284299,baseline,1,200,0.013057 +1793284299,baseline,1,200,0.020779 +1793284300,baseline,1,200,0.006621 +1793284300,baseline,1,200,0.014095 +1793284300,baseline,1,200,0.008624 +1793284301,baseline,1,200,0.013985 +1793284301,baseline,1,200,0.015882 +1793284301,baseline,1,200,0.018344 +1793284302,baseline,1,200,0.018817 +1793284302,baseline,1,200,0.018404 +1793284302,baseline,1,200,0.010623 +1793284302,baseline,1,200,0.012114 +1793284303,baseline,1,200,0.011435 +1793284303,baseline,1,200,0.016791 +1793284303,baseline,1,200,0.010777 +1793284304,baseline,1,200,0.008641 +1793284304,baseline,1,200,0.013420 +1793284304,baseline,1,200,0.008797 +1793284305,baseline,1,200,0.013438 +1793284305,baseline,1,200,0.018079 +1793284305,baseline,1,200,0.014375 +1793284306,baseline,1,200,0.015807 +1793284306,baseline,1,200,0.015707 +1793284306,baseline,1,200,0.016121 +1793284306,baseline,1,200,0.014934 +1793284307,baseline,1,200,0.006719 +1793284307,baseline,1,200,1.017419 +1793284308,baseline,1,200,0.010418 +1793284309,baseline,1,200,0.013196 +1793284309,baseline,1,200,0.011627 +1793284309,baseline,1,200,0.005214 +1793284310,baseline,1,200,0.006921 +1793284310,baseline,1,200,0.005442 +1793284310,baseline,1,200,0.013785 +1793284310,baseline,1,200,0.008313 +1793284311,baseline,1,200,0.014551 +1793284311,baseline,1,200,0.014727 +1793284311,baseline,1,200,0.017248 +1793284312,baseline,1,200,0.010939 +1793284312,baseline,1,200,0.008560 +1793284312,baseline,1,200,0.010881 +1793284313,baseline,1,200,0.008502 +1793284313,baseline,1,200,0.007017 +1793284313,baseline,1,200,0.009962 +1793284313,baseline,1,200,0.008113 +1793284314,baseline,1,200,0.010254 +1793284314,baseline,1,200,1.012908 +1793284315,baseline,1,200,0.011998 +1793284316,baseline,1,200,0.006147 +1793284316,baseline,1,200,0.010836 +1793284316,baseline,1,200,0.013878 +1793284317,baseline,1,200,0.016923 +1793284317,baseline,1,200,0.017221 +1793284317,baseline,1,200,0.015011 +1793284317,baseline,1,200,0.013940 +1793284318,baseline,1,200,0.012721 +1793284318,baseline,1,200,0.013702 +1793284318,baseline,1,200,0.013865 +1793284319,baseline,1,200,0.013481 +1793284319,baseline,1,200,0.011307 +1793284319,baseline,1,200,0.013599 +1793284320,baseline,1,200,0.011507 +1793284320,baseline,1,200,0.012123 +1793284320,baseline,1,200,0.013040 +1793284321,baseline,1,200,0.012402 +1793284321,baseline,1,200,0.013329 +1793284321,baseline,1,200,0.005570 +1793284321,baseline,1,200,0.013759 +1793284322,baseline,1,200,0.013616 +1793284322,baseline,1,200,0.006867 +1793284322,baseline,1,200,0.009542 +1793284323,baseline,1,200,0.008324 +1793284323,baseline,1,200,0.008124 +1793284323,baseline,1,200,0.007990 +1793284323,baseline,1,200,0.014461 +1793284324,baseline,1,200,0.008573 +1793284324,baseline,1,200,0.006984 +1793284324,baseline,1,200,0.010936 +1793284325,baseline,1,200,0.009118 +1793284325,baseline,1,200,0.004715 +1793284325,baseline,1,200,0.010917 +1793284326,baseline,1,200,0.013067 +1793284326,baseline,1,200,0.011938 +1793284326,baseline,1,200,0.006913 +1793284326,baseline,1,200,0.016407 +1793284327,baseline,1,200,0.013533 +1793284327,baseline,1,200,0.017555 +1793284327,baseline,1,200,0.011519 +1793284328,baseline,1,200,0.015217 +1793284328,baseline,1,200,0.005664 +1793284328,baseline,1,200,0.010856 +1793284329,baseline,1,200,0.013862 +1793284329,baseline,1,200,0.013613 +1793284329,baseline,1,200,0.009523 +1793284329,baseline,1,200,0.009775 +1793284330,baseline,1,200,0.015116 +1793284330,baseline,1,200,0.009338 +1793284330,baseline,1,200,0.012435 +1793284331,baseline,1,503,2.015222 +1793284333,short-db-spike,1,502,0.003399 +1793284333,short-db-spike,1,502,0.002697 +1793284334,short-db-spike,1,502,1.010382 +1793284335,short-db-spike,1,502,0.002988 +1793284335,short-db-spike,1,502,0.001763 +1793284335,short-db-spike,1,502,0.001067 +1793284336,short-db-spike,1,502,0.002828 +1793284336,short-db-spike,1,502,0.001227 +1793284336,spike-recovery,1,502,0.004466 +1793284337,spike-recovery,1,200,0.028157 +1793284337,spike-recovery,1,200,0.009540 +1793284337,spike-recovery,1,200,0.007043 +1793284337,spike-recovery,1,200,0.009750 +1793284338,spike-recovery,1,200,0.009129 +1793284338,spike-recovery,1,200,0.010747 +1793284338,spike-recovery,1,200,0.011200 +1793284339,spike-recovery,1,200,0.010356 +1793284339,spike-recovery,1,200,0.005623 +1793284339,spike-recovery,1,200,0.006443 +1793284340,spike-recovery,1,200,0.016335 +1793284340,spike-recovery,1,200,0.014772 +1793284340,spike-recovery,1,200,0.028289 +1793284340,spike-recovery,1,200,0.006452 +1793284341,spike-recovery,1,200,0.016275 +1793284341,spike-recovery,1,200,0.011562 +1793284341,spike-recovery,1,200,0.022646 +1793284342,spike-recovery,1,200,0.017987 +1793284342,spike-recovery,1,200,0.016312 +1793284342,spike-recovery,1,200,0.011285 +1793284343,spike-recovery,1,200,0.021915 +1793284343,spike-recovery,1,200,0.012904 +1793284343,spike-recovery,1,200,0.009503 +1793284344,spike-recovery,1,200,0.013804 +1793284344,spike-recovery,1,200,0.016190 +1793284344,spike-recovery,1,200,0.014793 +1793284345,spike-recovery,1,200,0.008171 +1793284345,spike-recovery,1,200,0.018180 +1793284345,spike-recovery,1,200,0.019813 +1793284345,spike-recovery,1,200,0.009241 +1793284346,spike-recovery,1,200,0.017788 +1793284346,spike-recovery,1,200,0.017034 +1793284346,spike-recovery,1,200,0.008296 +1793284347,spike-recovery,1,200,0.012816 +1793284347,spike-recovery,1,200,0.013771 +1793284347,spike-recovery,1,200,0.018987 +1793284348,spike-recovery,1,200,0.013540 +1793284348,spike-recovery,1,200,0.018010 +1793284348,spike-recovery,1,200,0.022321 +1793284349,spike-recovery,1,200,0.010859 +1793284349,spike-recovery,1,200,0.016032 +1793284349,spike-recovery,1,200,0.010474 +1793284350,spike-recovery,1,200,0.006752 +1793284350,spike-recovery,1,200,0.008863 +1793284350,spike-recovery,1,200,0.007697 +1793284350,spike-recovery,1,200,0.011614 +1793284351,spike-recovery,1,200,0.012900 +1793284351,spike-recovery,1,200,0.011498 +1793284351,spike-recovery,1,200,0.009382 +1793284352,spike-recovery,1,200,0.005831 +1793284352,spike-recovery,1,200,0.015206 +1793284352,spike-recovery,1,200,0.014452 +1793284353,spike-recovery,1,200,0.010664 +1793284353,spike-recovery,1,200,0.020393 +1793284353,spike-recovery,1,200,0.017481 +1793284353,spike-recovery,1,200,0.017405 +1793284354,spike-recovery,1,200,0.021901 +1793284354,spike-recovery,1,200,0.015941 +1793284354,spike-recovery,1,200,0.008027 +1793284355,spike-recovery,1,200,1.016556 +1793284356,spike-recovery,1,200,0.006082 +1793284356,spike-recovery,1,200,0.008284 +1793284357,spike-recovery,1,200,0.005838 +1793284357,spike-recovery,1,200,0.016461 +1793284357,spike-recovery,1,200,0.021079 +1793284357,spike-recovery,1,200,0.008747 +1793284358,spike-recovery,1,200,0.015244 +1793284358,spike-recovery,1,200,0.019670 +1793284358,spike-recovery,1,200,0.018100 +1793284359,spike-recovery,1,200,0.010704 +1793284359,spike-recovery,1,200,0.011977 +1793284359,spike-recovery,1,200,0.015021 +1793284360,spike-recovery,1,200,0.019916 +1793284360,spike-recovery,1,200,0.025136 +1793284360,spike-recovery,1,200,0.011404 +1793284361,spike-recovery,1,200,0.009888 +1793284361,spike-recovery,1,200,0.013703 +1793284361,spike-recovery,1,200,0.017516 +1793284362,spike-recovery,1,200,0.026763 +1793284362,spike-recovery,1,200,0.014323 +1793284362,spike-recovery,1,200,0.023786 +1793284363,spike-recovery,1,200,0.021130 +1793284363,spike-recovery,1,200,0.018941 +1793284363,spike-recovery,1,200,0.014715 +1793284364,spike-recovery,1,200,0.016336 +1793284364,spike-recovery,1,200,0.016046 +1793284364,spike-recovery,1,200,0.018118 +1793284364,spike-recovery,1,200,0.018013 +1793284365,spike-recovery,1,200,0.011003 +1793284365,spike-recovery,1,200,0.017926 +1793284365,spike-recovery,1,200,0.019289 +1793284366,spike-recovery,1,200,0.011023 +1793284366,spike-recovery,1,200,0.020328 +1793284366,spike-recovery,1,200,0.010390 +1793284367,spike-recovery,1,200,0.014154 +1793284367,spike-recovery,1,200,0.011185 +1793284367,spike-recovery,1,200,0.034378 +1793284368,spike-recovery,1,200,0.011092 +1793284368,spike-recovery,1,200,0.018519 +1793284368,spike-recovery,1,200,0.014776 +1793284369,spike-recovery,1,200,0.027623 +1793284369,spike-recovery,1,200,0.011995 +1793284369,spike-recovery,1,200,0.012136 +1793284370,spike-recovery,1,200,0.016933 +1793284370,spike-recovery,1,200,0.015057 +1793284370,spike-recovery,1,200,0.011524 +1793284370,spike-recovery,1,200,0.014605 +1793284371,spike-recovery,1,200,0.010645 +1793284371,spike-recovery,1,200,0.010700 +1793284371,spike-recovery,1,200,0.015575 +1793284372,spike-recovery,1,200,0.021950 +1793284372,spike-recovery,1,200,0.044464 +1793284372,spike-recovery,1,200,0.013123 +1793284373,spike-recovery,1,200,0.014680 +1793284373,spike-recovery,1,200,0.012844 +1793284373,spike-recovery,1,200,0.009483 +1793284374,spike-recovery,1,200,0.016597 +1793284374,spike-recovery,1,200,0.016241 +1793284374,spike-recovery,1,200,0.010932 +1793284375,spike-recovery,1,200,0.020033 +1793284375,spike-recovery,1,200,0.021300 +1793284375,spike-recovery,1,200,0.035352 +1793284376,spike-recovery,1,200,0.017826 +1793284376,spike-recovery,1,200,0.020901 +1793284376,spike-recovery,1,200,0.016839 +1793284377,spike-recovery,1,200,0.011726 +1793284377,spike-recovery,1,200,0.010604 +1793284377,spike-recovery,1,200,0.010186 +1793284377,spike-recovery,1,200,0.012808 +1793284378,spike-recovery,1,200,0.013265 +1793284378,spike-recovery,1,200,0.011497 +1793284378,spike-recovery,1,200,0.013054 +1793284379,spike-recovery,1,200,0.019425 +1793284379,spike-recovery,1,200,0.014884 +1793284379,spike-recovery,1,200,0.011160 +1793284380,spike-recovery,1,200,0.016777 +1793284380,spike-recovery,1,200,0.006882 +1793284380,spike-recovery,1,200,0.011200 +1793284380,spike-recovery,1,200,0.004975 +1793284381,spike-recovery,1,200,0.006863 +1793284381,spike-recovery,1,200,0.009292 +1793284381,spike-recovery,1,200,0.016119 +1793284382,spike-recovery,1,200,0.013921 +1793284382,spike-recovery,1,200,0.017972 +1793284382,spike-recovery,1,200,0.010129 +1793284383,spike-recovery,1,200,0.007392 +1793284383,spike-recovery,1,200,0.007157 +1793284383,spike-recovery,1,200,0.008867 +1793284383,spike-recovery,1,200,0.008623 +1793284384,spike-recovery,1,200,0.010744 +1793284384,spike-recovery,1,200,0.011679 +1793284384,spike-recovery,1,200,0.008001 +1793284385,spike-recovery,1,200,0.018207 +1793284385,spike-recovery,1,200,0.010924 +1793284385,spike-recovery,1,200,0.008999 +1793284386,spike-recovery,1,200,0.016694 +1793284386,spike-recovery,1,200,0.014895 +1793284386,spike-recovery,1,200,0.006507 +1793284386,spike-recovery,1,200,0.018489 +1793284387,spike-recovery,1,200,0.017242 +1793284387,spike-recovery,1,200,0.013409 +1793284387,spike-recovery,1,200,0.009351 +1793284388,spike-recovery,1,200,0.017868 +1793284388,spike-recovery,1,200,0.016843 +1793284388,spike-recovery,1,200,0.019183 +1793284389,spike-recovery,1,200,0.020261 +1793284389,spike-recovery,1,200,0.018628 +1793284389,spike-recovery,1,200,0.009130 +1793284390,spike-recovery,1,200,0.020546 +1793284390,spike-recovery,1,200,0.016328 +1793284390,spike-recovery,1,200,0.018960 +1793284390,spike-recovery,1,200,0.018478 +1793284391,spike-recovery,1,200,0.016081 +1793284391,spike-recovery,1,200,0.013225 +1793284391,spike-recovery,1,200,0.007719 +1793284392,spike-recovery,1,200,0.013404 +1793284392,spike-recovery,1,200,0.013945 +1793284392,spike-recovery,1,200,0.011190 +1793284393,spike-recovery,1,200,0.015428 +1793284393,spike-recovery,1,200,0.011796 +1793284393,spike-recovery,1,200,0.011947 +1793284393,spike-recovery,1,200,0.017580 +1793284394,spike-recovery,1,200,0.006344 +1793284394,spike-recovery,1,200,0.006327 +1793284394,spike-recovery,1,200,0.007848 +1793284395,spike-recovery,1,200,0.008958 +1793284395,spike-recovery,1,200,0.007929 +1793284395,spike-recovery,1,200,0.007080 +1793284395,spike-recovery,1,200,0.007751 +1793284396,spike-recovery,1,200,0.014059 +1793284396,spike-recovery,1,200,0.012777 +1793284396,spike-recovery,1,200,0.014160 +1793284397,spike-recovery,1,200,0.014943 +1793284397,spike-recovery,1,200,0.012189 +1793284397,spike-recovery,1,200,0.016321 +1793284398,spike-recovery,1,200,0.014631 +1793284398,spike-recovery,1,200,0.012351 +1793284398,spike-recovery,1,200,0.019703 +1793284399,spike-recovery,1,200,0.014605 +1793284399,spike-recovery,1,200,0.014688 +1793284399,spike-recovery,1,200,0.010297 +1793284399,spike-recovery,1,200,0.008868 +1793284400,spike-recovery,1,200,0.016193 +1793284400,spike-recovery,1,200,0.011255 +1793284400,spike-recovery,1,200,0.010149 +1793284401,spike-recovery,1,200,0.014930 +1793284401,spike-recovery,1,200,0.024563 +1793284401,spike-recovery,1,200,0.005797 +1793284402,spike-recovery,1,200,0.010609 +1793284402,spike-recovery,1,200,0.013326 +1793284402,spike-recovery,1,200,0.013783 +1793284402,spike-recovery,1,200,0.016514 +1793284403,spike-recovery,1,200,0.015547 +1793284403,spike-recovery,1,200,0.023804 +1793284403,spike-recovery,1,200,0.019825 +1793284404,spike-recovery,1,200,0.008943 +1793284404,spike-recovery,1,200,0.009778 +1793284404,spike-recovery,1,200,0.007024 +1793284405,spike-recovery,1,200,0.012168 +1793284405,spike-recovery,1,200,0.013431 +1793284405,spike-recovery,1,200,0.011402 +1793284405,spike-recovery,1,200,0.016186 +1793284406,spike-recovery,1,200,0.016082 +1793284406,spike-recovery,1,200,0.016205 +1793284406,spike-recovery,1,200,0.017816 +1793284407,spike-recovery,1,200,0.018472 +1793284407,spike-recovery,1,200,0.016322 +1793284407,spike-recovery,1,200,0.014737 +1793284408,spike-recovery,1,200,0.011699 +1793284408,spike-recovery,1,200,0.008700 +1793284408,spike-recovery,1,200,0.010367 +1793284409,spike-recovery,1,200,0.013969 +1793284409,spike-recovery,1,200,0.009533 +1793284409,spike-recovery,1,200,0.012857 +1793284409,spike-recovery,1,200,0.017201 +1793284410,spike-recovery,1,200,0.012550 +1793284410,spike-recovery,1,200,0.013192 +1793284410,spike-recovery,1,200,0.011053 +1793284411,spike-recovery,1,200,0.015301 +1793284411,spike-recovery,1,200,0.010989 +1793284411,spike-recovery,1,200,0.015610 +1793284412,spike-recovery,1,200,0.009403 +1793284412,spike-recovery,1,200,0.014549 +1793284412,spike-recovery,1,200,0.011856 +1793284412,spike-recovery,1,200,0.013965 +1793284413,spike-recovery,1,200,0.015375 +1793284413,spike-recovery,1,200,0.016047 +1793284413,spike-recovery,1,200,0.009631 +1793284414,spike-recovery,1,200,0.012551 +1793284414,spike-recovery,1,200,0.014293 +1793284414,spike-recovery,1,200,0.009271 +1793284415,spike-recovery,1,200,0.011340 +1793284415,spike-recovery,1,200,0.019860 +1793284415,spike-recovery,1,200,0.010362 +1793284416,spike-recovery,1,200,0.015708 +1793284416,spike-recovery,1,200,0.010547 +1793284416,spike-recovery,1,200,0.021747 +1793284416,spike-recovery,1,200,0.012352 +1793284417,spike-recovery,1,200,0.017228 +1793284417,spike-recovery,1,200,0.009565 +1793284417,spike-recovery,1,200,0.013907 +1793284418,spike-recovery,1,200,0.019124 +1793284418,spike-recovery,1,200,0.010971 +1793284418,spike-recovery,1,200,0.015985 +1793284419,spike-recovery,1,200,0.023132 +1793284419,spike-recovery,1,200,0.017499 +1793284419,spike-recovery,1,200,0.014196 +1793284420,spike-recovery,1,200,0.016499 +1793284420,spike-recovery,1,200,0.022213 +1793284420,spike-recovery,1,200,0.015383 +1793284421,spike-recovery,1,200,0.008779 +1793284421,spike-recovery,1,200,0.019113 +1793284421,spike-recovery,1,200,0.015316 +1793284421,spike-recovery,1,200,0.017906 +1793284422,spike-recovery,1,200,0.017798 +1793284422,spike-recovery,1,200,0.010208 +1793284422,spike-recovery,1,200,0.014859 +1793284423,spike-recovery,1,200,0.018253 +1793284423,spike-recovery,1,200,0.019519 +1793284423,spike-recovery,1,200,0.023153 +1793284424,spike-recovery,1,200,0.014903 +1793284424,spike-recovery,1,200,0.014287 +1793284424,spike-recovery,1,200,0.011185 +1793284425,spike-recovery,1,200,0.019423 +1793284425,spike-recovery,1,200,0.016703 +1793284425,spike-recovery,1,200,0.018826 +1793284425,spike-recovery,1,200,0.017277 +1793284426,spike-recovery,1,200,0.016380 +1793284426,spike-recovery,1,200,0.008698 +1793284426,spike-recovery,1,200,0.006127 +1793284427,spike-recovery,1,200,0.006392 +1793284427,spike-recovery,1,200,0.008502 +1793284427,spike-recovery,1,503,2.019423 +1793284430,sustained-db-outage,1,502,1.012823 +1793284431,sustained-db-outage,1,502,0.004426 +1793284431,sustained-db-outage,1,502,0.001010 +1793284431,sustained-db-outage,1,502,0.003042 +1793284432,sustained-db-outage,1,502,0.002527 +1793284432,sustained-db-outage,1,502,0.002433 +1793284432,sustained-db-outage,1,502,0.002352 +1793284433,sustained-db-outage,1,502,1.004606 +1793284434,sustained-db-outage,1,502,0.002482 +1793284434,sustained-db-outage,1,502,0.001607 +1793284434,sustained-db-outage,1,502,0.001359 +1793284435,sustained-db-outage,1,502,0.001025 +1793284435,sustained-db-outage,1,502,0.001205 +1793284435,sustained-db-outage,1,502,0.002711 +1793284436,sustained-db-outage,1,502,1.005687 +1793284437,sustained-db-outage,1,502,0.001610 +1793284437,sustained-db-outage,1,502,0.001657 +1793284437,sustained-db-outage,1,502,0.001607 +1793284438,sustained-db-outage,1,502,0.002191 +1793284438,sustained-db-outage,1,502,0.004878 +1793284438,sustained-db-outage,1,502,0.001848 +1793284439,sustained-db-outage,1,502,1.005549 +1793284440,sustained-db-outage,1,502,0.001523 +1793284440,sustained-db-outage,1,502,0.002013 +1793284440,sustained-db-outage,1,502,0.002069 +1793284441,sustained-db-outage,1,502,0.005018 +1793284441,sustained-db-outage,1,502,0.003566 +1793284441,sustained-db-outage,1,502,0.001892 +1793284442,sustained-db-outage,1,502,1.005866 +1793284443,sustained-db-outage,1,502,0.001733 +1793284443,sustained-db-outage,1,502,0.001391 +1793284443,sustained-db-outage,1,502,0.001103 +1793284444,sustained-db-outage,1,502,0.001566 +1793284444,sustained-db-outage,1,502,0.002342 +1793284444,sustained-db-outage,1,502,0.001765 +1793284444,sustained-db-outage,1,502,1.016092 +1793284446,sustained-db-outage,1,502,0.001513 +1793284446,sustained-db-outage,1,502,0.005786 +1793284446,sustained-db-outage,1,502,0.001096 +1793284447,sustained-db-outage,1,502,0.001439 +1793284447,sustained-db-outage,1,502,0.001467 +1793284447,sustained-db-outage,1,502,0.001842 +1793284447,sustained-db-outage,1,502,1.005288 +1793284449,sustained-db-outage,1,502,0.003024 +1793284449,sustained-db-outage,1,502,0.002548 +1793284449,sustained-db-outage,1,502,0.001075 +1793284450,sustained-db-outage,1,502,0.000903 +1793284450,sustained-db-outage,1,502,0.001713 +1793284450,sustained-db-outage,1,502,0.001362 +1793284450,sustained-db-outage,1,502,1.015900 +1793284452,sustained-db-outage,1,502,0.002461 +1793284452,sustained-db-outage,1,502,0.003656 +1793284452,sustained-db-outage,1,502,0.001118 +1793284453,sustained-db-outage,1,502,0.001243 +1793284453,sustained-db-outage,1,502,0.001502 +1793284453,sustained-db-outage,1,502,0.001806 +1793284453,sustained-db-outage,1,502,1.006009 +1793284455,sustained-db-outage,1,502,0.001024 +1793284455,sustained-db-outage,1,502,0.002616 +1793284455,sustained-db-outage,1,502,0.001183 +1793284456,sustained-db-outage,1,502,0.001763 +1793284456,sustained-db-outage,1,502,0.003814 +1793284456,sustained-db-outage,1,502,0.002816 +1793284456,sustained-db-outage,1,502,0.001221 +1793284457,sustained-db-outage,1,502,1.005702 +1793284458,sustained-db-outage,1,502,0.001057 +1793284458,sustained-db-outage,1,502,0.001237 +1793284459,sustained-db-outage,1,502,0.001243 +1793284459,sustained-db-outage,1,502,0.001766 +1793284459,sustained-db-outage,1,502,0.001616 +1793284459,sustained-db-outage,1,502,1.004371 +1793284461,sustained-db-outage,1,502,0.004895 +1793284461,sustained-db-outage,1,502,0.002087 +1793284461,sustained-db-outage,1,502,0.001164 +1793284462,sustained-db-outage,1,502,0.002822 +1793284462,sustained-db-outage,1,502,0.001156 +1793284462,sustained-db-outage,1,502,0.001047 +1793284462,sustained-db-outage,1,502,0.003434 +1793284463,sustained-db-outage,1,502,1.005345 +1793284464,sustained-db-outage,1,502,0.001210 +1793284464,sustained-db-outage,1,502,0.001625 +1793284465,sustained-db-outage,1,502,0.001969 +1793284465,sustained-db-outage,1,502,0.002607 +1793284465,sustained-db-outage,1,502,0.001545 +1793284465,sustained-db-outage,1,502,0.001418 +1793284466,sustained-db-outage,1,502,1.003834 +1793284467,sustained-db-outage,1,502,0.001483 +1793284467,sustained-db-outage,1,502,0.002722 +1793284468,sustained-db-outage,1,502,0.006145 +1793284468,sustained-db-outage,1,502,0.002842 +1793284468,sustained-db-outage,1,502,0.001820 +1793284468,sustained-db-outage,1,502,0.001232 +1793284469,sustained-db-outage,1,502,1.009660 +1793284470,sustained-db-outage,1,502,0.001438 +1793284470,sustained-db-outage,1,502,0.002617 +1793284471,sustained-db-outage,1,502,0.001313 +1793284471,sustained-db-outage,1,502,0.002692 +1793284471,sustained-db-outage,1,502,0.002254 +1793284471,sustained-db-outage,1,502,0.002915 +1793284472,sustained-db-outage,1,502,1.004783 +1793284473,sustained-db-outage,1,502,0.002011 +1793284473,sustained-db-outage,1,502,0.001901 +1793284474,sustained-db-outage,1,502,0.001785 +1793284474,sustained-db-outage,1,502,0.002297 +1793284474,sustained-db-outage,1,502,0.001170 +1793284474,sustained-db-outage,1,502,0.002356 +1793284475,sustained-db-outage,1,502,1.005104 +1793284476,sustained-db-outage,1,502,0.002964 +1793284476,sustained-db-outage,1,502,0.001347 +1793284477,sustained-db-outage,1,502,0.002555 +1793284477,sustained-db-outage,1,502,0.001385 +1793284477,sustained-db-outage,1,502,0.003134 +1793284477,sustained-db-outage,1,502,0.003448 +1793284478,sustained-db-outage,1,502,1.004188 +1793284479,sustained-db-outage,1,502,0.003693 +1793284479,sustained-db-outage,1,502,0.001836 +1793284480,sustained-db-outage,1,502,0.001789 +1793284480,sustained-db-outage,1,502,0.002147 +1793284480,sustained-db-outage,1,502,0.002258 +1793284480,sustained-db-outage,1,502,0.001799 +1793284481,sustained-db-outage,1,502,1.006269 +1793284482,sustained-db-outage,1,502,0.003804 +1793284482,sustained-db-outage,1,502,0.002203 +1793284483,sustained-db-outage,1,502,0.002588 +1793284483,sustained-db-outage,1,502,0.001437 +1793284483,sustained-db-outage,1,502,0.001982 +1793284483,sustained-db-outage,1,502,0.002126 +1793284484,sustained-db-outage,1,502,1.005099 +1793284485,sustained-db-outage,1,502,0.001972 +1793284485,sustained-db-outage,1,502,0.002698 +1793284486,sustained-db-outage,1,502,0.001423 +1793284486,sustained-db-outage,1,502,0.002840 +1793284486,sustained-db-outage,1,502,0.001630 +1793284486,sustained-db-outage,1,502,0.002168 +1793284487,sustained-db-outage,1,502,1.006108 +1793284488,sustained-db-outage,1,502,0.002286 +1793284488,sustained-db-outage,1,502,0.002309 +1793284489,sustained-db-outage,1,502,0.002554 +1793284489,sustained-db-outage,1,502,0.002811 +1793284489,sustained-db-outage,1,502,0.001430 +1793284489,sustained-db-outage,1,502,1.012716 +1793284491,sustained-db-outage,1,502,0.002385 +1793284491,sustained-db-outage,1,502,0.001780 +1793284491,sustained-db-outage,1,502,0.003751 +1793284492,sustained-db-outage,1,502,0.002084 +1793284492,sustained-db-outage,1,502,0.002583 +1793284492,sustained-db-outage,1,502,0.001573 +1793284493,sustained-db-outage,1,502,1.004664 +1793284494,sustained-db-outage,1,502,0.003391 +1793284494,sustained-db-outage,1,502,0.001912 +1793284494,sustained-db-outage,1,502,0.002228 +1793284495,sustained-db-outage,1,502,0.002456 +1793284495,sustained-db-outage,1,502,0.001568 +1793284495,sustained-db-outage,1,502,0.002044 +1793284496,sustained-db-outage,1,502,1.011156 +1793284497,sustained-db-outage,1,502,0.002015 +1793284497,sustained-db-outage,1,502,0.001744 +1793284497,sustained-db-outage,1,502,0.001233 +1793284498,sustained-db-outage,1,502,0.001353 +1793284498,sustained-db-outage,1,502,0.001066 +1793284498,sustained-db-outage,1,502,0.001217 +1793284499,sustained-db-outage,1,502,1.006174 +1793284500,sustained-db-outage,1,502,0.002276 +1793284500,sustained-db-outage,1,502,0.002243 +1793284500,sustained-db-outage,1,502,0.001890 +1793284501,sustained-db-outage,1,502,0.002915 +1793284501,sustained-db-outage,1,502,0.001956 +1793284501,sustained-db-outage,1,502,0.001157 +1793284501,sustained-db-outage,1,502,1.007219 +1793284503,sustained-db-outage,1,502,0.002409 +1793284503,sustained-db-outage,1,502,0.002649 +1793284503,sustained-db-outage,1,502,0.002377 +1793284504,sustained-db-outage,1,502,0.000915 +1793284504,sustained-db-outage,1,502,0.002413 +1793284504,sustained-db-outage,1,502,0.002604 +1793284505,sustained-db-outage,1,502,1.014774 +1793284506,sustained-db-outage,1,502,0.003064 +1793284506,sustained-db-outage,1,502,0.003772 +1793284506,sustained-db-outage,1,502,0.004106 +1793284507,sustained-db-outage,1,502,0.005178 +1793284507,sustained-db-outage,1,502,0.002045 +1793284507,sustained-db-outage,1,502,0.004274 +1793284508,sustained-db-outage,1,502,1.014937 +1793284509,sustained-db-outage,1,502,0.004413 +1793284509,sustained-db-outage,1,502,0.002085 +1793284510,sustained-db-outage,1,502,0.004045 +1793284510,sustained-db-outage,1,502,0.004010 +1793284510,sustained-db-outage,1,502,0.003511 +1793284510,sustained-db-outage,1,502,1.012457 +1793284512,sustained-db-outage,1,502,0.002499 +1793284512,sustained-db-outage,1,502,0.003151 +1793284512,sustained-db-outage,1,502,0.002435 +1793284513,sustained-db-outage,1,502,0.005896 +1793284513,sustained-db-outage,1,502,0.003896 +1793284513,sustained-db-outage,1,502,0.004019 +1793284514,sustained-db-outage,1,502,1.011162 +1793284515,sustained-db-outage,1,502,0.003001 +1793284515,sustained-db-outage,1,502,0.003014 +1793284515,sustained-db-outage,1,502,0.002973 +1793284516,sustained-db-outage,1,502,0.005998 +1793284516,sustained-db-outage,1,502,0.005241 +1793284516,sustained-db-outage,1,502,0.001465 +1793284517,sustained-db-outage,1,502,1.010972 +1793284518,sustained-db-outage,1,502,0.003517 +1793284518,sustained-db-outage,1,502,0.002477 +1793284519,sustained-db-outage,1,502,0.004098 +1793284519,sustained-db-outage,1,502,0.004679 +1793284519,sustained-db-outage,1,502,0.002054 +1793284519,sustained-db-outage,1,502,0.003370 +1793284520,sustained-db-outage,1,502,1.015470 +1793284521,sustained-db-outage,1,502,0.000761 +1793284521,sustained-db-outage,1,502,0.003891 +1793284522,sustained-db-outage,1,502,0.002843 +1793284522,sustained-db-outage,1,502,0.002216 +1793284522,sustained-db-outage,1,502,0.003073 +1793284522,sustained-db-outage,1,502,1.006049 +1793284524,sustained-db-outage,1,502,0.004085 +1793284524,sustained-db-outage,1,502,0.001928 +1793284524,sustained-db-outage,1,502,0.003407 +1793284525,sustained-db-outage,1,502,0.008528 +1793284525,sustained-db-outage,1,502,0.005427 +1793284525,sustained-db-outage,1,502,0.004774 +1793284526,sustained-db-outage,1,502,1.010757 +1793284527,sustained-db-outage,1,502,0.002048 +1793284527,sustained-db-outage,1,502,0.002795 +1793284527,sustained-db-outage,1,502,0.002674 +1793284528,sustained-db-outage,1,502,0.001153 +1793284528,sustained-db-outage,1,502,0.002700 +1793284528,sustained-db-outage,1,502,0.003542 +1793284529,sustained-db-outage,1,502,1.009450 +1793284530,sustained-db-outage,1,502,0.003668 +1793284530,sustained-db-outage,1,502,0.002201 +1793284531,sustained-db-outage,1,502,0.002478 +1793284531,sustained-db-outage,1,502,0.003792 +1793284531,sustained-db-outage,1,502,0.001344 +1793284531,sustained-db-outage,1,502,0.002333 +1793284532,sustained-db-outage,1,502,1.009441 +1793284533,sustained-db-outage,1,502,0.001628 +1793284533,sustained-db-outage,1,502,0.003270 +1793284534,sustained-db-outage,1,502,0.002050 +1793284534,sustained-db-outage,1,502,0.002698 +1793284534,sustained-db-outage,1,502,0.003624 +1793284534,sustained-db-outage,1,502,0.000812 +1793284535,sustained-db-outage,1,502,1.010864 +1793284536,sustained-db-outage,1,502,0.002652 +1793284536,sustained-db-outage,1,502,0.004523 +1793284537,sustained-db-outage,1,502,0.003737 +1793284537,sustained-db-outage,1,502,0.003822 +1793284537,sustained-db-outage,1,502,0.002001 +1793284538,sustained-db-outage,1,502,1.006082 +1793284539,sustained-db-outage,1,502,0.002411 +1793284539,sustained-db-outage,1,502,0.001840 +1793284539,sustained-db-outage,1,502,0.001367 +1793284540,sustained-db-outage,1,502,0.006517 +1793284540,sustained-db-outage,1,502,0.003235 +1793284540,sustained-db-outage,1,502,0.003755 +1793284541,sustained-db-outage,1,502,1.008711 +1793284542,sustained-db-outage,1,502,0.004007 +1793284542,sustained-db-outage,1,502,0.001841 +1793284542,sustained-db-outage,1,502,0.003738 +1793284543,sustained-db-outage,1,502,0.002048 +1793284543,sustained-db-outage,1,502,0.006195 +1793284543,sustained-db-outage,1,502,0.004303 +1793284544,sustained-db-outage,1,502,1.009775 +1793284545,sustained-db-outage,1,502,0.003113 +1793284545,sustained-db-outage,1,502,0.002524 +1793284546,sustained-db-outage,1,502,0.003313 +1793284546,sustained-db-outage,1,502,0.006449 +1793284546,sustained-db-outage,1,502,0.004787 +1793284546,sustained-db-outage,1,502,0.004726 +1793284547,sustained-db-outage,1,502,1.016913 +1793284548,sustained-db-outage,1,502,0.003368 +1793284548,sustained-db-outage,1,502,0.006271 +1793284549,sustained-db-outage,1,502,0.004123 +1793284549,db-recovery,1,502,0.007296 +1793284549,db-recovery,1,502,0.004473 +1793284550,db-recovery,1,200,0.029350 +1793284550,db-recovery,1,200,0.022312 +1793284550,db-recovery,1,200,0.011682 +1793284551,db-recovery,1,200,0.020428 +1793284551,db-recovery,1,200,0.020278 +1793284551,db-recovery,1,200,0.009133 +1793284552,db-recovery,1,200,0.017974 +1793284552,db-recovery,1,200,0.042636 +1793284552,db-recovery,1,200,0.011932 +1793284552,db-recovery,1,200,0.019891 +1793284553,db-recovery,1,200,0.017467 +1793284553,db-recovery,1,200,0.013541 +1793284553,db-recovery,1,200,0.018290 +1793284554,db-recovery,1,200,0.027590 +1793284554,db-recovery,1,200,0.016646 +1793284554,db-recovery,1,200,0.019686 +1793284555,db-recovery,1,200,0.016465 +1793284555,db-recovery,1,200,0.018894 +1793284555,db-recovery,1,200,0.024984 +1793284556,db-recovery,1,200,0.018335 +1793284556,db-recovery,1,200,0.017446 +1793284556,db-recovery,1,200,0.012546 +1793284557,db-recovery,1,200,0.021799 +1793284557,db-recovery,1,200,0.019778 +1793284557,db-recovery,1,200,0.024394 +1793284558,db-recovery,1,200,0.019666 +1793284558,db-recovery,1,200,0.016715 +1793284558,db-recovery,1,200,0.010218 +1793284559,db-recovery,1,200,0.011460 +1793284559,db-recovery,1,200,0.020376 +1793284559,db-recovery,1,200,0.009481 +1793284559,db-recovery,1,200,0.018170 +1793284560,db-recovery,1,200,0.019300 +1793284560,db-recovery,1,200,0.015525 +1793284560,db-recovery,1,200,0.015943 +1793284561,db-recovery,1,200,0.019770 +1793284561,db-recovery,1,200,0.019412 +1793284561,db-recovery,1,200,0.011502 +1793284562,db-recovery,1,200,0.029974 +1793284562,db-recovery,1,200,0.028566 +1793284562,db-recovery,1,200,0.010144 +1793284563,db-recovery,1,200,0.018110 +1793284563,db-recovery,1,200,0.024600 +1793284563,db-recovery,1,200,0.014151 +1793284564,db-recovery,1,200,0.019448 +1793284564,db-recovery,1,200,0.019519 +1793284564,db-recovery,1,200,0.016363 +1793284565,db-recovery,1,200,0.019023 +1793284565,db-recovery,1,200,0.016189 +1793284565,db-recovery,1,200,0.021790 +1793284566,db-recovery,1,200,0.016916 +1793284566,db-recovery,1,200,0.018911 +1793284566,db-recovery,1,200,0.015062 +1793284566,db-recovery,1,200,0.012499 +1793284567,db-recovery,1,200,0.012419 +1793284567,db-recovery,1,200,0.020094 +1793284567,db-recovery,1,200,0.012227 +1793284568,db-recovery,1,200,0.019360 +1793284568,db-recovery,1,200,0.020712 +1793284568,db-recovery,1,200,0.012232 +1793284569,db-recovery,1,200,0.018165 +1793284569,db-recovery,1,200,0.022422 +1793284569,db-recovery,1,200,0.010348 +1793284570,db-recovery,1,200,0.019269 +1793284570,db-recovery,1,200,0.021479 +1793284570,db-recovery,1,200,0.022156 +1793284571,db-recovery,1,200,0.011526 +1793284571,db-recovery,1,200,0.017563 +1793284571,db-recovery,1,200,0.025780 +1793284572,db-recovery,1,200,0.015955 +1793284572,db-recovery,1,200,0.018164 +1793284572,db-recovery,1,200,0.014097 +1793284572,db-recovery,1,200,0.022300 +1793284573,db-recovery,1,200,0.009239 +1793284573,db-recovery,1,200,0.014024 +1793284573,db-recovery,1,200,0.017652 +1793284574,db-recovery,1,200,0.021549 +1793284574,db-recovery,1,200,0.011794 +1793284574,db-recovery,1,200,0.017962 +1793284575,db-recovery,1,200,0.013659 +1793284575,db-recovery,1,200,0.015853 +1793284575,db-recovery,1,200,0.014625 +1793284576,db-recovery,1,200,0.014215 +1793284576,db-recovery,1,200,0.013554 +1793284576,db-recovery,1,200,0.026332 +1793284577,db-recovery,1,200,0.016884 +1793284577,db-recovery,1,200,0.013352 +1793284577,db-recovery,1,200,0.018720 +1793284577,db-recovery,1,200,0.009572 +1793284578,db-recovery,1,200,0.026382 +1793284578,db-recovery,1,200,0.017006 +1793284578,db-recovery,1,200,0.024567 +1793284579,db-recovery,1,200,0.008062 +1793284579,db-recovery,1,200,0.014425 +1793284579,db-recovery,1,200,0.015624 +1793284580,db-recovery,1,200,0.015735 +1793284580,db-recovery,1,200,0.019029 +1793284580,db-recovery,1,200,0.016180 +1793284581,db-recovery,1,200,0.015932 +1793284581,db-recovery,1,200,0.012405 +1793284581,db-recovery,1,200,0.016321 +1793284581,db-recovery,1,200,0.007779 +1793284582,db-recovery,1,200,0.016232 +1793284582,db-recovery,1,200,0.014671 +1793284582,db-recovery,1,200,0.015204 +1793284583,db-recovery,1,200,0.016478 +1793284583,db-recovery,1,200,0.017012 +1793284583,db-recovery,1,200,0.018305 +1793284584,db-recovery,1,200,0.019715 +1793284584,db-recovery,1,200,0.024672 +1793284584,db-recovery,1,200,0.015538 +1793284585,db-recovery,1,200,0.016691 +1793284585,db-recovery,1,200,0.017881 +1793284585,db-recovery,1,200,0.024086 +1793284585,db-recovery,1,200,0.018174 +1793284586,db-recovery,1,200,0.015730 +1793284586,db-recovery,1,200,0.015054 +1793284586,db-recovery,1,200,0.023556 +1793284587,db-recovery,1,200,0.011617 +1793284587,db-recovery,1,200,0.021108 +1793284587,db-recovery,1,200,0.019907 +1793284588,db-recovery,1,200,0.013024 +1793284588,db-recovery,1,200,0.025571 +1793284588,db-recovery,1,200,0.016597 +1793284589,db-recovery,1,200,0.015220 +1793284589,db-recovery,1,200,0.017436 +1793284589,db-recovery,1,200,0.017613 +1793284590,db-recovery,1,200,0.013606 +1793284590,db-recovery,1,200,0.019219 +1793284590,db-recovery,1,200,0.015854 +1793284591,db-recovery,1,200,0.021559 +1793284591,db-recovery,1,200,0.017383 +1793284591,db-recovery,1,200,0.018858 +1793284591,db-recovery,1,200,0.008925 +1793284592,db-recovery,1,200,0.017628 +1793284592,db-recovery,1,200,0.017848 +1793284592,db-recovery,1,200,0.018590 +1793284593,db-recovery,1,200,0.018241 +1793284593,db-recovery,1,200,0.015678 +1793284593,db-recovery,1,200,0.016018 +1793284594,db-recovery,1,200,0.017724 +1793284594,db-recovery,1,200,0.017340 +1793284594,db-recovery,1,200,0.010801 +1793284595,db-recovery,1,200,0.008456 +1793284595,db-recovery,1,200,0.011279 +1793284595,db-recovery,1,200,0.018112 +1793284596,db-recovery,1,200,0.021752 +1793284596,db-recovery,1,200,0.016237 +1793284596,db-recovery,1,200,0.017656 +1793284597,db-recovery,1,200,0.009837 +1793284597,db-recovery,1,200,0.015666 +1793284597,db-recovery,1,200,0.018534 +1793284597,db-recovery,1,200,0.013189 +1793284598,db-recovery,1,200,0.020802 +1793284598,db-recovery,1,200,0.019280 +1793284598,db-recovery,1,200,0.015810 +1793284599,db-recovery,1,200,0.012171 +1793284599,db-recovery,1,200,0.023552 +1793284599,db-recovery,1,200,0.021258 +1793284600,db-recovery,1,200,0.019847 +1793284600,db-recovery,1,200,0.018393 +1793284600,db-recovery,1,200,0.020147 +1793284601,db-recovery,1,200,0.018118 +1793284601,db-recovery,1,200,0.018591 +1793284601,db-recovery,1,200,0.015789 +1793284602,db-recovery,1,200,0.011120 +1793284602,db-recovery,1,200,0.009268 +1793284602,db-recovery,1,200,0.016543 +1793284602,db-recovery,1,200,0.019992 +1793284603,db-recovery,1,200,0.017717 +1793284603,db-recovery,1,200,0.013315 +1793284603,db-recovery,1,200,0.018883 +1793284604,db-recovery,1,200,0.020808 +1793284604,db-recovery,1,200,0.013099 +1793284604,db-recovery,1,200,0.012238 +1793284605,db-recovery,1,200,0.014082 +1793284605,db-recovery,1,200,0.034667 +1793284605,db-recovery,1,200,0.017741 +1793284606,db-recovery,1,200,0.014860 +1793284606,db-recovery,1,200,0.019190 +1793284606,db-recovery,1,200,0.015116 +1793284607,db-recovery,1,200,0.017986 +1793284607,db-recovery,1,200,0.018649 +1793284607,db-recovery,1,200,0.008797 +1793284608,db-recovery,1,200,0.016051 +1793284608,db-recovery,1,200,0.021284 +1793284608,db-recovery,1,200,0.015269 +1793284608,db-recovery,1,200,0.017627 +1793284609,db-recovery,1,200,0.020219 +1793284609,db-recovery,1,200,0.019998 +1793284609,db-recovery,1,200,0.012289 +1793284610,db-recovery,1,200,0.016896 +1793284610,db-recovery,1,200,0.020369 +1793284610,db-recovery,1,200,0.018103 +1793284611,db-recovery,1,200,0.018114 +1793284611,db-recovery,1,200,0.010087 +1793284611,db-recovery,1,200,0.016092 +1793284612,db-recovery,1,200,0.015031 +1793284612,db-recovery,1,200,0.018442 +1793284612,db-recovery,1,200,0.016405 +1793284612,db-recovery,1,200,0.018408 +1793284613,db-recovery,1,200,0.017589 +1793284613,db-recovery,1,200,0.017269 +1793284613,db-recovery,1,200,0.012811 +1793284614,db-recovery,1,200,0.020019 +1793284614,db-recovery,1,200,0.015260 +1793284614,db-recovery,1,200,0.020041 +1793284615,db-recovery,1,200,0.019233 +1793284615,db-recovery,1,200,0.018559 +1793284615,db-recovery,1,200,0.018191 +1793284616,db-recovery,1,200,0.014857 +1793284616,db-recovery,1,200,0.015904 +1793284616,db-recovery,1,200,0.015286 +1793284617,db-recovery,1,200,0.016355 +1793284617,db-recovery,1,200,0.027177 +1793284617,db-recovery,1,200,0.015382 +1793284617,db-recovery,1,200,0.007893 +1793284618,db-recovery,1,200,0.009215 +1793284618,db-recovery,1,200,0.017725 +1793284618,db-recovery,1,200,0.015499 +1793284619,db-recovery,1,200,1.022469 +1793284620,db-recovery,1,200,0.016228 +1793284620,db-recovery,1,200,0.009862 +1793284621,db-recovery,1,200,0.010628 +1793284621,db-recovery,1,200,0.033551 +1793284621,db-recovery,1,200,0.012637 +1793284622,db-recovery,1,200,0.007430 +1793284622,db-recovery,1,200,0.011399 +1793284622,db-recovery,1,200,0.014267 +1793284622,db-recovery,1,200,0.018570 +1793284623,db-recovery,1,200,0.021365 +1793284623,db-recovery,1,200,0.013889 +1793284623,db-recovery,1,200,0.018594 +1793284624,db-recovery,1,200,0.024859 +1793284624,db-recovery,1,200,0.013138 +1793284624,db-recovery,1,200,0.017815 +1793284625,db-recovery,1,200,0.017225 +1793284625,db-recovery,1,200,0.021207 +1793284625,db-recovery,1,200,0.014271 +1793284626,db-recovery,1,200,0.010241 +1793284626,db-recovery,1,200,0.018597 +1793284626,db-recovery,1,200,0.008809 +1793284627,db-recovery,1,200,0.016700 +1793284627,db-recovery,1,200,0.016510 +1793284627,db-recovery,1,200,0.019144 +1793284627,db-recovery,1,200,0.011446 +1793284628,db-recovery,1,200,0.017988 +1793284628,db-recovery,1,200,0.015331 +1793284628,db-recovery,1,200,0.019127 +1793284629,db-recovery,1,200,0.009499 +1793284629,db-recovery,1,200,0.019231 +1793284629,db-recovery,1,200,0.018929 +1793284630,db-recovery,1,200,0.010730 +1793284630,db-recovery,1,200,0.013689 +1793284630,db-recovery,1,200,0.016656 +1793284631,db-recovery,1,200,0.009680 +1793284631,db-recovery,1,200,0.011734 +1793284631,db-recovery,1,200,0.019789 +1793284632,db-recovery,1,200,0.016608 +1793284632,db-recovery,1,200,0.011219 +1793284632,db-recovery,1,200,0.015266 +1793284632,db-recovery,1,200,0.016040 +1793284633,db-recovery,1,200,0.018177 +1793284633,db-recovery,1,200,0.013544 +1793284633,db-recovery,1,200,0.015541 +1793284634,db-recovery,1,200,0.007296 +1793284634,db-recovery,1,200,0.016830 +1793284634,db-recovery,1,200,0.015660 +1793284635,db-recovery,1,200,0.017401 +1793284635,db-recovery,1,200,0.019793 +1793284635,db-recovery,1,200,0.011976 +1793284636,db-recovery,1,200,0.019113 +1793284636,db-recovery,1,200,0.017584 +1793284636,db-recovery,1,200,0.015828 +1793284637,db-recovery,1,200,0.010781 +1793284637,db-recovery,1,200,0.011184 +1793284637,db-recovery,1,200,0.019990 +1793284637,db-recovery,1,200,0.007853 +1793284638,db-recovery,1,200,0.017539 +1793284638,db-recovery,1,200,1.019786 +1793284639,db-recovery,1,200,0.013800 +1793284640,db-recovery,1,200,0.014219 +1793284640,db-recovery,1,200,0.008346 +1793284640,db-recovery,1,200,0.018686 +1793284641,worker-stopped,1,200,0.014155 +1793284641,worker-stopped,1,200,0.017528 +1793284641,worker-stopped,1,200,0.014676 +1793284641,worker-stopped,1,200,0.010304 +1793284642,worker-stopped,1,200,0.018577 +1793284642,worker-stopped,1,200,0.012625 +1793284642,worker-stopped,1,200,0.013394 +1793284643,worker-stopped,1,200,0.016368 +1793284643,worker-stopped,1,200,0.022763 +1793284643,worker-stopped,1,200,0.015035 +1793284644,worker-stopped,1,200,0.008335 +1793284644,worker-stopped,1,200,0.013334 +1793284644,worker-stopped,1,200,0.015434 +1793284644,worker-stopped,1,200,0.009661 +1793284645,worker-stopped,1,200,0.020513 +1793284645,worker-stopped,1,200,0.012479 +1793284645,worker-stopped,1,200,0.017519 +1793284646,worker-stopped,1,200,0.006793 +1793284646,worker-stopped,1,200,0.008316 +1793284646,worker-stopped,1,200,0.013682 +1793284647,worker-stopped,1,200,0.017016 +1793284647,worker-stopped,1,200,0.013544 +1793284647,worker-stopped,1,200,0.016560 +1793284647,worker-stopped,1,200,0.004040 +1793284648,worker-stopped,1,200,0.005595 +1793284648,worker-stopped,1,200,0.008805 +1793284648,worker-stopped,1,200,0.008861 +1793284649,worker-stopped,1,200,0.014601 +1793284649,worker-stopped,1,200,0.009125 +1793284649,worker-stopped,1,200,0.008782 +1793284649,worker-stopped,1,200,0.008820 +1793284650,worker-stopped,1,200,0.010816 +1793284650,worker-stopped,1,200,0.014111 +1793284650,worker-stopped,1,200,0.010510 +1793284651,worker-stopped,1,200,1.015835 +1793284652,worker-stopped,1,200,0.008506 +1793284652,worker-stopped,1,200,0.007653 +1793284652,worker-stopped,1,200,0.017379 +1793284653,worker-stopped,1,200,0.014896 +1793284653,worker-stopped,1,200,0.011902 +1793284653,worker-stopped,1,200,0.016364 +1793284654,worker-stopped,1,200,0.016205 +1793284654,worker-stopped,1,200,0.012478 +1793284654,worker-stopped,1,200,1.012619 +1793284656,worker-stopped,1,200,0.013583 +1793284656,worker-stopped,1,200,0.009906 +1793284656,worker-stopped,1,200,0.013870 +1793284656,worker-stopped,1,200,0.022325 +1793284657,worker-stopped,1,200,0.012196 +1793284657,worker-stopped,1,200,0.014969 +1793284657,worker-stopped,1,200,0.005259 +1793284658,worker-stopped,1,200,0.009548 +1793284658,worker-stopped,1,200,0.008708 +1793284658,worker-stopped,1,200,0.009981 +1793284658,worker-stopped,1,200,0.017814 +1793284659,worker-stopped,1,200,0.016176 +1793284659,worker-stopped,1,200,0.011695 +1793284659,worker-stopped,1,200,0.009274 +1793284660,worker-stopped,1,200,0.006202 +1793284660,worker-stopped,1,200,0.012027 +1793284660,worker-stopped,1,200,0.011342 +1793284661,worker-stopped,1,200,0.017146 +1793284661,worker-stopped,1,200,0.019579 +1793284661,worker-stopped,1,200,0.011983 +1793284661,worker-stopped,1,200,0.020931 +1793284662,worker-stopped,1,200,0.014519 +1793284662,worker-stopped,1,200,0.010709 +1793284662,worker-stopped,1,200,0.026265 +1793284663,worker-stopped,1,200,0.009982 +1793284663,worker-stopped,1,200,0.009383 +1793284663,worker-stopped,1,200,0.009499 +1793284664,worker-stopped,1,200,0.010191 +1793284664,worker-stopped,1,200,0.015340 +1793284664,worker-stopped,1,200,0.011967 +1793284665,worker-stopped,1,200,0.010183 +1793284665,worker-stopped,1,200,0.016046 +1793284665,worker-stopped,1,200,0.011989 +1793284665,worker-stopped,1,200,0.020189 +1793284666,worker-stopped,1,200,0.005580 +1793284666,worker-stopped,1,200,0.010489 +1793284666,worker-stopped,1,200,0.019952 +1793284667,worker-stopped,1,200,0.013537 +1793284667,worker-stopped,1,200,0.013595 +1793284667,worker-stopped,1,200,0.011323 +1793284668,worker-stopped,1,200,0.011938 +1793284668,worker-stopped,1,200,0.011636 +1793284668,worker-stopped,1,200,0.007957 +1793284668,worker-stopped,1,200,0.005310 +1793284669,worker-stopped,1,200,0.013691 +1793284669,worker-stopped,1,200,0.013580 +1793284669,worker-stopped,1,200,0.010994 +1793284670,worker-stopped,1,200,0.014115 +1793284670,worker-stopped,1,200,0.015746 +1793284670,worker-stopped,1,200,0.014352 +1793284671,worker-stopped,1,200,0.013845 +1793284671,worker-stopped,1,200,0.013416 +1793284671,worker-stopped,1,200,0.016509 +1793284671,worker-stopped,1,200,0.014801 +1793284672,worker-stopped,1,200,0.022645 +1793284672,worker-stopped,1,200,0.008663 +1793284672,worker-stopped,1,200,0.015996 +1793284673,worker-stopped,1,200,0.013792 +1793284673,worker-stopped,1,200,0.013770 +1793284673,worker-stopped,1,200,0.009806 +1793284674,worker-stopped,1,200,0.015879 +1793284674,worker-stopped,1,200,0.015063 +1793284674,worker-stopped,1,200,0.012954 +1793284675,worker-stopped,1,200,0.008419 +1793284675,worker-stopped,1,200,0.009689 +1793284675,worker-stopped,1,200,0.011603 +1793284675,worker-stopped,1,200,0.013768 +1793284676,worker-stopped,1,200,0.017622 +1793284676,worker-stopped,1,200,0.010720 +1793284676,worker-stopped,1,200,0.012562 +1793284677,worker-stopped,1,200,0.013052 +1793284677,worker-stopped,1,200,0.013695 +1793284677,worker-stopped,1,200,0.014293 +1793284678,worker-stopped,1,200,0.012400 +1793284678,worker-stopped,1,200,0.013887 +1793284678,worker-stopped,1,200,0.014046 +1793284678,worker-stopped,1,200,0.017807 +1793284679,worker-stopped,1,200,0.014535 +1793284679,worker-stopped,1,200,0.012404 +1793284679,worker-stopped,1,200,0.006823 +1793284680,worker-stopped,1,200,0.011124 +1793284680,worker-stopped,1,200,0.009851 +1793284680,worker-stopped,1,200,0.015791 +1793284681,worker-stopped,1,200,0.019259 +1793284681,worker-stopped,1,200,0.009288 +1793284681,worker-stopped,1,200,0.019564 +1793284682,worker-stopped,1,200,0.015571 +1793284682,worker-stopped,1,200,0.013420 +1793284682,worker-stopped,1,200,0.007239 +1793284682,worker-stopped,1,200,0.014144 +1793284683,worker-stopped,1,200,0.007624 +1793284683,worker-stopped,1,200,0.009673 +1793284683,worker-stopped,1,200,0.008073 +1793284684,worker-stopped,1,200,0.013017 +1793284684,worker-stopped,1,200,0.007833 +1793284684,worker-stopped,1,200,0.012452 +1793284685,worker-stopped,1,200,0.010766 +1793284685,worker-stopped,1,200,0.011903 +1793284685,worker-stopped,1,200,0.013391 +1793284685,worker-stopped,1,200,0.014791 +1793284686,worker-stopped,1,200,0.013288 +1793284686,worker-stopped,1,200,0.007344 +1793284686,worker-stopped,1,200,0.011368 +1793284687,worker-stopped,1,200,0.015435 +1793284687,worker-stopped,1,200,0.006217 +1793284687,worker-stopped,1,200,0.012646 +1793284688,worker-stopped,1,200,0.016370 +1793284688,worker-stopped,1,200,0.011472 +1793284688,worker-stopped,1,200,0.011398 +1793284689,worker-stopped,1,200,0.016175 +1793284689,worker-stopped,1,200,0.010110 +1793284689,worker-stopped,1,200,0.013390 +1793284689,worker-stopped,1,200,0.008208 +1793284690,worker-stopped,1,200,0.014141 +1793284690,worker-stopped,1,200,0.008368 +1793284690,worker-stopped,1,200,0.011884 +1793284691,worker-stopped,1,200,0.011483 +1793284691,worker-stopped,1,200,0.017517 +1793284691,worker-stopped,1,200,0.013562 +1793284692,worker-stopped,1,200,0.010170 +1793284692,worker-stopped,1,200,0.008529 +1793284692,worker-stopped,1,200,0.007894 +1793284692,worker-stopped,1,200,0.014299 +1793284693,worker-stopped,1,200,0.012550 +1793284693,worker-stopped,1,200,0.008443 +1793284693,worker-stopped,1,200,0.013044 +1793284694,worker-stopped,1,200,0.016223 +1793284694,worker-stopped,1,200,0.011274 +1793284694,worker-stopped,1,200,0.011237 +1793284695,worker-stopped,1,200,0.010961 +1793284695,worker-stopped,1,200,0.016506 +1793284695,worker-stopped,1,200,0.014924 +1793284695,worker-stopped,1,200,0.010958 +1793284696,worker-stopped,1,200,0.015711 +1793284696,worker-stopped,1,200,0.012021 +1793284696,worker-stopped,1,200,0.014635 +1793284697,worker-stopped,1,200,0.012610 +1793284697,worker-stopped,1,200,0.014340 +1793284697,worker-stopped,1,200,0.018249 +1793284697,worker-stopped,1,200,0.013144 +1793284698,worker-stopped,1,200,0.014337 +1793284698,worker-stopped,1,200,0.012878 +1793284698,worker-stopped,1,200,1.013434 +1793284700,worker-stopped,1,200,0.012431 +1793284700,worker-stopped,1,200,0.010874 +1793284700,worker-stopped,1,200,0.010470 +1793284700,worker-stopped,1,200,0.010650 +1793284701,worker-stopped,1,200,0.011779 +1793284701,worker-stopped,1,200,0.010463 +1793284701,worker-stopped,1,200,0.012803 +1793284702,worker-stopped,1,200,0.010129 +1793284702,worker-stopped,1,200,0.010544 +1793284702,worker-stopped,1,200,0.018595 +1793284702,worker-stopped,1,200,0.009194 +1793284703,worker-stopped,1,200,0.009844 +1793284703,worker-stopped,1,200,0.013339 +1793284703,worker-stopped,1,200,0.008382 +1793284704,worker-stopped,1,200,0.014389 +1793284704,worker-stopped,1,200,0.023442 +1793284704,worker-stopped,1,200,0.008726 +1793284704,worker-stopped,1,200,0.008283 +1793284705,worker-stopped,1,200,0.010299 +1793284705,worker-stopped,1,200,0.008118 +1793284705,worker-stopped,1,200,0.011135 +1793284706,worker-stopped,1,200,0.011842 +1793284706,worker-stopped,1,200,0.009565 +1793284706,worker-stopped,1,200,0.010218 +1793284706,worker-stopped,1,200,0.007199 +1793284707,worker-stopped,1,200,0.012728 +1793284707,worker-stopped,1,200,0.005468 +1793284707,worker-stopped,1,200,0.011937 +1793284708,worker-stopped,1,200,0.005475 +1793284708,worker-stopped,1,200,0.006421 +1793284708,worker-stopped,1,200,0.006374 +1793284708,worker-stopped,1,200,0.006285 +1793284709,worker-stopped,1,200,0.004284 +1793284709,worker-stopped,1,200,0.003831 +1793284709,worker-stopped,1,200,0.007520 +1793284709,worker-stopped,1,200,0.006033 +1793284710,worker-stopped,1,200,0.003912 +1793284710,worker-stopped,1,200,0.004025 +1793284710,worker-stopped,1,200,0.006417 +1793284711,worker-stopped,1,200,0.005735 +1793284711,worker-stopped,1,200,0.004266 +1793284711,worker-stopped,1,200,0.005693 +1793284711,worker-stopped,1,200,0.003929 +1793284712,worker-stopped,1,200,0.003299 +1793284712,worker-stopped,1,200,0.008932 +1793284712,worker-stopped,1,200,0.015726 +1793284713,worker-stopped,1,200,0.010862 +1793284713,worker-stopped,1,200,0.012034 +1793284713,worker-stopped,1,200,0.009224 +1793284713,worker-stopped,1,200,0.014382 +1793284714,worker-stopped,1,200,0.013655 +1793284714,worker-stopped,1,200,0.009601 +1793284714,worker-stopped,1,200,0.006621 +1793284715,worker-stopped,1,200,0.006746 +1793284715,worker-stopped,1,200,0.013506 +1793284715,worker-stopped,1,200,0.016583 +1793284716,worker-stopped,1,200,0.009139 +1793284716,worker-stopped,1,200,0.009464 +1793284716,worker-stopped,1,200,0.010917 +1793284716,worker-stopped,1,200,0.011002 +1793284717,worker-stopped,1,200,0.012120 +1793284717,worker-stopped,1,200,0.011973 +1793284717,worker-stopped,1,200,0.009418 +1793284717,worker-stopped,1,200,0.007395 +1793284718,worker-stopped,1,200,0.003799 +1793284718,worker-stopped,1,200,0.010616 +1793284718,worker-stopped,1,200,0.012847 +1793284719,worker-stopped,1,200,0.005648 +1793284719,worker-stopped,1,200,0.013693 +1793284719,worker-stopped,1,200,0.009650 +1793284720,worker-stopped,1,200,0.009727 +1793284720,worker-stopped,1,200,0.014185 +1793284720,worker-stopped,1,200,0.013882 +1793284720,worker-stopped,1,200,0.008374 +1793284721,worker-stopped,1,200,0.013354 +1793284721,worker-stopped,1,200,0.010069 +1793284721,worker-stopped,1,200,0.012079 +1793284722,worker-stopped,1,200,0.010210 +1793284722,worker-stopped,1,200,0.009292 +1793284722,worker-stopped,1,200,0.012190 +1793284723,worker-stopped,1,200,0.007577 +1793284723,worker-stopped,1,200,0.010421 +1793284723,worker-stopped,1,200,0.007376 +1793284723,worker-stopped,1,200,0.009566 +1793284724,worker-stopped,1,200,0.008576 +1793284724,worker-stopped,1,200,0.013243 +1793284724,worker-stopped,1,200,0.016405 +1793284724,worker-stopped,1,200,0.008943 +1793284725,worker-stopped,1,200,0.011648 +1793284725,worker-stopped,1,200,0.010939 +1793284725,worker-stopped,1,200,0.035198 +1793284726,worker-stopped,1,200,0.011010 +1793284726,worker-stopped,1,200,0.010588 +1793284726,worker-stopped,1,200,0.016497 +1793284727,worker-stopped,1,200,0.007266 +1793284727,worker-stopped,1,200,0.010445 +1793284727,worker-stopped,1,200,0.010721 +1793284728,worker-stopped,1,200,0.016974 +1793284728,worker-stopped,1,200,0.010173 +1793284728,worker-stopped,1,200,0.014606 +1793284728,worker-stopped,1,200,0.012050 +1793284729,worker-stopped,1,200,0.011478 +1793284729,worker-stopped,1,200,0.021747 +1793284729,worker-stopped,1,200,0.016157 +1793284730,worker-stopped,1,200,0.013543 +1793284730,worker-stopped,1,200,0.012689 +1793284730,worker-stopped,1,200,0.012343 +1793284730,worker-stopped,1,200,0.014446 +1793284731,worker-stopped,1,200,0.011611 +1793284731,worker-stopped,1,200,0.015922 +1793284731,worker-stopped,1,200,0.028301 +1793284732,worker-stopped,1,200,0.007004 +1793284732,worker-stopped,1,200,0.012567 +1793284732,worker-stopped,1,200,0.019845 +1793284733,worker-stopped,1,200,0.011730 +1793284733,worker-stopped,1,200,0.014983 +1793284733,worker-stopped,1,200,0.011826 +1793284733,worker-stopped,1,200,0.018628 +1793284734,worker-stopped,1,200,1.010526 +1793284735,worker-stopped,1,200,0.012143 +1793284735,worker-stopped,1,200,0.012531 +1793284736,worker-stopped,1,200,0.008352 +1793284736,worker-stopped,1,200,0.011316 +1793284736,worker-stopped,1,200,0.011217 +1793284736,worker-stopped,1,200,0.012013 +1793284737,worker-stopped,1,200,0.009795 +1793284737,worker-stopped,1,200,0.012824 +1793284737,worker-stopped,1,200,0.013085 +1793284738,worker-stopped,1,200,0.010932 +1793284738,worker-stopped,1,200,0.010279 +1793284738,worker-stopped,1,200,0.014130 +1793284739,worker-stopped,1,200,0.013467 +1793284739,worker-stopped,1,200,0.009842 +1793284739,worker-stopped,1,200,0.015344 +1793284739,worker-stopped,1,200,0.009191 +1793284740,worker-stopped,1,200,0.016097 +1793284740,worker-stopped,1,200,0.008361 +1793284740,worker-stopped,1,200,0.013803 +1793284741,worker-stopped,1,200,0.014136 +1793284741,worker-stopped,1,200,0.014525 +1793284741,worker-stopped,1,200,0.016526 +1793284741,worker-stopped,1,200,0.014749 +1793284742,worker-stopped,1,200,0.007356 +1793284742,worker-stopped,1,200,0.011315 +1793284742,worker-stopped,1,200,0.016777 +1793284743,worker-stopped,1,200,0.009801 +1793284743,worker-stopped,1,200,0.006439 +1793284743,worker-stopped,1,200,0.005960 +1793284743,worker-stopped,1,200,0.006404 +1793284744,worker-stopped,1,200,0.004975 +1793284744,worker-stopped,1,200,0.006741 +1793284744,worker-stopped,1,200,0.006990 +1793284745,worker-stopped,1,200,0.006193 +1793284745,worker-stopped,1,200,0.005009 +1793284745,worker-stopped,1,200,0.011848 +1793284745,worker-stopped,1,200,0.011024 +1793284746,worker-stopped,1,200,0.012883 +1793284746,worker-stopped,1,200,0.008822 +1793284746,worker-stopped,1,200,0.008857 +1793284747,worker-stopped,1,200,0.013985 +1793284747,worker-stopped,1,200,0.013608 +1793284747,worker-stopped,1,200,0.013786 +1793284748,worker-stopped,1,200,0.014495 +1793284748,worker-stopped,1,200,0.012895 +1793284748,worker-stopped,1,200,0.017846 +1793284748,worker-stopped,1,200,0.012580 +1793284749,worker-stopped,1,200,0.010555 +1793284749,worker-stopped,1,200,0.011107 +1793284749,worker-stopped,1,200,0.011788 +1793284750,worker-stopped,1,200,0.020163 +1793284750,worker-stopped,1,200,0.020057 +1793284750,worker-stopped,1,200,0.015509 +1793284751,worker-stopped,1,200,0.010826 +1793284751,worker-stopped,1,200,0.009176 +1793284751,worker-stopped,1,200,0.012134 +1793284752,worker-stopped,1,200,0.016806 +1793284752,worker-stopped,1,200,0.011760 +1793284752,worker-stopped,1,200,0.010094 +1793284752,worker-stopped,1,200,0.019272 +1793284753,worker-recovery,1,200,0.007938 +1793284753,worker-recovery,1,200,0.004768 +1793284753,worker-recovery,1,200,0.007021 +1793284754,worker-recovery,1,200,0.015535 +1793284754,worker-recovery,1,200,0.015584 +1793284754,worker-recovery,1,200,0.009163 +1793284754,worker-recovery,1,200,0.015243 +1793284755,worker-recovery,1,200,0.019047 +1793284755,worker-recovery,1,200,0.017591 +1793284755,worker-recovery,1,200,0.014631 +1793284756,worker-recovery,1,200,0.018456 +1793284756,worker-recovery,1,200,0.018973 +1793284756,worker-recovery,1,200,0.009098 +1793284756,worker-recovery,1,200,0.012535 +1793284757,worker-recovery,1,200,0.013318 +1793284757,worker-recovery,1,200,0.017025 +1793284757,worker-recovery,1,200,0.007758 +1793284758,worker-recovery,1,200,0.009051 +1793284758,worker-recovery,1,200,0.012977 +1793284758,worker-recovery,1,200,0.008424 +1793284758,worker-recovery,1,200,0.013525 +1793284759,worker-recovery,1,200,0.011598 +1793284759,worker-recovery,1,200,0.015328 +1793284759,worker-recovery,1,200,0.033218 +1793284760,worker-recovery,1,200,0.016334 +1793284760,worker-recovery,1,200,0.011001 +1793284760,worker-recovery,1,200,0.009533 +1793284761,worker-recovery,1,200,0.020263 +1793284761,worker-recovery,1,200,0.011918 +1793284761,worker-recovery,1,200,0.013716 +1793284761,worker-recovery,1,200,0.014195 +1793284762,worker-recovery,1,200,0.010583 +1793284762,worker-recovery,1,200,0.011890 +1793284762,worker-recovery,1,200,0.012500 +1793284763,worker-recovery,1,200,0.008335 +1793284763,worker-recovery,1,200,0.011542 +1793284763,worker-recovery,1,200,0.013806 +1793284764,worker-recovery,1,200,0.016059 +1793284764,worker-recovery,1,200,0.011972 +1793284764,worker-recovery,1,200,0.012024 +1793284764,worker-recovery,1,200,0.014120 +1793284765,worker-recovery,1,200,0.012598 +1793284765,worker-recovery,1,200,0.014892 +1793284765,worker-recovery,1,200,0.012612 +1793284766,worker-recovery,1,200,0.014990 +1793284766,worker-recovery,1,200,0.013383 +1793284766,worker-recovery,1,200,0.016486 +1793284766,worker-recovery,1,200,0.007896 +1793284767,worker-recovery,1,200,0.011079 +1793284767,worker-recovery,1,200,0.013272 +1793284767,worker-recovery,1,200,0.013450 +1793284768,worker-recovery,1,200,0.011011 +1793284768,worker-recovery,1,200,0.007013 +1793284768,worker-recovery,1,200,0.012322 +1793284769,worker-recovery,1,200,0.005323 +1793284769,worker-recovery,1,200,0.010054 +1793284769,worker-recovery,1,200,0.012121 +1793284769,worker-recovery,1,200,0.009108 +1793284770,worker-recovery,1,200,0.015288 +1793284770,worker-recovery,1,200,0.013756 +1793284770,worker-recovery,1,200,0.013667 +1793284771,worker-recovery,1,200,0.006255 +1793284771,worker-recovery,1,200,0.013749 +1793284771,worker-recovery,1,200,0.014691 +1793284771,worker-recovery,1,200,0.013399 +1793284772,worker-recovery,1,200,0.015471 +1793284772,worker-recovery,1,200,0.014085 +1793284772,worker-recovery,1,200,0.011712 +1793284773,worker-recovery,1,200,0.007819 +1793284773,worker-recovery,1,200,0.006129 +1793284773,worker-recovery,1,200,0.021276 +1793284773,worker-recovery,1,200,0.017881 +1793284774,worker-recovery,1,200,0.017069 +1793284774,worker-recovery,1,200,0.016972 +1793284774,worker-recovery,1,200,0.010324 +1793284775,worker-recovery,1,200,0.004211 +1793284775,worker-recovery,1,200,0.013389 +1793284775,worker-recovery,1,200,0.010043 +1793284776,worker-recovery,1,200,0.012164 +1793284776,worker-recovery,1,200,0.010818 +1793284776,worker-recovery,1,200,0.010395 +1793284776,worker-recovery,1,200,0.012069 +1793284777,worker-recovery,1,200,0.007383 +1793284777,worker-recovery,1,200,0.011689 +1793284777,worker-recovery,1,200,0.013912 +1793284778,worker-recovery,1,200,0.011415 +1793284778,worker-recovery,1,200,0.009114 +1793284778,worker-recovery,1,200,0.014638 +1793284778,worker-recovery,1,200,0.010028 +1793284779,worker-recovery,1,200,0.015610 +1793284779,worker-recovery,1,200,0.009494 +1793284779,worker-recovery,1,200,0.009881 +1793284780,worker-recovery,1,200,0.010973 +1793284780,worker-recovery,1,200,0.034025 +1793284780,worker-recovery,1,200,0.012741 +1793284780,worker-recovery,1,200,0.008385 +1793284781,worker-recovery,1,200,0.007084 +1793284781,worker-recovery,1,200,0.010868 +1793284781,worker-recovery,1,200,0.009186 +1793284782,worker-recovery,1,200,0.010961 +1793284782,worker-recovery,1,200,0.011214 +1793284782,worker-recovery,1,200,0.011666 +1793284782,worker-recovery,1,200,0.009267 +1793284783,worker-recovery,1,200,0.033875 +1793284783,worker-recovery,1,200,0.013426 +1793284783,worker-recovery,1,200,0.008509 +1793284784,worker-recovery,1,200,0.009217 +1793284784,worker-recovery,1,200,0.009286 +1793284784,worker-recovery,1,200,0.008575 +1793284784,worker-recovery,1,200,0.010301 +1793284785,worker-recovery,1,200,0.013101 +1793284785,worker-recovery,1,200,0.013873 +1793284785,worker-recovery,1,200,0.010633 +1793284786,worker-recovery,1,200,0.011059 +1793284786,worker-recovery,1,200,0.009325 +1793284786,worker-recovery,1,200,0.033810 +1793284787,worker-recovery,1,200,0.013081 +1793284787,worker-recovery,1,200,0.006510 +1793284787,worker-recovery,1,200,0.009371 +1793284787,worker-recovery,1,200,0.037242 +1793284788,worker-recovery,1,200,0.014314 +1793284788,worker-recovery,1,200,0.012323 +1793284788,worker-recovery,1,200,0.032680 +1793284789,worker-recovery,1,200,0.018990 +1793284789,worker-recovery,1,200,0.007101 +1793284789,worker-recovery,1,200,0.010795 +1793284789,worker-recovery,1,200,0.010743 +1793284790,worker-recovery,1,200,0.006244 +1793284790,worker-recovery,1,200,0.015542 +1793284790,worker-recovery,1,200,0.013658 +1793284791,worker-recovery,1,200,0.012586 +1793284791,worker-recovery,1,200,0.007304 +1793284791,worker-recovery,1,200,0.025574 +1793284792,worker-recovery,1,200,0.012266 +1793284792,worker-recovery,1,200,0.010493 +1793284792,worker-recovery,1,200,0.021457 +1793284792,worker-recovery,1,200,0.005587 +1793284793,worker-recovery,1,200,0.016551 +1793284793,worker-recovery,1,200,0.011773 +1793284793,worker-recovery,1,200,0.021989 +1793284794,worker-recovery,1,200,0.029036 +1793284794,worker-recovery,1,200,0.015901 +1793284794,worker-recovery,1,200,0.008164 +1793284794,worker-recovery,1,200,0.012547 +1793284795,worker-recovery,1,200,0.009671 +1793284795,worker-recovery,1,200,0.006796 +1793284795,worker-recovery,1,200,0.013073 +1793284796,worker-recovery,1,200,0.010658 +1793284796,worker-recovery,1,200,0.013827 +1793284796,worker-recovery,1,200,0.017912 +1793284797,worker-recovery,1,200,0.018935 +1793284797,worker-recovery,1,200,0.013032 +1793284797,worker-recovery,1,200,0.007874 +1793284797,worker-recovery,1,200,0.008192 +1793284798,worker-recovery,1,200,0.006908 +1793284798,worker-recovery,1,200,0.007863 +1793284798,worker-recovery,1,200,0.009067 +1793284799,worker-recovery,1,200,0.030957 +1793284799,worker-recovery,1,200,0.008950 +1793284799,worker-recovery,1,200,0.011121 +1793284799,worker-recovery,1,200,0.017152 +1793284800,worker-recovery,1,200,0.009661 +1793284800,worker-recovery,1,200,0.010101 +1793284800,worker-recovery,1,200,0.016054 +1793284801,worker-recovery,1,200,0.009749 +1793284801,worker-recovery,1,200,0.007315 +1793284801,worker-recovery,1,200,0.006744 +1793284802,worker-recovery,1,200,0.007395 +1793284802,worker-recovery,1,200,0.006401 +1793284802,worker-recovery,1,200,0.007067 +1793284802,worker-recovery,1,200,0.009230 +1793284803,worker-recovery,1,200,0.028377 +1793284803,worker-recovery,1,200,0.008341 +1793284803,worker-recovery,1,200,0.007821 +1793284804,worker-recovery,1,200,0.013512 +1793284804,worker-recovery,1,200,0.011958 +1793284804,worker-recovery,1,200,0.018057 +1793284805,worker-recovery,1,200,0.008796 +1793284805,worker-recovery,1,200,0.014926 +1793284805,worker-recovery,1,200,0.006146 +1793284805,worker-recovery,1,200,0.016257 +1793284806,worker-recovery,1,200,0.012948 +1793284806,worker-recovery,1,200,0.005872 +1793284806,worker-recovery,1,200,0.026116 +1793284807,worker-recovery,1,200,0.008950 +1793284807,worker-recovery,1,200,0.007510 +1793284807,worker-recovery,1,200,0.019339 +1793284807,worker-recovery,1,200,0.011556 +1793284808,worker-recovery,1,200,0.007972 +1793284808,worker-recovery,1,200,0.009687 +1793284808,worker-recovery,1,200,0.011743 +1793284809,worker-recovery,1,200,0.009819 +1793284809,worker-recovery,1,200,0.005987 +1793284809,worker-recovery,1,200,0.007475 +1793284809,worker-recovery,1,200,0.011299 +1793284810,worker-recovery,1,200,0.006615 +1793284810,worker-recovery,1,200,0.008419 +1793284810,worker-recovery,1,200,0.009314 +1793284811,worker-recovery,1,200,0.012769 +1793284811,worker-recovery,1,200,0.014890 +1793284811,worker-recovery,1,200,0.010221 +1793284811,worker-recovery,1,200,0.014596 +1793284812,worker-recovery,1,200,0.007879 +1793284812,worker-recovery,1,200,0.009963 +1793284812,worker-recovery,1,200,0.007814 +1793284813,worker-recovery,1,200,0.007646 +1793284813,worker-recovery,1,200,0.005167 +1793284813,worker-recovery,1,200,0.008224 +1793284814,worker-recovery,1,200,0.010916 +1793284814,worker-recovery,1,200,0.009203 +1793284814,worker-recovery,1,200,0.007899 +1793284814,worker-recovery,1,200,0.006628 +1793284815,worker-recovery,1,200,0.007019 +1793284815,worker-recovery,1,200,0.008512 +1793284815,worker-recovery,1,200,0.006384 +1793284816,worker-recovery,1,200,0.016140 +1793284816,worker-recovery,1,200,0.012582 +1793284816,worker-recovery,1,200,0.010257 +1793284816,worker-recovery,1,200,0.009649 +1793284817,worker-recovery,1,200,0.006481 +1793284817,worker-recovery,1,200,0.008699 +1793284817,worker-recovery,1,200,0.009136 +1793284818,worker-recovery,1,200,0.015687 +1793284818,worker-recovery,1,200,0.004177 +1793284818,worker-recovery,1,200,0.011194 +1793284818,worker-recovery,1,200,0.009351 +1793284819,worker-recovery,1,200,0.009207 +1793284819,worker-recovery,1,200,0.005319 +1793284819,worker-recovery,1,200,0.009369 +1793284820,worker-recovery,1,200,0.011829 +1793284820,worker-recovery,1,200,0.007577 +1793284820,worker-recovery,1,200,0.011103 +1793284820,worker-recovery,1,200,0.009158 +1793284821,worker-recovery,1,200,0.016067 +1793284821,worker-recovery,1,200,0.018971 +1793284821,worker-recovery,1,200,0.008095 +1793284822,worker-recovery,1,200,0.010580 +1793284822,worker-recovery,1,200,0.013776 +1793284822,worker-recovery,1,200,0.009252 +1793284822,worker-recovery,1,200,0.009329 +1793284823,worker-recovery,1,200,0.009450 +1793284823,worker-recovery,1,200,0.008152 +1793284823,worker-recovery,1,200,0.021793 +1793284824,worker-recovery,1,200,0.007484 +1793284824,worker-recovery,1,200,0.007781 +1793284824,worker-recovery,1,200,0.018737 +1793284825,worker-recovery,1,200,0.007943 +1793284825,worker-recovery,1,200,0.006734 +1793284825,worker-recovery,1,200,0.011539 +1793284825,worker-recovery,1,200,0.010859 +1793284826,worker-recovery,1,200,0.013780 +1793284826,worker-recovery,1,200,0.009282 +1793284826,worker-recovery,1,200,0.007548 +1793284827,worker-recovery,1,200,0.006187 +1793284827,worker-recovery,1,200,0.013068 +1793284827,worker-recovery,1,200,0.009419 +1793284827,worker-recovery,1,200,0.007920 +1793284828,worker-recovery,1,200,0.012338 +1793284828,worker-recovery,1,200,0.010215 +1793284828,worker-recovery,1,200,0.010234 +1793284829,worker-recovery,1,200,0.015216 +1793284829,worker-recovery,1,200,0.008253 +1793284829,worker-recovery,1,200,0.018897 +1793284830,worker-recovery,1,200,0.020451 +1793284830,worker-recovery,1,200,0.010836 +1793284830,worker-recovery,1,200,0.013089 +1793284830,worker-recovery,1,200,0.006631 +1793284831,worker-recovery,1,200,0.010077 +1793284831,worker-recovery,1,200,0.010787 +1793284831,worker-recovery,1,200,0.007833 +1793284832,worker-recovery,1,200,0.020161 +1793284832,worker-recovery,1,200,0.010642 +1793284832,worker-recovery,1,200,0.016337 +1793284832,worker-recovery,1,200,0.022559 +1793284833,worker-recovery,1,200,0.009532 +1793284833,worker-recovery,1,200,0.010244 +1793284833,worker-recovery,1,200,0.008437 +1793284834,worker-recovery,1,200,0.010585 +1793284834,worker-recovery,1,200,0.023702 +1793284834,worker-recovery,1,200,0.016761 +1793284835,worker-recovery,1,200,0.012819 +1793284835,worker-recovery,1,200,0.023547 +1793284835,worker-recovery,1,200,0.010593 +1793284835,worker-recovery,1,200,0.006287 +1793284836,worker-recovery,1,200,0.008743 +1793284836,worker-recovery,1,200,0.007366 +1793284836,worker-recovery,1,200,0.009249 +1793284837,worker-recovery,1,200,0.014501 +1793284837,worker-recovery,1,200,0.022122 +1793284837,worker-recovery,1,200,0.044631 +1793284838,worker-recovery,1,200,0.008514 +1793284838,worker-recovery,1,200,0.008176 +1793284838,worker-recovery,1,200,0.009648 +1793284838,worker-recovery,1,200,0.007009 +1793284839,worker-recovery,1,200,0.019722 +1793284839,worker-recovery,1,200,0.007046 +1793284839,worker-recovery,1,200,0.009661 +1793284840,worker-recovery,1,200,0.015826 +1793284840,worker-recovery,1,200,0.008732 +1793284840,worker-recovery,1,200,0.006243 +1793284840,worker-recovery,1,200,0.006360 +1793284841,worker-recovery,1,200,0.023070 +1793284841,worker-recovery,1,200,0.009570 +1793284841,worker-recovery,1,200,0.008731 +1793284842,worker-recovery,1,200,0.020930 +1793284842,worker-recovery,1,200,0.008364 +1793284842,worker-recovery,1,200,0.010726 +1793284842,worker-recovery,1,200,0.009131 +1793284843,worker-recovery,1,200,0.010306 +1793284843,worker-recovery,1,200,0.007819 +1793284843,worker-recovery,1,200,0.007213 +1793284844,worker-recovery,1,200,0.007353 +1793284844,worker-recovery,1,200,0.035055 +1793284844,trace-backend-outage,1,200,0.029665 +1793284845,trace-backend-outage,1,200,0.012397 +1793284845,trace-backend-outage,1,200,0.006754 +1793284845,trace-backend-outage,1,200,0.015063 +1793284845,trace-backend-outage,1,200,0.007997 +1793284846,trace-backend-outage,1,200,0.011252 +1793284846,trace-backend-outage,1,200,0.008634 +1793284846,trace-backend-outage,1,200,0.014271 +1793284847,trace-backend-outage,1,200,0.022467 +1793284847,trace-backend-outage,1,200,0.010625 +1793284847,trace-backend-outage,1,200,0.022981 +1793284847,trace-backend-outage,1,200,0.030242 +1793284848,trace-backend-outage,1,200,0.006765 +1793284848,trace-backend-outage,1,200,0.013247 +1793284848,trace-backend-outage,1,200,0.012057 +1793284849,trace-backend-outage,1,200,0.009603 +1793284849,trace-backend-outage,1,200,0.017727 +1793284849,trace-backend-outage,1,200,0.010139 +1793284850,trace-backend-outage,1,200,0.020499 +1793284850,trace-backend-outage,1,200,0.008178 +1793284850,trace-backend-outage,1,200,0.015384 +1793284850,trace-backend-outage,1,200,0.016699 +1793284851,trace-backend-outage,1,200,0.024025 +1793284851,trace-backend-outage,1,200,0.042234 +1793284851,trace-backend-outage,1,200,0.017770 +1793284852,trace-backend-outage,1,200,0.010150 +1793284852,trace-backend-outage,1,200,0.009662 +1793284852,trace-backend-outage,1,200,0.007844 +1793284852,trace-backend-outage,1,200,0.016889 +1793284853,trace-backend-outage,1,200,0.023360 +1793284853,trace-backend-outage,1,200,0.014489 +1793284853,trace-backend-outage,1,200,0.010382 +1793284854,trace-backend-outage,1,200,0.008583 +1793284854,trace-backend-outage,1,200,0.008679 +1793284854,trace-backend-outage,1,200,0.014170 +1793284855,trace-backend-outage,1,200,0.005482 +1793284855,trace-backend-outage,1,200,0.007158 +1793284855,trace-backend-outage,1,200,0.012757 +1793284855,trace-backend-outage,1,200,0.008189 +1793284856,trace-backend-outage,1,200,0.010249 +1793284856,trace-backend-outage,1,200,0.009144 +1793284856,trace-backend-outage,1,200,0.009479 +1793284857,trace-backend-outage,1,200,0.007898 +1793284857,trace-backend-outage,1,200,0.010440 +1793284857,trace-backend-outage,1,200,0.008341 +1793284857,trace-backend-outage,1,200,0.006532 +1793284858,trace-backend-outage,1,200,0.018171 +1793284858,trace-backend-outage,1,200,0.012539 +1793284858,trace-backend-outage,1,200,0.008085 +1793284858,trace-backend-outage,1,200,0.010920 +1793284859,trace-backend-outage,1,200,0.012785 +1793284859,trace-backend-outage,1,200,0.008482 +1793284859,trace-backend-outage,1,200,0.014323 +1793284860,trace-backend-outage,1,200,0.005700 +1793284860,trace-backend-outage,1,200,0.004542 +1793284860,trace-backend-outage,1,200,0.008563 +1793284860,trace-backend-outage,1,200,0.006382 +1793284861,trace-backend-outage,1,200,0.005597 +1793284861,trace-backend-outage,1,200,0.011627 +1793284861,trace-backend-outage,1,200,0.005071 +1793284862,trace-backend-outage,1,200,0.017243 +1793284862,trace-backend-outage,1,200,0.007438 +1793284862,trace-backend-outage,1,200,0.006735 +1793284862,trace-backend-outage,1,200,0.007067 +1793284863,trace-backend-outage,1,200,0.008513 +1793284863,trace-backend-outage,1,200,0.008697 +1793284863,trace-backend-outage,1,200,0.010821 +1793284864,trace-backend-outage,1,200,0.010172 +1793284864,trace-backend-outage,1,200,0.007832 +1793284864,trace-backend-outage,1,200,0.011005 +1793284864,trace-backend-outage,1,200,0.007833 +1793284865,trace-backend-outage,1,200,0.015246 +1793284865,trace-backend-outage,1,200,0.017498 +1793284865,trace-backend-outage,1,200,0.014236 +1793284866,trace-backend-outage,1,200,0.022186 +1793284866,trace-backend-outage,1,200,0.011622 +1793284866,trace-backend-outage,1,200,0.009243 +1793284867,trace-backend-outage,1,200,0.016878 +1793284867,trace-backend-outage,1,200,0.016400 +1793284867,trace-backend-outage,1,200,0.013197 +1793284867,trace-backend-outage,1,200,0.009315 +1793284868,trace-backend-outage,1,200,0.021064 +1793284868,trace-backend-outage,1,200,0.016319 +1793284868,trace-backend-outage,1,200,0.009906 +1793284869,trace-backend-outage,1,200,0.016248 +1793284869,trace-backend-outage,1,200,0.013465 +1793284869,trace-backend-outage,1,200,0.011063 +1793284870,trace-backend-outage,1,200,0.019590 +1793284870,trace-backend-outage,1,200,0.013798 +1793284870,trace-backend-outage,1,200,0.013147 +1793284870,trace-backend-outage,1,200,0.010959 +1793284871,trace-backend-outage,1,200,0.014419 +1793284871,trace-backend-outage,1,200,0.017089 +1793284871,trace-backend-outage,1,200,0.009348 +1793284872,trace-backend-outage,1,200,0.019075 +1793284872,trace-backend-outage,1,200,0.010341 +1793284872,trace-backend-outage,1,200,0.023836 +1793284873,trace-backend-outage,1,200,0.019568 +1793284873,trace-backend-outage,1,200,0.019834 +1793284873,trace-backend-outage,1,200,0.015018 +1793284874,trace-backend-outage,1,200,0.010356 +1793284874,trace-backend-outage,1,200,0.018357 +1793284874,trace-backend-outage,1,200,0.016828 +1793284874,trace-backend-outage,1,200,0.008542 +1793284875,trace-backend-outage,1,200,0.025113 +1793284875,trace-backend-outage,1,200,0.024282 +1793284875,trace-backend-outage,1,200,0.010071 +1793284876,trace-backend-outage,1,200,0.015899 +1793284876,trace-backend-outage,1,200,0.016565 +1793284876,trace-backend-outage,1,200,0.017677 +1793284877,trace-backend-outage,1,200,0.016929 +1793284877,trace-backend-outage,1,200,0.017117 +1793284877,trace-backend-outage,1,200,0.018649 +1793284877,trace-backend-outage,1,200,0.015324 +1793284878,trace-backend-outage,1,200,0.015054 +1793284878,trace-backend-outage,1,200,0.018863 +1793284878,trace-backend-outage,1,200,0.010008 +1793284879,trace-backend-outage,1,200,0.017199 +1793284879,trace-backend-outage,1,200,0.016735 +1793284879,trace-backend-outage,1,200,0.021094 +1793284880,trace-backend-outage,1,200,0.017068 +1793284880,trace-backend-outage,1,200,0.017467 +1793284880,trace-backend-outage,1,200,0.018922 +1793284881,trace-backend-outage,1,200,0.020090 +1793284881,trace-backend-outage,1,200,0.018789 +1793284881,trace-backend-outage,1,200,0.014446 +1793284881,trace-backend-outage,1,200,0.010775 +1793284882,trace-backend-outage,1,200,0.016438 +1793284882,trace-backend-outage,1,200,0.012605 +1793284882,trace-backend-outage,1,200,0.021813 +1793284883,trace-backend-outage,1,200,0.018392 +1793284883,trace-backend-outage,1,200,0.016527 +1793284883,trace-backend-outage,1,200,0.009615 +1793284884,trace-backend-outage,1,200,0.008502 +1793284884,trace-backend-outage,1,200,0.025718 +1793284884,trace-backend-outage,1,200,0.018461 +1793284884,trace-backend-outage,1,200,0.008219 +1793284885,trace-backend-outage,1,200,0.013857 +1793284885,trace-backend-outage,1,200,0.018340 +1793284885,trace-backend-outage,1,200,0.020452 +1793284886,trace-backend-outage,1,200,0.020575 +1793284886,trace-backend-outage,1,200,0.020224 +1793284886,trace-backend-outage,1,200,0.020652 +1793284887,trace-backend-outage,1,200,0.020727 +1793284887,trace-backend-outage,1,200,0.020496 +1793284887,trace-backend-outage,1,200,0.016747 +1793284887,trace-backend-outage,1,200,0.009260 +1793284888,trace-backend-outage,1,200,0.011636 +1793284888,trace-backend-outage,1,200,0.010135 +1793284888,trace-backend-outage,1,200,0.011420 +1793284889,trace-backend-outage,1,200,0.008006 +1793284889,trace-backend-outage,1,200,0.021722 +1793284889,trace-backend-outage,1,200,0.014604 +1793284890,trace-backend-outage,1,200,0.020530 +1793284890,trace-backend-outage,1,200,0.014803 +1793284890,trace-backend-outage,1,200,0.015356 +1793284890,trace-backend-outage,1,200,0.011917 +1793284891,trace-backend-outage,1,200,0.011477 +1793284891,trace-backend-outage,1,200,0.011297 +1793284891,trace-backend-outage,1,200,0.009457 +1793284892,trace-backend-outage,1,200,0.009550 +1793284892,trace-backend-outage,1,200,0.007725 +1793284892,trace-backend-outage,1,200,0.021497 +1793284893,trace-backend-outage,1,200,0.045668 +1793284893,telemetry-recovery,1,200,0.030253 +1793284893,telemetry-recovery,1,200,0.010026 +1793284894,telemetry-recovery,1,200,0.011613 +1793284894,telemetry-recovery,1,200,0.016338 +1793284894,telemetry-recovery,1,200,0.011531 +1793284894,telemetry-recovery,1,200,0.008878 +1793284895,telemetry-recovery,1,200,0.009858 +1793284895,telemetry-recovery,1,200,0.008119 +1793284895,telemetry-recovery,1,200,0.008108 +1793284896,telemetry-recovery,1,200,0.014278 +1793284896,telemetry-recovery,1,200,0.017264 +1793284896,telemetry-recovery,1,200,0.015195 +1793284897,telemetry-recovery,1,200,0.014451 +1793284897,telemetry-recovery,1,200,0.014512 +1793284897,telemetry-recovery,1,200,0.016762 +1793284897,telemetry-recovery,1,200,0.009617 +1793284898,telemetry-recovery,1,200,0.017183 +1793284898,telemetry-recovery,1,200,0.022359 +1793284898,telemetry-recovery,1,200,0.017843 +1793284899,telemetry-recovery,1,200,0.014207 +1793284899,telemetry-recovery,1,200,0.013108 +1793284899,telemetry-recovery,1,200,0.012883 +1793284900,telemetry-recovery,1,200,0.021735 +1793284900,telemetry-recovery,1,200,0.014230 +1793284900,telemetry-recovery,1,200,0.016118 +1793284901,telemetry-recovery,1,200,0.018506 +1793284901,telemetry-recovery,1,200,0.011194 +1793284901,telemetry-recovery,1,200,0.009908 +1793284901,telemetry-recovery,1,200,0.012124 +1793284902,telemetry-recovery,1,200,0.014793 +1793284902,telemetry-recovery,1,200,0.013646 +1793284902,telemetry-recovery,1,200,0.013525 +1793284903,telemetry-recovery,1,200,0.014725 +1793284903,telemetry-recovery,1,200,0.010449 +1793284903,telemetry-recovery,1,200,0.014946 +1793284904,telemetry-recovery,1,200,0.011881 +1793284904,telemetry-recovery,1,200,0.016154 +1793284904,telemetry-recovery,1,200,0.016677 +1793284904,telemetry-recovery,1,200,0.012779 +1793284905,telemetry-recovery,1,200,0.027434 +1793284905,telemetry-recovery,1,200,0.008935 +1793284905,telemetry-recovery,1,200,0.015557 +1793284906,telemetry-recovery,1,200,0.009635 +1793284906,telemetry-recovery,1,200,0.015732 +1793284906,telemetry-recovery,1,200,0.009787 +1793284907,telemetry-recovery,1,200,1.011980 +1793284908,telemetry-recovery,1,200,0.010250 +1793284908,telemetry-recovery,1,200,0.019536 +1793284909,telemetry-recovery,1,200,0.014864 +1793284909,telemetry-recovery,1,200,0.010597 +1793284909,telemetry-recovery,1,200,0.018211 +1793284909,telemetry-recovery,1,200,0.013224 +1793284910,telemetry-recovery,1,200,0.012471 +1793284910,telemetry-recovery,1,200,0.014367 +1793284910,telemetry-recovery,1,200,0.014202 +1793284911,telemetry-recovery,1,200,0.016788 +1793284911,telemetry-recovery,1,200,0.008131 +1793284911,telemetry-recovery,1,200,0.011913 +1793284912,telemetry-recovery,1,200,0.015702 +1793284912,telemetry-recovery,1,200,0.015186 +1793284912,telemetry-recovery,1,200,0.014230 +1793284913,telemetry-recovery,1,200,0.017327 +1793284913,telemetry-recovery,1,200,0.017655 diff --git a/evidence/incidents/http-2.csv b/evidence/incidents/http-2.csv new file mode 100644 index 0000000..55e5901 --- /dev/null +++ b/evidence/incidents/http-2.csv @@ -0,0 +1,2160 @@ +1793284209,baseline,2,200,0.018693 +1793284209,baseline,2,200,0.010655 +1793284210,baseline,2,200,0.018710 +1793284210,baseline,2,200,0.023239 +1793284210,baseline,2,200,0.016429 +1793284211,baseline,2,200,0.012891 +1793284211,baseline,2,200,0.017573 +1793284211,baseline,2,200,0.008094 +1793284211,baseline,2,200,0.026398 +1793284212,baseline,2,200,0.010248 +1793284212,baseline,2,200,0.022264 +1793284212,baseline,2,200,0.009499 +1793284213,baseline,2,200,0.021712 +1793284213,baseline,2,200,0.014201 +1793284213,baseline,2,200,0.009614 +1793284214,baseline,2,200,0.019519 +1793284214,baseline,2,200,0.009681 +1793284214,baseline,2,200,0.011404 +1793284214,baseline,2,200,0.015267 +1793284215,baseline,2,200,0.017955 +1793284215,baseline,2,200,0.008357 +1793284215,baseline,2,200,0.011994 +1793284216,baseline,2,200,0.010117 +1793284216,baseline,2,200,0.009494 +1793284216,baseline,2,200,0.008494 +1793284217,baseline,2,200,0.007537 +1793284217,baseline,2,200,0.015091 +1793284217,baseline,2,200,0.025454 +1793284217,baseline,2,200,0.010494 +1793284218,baseline,2,200,0.018750 +1793284218,baseline,2,200,0.019484 +1793284218,baseline,2,200,0.016795 +1793284219,baseline,2,200,0.022487 +1793284219,baseline,2,200,0.021889 +1793284219,baseline,2,200,0.010765 +1793284220,baseline,2,200,0.019446 +1793284220,baseline,2,200,0.029234 +1793284220,baseline,2,200,0.015098 +1793284220,baseline,2,200,0.021032 +1793284221,baseline,2,200,0.010249 +1793284221,baseline,2,200,0.013343 +1793284221,baseline,2,200,0.015617 +1793284222,baseline,2,200,0.016550 +1793284222,baseline,2,200,0.010445 +1793284222,baseline,2,200,0.018098 +1793284223,baseline,2,200,0.019241 +1793284223,baseline,2,200,0.015532 +1793284223,baseline,2,200,0.018369 +1793284224,baseline,2,200,0.010323 +1793284224,baseline,2,200,0.014543 +1793284224,baseline,2,200,0.009713 +1793284224,baseline,2,200,0.012257 +1793284225,baseline,2,200,0.014872 +1793284225,baseline,2,200,0.011126 +1793284225,baseline,2,200,0.014190 +1793284226,baseline,2,200,0.011280 +1793284226,baseline,2,200,0.021564 +1793284226,baseline,2,200,0.013616 +1793284226,baseline,2,200,0.021487 +1793284227,baseline,2,200,0.018705 +1793284227,baseline,2,200,0.014699 +1793284227,baseline,2,200,0.015757 +1793284228,baseline,2,200,0.010129 +1793284228,baseline,2,200,0.014870 +1793284228,baseline,2,200,0.012969 +1793284229,baseline,2,200,0.018549 +1793284229,baseline,2,200,0.018118 +1793284229,baseline,2,200,0.015223 +1793284229,baseline,2,200,0.015431 +1793284230,baseline,2,200,0.013762 +1793284230,baseline,2,200,0.015439 +1793284230,baseline,2,200,0.015086 +1793284231,baseline,2,200,0.013603 +1793284231,baseline,2,200,0.012774 +1793284231,baseline,2,200,0.017591 +1793284232,baseline,2,200,0.015716 +1793284232,baseline,2,200,0.016416 +1793284232,baseline,2,200,0.006970 +1793284233,baseline,2,200,0.015258 +1793284233,baseline,2,200,0.015911 +1793284233,baseline,2,200,0.014058 +1793284233,baseline,2,200,0.016444 +1793284234,baseline,2,200,0.012515 +1793284234,baseline,2,200,0.018804 +1793284234,baseline,2,200,0.023287 +1793284235,baseline,2,200,0.005658 +1793284235,baseline,2,200,0.011763 +1793284235,baseline,2,200,0.011729 +1793284235,baseline,2,200,0.012420 +1793284236,baseline,2,200,0.022444 +1793284236,baseline,2,200,0.010847 +1793284236,baseline,2,200,0.012006 +1793284237,baseline,2,200,0.012107 +1793284237,baseline,2,200,0.011729 +1793284237,baseline,2,200,0.009740 +1793284238,baseline,2,200,0.006151 +1793284238,baseline,2,200,0.007726 +1793284238,baseline,2,200,0.012791 +1793284238,baseline,2,200,0.008726 +1793284239,baseline,2,200,0.013737 +1793284239,baseline,2,200,0.014962 +1793284239,baseline,2,200,0.008506 +1793284240,baseline,2,200,0.013803 +1793284240,baseline,2,200,0.008093 +1793284240,baseline,2,200,0.013075 +1793284241,baseline,2,200,0.014396 +1793284241,baseline,2,200,0.012761 +1793284241,baseline,2,200,0.030027 +1793284241,baseline,2,200,0.018229 +1793284242,baseline,2,200,0.014909 +1793284242,baseline,2,200,0.015599 +1793284242,baseline,2,200,0.015773 +1793284243,baseline,2,200,0.014659 +1793284243,baseline,2,200,0.011247 +1793284243,baseline,2,200,0.011846 +1793284244,baseline,2,200,0.016375 +1793284244,baseline,2,200,0.011263 +1793284244,baseline,2,200,0.017112 +1793284244,baseline,2,200,0.013717 +1793284245,baseline,2,200,0.011214 +1793284245,baseline,2,200,0.013342 +1793284245,baseline,2,200,0.011797 +1793284246,baseline,2,200,0.011703 +1793284246,baseline,2,200,0.015755 +1793284246,baseline,2,200,0.013746 +1793284247,baseline,2,200,0.010554 +1793284247,baseline,2,200,0.014806 +1793284247,baseline,2,200,0.012417 +1793284247,baseline,2,200,0.007945 +1793284248,baseline,2,200,0.016807 +1793284248,baseline,2,200,0.005933 +1793284248,baseline,2,200,0.018660 +1793284249,baseline,2,200,0.016404 +1793284249,baseline,2,200,0.013336 +1793284249,baseline,2,200,0.015248 +1793284250,baseline,2,200,0.015180 +1793284250,baseline,2,200,0.020239 +1793284250,baseline,2,200,0.017226 +1793284251,baseline,2,200,0.015719 +1793284251,baseline,2,200,0.019861 +1793284251,baseline,2,200,0.019239 +1793284251,baseline,2,200,0.016588 +1793284252,baseline,2,200,0.015548 +1793284252,baseline,2,200,0.011454 +1793284252,baseline,2,200,0.018685 +1793284253,baseline,2,200,0.007569 +1793284253,baseline,2,200,0.006562 +1793284253,baseline,2,200,0.007547 +1793284253,baseline,2,200,0.011644 +1793284254,baseline,2,200,0.007700 +1793284254,baseline,2,200,0.010786 +1793284254,baseline,2,200,0.007238 +1793284255,baseline,2,200,0.019465 +1793284255,baseline,2,200,0.021234 +1793284255,baseline,2,200,0.008246 +1793284256,baseline,2,200,0.006967 +1793284256,baseline,2,200,0.012325 +1793284256,baseline,2,200,0.011523 +1793284256,baseline,2,200,0.012328 +1793284257,baseline,2,200,0.006539 +1793284257,baseline,2,200,0.012273 +1793284257,baseline,2,200,0.011176 +1793284258,baseline,2,200,0.013866 +1793284258,baseline,2,200,0.008131 +1793284258,baseline,2,200,0.020024 +1793284259,baseline,2,200,0.019860 +1793284259,baseline,2,200,0.015355 +1793284259,baseline,2,200,0.017722 +1793284260,baseline,2,200,0.010947 +1793284260,baseline,2,200,0.011461 +1793284260,baseline,2,200,0.013938 +1793284260,baseline,2,200,0.011724 +1793284261,baseline,2,200,0.016363 +1793284261,baseline,2,200,0.025357 +1793284261,baseline,2,200,0.014469 +1793284262,baseline,2,200,0.018861 +1793284262,baseline,2,200,0.010959 +1793284262,baseline,2,200,0.015048 +1793284263,baseline,2,200,0.017713 +1793284263,baseline,2,200,0.013396 +1793284263,baseline,2,200,0.011835 +1793284264,baseline,2,200,0.014644 +1793284264,baseline,2,200,1.006877 +1793284265,baseline,2,200,0.011641 +1793284266,baseline,2,200,0.011621 +1793284266,baseline,2,200,0.010199 +1793284266,baseline,2,200,0.013774 +1793284266,baseline,2,200,0.007983 +1793284267,baseline,2,200,0.012295 +1793284267,baseline,2,200,0.011399 +1793284267,baseline,2,200,0.013299 +1793284268,baseline,2,200,0.007899 +1793284268,baseline,2,200,0.007338 +1793284268,baseline,2,200,0.008131 +1793284268,baseline,2,200,0.006553 +1793284269,baseline,2,200,0.015170 +1793284269,baseline,2,200,0.009151 +1793284269,baseline,2,200,0.004750 +1793284270,baseline,2,200,0.003825 +1793284270,baseline,2,200,0.003397 +1793284270,baseline,2,200,0.006087 +1793284270,baseline,2,200,0.005024 +1793284271,baseline,2,200,0.004046 +1793284271,baseline,2,200,0.005663 +1793284271,baseline,2,200,0.005444 +1793284272,baseline,2,200,0.006162 +1793284272,baseline,2,200,0.005511 +1793284272,baseline,2,200,0.007518 +1793284272,baseline,2,200,0.006800 +1793284273,baseline,2,200,0.005321 +1793284273,baseline,2,200,0.003815 +1793284273,baseline,2,200,0.008134 +1793284274,baseline,2,200,0.117932 +1793284274,baseline,2,200,0.026632 +1793284274,baseline,2,200,0.005139 +1793284274,baseline,2,200,0.010307 +1793284275,baseline,2,200,0.006101 +1793284275,baseline,2,200,0.016739 +1793284275,baseline,2,200,0.011412 +1793284276,baseline,2,200,0.013007 +1793284276,baseline,2,200,0.015028 +1793284276,baseline,2,200,0.010325 +1793284277,baseline,2,200,0.015135 +1793284277,baseline,2,200,0.013796 +1793284277,baseline,2,200,0.009042 +1793284278,baseline,2,200,0.010897 +1793284278,baseline,2,200,0.012425 +1793284278,baseline,2,200,0.013434 +1793284278,baseline,2,200,0.016142 +1793284279,baseline,2,200,0.012784 +1793284279,baseline,2,200,0.011381 +1793284279,baseline,2,200,0.007490 +1793284280,baseline,2,200,0.008117 +1793284280,baseline,2,200,0.035405 +1793284280,baseline,2,200,0.010861 +1793284281,baseline,2,200,0.010241 +1793284281,baseline,2,200,0.009315 +1793284281,baseline,2,200,0.012964 +1793284281,baseline,2,200,0.011487 +1793284282,baseline,2,200,0.013614 +1793284282,baseline,2,200,0.010735 +1793284282,baseline,2,200,0.012591 +1793284283,baseline,2,200,0.011361 +1793284283,baseline,2,200,0.008830 +1793284283,baseline,2,200,0.013950 +1793284284,baseline,2,200,0.014877 +1793284284,baseline,2,200,0.013167 +1793284284,baseline,2,200,0.008883 +1793284284,baseline,2,200,0.009902 +1793284285,baseline,2,200,0.016231 +1793284285,baseline,2,200,0.008990 +1793284285,baseline,2,200,0.007839 +1793284286,baseline,2,200,0.009026 +1793284286,baseline,2,200,0.020422 +1793284286,baseline,2,200,0.009194 +1793284286,baseline,2,200,0.007999 +1793284287,baseline,2,200,0.012347 +1793284287,baseline,2,200,0.007727 +1793284287,baseline,2,200,0.013211 +1793284288,baseline,2,200,0.009111 +1793284288,baseline,2,200,0.014715 +1793284288,baseline,2,200,0.014316 +1793284289,baseline,2,200,0.010721 +1793284289,baseline,2,200,0.007576 +1793284289,baseline,2,200,0.009765 +1793284289,baseline,2,200,0.010929 +1793284290,baseline,2,200,0.010140 +1793284290,baseline,2,200,0.008077 +1793284290,baseline,2,200,0.016836 +1793284291,baseline,2,200,0.011545 +1793284291,baseline,2,200,0.015947 +1793284291,baseline,2,200,0.015127 +1793284292,baseline,2,200,0.006509 +1793284292,baseline,2,200,0.008423 +1793284292,baseline,2,200,0.009194 +1793284292,baseline,2,200,0.013441 +1793284293,baseline,2,200,0.015785 +1793284293,baseline,2,200,0.012696 +1793284293,baseline,2,200,0.013960 +1793284294,baseline,2,200,0.014635 +1793284294,baseline,2,200,0.007746 +1793284294,baseline,2,200,0.007297 +1793284295,baseline,2,200,0.009241 +1793284295,baseline,2,200,0.013209 +1793284295,baseline,2,200,0.004528 +1793284295,baseline,2,200,0.013184 +1793284296,baseline,2,200,0.016398 +1793284296,baseline,2,200,0.011544 +1793284296,baseline,2,200,0.014447 +1793284297,baseline,2,200,0.018116 +1793284297,baseline,2,200,0.013319 +1793284297,baseline,2,200,0.008013 +1793284298,baseline,2,200,0.009580 +1793284298,baseline,2,200,0.014049 +1793284298,baseline,2,200,0.011221 +1793284299,baseline,2,200,0.011411 +1793284299,baseline,2,200,0.013329 +1793284299,baseline,2,200,0.012876 +1793284299,baseline,2,200,0.020973 +1793284300,baseline,2,200,0.006188 +1793284300,baseline,2,200,0.013524 +1793284300,baseline,2,200,0.009056 +1793284301,baseline,2,200,0.014413 +1793284301,baseline,2,200,0.016210 +1793284301,baseline,2,200,0.018609 +1793284302,baseline,2,200,0.019060 +1793284302,baseline,2,200,0.018312 +1793284302,baseline,2,200,0.010898 +1793284302,baseline,2,200,0.009630 +1793284303,baseline,2,200,0.011882 +1793284303,baseline,2,200,0.016408 +1793284303,baseline,2,200,0.010270 +1793284304,baseline,2,200,0.008670 +1793284304,baseline,2,200,0.012885 +1793284304,baseline,2,200,0.008556 +1793284305,baseline,2,200,0.014371 +1793284305,baseline,2,200,0.018027 +1793284305,baseline,2,200,0.014769 +1793284306,baseline,2,200,0.015584 +1793284306,baseline,2,200,0.015146 +1793284306,baseline,2,200,0.015662 +1793284306,baseline,2,200,0.014180 +1793284307,baseline,2,200,0.006358 +1793284307,baseline,2,200,0.011744 +1793284307,baseline,2,200,0.011914 +1793284308,baseline,2,200,0.014894 +1793284308,baseline,2,200,0.015440 +1793284308,baseline,2,200,0.015329 +1793284309,baseline,2,200,0.013602 +1793284309,baseline,2,200,0.016575 +1793284309,baseline,2,200,0.009716 +1793284309,baseline,2,200,1.015178 +1793284311,baseline,2,200,0.015325 +1793284311,baseline,2,200,0.012076 +1793284311,baseline,2,200,0.011070 +1793284312,baseline,2,200,0.011231 +1793284312,baseline,2,200,0.006883 +1793284312,baseline,2,200,0.010207 +1793284313,baseline,2,200,0.004870 +1793284313,baseline,2,200,0.006012 +1793284313,baseline,2,200,0.009456 +1793284313,baseline,2,200,1.009408 +1793284315,baseline,2,200,0.013467 +1793284315,baseline,2,200,0.009035 +1793284315,baseline,2,200,0.008144 +1793284316,baseline,2,200,0.005837 +1793284316,baseline,2,200,0.011276 +1793284316,baseline,2,200,0.013818 +1793284317,baseline,2,200,0.016757 +1793284317,baseline,2,200,0.017332 +1793284317,baseline,2,200,0.014747 +1793284317,baseline,2,200,0.013669 +1793284318,baseline,2,200,0.012275 +1793284318,baseline,2,200,0.013648 +1793284318,baseline,2,200,0.013604 +1793284319,baseline,2,200,0.013296 +1793284319,baseline,2,200,0.011357 +1793284319,baseline,2,200,0.013159 +1793284320,baseline,2,200,0.011492 +1793284320,baseline,2,200,0.012396 +1793284320,baseline,2,200,0.013293 +1793284321,baseline,2,200,0.010557 +1793284321,baseline,2,200,0.012917 +1793284321,baseline,2,200,0.005350 +1793284321,baseline,2,200,0.013068 +1793284322,baseline,2,200,0.013958 +1793284322,baseline,2,200,0.006741 +1793284322,baseline,2,200,0.008890 +1793284323,baseline,2,200,0.008518 +1793284323,baseline,2,200,0.008322 +1793284323,baseline,2,200,0.008306 +1793284323,baseline,2,200,0.014605 +1793284324,baseline,2,200,0.008840 +1793284324,baseline,2,200,0.007513 +1793284324,baseline,2,200,0.010367 +1793284325,baseline,2,200,0.009367 +1793284325,baseline,2,200,0.004590 +1793284325,baseline,2,200,0.011294 +1793284326,baseline,2,200,0.013108 +1793284326,baseline,2,200,0.011656 +1793284326,baseline,2,200,0.006716 +1793284326,baseline,2,200,0.016678 +1793284327,baseline,2,200,0.013019 +1793284327,baseline,2,200,0.017348 +1793284327,baseline,2,200,1.014786 +1793284329,baseline,2,200,0.016041 +1793284329,baseline,2,200,0.013410 +1793284329,baseline,2,200,0.006280 +1793284330,baseline,2,200,0.011096 +1793284330,baseline,2,200,0.007435 +1793284330,baseline,2,200,0.008936 +1793284330,baseline,2,200,0.011524 +1793284331,baseline,2,502,1.011914 +1793284332,short-db-spike,2,502,0.001399 +1793284332,short-db-spike,2,502,0.001683 +1793284333,short-db-spike,2,502,0.004253 +1793284333,short-db-spike,2,502,0.002289 +1793284333,short-db-spike,2,502,0.003435 +1793284333,short-db-spike,2,502,1.007399 +1793284335,short-db-spike,2,502,0.003852 +1793284335,short-db-spike,2,502,0.001602 +1793284335,short-db-spike,2,502,0.001426 +1793284336,short-db-spike,2,502,0.002235 +1793284336,short-db-spike,2,502,0.001583 +1793284336,spike-recovery,2,502,0.004834 +1793284337,spike-recovery,2,200,0.015804 +1793284337,spike-recovery,2,200,0.010573 +1793284337,spike-recovery,2,200,0.009350 +1793284337,spike-recovery,2,200,0.010826 +1793284338,spike-recovery,2,200,0.009375 +1793284338,spike-recovery,2,200,0.010555 +1793284338,spike-recovery,2,200,0.010809 +1793284339,spike-recovery,2,200,0.010274 +1793284339,spike-recovery,2,200,0.005881 +1793284339,spike-recovery,2,200,0.006460 +1793284340,spike-recovery,2,200,0.015978 +1793284340,spike-recovery,2,200,0.014746 +1793284340,spike-recovery,2,200,0.020631 +1793284340,spike-recovery,2,200,0.007820 +1793284341,spike-recovery,2,200,0.016270 +1793284341,spike-recovery,2,200,0.009527 +1793284341,spike-recovery,2,200,0.022815 +1793284342,spike-recovery,2,200,0.017802 +1793284342,spike-recovery,2,200,0.016557 +1793284342,spike-recovery,2,200,0.011285 +1793284343,spike-recovery,2,200,0.022420 +1793284343,spike-recovery,2,200,0.013210 +1793284343,spike-recovery,2,200,0.009669 +1793284344,spike-recovery,2,200,0.014012 +1793284344,spike-recovery,2,200,0.015900 +1793284344,spike-recovery,2,200,0.014220 +1793284345,spike-recovery,2,200,0.007782 +1793284345,spike-recovery,2,200,0.018872 +1793284345,spike-recovery,2,200,0.019358 +1793284345,spike-recovery,2,200,0.010861 +1793284346,spike-recovery,2,200,0.017234 +1793284346,spike-recovery,2,200,0.016938 +1793284346,spike-recovery,2,200,0.008197 +1793284347,spike-recovery,2,200,0.012082 +1793284347,spike-recovery,2,200,0.013471 +1793284347,spike-recovery,2,200,0.018701 +1793284348,spike-recovery,2,200,0.013539 +1793284348,spike-recovery,2,200,0.018257 +1793284348,spike-recovery,2,200,0.022608 +1793284349,spike-recovery,2,200,0.010735 +1793284349,spike-recovery,2,200,0.015989 +1793284349,spike-recovery,2,200,0.010229 +1793284350,spike-recovery,2,200,0.006515 +1793284350,spike-recovery,2,200,0.009605 +1793284350,spike-recovery,2,200,0.007698 +1793284350,spike-recovery,2,200,0.012897 +1793284351,spike-recovery,2,200,0.012711 +1793284351,spike-recovery,2,200,0.012021 +1793284351,spike-recovery,2,200,0.009554 +1793284352,spike-recovery,2,200,0.006159 +1793284352,spike-recovery,2,200,0.015636 +1793284352,spike-recovery,2,200,0.015079 +1793284353,spike-recovery,2,200,0.010603 +1793284353,spike-recovery,2,200,0.020203 +1793284353,spike-recovery,2,200,0.017813 +1793284353,spike-recovery,2,200,0.017795 +1793284354,spike-recovery,2,200,0.022174 +1793284354,spike-recovery,2,200,0.015607 +1793284354,spike-recovery,2,200,0.008252 +1793284355,spike-recovery,2,200,0.007268 +1793284355,spike-recovery,2,200,0.014259 +1793284355,spike-recovery,2,200,0.015905 +1793284356,spike-recovery,2,200,0.011408 +1793284356,spike-recovery,2,200,0.007794 +1793284356,spike-recovery,2,200,0.015478 +1793284357,spike-recovery,2,200,0.008625 +1793284357,spike-recovery,2,200,0.031910 +1793284357,spike-recovery,2,200,0.019251 +1793284357,spike-recovery,2,200,0.008737 +1793284358,spike-recovery,2,200,0.014073 +1793284358,spike-recovery,2,200,0.017439 +1793284358,spike-recovery,2,200,0.018073 +1793284359,spike-recovery,2,200,0.010417 +1793284359,spike-recovery,2,200,0.011809 +1793284359,spike-recovery,2,200,0.014915 +1793284360,spike-recovery,2,200,0.020181 +1793284360,spike-recovery,2,200,0.025349 +1793284360,spike-recovery,2,200,0.011247 +1793284361,spike-recovery,2,200,0.009977 +1793284361,spike-recovery,2,200,0.014067 +1793284361,spike-recovery,2,200,0.017369 +1793284362,spike-recovery,2,200,0.024720 +1793284362,spike-recovery,2,200,0.014658 +1793284362,spike-recovery,2,200,0.023795 +1793284363,spike-recovery,2,200,0.021184 +1793284363,spike-recovery,2,200,0.019094 +1793284363,spike-recovery,2,200,0.014695 +1793284364,spike-recovery,2,200,0.016337 +1793284364,spike-recovery,2,200,0.016077 +1793284364,spike-recovery,2,200,0.017832 +1793284364,spike-recovery,2,200,0.017787 +1793284365,spike-recovery,2,200,0.011191 +1793284365,spike-recovery,2,200,0.018332 +1793284365,spike-recovery,2,200,0.018755 +1793284366,spike-recovery,2,200,0.010979 +1793284366,spike-recovery,2,200,0.020607 +1793284366,spike-recovery,2,200,0.010250 +1793284367,spike-recovery,2,200,0.013532 +1793284367,spike-recovery,2,200,0.010910 +1793284367,spike-recovery,2,200,0.032537 +1793284368,spike-recovery,2,200,0.010735 +1793284368,spike-recovery,2,200,0.018758 +1793284368,spike-recovery,2,200,0.015123 +1793284369,spike-recovery,2,200,0.027025 +1793284369,spike-recovery,2,200,0.012104 +1793284369,spike-recovery,2,200,0.013125 +1793284370,spike-recovery,2,200,0.016951 +1793284370,spike-recovery,2,200,0.014815 +1793284370,spike-recovery,2,200,0.011526 +1793284370,spike-recovery,2,200,0.014975 +1793284371,spike-recovery,2,200,0.010800 +1793284371,spike-recovery,2,200,0.010480 +1793284371,spike-recovery,2,200,0.015645 +1793284372,spike-recovery,2,200,0.020937 +1793284372,spike-recovery,2,200,0.044755 +1793284372,spike-recovery,2,200,0.013017 +1793284373,spike-recovery,2,200,0.015022 +1793284373,spike-recovery,2,200,0.012671 +1793284373,spike-recovery,2,200,0.009762 +1793284374,spike-recovery,2,200,0.016909 +1793284374,spike-recovery,2,200,0.016757 +1793284374,spike-recovery,2,200,0.011295 +1793284375,spike-recovery,2,200,0.020259 +1793284375,spike-recovery,2,200,0.021115 +1793284375,spike-recovery,2,200,0.035912 +1793284376,spike-recovery,2,200,0.018165 +1793284376,spike-recovery,2,200,0.020912 +1793284376,spike-recovery,2,200,0.017037 +1793284377,spike-recovery,2,200,0.011651 +1793284377,spike-recovery,2,200,0.010732 +1793284377,spike-recovery,2,200,0.010337 +1793284377,spike-recovery,2,200,0.012857 +1793284378,spike-recovery,2,200,0.013334 +1793284378,spike-recovery,2,200,0.011230 +1793284378,spike-recovery,2,200,0.013584 +1793284379,spike-recovery,2,200,0.019445 +1793284379,spike-recovery,2,200,0.015145 +1793284379,spike-recovery,2,200,0.011095 +1793284380,spike-recovery,2,200,0.016442 +1793284380,spike-recovery,2,200,0.007255 +1793284380,spike-recovery,2,200,0.011069 +1793284380,spike-recovery,2,200,0.005672 +1793284381,spike-recovery,2,200,0.006648 +1793284381,spike-recovery,2,200,0.010161 +1793284381,spike-recovery,2,200,0.015657 +1793284382,spike-recovery,2,200,0.014108 +1793284382,spike-recovery,2,200,0.018302 +1793284382,spike-recovery,2,200,0.010327 +1793284383,spike-recovery,2,200,0.007560 +1793284383,spike-recovery,2,200,0.006846 +1793284383,spike-recovery,2,200,0.008724 +1793284383,spike-recovery,2,200,0.008807 +1793284384,spike-recovery,2,200,0.010863 +1793284384,spike-recovery,2,200,0.010729 +1793284384,spike-recovery,2,200,0.006995 +1793284385,spike-recovery,2,200,0.018652 +1793284385,spike-recovery,2,200,0.011056 +1793284385,spike-recovery,2,200,0.009169 +1793284386,spike-recovery,2,200,0.017041 +1793284386,spike-recovery,2,200,0.014731 +1793284386,spike-recovery,2,200,0.006911 +1793284386,spike-recovery,2,200,0.018626 +1793284387,spike-recovery,2,200,0.017352 +1793284387,spike-recovery,2,200,0.013706 +1793284387,spike-recovery,2,200,0.009567 +1793284388,spike-recovery,2,200,0.017794 +1793284388,spike-recovery,2,200,0.016867 +1793284388,spike-recovery,2,200,0.019500 +1793284389,spike-recovery,2,200,0.020683 +1793284389,spike-recovery,2,200,0.018738 +1793284389,spike-recovery,2,200,0.010070 +1793284390,spike-recovery,2,200,0.020001 +1793284390,spike-recovery,2,200,0.016213 +1793284390,spike-recovery,2,200,0.019534 +1793284390,spike-recovery,2,200,0.018721 +1793284391,spike-recovery,2,200,0.016289 +1793284391,spike-recovery,2,200,0.013519 +1793284391,spike-recovery,2,200,0.005995 +1793284392,spike-recovery,2,200,0.012955 +1793284392,spike-recovery,2,200,0.014355 +1793284392,spike-recovery,2,200,0.010986 +1793284393,spike-recovery,2,200,0.015872 +1793284393,spike-recovery,2,200,0.012079 +1793284393,spike-recovery,2,200,0.011594 +1793284393,spike-recovery,2,200,0.017566 +1793284394,spike-recovery,2,200,0.005775 +1793284394,spike-recovery,2,200,0.006208 +1793284394,spike-recovery,2,200,0.007706 +1793284395,spike-recovery,2,200,0.008671 +1793284395,spike-recovery,2,200,0.007769 +1793284395,spike-recovery,2,200,0.006913 +1793284395,spike-recovery,2,200,0.007995 +1793284396,spike-recovery,2,200,0.013608 +1793284396,spike-recovery,2,200,0.013451 +1793284396,spike-recovery,2,200,0.013658 +1793284397,spike-recovery,2,200,0.015680 +1793284397,spike-recovery,2,200,0.012829 +1793284397,spike-recovery,2,200,0.016025 +1793284398,spike-recovery,2,200,0.014095 +1793284398,spike-recovery,2,200,0.012207 +1793284398,spike-recovery,2,200,0.020156 +1793284399,spike-recovery,2,200,0.014548 +1793284399,spike-recovery,2,200,0.014823 +1793284399,spike-recovery,2,200,0.010044 +1793284399,spike-recovery,2,200,0.009922 +1793284400,spike-recovery,2,200,0.016365 +1793284400,spike-recovery,2,200,0.011606 +1793284400,spike-recovery,2,200,0.010051 +1793284401,spike-recovery,2,200,0.014895 +1793284401,spike-recovery,2,200,0.031456 +1793284401,spike-recovery,2,200,0.005830 +1793284402,spike-recovery,2,200,0.010235 +1793284402,spike-recovery,2,200,0.013036 +1793284402,spike-recovery,2,200,0.014257 +1793284402,spike-recovery,2,200,0.016924 +1793284403,spike-recovery,2,200,0.015003 +1793284403,spike-recovery,2,200,0.023530 +1793284403,spike-recovery,2,200,0.019750 +1793284404,spike-recovery,2,200,0.008811 +1793284404,spike-recovery,2,200,0.008925 +1793284404,spike-recovery,2,200,0.008244 +1793284405,spike-recovery,2,200,0.012489 +1793284405,spike-recovery,2,200,0.013709 +1793284405,spike-recovery,2,200,0.011060 +1793284405,spike-recovery,2,200,0.015857 +1793284406,spike-recovery,2,200,0.016219 +1793284406,spike-recovery,2,200,0.015743 +1793284406,spike-recovery,2,200,0.017651 +1793284407,spike-recovery,2,200,0.018905 +1793284407,spike-recovery,2,200,0.016435 +1793284407,spike-recovery,2,200,0.013041 +1793284408,spike-recovery,2,200,0.011403 +1793284408,spike-recovery,2,200,0.008539 +1793284408,spike-recovery,2,200,0.009792 +1793284409,spike-recovery,2,200,0.013727 +1793284409,spike-recovery,2,200,0.009161 +1793284409,spike-recovery,2,200,0.012222 +1793284409,spike-recovery,2,200,0.017568 +1793284410,spike-recovery,2,200,0.012497 +1793284410,spike-recovery,2,200,0.013352 +1793284410,spike-recovery,2,200,0.010678 +1793284411,spike-recovery,2,200,0.014897 +1793284411,spike-recovery,2,200,0.011461 +1793284411,spike-recovery,2,200,0.014544 +1793284412,spike-recovery,2,200,0.010231 +1793284412,spike-recovery,2,200,0.015014 +1793284412,spike-recovery,2,200,0.011434 +1793284412,spike-recovery,2,200,0.013893 +1793284413,spike-recovery,2,200,0.015435 +1793284413,spike-recovery,2,200,0.015734 +1793284413,spike-recovery,2,200,0.009239 +1793284414,spike-recovery,2,200,0.012670 +1793284414,spike-recovery,2,200,0.013984 +1793284414,spike-recovery,2,200,0.009475 +1793284415,spike-recovery,2,200,0.011015 +1793284415,spike-recovery,2,200,0.019364 +1793284415,spike-recovery,2,200,0.010937 +1793284416,spike-recovery,2,200,0.015595 +1793284416,spike-recovery,2,200,0.010546 +1793284416,spike-recovery,2,200,0.021372 +1793284416,spike-recovery,2,200,0.012773 +1793284417,spike-recovery,2,200,0.017471 +1793284417,spike-recovery,2,200,0.010100 +1793284417,spike-recovery,2,200,0.013637 +1793284418,spike-recovery,2,200,0.019093 +1793284418,spike-recovery,2,200,0.011256 +1793284418,spike-recovery,2,200,0.015837 +1793284419,spike-recovery,2,200,0.022453 +1793284419,spike-recovery,2,200,0.017869 +1793284419,spike-recovery,2,200,0.013998 +1793284420,spike-recovery,2,200,0.016781 +1793284420,spike-recovery,2,200,0.022218 +1793284420,spike-recovery,2,200,0.014934 +1793284421,spike-recovery,2,200,0.008723 +1793284421,spike-recovery,2,200,0.019467 +1793284421,spike-recovery,2,200,0.015580 +1793284421,spike-recovery,2,200,0.018437 +1793284422,spike-recovery,2,200,0.018126 +1793284422,spike-recovery,2,200,0.009989 +1793284422,spike-recovery,2,200,0.014816 +1793284423,spike-recovery,2,200,0.017912 +1793284423,spike-recovery,2,200,0.019621 +1793284423,spike-recovery,2,200,0.023247 +1793284424,spike-recovery,2,200,0.014589 +1793284424,spike-recovery,2,200,0.014611 +1793284424,spike-recovery,2,200,0.010958 +1793284425,spike-recovery,2,200,0.018826 +1793284425,spike-recovery,2,200,0.016871 +1793284425,spike-recovery,2,200,0.018440 +1793284425,spike-recovery,2,200,0.017054 +1793284426,spike-recovery,2,200,0.016323 +1793284426,spike-recovery,2,200,0.008302 +1793284426,spike-recovery,2,200,0.006016 +1793284427,spike-recovery,2,200,0.005567 +1793284427,spike-recovery,2,200,0.008718 +1793284427,spike-recovery,2,502,1.011096 +1793284429,sustained-db-outage,2,502,0.002826 +1793284429,sustained-db-outage,2,502,0.001008 +1793284429,sustained-db-outage,2,502,0.001526 +1793284429,sustained-db-outage,2,502,0.001494 +1793284430,sustained-db-outage,2,502,1.009924 +1793284431,sustained-db-outage,2,502,0.003175 +1793284431,sustained-db-outage,2,502,0.001298 +1793284432,sustained-db-outage,2,502,0.003927 +1793284432,sustained-db-outage,2,502,0.001671 +1793284432,sustained-db-outage,2,502,0.001338 +1793284432,sustained-db-outage,2,502,0.002089 +1793284433,sustained-db-outage,2,502,1.004637 +1793284434,sustained-db-outage,2,502,0.001978 +1793284434,sustained-db-outage,2,502,0.001129 +1793284435,sustained-db-outage,2,502,0.002372 +1793284435,sustained-db-outage,2,502,0.001917 +1793284435,sustained-db-outage,2,502,0.004306 +1793284435,sustained-db-outage,2,502,0.002604 +1793284436,sustained-db-outage,2,502,1.009275 +1793284437,sustained-db-outage,2,502,0.002226 +1793284437,sustained-db-outage,2,502,0.002151 +1793284438,sustained-db-outage,2,502,0.001411 +1793284438,sustained-db-outage,2,502,0.000973 +1793284438,sustained-db-outage,2,502,0.004394 +1793284438,sustained-db-outage,2,502,0.001086 +1793284439,sustained-db-outage,2,502,1.003967 +1793284440,sustained-db-outage,2,502,0.002740 +1793284440,sustained-db-outage,2,502,0.002102 +1793284440,sustained-db-outage,2,502,0.001328 +1793284441,sustained-db-outage,2,502,0.003212 +1793284441,sustained-db-outage,2,502,0.001025 +1793284441,sustained-db-outage,2,502,0.002072 +1793284442,sustained-db-outage,2,502,1.004533 +1793284443,sustained-db-outage,2,502,0.002343 +1793284443,sustained-db-outage,2,502,0.001923 +1793284443,sustained-db-outage,2,502,0.002634 +1793284444,sustained-db-outage,2,502,0.001269 +1793284444,sustained-db-outage,2,502,0.001140 +1793284444,sustained-db-outage,2,502,0.001536 +1793284445,sustained-db-outage,2,502,1.007982 +1793284446,sustained-db-outage,2,502,0.001998 +1793284446,sustained-db-outage,2,502,0.005943 +1793284446,sustained-db-outage,2,502,0.000994 +1793284447,sustained-db-outage,2,502,0.002368 +1793284447,sustained-db-outage,2,502,0.001063 +1793284447,sustained-db-outage,2,502,0.001211 +1793284448,sustained-db-outage,2,502,1.004185 +1793284449,sustained-db-outage,2,502,0.001861 +1793284449,sustained-db-outage,2,502,0.001229 +1793284449,sustained-db-outage,2,502,0.002650 +1793284450,sustained-db-outage,2,502,0.001199 +1793284450,sustained-db-outage,2,502,0.001899 +1793284450,sustained-db-outage,2,502,0.001506 +1793284450,sustained-db-outage,2,502,1.014836 +1793284452,sustained-db-outage,2,502,0.002456 +1793284452,sustained-db-outage,2,502,0.003664 +1793284452,sustained-db-outage,2,502,0.001055 +1793284453,sustained-db-outage,2,502,0.000958 +1793284453,sustained-db-outage,2,502,0.001381 +1793284453,sustained-db-outage,2,502,0.002018 +1793284453,sustained-db-outage,2,502,1.005582 +1793284455,sustained-db-outage,2,502,0.000867 +1793284455,sustained-db-outage,2,502,0.002519 +1793284455,sustained-db-outage,2,502,0.000930 +1793284456,sustained-db-outage,2,502,0.001680 +1793284456,sustained-db-outage,2,502,0.003821 +1793284456,sustained-db-outage,2,502,0.002877 +1793284456,sustained-db-outage,2,502,0.001356 +1793284457,sustained-db-outage,2,502,1.004992 +1793284458,sustained-db-outage,2,502,0.001115 +1793284458,sustained-db-outage,2,502,0.002012 +1793284459,sustained-db-outage,2,502,0.001443 +1793284459,sustained-db-outage,2,502,0.001740 +1793284459,sustained-db-outage,2,502,0.001678 +1793284459,sustained-db-outage,2,502,1.005419 +1793284461,sustained-db-outage,2,502,0.006320 +1793284461,sustained-db-outage,2,502,0.002085 +1793284461,sustained-db-outage,2,502,0.001240 +1793284462,sustained-db-outage,2,502,0.002826 +1793284462,sustained-db-outage,2,502,0.002496 +1793284462,sustained-db-outage,2,502,0.001234 +1793284462,sustained-db-outage,2,502,0.003145 +1793284463,sustained-db-outage,2,502,1.005478 +1793284464,sustained-db-outage,2,502,0.001937 +1793284464,sustained-db-outage,2,502,0.001986 +1793284465,sustained-db-outage,2,502,0.002052 +1793284465,sustained-db-outage,2,502,0.002632 +1793284465,sustained-db-outage,2,502,0.001449 +1793284465,sustained-db-outage,2,502,0.002279 +1793284466,sustained-db-outage,2,502,1.003387 +1793284467,sustained-db-outage,2,502,0.001347 +1793284467,sustained-db-outage,2,502,0.002354 +1793284468,sustained-db-outage,2,502,0.006096 +1793284468,sustained-db-outage,2,502,0.003592 +1793284468,sustained-db-outage,2,502,0.001821 +1793284468,sustained-db-outage,2,502,0.001192 +1793284469,sustained-db-outage,2,502,1.010233 +1793284470,sustained-db-outage,2,502,0.001434 +1793284470,sustained-db-outage,2,502,0.001710 +1793284471,sustained-db-outage,2,502,0.001377 +1793284471,sustained-db-outage,2,502,0.002725 +1793284471,sustained-db-outage,2,502,0.001471 +1793284471,sustained-db-outage,2,502,0.002264 +1793284472,sustained-db-outage,2,502,1.004826 +1793284473,sustained-db-outage,2,502,0.001990 +1793284473,sustained-db-outage,2,502,0.001896 +1793284474,sustained-db-outage,2,502,0.001941 +1793284474,sustained-db-outage,2,502,0.002025 +1793284474,sustained-db-outage,2,502,0.002108 +1793284474,sustained-db-outage,2,502,0.002444 +1793284475,sustained-db-outage,2,502,1.003995 +1793284476,sustained-db-outage,2,502,0.002992 +1793284476,sustained-db-outage,2,502,0.001603 +1793284477,sustained-db-outage,2,502,0.002536 +1793284477,sustained-db-outage,2,502,0.001428 +1793284477,sustained-db-outage,2,502,0.003281 +1793284477,sustained-db-outage,2,502,0.003253 +1793284478,sustained-db-outage,2,502,1.004601 +1793284479,sustained-db-outage,2,502,0.003695 +1793284479,sustained-db-outage,2,502,0.001951 +1793284480,sustained-db-outage,2,502,0.001278 +1793284480,sustained-db-outage,2,502,0.002276 +1793284480,sustained-db-outage,2,502,0.001891 +1793284480,sustained-db-outage,2,502,0.001234 +1793284481,sustained-db-outage,2,502,1.006056 +1793284482,sustained-db-outage,2,502,0.004956 +1793284482,sustained-db-outage,2,502,0.001815 +1793284483,sustained-db-outage,2,502,0.002592 +1793284483,sustained-db-outage,2,502,0.002274 +1793284483,sustained-db-outage,2,502,0.002133 +1793284483,sustained-db-outage,2,502,0.001376 +1793284484,sustained-db-outage,2,502,1.005296 +1793284485,sustained-db-outage,2,502,0.001831 +1793284485,sustained-db-outage,2,502,0.001576 +1793284486,sustained-db-outage,2,502,0.001542 +1793284486,sustained-db-outage,2,502,0.002687 +1793284486,sustained-db-outage,2,502,0.001688 +1793284486,sustained-db-outage,2,502,0.002464 +1793284487,sustained-db-outage,2,502,1.006028 +1793284488,sustained-db-outage,2,502,0.001947 +1793284488,sustained-db-outage,2,502,0.001682 +1793284489,sustained-db-outage,2,502,0.003021 +1793284489,sustained-db-outage,2,502,0.004054 +1793284489,sustained-db-outage,2,502,0.002892 +1793284489,sustained-db-outage,2,502,1.012526 +1793284491,sustained-db-outage,2,502,0.002405 +1793284491,sustained-db-outage,2,502,0.001818 +1793284491,sustained-db-outage,2,502,0.003745 +1793284492,sustained-db-outage,2,502,0.002084 +1793284492,sustained-db-outage,2,502,0.002781 +1793284492,sustained-db-outage,2,502,0.001684 +1793284493,sustained-db-outage,2,502,1.004416 +1793284494,sustained-db-outage,2,502,0.003501 +1793284494,sustained-db-outage,2,502,0.001906 +1793284494,sustained-db-outage,2,502,0.002233 +1793284495,sustained-db-outage,2,502,0.002581 +1793284495,sustained-db-outage,2,502,0.001473 +1793284495,sustained-db-outage,2,502,0.001224 +1793284496,sustained-db-outage,2,502,1.011192 +1793284497,sustained-db-outage,2,502,0.002046 +1793284497,sustained-db-outage,2,502,0.001689 +1793284497,sustained-db-outage,2,502,0.001205 +1793284498,sustained-db-outage,2,502,0.001038 +1793284498,sustained-db-outage,2,502,0.001045 +1793284498,sustained-db-outage,2,502,0.001555 +1793284499,sustained-db-outage,2,502,1.006452 +1793284500,sustained-db-outage,2,502,0.002234 +1793284500,sustained-db-outage,2,502,0.002094 +1793284500,sustained-db-outage,2,502,0.001699 +1793284501,sustained-db-outage,2,502,0.001481 +1793284501,sustained-db-outage,2,502,0.001874 +1793284501,sustained-db-outage,2,502,0.000988 +1793284501,sustained-db-outage,2,502,1.007410 +1793284503,sustained-db-outage,2,502,0.002230 +1793284503,sustained-db-outage,2,502,0.002364 +1793284503,sustained-db-outage,2,502,0.002208 +1793284504,sustained-db-outage,2,502,0.001158 +1793284504,sustained-db-outage,2,502,0.002922 +1793284504,sustained-db-outage,2,502,0.003112 +1793284505,sustained-db-outage,2,502,1.014286 +1793284506,sustained-db-outage,2,502,0.003828 +1793284506,sustained-db-outage,2,502,0.003179 +1793284506,sustained-db-outage,2,502,0.003829 +1793284507,sustained-db-outage,2,502,0.004848 +1793284507,sustained-db-outage,2,502,0.002046 +1793284507,sustained-db-outage,2,502,0.003731 +1793284508,sustained-db-outage,2,502,1.014797 +1793284509,sustained-db-outage,2,502,0.004058 +1793284509,sustained-db-outage,2,502,0.002403 +1793284510,sustained-db-outage,2,502,0.003488 +1793284510,sustained-db-outage,2,502,0.003752 +1793284510,sustained-db-outage,2,502,0.003774 +1793284510,sustained-db-outage,2,502,1.012893 +1793284512,sustained-db-outage,2,502,0.002735 +1793284512,sustained-db-outage,2,502,0.002828 +1793284512,sustained-db-outage,2,502,0.002584 +1793284513,sustained-db-outage,2,502,0.006144 +1793284513,sustained-db-outage,2,502,0.003345 +1793284513,sustained-db-outage,2,502,0.004199 +1793284514,sustained-db-outage,2,502,1.010966 +1793284515,sustained-db-outage,2,502,0.002639 +1793284515,sustained-db-outage,2,502,0.002685 +1793284515,sustained-db-outage,2,502,0.002724 +1793284516,sustained-db-outage,2,502,0.005709 +1793284516,sustained-db-outage,2,502,0.005233 +1793284516,sustained-db-outage,2,502,0.001091 +1793284517,sustained-db-outage,2,502,1.011301 +1793284518,sustained-db-outage,2,502,0.003035 +1793284518,sustained-db-outage,2,502,0.002855 +1793284519,sustained-db-outage,2,502,0.004456 +1793284519,sustained-db-outage,2,502,0.004308 +1793284519,sustained-db-outage,2,502,0.001700 +1793284519,sustained-db-outage,2,502,0.002650 +1793284520,sustained-db-outage,2,502,1.015060 +1793284521,sustained-db-outage,2,502,0.001195 +1793284521,sustained-db-outage,2,502,0.004247 +1793284522,sustained-db-outage,2,502,0.002479 +1793284522,sustained-db-outage,2,502,0.002744 +1793284522,sustained-db-outage,2,502,0.002945 +1793284522,sustained-db-outage,2,502,1.006484 +1793284524,sustained-db-outage,2,502,0.004542 +1793284524,sustained-db-outage,2,502,0.001569 +1793284524,sustained-db-outage,2,502,0.003072 +1793284525,sustained-db-outage,2,502,0.008085 +1793284525,sustained-db-outage,2,502,0.005222 +1793284525,sustained-db-outage,2,502,0.005127 +1793284526,sustained-db-outage,2,502,1.010677 +1793284527,sustained-db-outage,2,502,0.002329 +1793284527,sustained-db-outage,2,502,0.002948 +1793284527,sustained-db-outage,2,502,0.002192 +1793284528,sustained-db-outage,2,502,0.001096 +1793284528,sustained-db-outage,2,502,0.003082 +1793284528,sustained-db-outage,2,502,0.003988 +1793284529,sustained-db-outage,2,502,1.009765 +1793284530,sustained-db-outage,2,502,0.003268 +1793284530,sustained-db-outage,2,502,0.002574 +1793284531,sustained-db-outage,2,502,0.002192 +1793284531,sustained-db-outage,2,502,0.003484 +1793284531,sustained-db-outage,2,502,0.000897 +1793284531,sustained-db-outage,2,502,0.002524 +1793284532,sustained-db-outage,2,502,1.009228 +1793284533,sustained-db-outage,2,502,0.002017 +1793284533,sustained-db-outage,2,502,0.002929 +1793284534,sustained-db-outage,2,502,0.002316 +1793284534,sustained-db-outage,2,502,0.003070 +1793284534,sustained-db-outage,2,502,0.003959 +1793284534,sustained-db-outage,2,502,0.000854 +1793284535,sustained-db-outage,2,502,1.010480 +1793284536,sustained-db-outage,2,502,0.002380 +1793284536,sustained-db-outage,2,502,0.004212 +1793284537,sustained-db-outage,2,502,0.004062 +1793284537,sustained-db-outage,2,502,0.004168 +1793284537,sustained-db-outage,2,502,0.001976 +1793284538,sustained-db-outage,2,502,1.005872 +1793284539,sustained-db-outage,2,502,0.002786 +1793284539,sustained-db-outage,2,502,0.002133 +1793284539,sustained-db-outage,2,502,0.001100 +1793284540,sustained-db-outage,2,502,0.006934 +1793284540,sustained-db-outage,2,502,0.003132 +1793284540,sustained-db-outage,2,502,0.003939 +1793284541,sustained-db-outage,2,502,1.008963 +1793284542,sustained-db-outage,2,502,0.003933 +1793284542,sustained-db-outage,2,502,0.004104 +1793284542,sustained-db-outage,2,502,0.003788 +1793284543,sustained-db-outage,2,502,0.001517 +1793284543,sustained-db-outage,2,502,0.006063 +1793284543,sustained-db-outage,2,502,0.004524 +1793284544,sustained-db-outage,2,502,1.009765 +1793284545,sustained-db-outage,2,502,0.002982 +1793284545,sustained-db-outage,2,502,0.001828 +1793284546,sustained-db-outage,2,502,0.002964 +1793284546,sustained-db-outage,2,502,0.006160 +1793284546,sustained-db-outage,2,502,0.004493 +1793284546,sustained-db-outage,2,502,0.004335 +1793284547,sustained-db-outage,2,502,1.017084 +1793284548,sustained-db-outage,2,502,0.002432 +1793284548,sustained-db-outage,2,502,0.006273 +1793284549,sustained-db-outage,2,502,0.004123 +1793284549,db-recovery,2,502,0.006910 +1793284549,db-recovery,2,502,0.004185 +1793284550,db-recovery,2,200,0.029171 +1793284550,db-recovery,2,200,0.022312 +1793284550,db-recovery,2,200,0.011213 +1793284551,db-recovery,2,200,0.020648 +1793284551,db-recovery,2,200,0.020465 +1793284551,db-recovery,2,200,0.009133 +1793284552,db-recovery,2,200,0.017679 +1793284552,db-recovery,2,200,0.040934 +1793284552,db-recovery,2,200,0.011542 +1793284552,db-recovery,2,200,0.020343 +1793284553,db-recovery,2,200,0.017896 +1793284553,db-recovery,2,200,0.013213 +1793284553,db-recovery,2,200,0.018624 +1793284554,db-recovery,2,200,0.028254 +1793284554,db-recovery,2,200,0.015804 +1793284554,db-recovery,2,200,0.019495 +1793284555,db-recovery,2,200,0.016475 +1793284555,db-recovery,2,200,0.018014 +1793284555,db-recovery,2,200,0.025220 +1793284556,db-recovery,2,200,0.018778 +1793284556,db-recovery,2,200,0.016806 +1793284556,db-recovery,2,200,0.012508 +1793284557,db-recovery,2,200,0.021996 +1793284557,db-recovery,2,200,0.019880 +1793284557,db-recovery,2,200,0.024630 +1793284558,db-recovery,2,200,0.020214 +1793284558,db-recovery,2,200,0.017005 +1793284558,db-recovery,2,200,0.011722 +1793284559,db-recovery,2,200,0.011547 +1793284559,db-recovery,2,200,0.020085 +1793284559,db-recovery,2,200,0.009371 +1793284559,db-recovery,2,200,0.018410 +1793284560,db-recovery,2,200,0.019504 +1793284560,db-recovery,2,200,0.015392 +1793284560,db-recovery,2,200,0.015820 +1793284561,db-recovery,2,200,0.019847 +1793284561,db-recovery,2,200,0.019373 +1793284561,db-recovery,2,200,0.011803 +1793284562,db-recovery,2,200,0.029670 +1793284562,db-recovery,2,200,0.028228 +1793284562,db-recovery,2,200,0.010106 +1793284563,db-recovery,2,200,0.018022 +1793284563,db-recovery,2,200,0.025216 +1793284563,db-recovery,2,200,0.014269 +1793284564,db-recovery,2,200,0.019083 +1793284564,db-recovery,2,200,0.018869 +1793284564,db-recovery,2,200,0.016083 +1793284565,db-recovery,2,200,0.018592 +1793284565,db-recovery,2,200,0.016511 +1793284565,db-recovery,2,200,0.019782 +1793284566,db-recovery,2,200,0.017399 +1793284566,db-recovery,2,200,0.018678 +1793284566,db-recovery,2,200,0.014755 +1793284566,db-recovery,2,200,0.012208 +1793284567,db-recovery,2,200,0.011827 +1793284567,db-recovery,2,200,0.020400 +1793284567,db-recovery,2,200,0.012197 +1793284568,db-recovery,2,200,0.019740 +1793284568,db-recovery,2,200,0.021067 +1793284568,db-recovery,2,200,0.012477 +1793284569,db-recovery,2,200,0.017992 +1793284569,db-recovery,2,200,0.022577 +1793284569,db-recovery,2,200,0.010356 +1793284570,db-recovery,2,200,0.019918 +1793284570,db-recovery,2,200,0.021057 +1793284570,db-recovery,2,200,0.021953 +1793284571,db-recovery,2,200,0.011490 +1793284571,db-recovery,2,200,0.017880 +1793284571,db-recovery,2,200,0.025725 +1793284572,db-recovery,2,200,0.016778 +1793284572,db-recovery,2,200,0.017798 +1793284572,db-recovery,2,200,0.013921 +1793284572,db-recovery,2,200,0.021500 +1793284573,db-recovery,2,200,0.009462 +1793284573,db-recovery,2,200,0.013943 +1793284573,db-recovery,2,200,0.017854 +1793284574,db-recovery,2,200,0.020705 +1793284574,db-recovery,2,200,0.011602 +1793284574,db-recovery,2,200,0.018353 +1793284575,db-recovery,2,200,0.013568 +1793284575,db-recovery,2,200,0.015719 +1793284575,db-recovery,2,200,0.014606 +1793284576,db-recovery,2,200,0.014338 +1793284576,db-recovery,2,200,0.013431 +1793284576,db-recovery,2,200,0.026802 +1793284577,db-recovery,2,200,0.016916 +1793284577,db-recovery,2,200,0.012679 +1793284577,db-recovery,2,200,0.018483 +1793284577,db-recovery,2,200,0.009473 +1793284578,db-recovery,2,200,0.026914 +1793284578,db-recovery,2,200,0.016534 +1793284578,db-recovery,2,200,0.024385 +1793284579,db-recovery,2,200,0.007926 +1793284579,db-recovery,2,200,0.014760 +1793284579,db-recovery,2,200,0.015845 +1793284580,db-recovery,2,200,0.015746 +1793284580,db-recovery,2,200,0.019214 +1793284580,db-recovery,2,200,0.016318 +1793284581,db-recovery,2,200,0.015898 +1793284581,db-recovery,2,200,0.012503 +1793284581,db-recovery,2,200,0.016675 +1793284581,db-recovery,2,200,0.007642 +1793284582,db-recovery,2,200,0.016457 +1793284582,db-recovery,2,200,0.015128 +1793284582,db-recovery,2,200,0.015804 +1793284583,db-recovery,2,200,0.016578 +1793284583,db-recovery,2,200,0.016716 +1793284583,db-recovery,2,200,0.018386 +1793284584,db-recovery,2,200,0.019678 +1793284584,db-recovery,2,200,0.024433 +1793284584,db-recovery,2,200,0.015022 +1793284585,db-recovery,2,200,0.016834 +1793284585,db-recovery,2,200,0.017685 +1793284585,db-recovery,2,200,0.024313 +1793284585,db-recovery,2,200,0.018020 +1793284586,db-recovery,2,200,0.016104 +1793284586,db-recovery,2,200,0.014797 +1793284586,db-recovery,2,200,0.023667 +1793284587,db-recovery,2,200,0.011906 +1793284587,db-recovery,2,200,0.021362 +1793284587,db-recovery,2,200,0.019907 +1793284588,db-recovery,2,200,0.013025 +1793284588,db-recovery,2,200,0.025792 +1793284588,db-recovery,2,200,0.016695 +1793284589,db-recovery,2,200,0.015516 +1793284589,db-recovery,2,200,0.017919 +1793284589,db-recovery,2,200,0.017373 +1793284590,db-recovery,2,200,0.011657 +1793284590,db-recovery,2,200,0.019428 +1793284590,db-recovery,2,200,0.015649 +1793284591,db-recovery,2,200,0.021234 +1793284591,db-recovery,2,200,0.017271 +1793284591,db-recovery,2,200,0.018680 +1793284591,db-recovery,2,200,0.008717 +1793284592,db-recovery,2,200,0.017523 +1793284592,db-recovery,2,200,0.017629 +1793284592,db-recovery,2,200,0.019037 +1793284593,db-recovery,2,200,0.018634 +1793284593,db-recovery,2,200,0.015972 +1793284593,db-recovery,2,200,0.015994 +1793284594,db-recovery,2,200,0.017245 +1793284594,db-recovery,2,200,0.017056 +1793284594,db-recovery,2,200,0.010456 +1793284595,db-recovery,2,200,0.008648 +1793284595,db-recovery,2,200,0.010813 +1793284595,db-recovery,2,200,0.017475 +1793284596,db-recovery,2,200,0.021872 +1793284596,db-recovery,2,200,0.016706 +1793284596,db-recovery,2,200,0.017180 +1793284597,db-recovery,2,200,0.010219 +1793284597,db-recovery,2,200,0.015654 +1793284597,db-recovery,2,200,0.018384 +1793284597,db-recovery,2,200,0.013300 +1793284598,db-recovery,2,200,0.021000 +1793284598,db-recovery,2,200,0.019764 +1793284598,db-recovery,2,200,0.015697 +1793284599,db-recovery,2,200,0.012302 +1793284599,db-recovery,2,200,0.023032 +1793284599,db-recovery,2,200,0.021065 +1793284600,db-recovery,2,200,0.019411 +1793284600,db-recovery,2,200,0.018824 +1793284600,db-recovery,2,200,0.020581 +1793284601,db-recovery,2,200,0.017899 +1793284601,db-recovery,2,200,0.018919 +1793284601,db-recovery,2,200,0.015669 +1793284602,db-recovery,2,200,0.011072 +1793284602,db-recovery,2,200,0.009266 +1793284602,db-recovery,2,200,0.016310 +1793284602,db-recovery,2,200,0.020282 +1793284603,db-recovery,2,200,0.017861 +1793284603,db-recovery,2,200,0.013181 +1793284603,db-recovery,2,200,0.019304 +1793284604,db-recovery,2,200,0.020270 +1793284604,db-recovery,2,200,0.012303 +1793284604,db-recovery,2,200,0.012062 +1793284605,db-recovery,2,200,0.014807 +1793284605,db-recovery,2,200,0.034529 +1793284605,db-recovery,2,200,0.018284 +1793284606,db-recovery,2,200,0.015340 +1793284606,db-recovery,2,200,0.019708 +1793284606,db-recovery,2,200,0.015555 +1793284607,db-recovery,2,200,0.018088 +1793284607,db-recovery,2,200,0.018786 +1793284607,db-recovery,2,200,0.008671 +1793284608,db-recovery,2,200,0.015800 +1793284608,db-recovery,2,200,0.021284 +1793284608,db-recovery,2,200,0.015144 +1793284608,db-recovery,2,200,0.017607 +1793284609,db-recovery,2,200,0.020399 +1793284609,db-recovery,2,200,0.019522 +1793284609,db-recovery,2,200,0.012224 +1793284610,db-recovery,2,200,0.016745 +1793284610,db-recovery,2,200,0.020312 +1793284610,db-recovery,2,200,0.017511 +1793284611,db-recovery,2,200,0.018704 +1793284611,db-recovery,2,200,0.010397 +1793284611,db-recovery,2,200,0.015714 +1793284612,db-recovery,2,200,0.014873 +1793284612,db-recovery,2,200,0.018202 +1793284612,db-recovery,2,200,0.016177 +1793284612,db-recovery,2,200,0.018145 +1793284613,db-recovery,2,200,0.017511 +1793284613,db-recovery,2,200,0.017859 +1793284613,db-recovery,2,200,0.012402 +1793284614,db-recovery,2,200,0.020413 +1793284614,db-recovery,2,200,0.015110 +1793284614,db-recovery,2,200,0.020460 +1793284615,db-recovery,2,200,0.019207 +1793284615,db-recovery,2,200,0.018774 +1793284615,db-recovery,2,200,0.018421 +1793284616,db-recovery,2,200,0.015153 +1793284616,db-recovery,2,200,0.015706 +1793284616,db-recovery,2,200,0.015583 +1793284617,db-recovery,2,200,0.016659 +1793284617,db-recovery,2,200,0.027531 +1793284617,db-recovery,2,200,0.015520 +1793284617,db-recovery,2,200,0.007583 +1793284618,db-recovery,2,200,0.009320 +1793284618,db-recovery,2,200,0.017540 +1793284618,db-recovery,2,200,0.015396 +1793284619,db-recovery,2,200,0.020217 +1793284619,db-recovery,2,200,0.013942 +1793284619,db-recovery,2,200,0.014598 +1793284620,db-recovery,2,200,0.012644 +1793284620,db-recovery,2,200,0.013620 +1793284620,db-recovery,2,200,0.015514 +1793284621,db-recovery,2,200,0.017692 +1793284621,db-recovery,2,200,0.014610 +1793284621,db-recovery,2,200,0.018490 +1793284622,db-recovery,2,200,1.018747 +1793284623,db-recovery,2,200,0.015028 +1793284623,db-recovery,2,200,0.007807 +1793284623,db-recovery,2,200,0.009344 +1793284624,db-recovery,2,200,0.012357 +1793284624,db-recovery,2,200,0.013852 +1793284624,db-recovery,2,200,0.017534 +1793284625,db-recovery,2,200,0.017420 +1793284625,db-recovery,2,200,0.021046 +1793284625,db-recovery,2,200,0.013653 +1793284626,db-recovery,2,200,0.010241 +1793284626,db-recovery,2,200,0.018801 +1793284626,db-recovery,2,200,0.008696 +1793284627,db-recovery,2,200,1.013446 +1793284628,db-recovery,2,200,0.033181 +1793284628,db-recovery,2,200,0.013225 +1793284628,db-recovery,2,200,0.018793 +1793284629,db-recovery,2,200,0.009410 +1793284629,db-recovery,2,200,0.019270 +1793284629,db-recovery,2,200,0.019534 +1793284630,db-recovery,2,200,0.010238 +1793284630,db-recovery,2,200,0.013507 +1793284630,db-recovery,2,200,1.021545 +1793284632,db-recovery,2,200,0.016178 +1793284632,db-recovery,2,200,0.013089 +1793284632,db-recovery,2,200,0.007521 +1793284633,db-recovery,2,200,1.016876 +1793284634,db-recovery,2,200,0.009032 +1793284634,db-recovery,2,200,0.015805 +1793284634,db-recovery,2,200,0.009143 +1793284635,db-recovery,2,200,0.015173 +1793284635,db-recovery,2,200,0.020223 +1793284635,db-recovery,2,200,0.011784 +1793284636,db-recovery,2,200,0.019154 +1793284636,db-recovery,2,200,0.019859 +1793284636,db-recovery,2,200,0.015474 +1793284637,db-recovery,2,200,0.010781 +1793284637,db-recovery,2,200,0.011731 +1793284637,db-recovery,2,200,0.019559 +1793284637,db-recovery,2,200,0.008189 +1793284638,db-recovery,2,200,0.017925 +1793284638,db-recovery,2,200,0.015676 +1793284638,db-recovery,2,200,0.007839 +1793284639,db-recovery,2,200,0.019535 +1793284639,db-recovery,2,200,0.014321 +1793284639,db-recovery,2,200,0.015429 +1793284640,db-recovery,2,200,0.017772 +1793284640,db-recovery,2,200,0.008170 +1793284640,db-recovery,2,200,0.023559 +1793284641,worker-stopped,2,200,0.015886 +1793284641,worker-stopped,2,200,0.016635 +1793284641,worker-stopped,2,200,0.014092 +1793284641,worker-stopped,2,200,0.008965 +1793284642,worker-stopped,2,200,0.018421 +1793284642,worker-stopped,2,200,0.012528 +1793284642,worker-stopped,2,200,0.014135 +1793284643,worker-stopped,2,200,0.016736 +1793284643,worker-stopped,2,200,0.022825 +1793284643,worker-stopped,2,200,0.014978 +1793284644,worker-stopped,2,200,0.008485 +1793284644,worker-stopped,2,200,0.012878 +1793284644,worker-stopped,2,200,0.015625 +1793284644,worker-stopped,2,200,0.009374 +1793284645,worker-stopped,2,200,0.020159 +1793284645,worker-stopped,2,200,0.012917 +1793284645,worker-stopped,2,200,0.017607 +1793284646,worker-stopped,2,200,0.006865 +1793284646,worker-stopped,2,200,0.008794 +1793284646,worker-stopped,2,200,0.013979 +1793284647,worker-stopped,2,200,0.017328 +1793284647,worker-stopped,2,200,0.013675 +1793284647,worker-stopped,2,200,0.016309 +1793284647,worker-stopped,2,200,0.004695 +1793284648,worker-stopped,2,200,0.005468 +1793284648,worker-stopped,2,200,1.025854 +1793284649,worker-stopped,2,200,0.009113 +1793284650,worker-stopped,2,200,0.011154 +1793284650,worker-stopped,2,200,0.006153 +1793284650,worker-stopped,2,200,0.007977 +1793284650,worker-stopped,2,200,0.017422 +1793284651,worker-stopped,2,200,0.016153 +1793284651,worker-stopped,2,200,0.020692 +1793284651,worker-stopped,2,200,0.011968 +1793284652,worker-stopped,2,200,0.005383 +1793284652,worker-stopped,2,200,0.010424 +1793284652,worker-stopped,2,200,0.007066 +1793284652,worker-stopped,2,200,0.013663 +1793284653,worker-stopped,2,200,0.014659 +1793284653,worker-stopped,2,200,0.022363 +1793284653,worker-stopped,2,200,0.015612 +1793284654,worker-stopped,2,200,0.014908 +1793284654,worker-stopped,2,200,0.010715 +1793284654,worker-stopped,2,200,0.005517 +1793284655,worker-stopped,2,200,0.005326 +1793284655,worker-stopped,2,200,0.014506 +1793284655,worker-stopped,2,200,0.013078 +1793284655,worker-stopped,2,200,0.016087 +1793284656,worker-stopped,2,200,0.010921 +1793284656,worker-stopped,2,200,0.013795 +1793284656,worker-stopped,2,200,0.010431 +1793284657,worker-stopped,2,200,0.008739 +1793284657,worker-stopped,2,200,0.014272 +1793284657,worker-stopped,2,200,0.008244 +1793284658,worker-stopped,2,200,0.006645 +1793284658,worker-stopped,2,200,0.016907 +1793284658,worker-stopped,2,200,0.010219 +1793284658,worker-stopped,2,200,0.017312 +1793284659,worker-stopped,2,200,0.015817 +1793284659,worker-stopped,2,200,0.011290 +1793284659,worker-stopped,2,200,0.010552 +1793284660,worker-stopped,2,200,0.005739 +1793284660,worker-stopped,2,200,0.012393 +1793284660,worker-stopped,2,200,0.012207 +1793284661,worker-stopped,2,200,0.016863 +1793284661,worker-stopped,2,200,0.019277 +1793284661,worker-stopped,2,200,0.011767 +1793284661,worker-stopped,2,200,0.021293 +1793284662,worker-stopped,2,200,0.014438 +1793284662,worker-stopped,2,200,0.010337 +1793284662,worker-stopped,2,200,0.026363 +1793284663,worker-stopped,2,200,0.010213 +1793284663,worker-stopped,2,200,0.009462 +1793284663,worker-stopped,2,200,0.008786 +1793284664,worker-stopped,2,200,0.009681 +1793284664,worker-stopped,2,200,0.014416 +1793284664,worker-stopped,2,200,0.011407 +1793284665,worker-stopped,2,200,0.010401 +1793284665,worker-stopped,2,200,0.016722 +1793284665,worker-stopped,2,200,0.014278 +1793284665,worker-stopped,2,200,0.020330 +1793284666,worker-stopped,2,200,0.005456 +1793284666,worker-stopped,2,200,0.009792 +1793284666,worker-stopped,2,200,0.020223 +1793284667,worker-stopped,2,200,0.013088 +1793284667,worker-stopped,2,200,0.013719 +1793284667,worker-stopped,2,200,0.011995 +1793284668,worker-stopped,2,200,0.012436 +1793284668,worker-stopped,2,200,0.012186 +1793284668,worker-stopped,2,200,0.007999 +1793284668,worker-stopped,2,200,0.005304 +1793284669,worker-stopped,2,200,0.013511 +1793284669,worker-stopped,2,200,0.013261 +1793284669,worker-stopped,2,200,0.011717 +1793284670,worker-stopped,2,200,0.014189 +1793284670,worker-stopped,2,200,0.016249 +1793284670,worker-stopped,2,200,0.014028 +1793284671,worker-stopped,2,200,0.013305 +1793284671,worker-stopped,2,200,0.013472 +1793284671,worker-stopped,2,200,0.016151 +1793284671,worker-stopped,2,200,0.015044 +1793284672,worker-stopped,2,200,0.024397 +1793284672,worker-stopped,2,200,0.008247 +1793284672,worker-stopped,2,200,0.015924 +1793284673,worker-stopped,2,200,0.013292 +1793284673,worker-stopped,2,200,0.013225 +1793284673,worker-stopped,2,200,0.010188 +1793284674,worker-stopped,2,200,0.015764 +1793284674,worker-stopped,2,200,0.015399 +1793284674,worker-stopped,2,200,0.013131 +1793284675,worker-stopped,2,200,0.007950 +1793284675,worker-stopped,2,200,0.009366 +1793284675,worker-stopped,2,200,0.011653 +1793284675,worker-stopped,2,200,0.013325 +1793284676,worker-stopped,2,200,0.017987 +1793284676,worker-stopped,2,200,0.010337 +1793284676,worker-stopped,2,200,0.012427 +1793284677,worker-stopped,2,200,0.013227 +1793284677,worker-stopped,2,200,0.013497 +1793284677,worker-stopped,2,200,0.014487 +1793284678,worker-stopped,2,200,0.012728 +1793284678,worker-stopped,2,200,0.013846 +1793284678,worker-stopped,2,200,0.013678 +1793284678,worker-stopped,2,200,0.017469 +1793284679,worker-stopped,2,200,0.014969 +1793284679,worker-stopped,2,200,0.011944 +1793284679,worker-stopped,2,200,0.007002 +1793284680,worker-stopped,2,200,0.011480 +1793284680,worker-stopped,2,200,0.010367 +1793284680,worker-stopped,2,200,0.016160 +1793284681,worker-stopped,2,200,0.019790 +1793284681,worker-stopped,2,200,0.008955 +1793284681,worker-stopped,2,200,0.019282 +1793284682,worker-stopped,2,200,0.015502 +1793284682,worker-stopped,2,200,0.018146 +1793284682,worker-stopped,2,200,0.007438 +1793284682,worker-stopped,2,200,0.014220 +1793284683,worker-stopped,2,200,0.007855 +1793284683,worker-stopped,2,200,0.009116 +1793284683,worker-stopped,2,200,0.007758 +1793284684,worker-stopped,2,200,0.013609 +1793284684,worker-stopped,2,200,0.008193 +1793284684,worker-stopped,2,200,0.012846 +1793284685,worker-stopped,2,200,0.010917 +1793284685,worker-stopped,2,200,0.012405 +1793284685,worker-stopped,2,200,0.013099 +1793284685,worker-stopped,2,200,0.014235 +1793284686,worker-stopped,2,200,0.013827 +1793284686,worker-stopped,2,200,0.006709 +1793284686,worker-stopped,2,200,0.011780 +1793284687,worker-stopped,2,200,0.014761 +1793284687,worker-stopped,2,200,0.006416 +1793284687,worker-stopped,2,200,0.012120 +1793284688,worker-stopped,2,200,0.016256 +1793284688,worker-stopped,2,200,0.011533 +1793284688,worker-stopped,2,200,0.011132 +1793284689,worker-stopped,2,200,0.016568 +1793284689,worker-stopped,2,200,0.010449 +1793284689,worker-stopped,2,200,0.012876 +1793284689,worker-stopped,2,200,0.008543 +1793284690,worker-stopped,2,200,0.014552 +1793284690,worker-stopped,2,200,0.007708 +1793284690,worker-stopped,2,200,0.011392 +1793284691,worker-stopped,2,200,0.011029 +1793284691,worker-stopped,2,200,0.016921 +1793284691,worker-stopped,2,200,0.013894 +1793284692,worker-stopped,2,200,0.009714 +1793284692,worker-stopped,2,200,0.008264 +1793284692,worker-stopped,2,200,0.008274 +1793284692,worker-stopped,2,200,0.013832 +1793284693,worker-stopped,2,200,0.012735 +1793284693,worker-stopped,2,200,0.008689 +1793284693,worker-stopped,2,200,0.012827 +1793284694,worker-stopped,2,200,0.016384 +1793284694,worker-stopped,2,200,0.011820 +1793284694,worker-stopped,2,200,0.010965 +1793284695,worker-stopped,2,200,0.010653 +1793284695,worker-stopped,2,200,0.016653 +1793284695,worker-stopped,2,200,0.014781 +1793284695,worker-stopped,2,200,0.011809 +1793284696,worker-stopped,2,200,0.015626 +1793284696,worker-stopped,2,200,0.011731 +1793284696,worker-stopped,2,200,0.014356 +1793284697,worker-stopped,2,200,0.012292 +1793284697,worker-stopped,2,200,0.014274 +1793284697,worker-stopped,2,200,0.018190 +1793284697,worker-stopped,2,200,0.012949 +1793284698,worker-stopped,2,200,0.014943 +1793284698,worker-stopped,2,200,0.013345 +1793284698,worker-stopped,2,200,0.012545 +1793284699,worker-stopped,2,200,0.010312 +1793284699,worker-stopped,2,200,0.008909 +1793284699,worker-stopped,2,200,0.009081 +1793284699,worker-stopped,2,200,0.009863 +1793284700,worker-stopped,2,200,0.010708 +1793284700,worker-stopped,2,200,0.012810 +1793284700,worker-stopped,2,200,0.010339 +1793284701,worker-stopped,2,200,0.007926 +1793284701,worker-stopped,2,200,0.011663 +1793284701,worker-stopped,2,200,1.013426 +1793284702,worker-stopped,2,200,0.015967 +1793284703,worker-stopped,2,200,0.012599 +1793284703,worker-stopped,2,200,0.010575 +1793284703,worker-stopped,2,200,0.011126 +1793284704,worker-stopped,2,200,0.011787 +1793284704,worker-stopped,2,200,0.008472 +1793284704,worker-stopped,2,200,1.005839 +1793284705,worker-stopped,2,200,0.012083 +1793284706,worker-stopped,2,200,0.009198 +1793284706,worker-stopped,2,200,0.008320 +1793284706,worker-stopped,2,200,0.008263 +1793284707,worker-stopped,2,200,0.008755 +1793284707,worker-stopped,2,200,0.011183 +1793284707,worker-stopped,2,200,0.009231 +1793284707,worker-stopped,2,200,0.043933 +1793284708,worker-stopped,2,200,0.007126 +1793284708,worker-stopped,2,200,0.003746 +1793284708,worker-stopped,2,200,0.004539 +1793284709,worker-stopped,2,200,0.003536 +1793284709,worker-stopped,2,200,0.004837 +1793284709,worker-stopped,2,200,0.005814 +1793284709,worker-stopped,2,200,0.009441 +1793284710,worker-stopped,2,200,0.007236 +1793284710,worker-stopped,2,200,0.004886 +1793284710,worker-stopped,2,200,0.004371 +1793284711,worker-stopped,2,200,0.005521 +1793284711,worker-stopped,2,200,0.003732 +1793284711,worker-stopped,2,200,0.004335 +1793284711,worker-stopped,2,200,0.003311 +1793284712,worker-stopped,2,200,0.005783 +1793284712,worker-stopped,2,200,0.008225 +1793284712,worker-stopped,2,200,0.015228 +1793284713,worker-stopped,2,200,0.010788 +1793284713,worker-stopped,2,200,0.011996 +1793284713,worker-stopped,2,200,0.008988 +1793284713,worker-stopped,2,200,0.014648 +1793284714,worker-stopped,2,200,0.013104 +1793284714,worker-stopped,2,200,0.009903 +1793284714,worker-stopped,2,200,0.007292 +1793284715,worker-stopped,2,200,0.006626 +1793284715,worker-stopped,2,200,0.013790 +1793284715,worker-stopped,2,200,0.016336 +1793284716,worker-stopped,2,200,0.009464 +1793284716,worker-stopped,2,200,0.009212 +1793284716,worker-stopped,2,200,0.010636 +1793284716,worker-stopped,2,200,0.010941 +1793284717,worker-stopped,2,200,0.012953 +1793284717,worker-stopped,2,200,0.010497 +1793284717,worker-stopped,2,200,0.009642 +1793284717,worker-stopped,2,200,0.007541 +1793284718,worker-stopped,2,200,0.004326 +1793284718,worker-stopped,2,200,0.010708 +1793284718,worker-stopped,2,200,0.012913 +1793284719,worker-stopped,2,200,0.005307 +1793284719,worker-stopped,2,200,0.013653 +1793284719,worker-stopped,2,200,0.009848 +1793284720,worker-stopped,2,200,0.009971 +1793284720,worker-stopped,2,200,0.014156 +1793284720,worker-stopped,2,200,0.014179 +1793284720,worker-stopped,2,200,0.008654 +1793284721,worker-stopped,2,200,0.014020 +1793284721,worker-stopped,2,200,0.009455 +1793284721,worker-stopped,2,200,0.011763 +1793284722,worker-stopped,2,200,0.010589 +1793284722,worker-stopped,2,200,0.009845 +1793284722,worker-stopped,2,200,0.012564 +1793284723,worker-stopped,2,200,0.006959 +1793284723,worker-stopped,2,200,0.010180 +1793284723,worker-stopped,2,200,0.007654 +1793284723,worker-stopped,2,200,0.009764 +1793284724,worker-stopped,2,200,0.007979 +1793284724,worker-stopped,2,200,0.012867 +1793284724,worker-stopped,2,200,0.016292 +1793284724,worker-stopped,2,200,0.009113 +1793284725,worker-stopped,2,200,0.012020 +1793284725,worker-stopped,2,200,0.010691 +1793284725,worker-stopped,2,200,0.030361 +1793284726,worker-stopped,2,200,0.014995 +1793284726,worker-stopped,2,200,0.013463 +1793284726,worker-stopped,2,200,0.016639 +1793284727,worker-stopped,2,200,0.005398 +1793284727,worker-stopped,2,200,0.010319 +1793284727,worker-stopped,2,200,0.010872 +1793284728,worker-stopped,2,200,0.016938 +1793284728,worker-stopped,2,200,0.009825 +1793284728,worker-stopped,2,200,0.013385 +1793284728,worker-stopped,2,200,0.012220 +1793284729,worker-stopped,2,200,0.011557 +1793284729,worker-stopped,2,200,0.021504 +1793284729,worker-stopped,2,200,0.016069 +1793284730,worker-stopped,2,200,0.013677 +1793284730,worker-stopped,2,200,0.012331 +1793284730,worker-stopped,2,200,0.012511 +1793284730,worker-stopped,2,200,0.014466 +1793284731,worker-stopped,2,200,0.011035 +1793284731,worker-stopped,2,200,0.016251 +1793284731,worker-stopped,2,200,0.022223 +1793284732,worker-stopped,2,200,0.009313 +1793284732,worker-stopped,2,200,0.015348 +1793284732,worker-stopped,2,200,0.020173 +1793284733,worker-stopped,2,200,0.011489 +1793284733,worker-stopped,2,200,0.014976 +1793284733,worker-stopped,2,200,0.012023 +1793284733,worker-stopped,2,200,0.018801 +1793284734,worker-stopped,2,200,0.008846 +1793284734,worker-stopped,2,200,0.016487 +1793284734,worker-stopped,2,200,0.016336 +1793284735,worker-stopped,2,200,0.007240 +1793284735,worker-stopped,2,200,0.014588 +1793284735,worker-stopped,2,200,0.013343 +1793284735,worker-stopped,2,200,0.012414 +1793284736,worker-stopped,2,200,0.006400 +1793284736,worker-stopped,2,200,0.013940 +1793284736,worker-stopped,2,200,0.011273 +1793284737,worker-stopped,2,200,0.010144 +1793284737,worker-stopped,2,200,0.018064 +1793284737,worker-stopped,2,200,0.011045 +1793284738,worker-stopped,2,200,0.010972 +1793284738,worker-stopped,2,200,0.006236 +1793284738,worker-stopped,2,200,0.012261 +1793284738,worker-stopped,2,200,0.014813 +1793284739,worker-stopped,2,200,0.006378 +1793284739,worker-stopped,2,200,0.007301 +1793284739,worker-stopped,2,200,0.012518 +1793284740,worker-stopped,2,200,0.012799 +1793284740,worker-stopped,2,200,0.006008 +1793284740,worker-stopped,2,200,0.013206 +1793284740,worker-stopped,2,200,0.010233 +1793284741,worker-stopped,2,200,0.015894 +1793284741,worker-stopped,2,200,0.018565 +1793284741,worker-stopped,2,200,0.016901 +1793284742,worker-stopped,2,200,0.014126 +1793284742,worker-stopped,2,200,0.009536 +1793284742,worker-stopped,2,200,0.013139 +1793284743,worker-stopped,2,200,0.007306 +1793284743,worker-stopped,2,200,0.013512 +1793284743,worker-stopped,2,200,0.005509 +1793284743,worker-stopped,2,200,0.018266 +1793284744,worker-stopped,2,200,0.005526 +1793284744,worker-stopped,2,200,0.004387 +1793284744,worker-stopped,2,200,0.006930 +1793284745,worker-stopped,2,200,0.004701 +1793284745,worker-stopped,2,200,0.006511 +1793284745,worker-stopped,2,200,0.013169 +1793284745,worker-stopped,2,200,0.011710 +1793284746,worker-stopped,2,200,0.013158 +1793284746,worker-stopped,2,200,0.008952 +1793284746,worker-stopped,2,200,0.008478 +1793284747,worker-stopped,2,200,0.013616 +1793284747,worker-stopped,2,200,0.013111 +1793284747,worker-stopped,2,200,0.013961 +1793284748,worker-stopped,2,200,0.014495 +1793284748,worker-stopped,2,200,0.012819 +1793284748,worker-stopped,2,200,0.017509 +1793284748,worker-stopped,2,200,0.013068 +1793284749,worker-stopped,2,200,0.010754 +1793284749,worker-stopped,2,200,0.011560 +1793284749,worker-stopped,2,200,0.011906 +1793284750,worker-stopped,2,200,0.019708 +1793284750,worker-stopped,2,200,0.019837 +1793284750,worker-stopped,2,200,0.016390 +1793284751,worker-stopped,2,200,0.009755 +1793284751,worker-stopped,2,200,0.009177 +1793284751,worker-stopped,2,200,0.016134 +1793284752,worker-stopped,2,200,0.016979 +1793284752,worker-stopped,2,200,0.011550 +1793284752,worker-stopped,2,200,0.010251 +1793284752,worker-stopped,2,200,0.019123 +1793284753,worker-recovery,2,200,0.007938 +1793284753,worker-recovery,2,200,0.005699 +1793284753,worker-recovery,2,200,0.007352 +1793284754,worker-recovery,2,200,0.015450 +1793284754,worker-recovery,2,200,0.015584 +1793284754,worker-recovery,2,200,0.009291 +1793284754,worker-recovery,2,200,0.014948 +1793284755,worker-recovery,2,200,0.018903 +1793284755,worker-recovery,2,200,0.017255 +1793284755,worker-recovery,2,200,0.014763 +1793284756,worker-recovery,2,200,0.018261 +1793284756,worker-recovery,2,200,0.017766 +1793284756,worker-recovery,2,200,0.009261 +1793284756,worker-recovery,2,200,0.012546 +1793284757,worker-recovery,2,200,0.013162 +1793284757,worker-recovery,2,200,0.007376 +1793284757,worker-recovery,2,200,0.008491 +1793284758,worker-recovery,2,200,0.012972 +1793284758,worker-recovery,2,200,0.015079 +1793284758,worker-recovery,2,200,0.010983 +1793284758,worker-recovery,2,200,0.013134 +1793284759,worker-recovery,2,200,0.010098 +1793284759,worker-recovery,2,200,0.014917 +1793284759,worker-recovery,2,200,0.032379 +1793284760,worker-recovery,2,200,0.016249 +1793284760,worker-recovery,2,200,0.011403 +1793284760,worker-recovery,2,200,0.009078 +1793284761,worker-recovery,2,200,0.020354 +1793284761,worker-recovery,2,200,0.014341 +1793284761,worker-recovery,2,200,0.013587 +1793284761,worker-recovery,2,200,0.013628 +1793284762,worker-recovery,2,200,0.010211 +1793284762,worker-recovery,2,200,0.012050 +1793284762,worker-recovery,2,200,0.012658 +1793284763,worker-recovery,2,200,0.009531 +1793284763,worker-recovery,2,200,0.011462 +1793284763,worker-recovery,2,200,0.013725 +1793284764,worker-recovery,2,200,0.015866 +1793284764,worker-recovery,2,200,0.011393 +1793284764,worker-recovery,2,200,0.012263 +1793284764,worker-recovery,2,200,0.013921 +1793284765,worker-recovery,2,200,0.011867 +1793284765,worker-recovery,2,200,0.015301 +1793284765,worker-recovery,2,200,0.012831 +1793284766,worker-recovery,2,200,0.014904 +1793284766,worker-recovery,2,200,0.013195 +1793284766,worker-recovery,2,200,0.016847 +1793284766,worker-recovery,2,200,1.012591 +1793284768,worker-recovery,2,200,0.008217 +1793284768,worker-recovery,2,200,0.017500 +1793284768,worker-recovery,2,200,0.034353 +1793284769,worker-recovery,2,200,0.007074 +1793284769,worker-recovery,2,200,0.010768 +1793284769,worker-recovery,2,200,0.007310 +1793284770,worker-recovery,2,200,0.009940 +1793284770,worker-recovery,2,200,0.013182 +1793284770,worker-recovery,2,200,0.013353 +1793284770,worker-recovery,2,200,0.014428 +1793284771,worker-recovery,2,200,0.011736 +1793284771,worker-recovery,2,200,0.012153 +1793284771,worker-recovery,2,200,0.022551 +1793284772,worker-recovery,2,200,1.013879 +1793284773,worker-recovery,2,200,0.013126 +1793284773,worker-recovery,2,200,0.018333 +1793284773,worker-recovery,2,200,0.016954 +1793284774,worker-recovery,2,200,0.016983 +1793284774,worker-recovery,2,200,0.016863 +1793284774,worker-recovery,2,200,0.011241 +1793284775,worker-recovery,2,200,0.005244 +1793284775,worker-recovery,2,200,0.013212 +1793284775,worker-recovery,2,200,0.010199 +1793284776,worker-recovery,2,200,0.012243 +1793284776,worker-recovery,2,200,0.010217 +1793284776,worker-recovery,2,200,0.010388 +1793284776,worker-recovery,2,200,0.012286 +1793284777,worker-recovery,2,200,0.008538 +1793284777,worker-recovery,2,200,0.011369 +1793284777,worker-recovery,2,200,0.011221 +1793284778,worker-recovery,2,200,0.011247 +1793284778,worker-recovery,2,200,0.009347 +1793284778,worker-recovery,2,200,0.014444 +1793284778,worker-recovery,2,200,0.010151 +1793284779,worker-recovery,2,200,0.015685 +1793284779,worker-recovery,2,200,0.010008 +1793284779,worker-recovery,2,200,0.010073 +1793284780,worker-recovery,2,200,0.011018 +1793284780,worker-recovery,2,200,0.037037 +1793284780,worker-recovery,2,200,0.012330 +1793284780,worker-recovery,2,200,0.005965 +1793284781,worker-recovery,2,200,0.008387 +1793284781,worker-recovery,2,200,0.010381 +1793284781,worker-recovery,2,200,0.009415 +1793284782,worker-recovery,2,200,0.009175 +1793284782,worker-recovery,2,200,0.007469 +1793284782,worker-recovery,2,200,0.021943 +1793284782,worker-recovery,2,200,0.009959 +1793284783,worker-recovery,2,200,0.034740 +1793284783,worker-recovery,2,200,0.010085 +1793284783,worker-recovery,2,200,0.008381 +1793284784,worker-recovery,2,200,0.010440 +1793284784,worker-recovery,2,200,0.008878 +1793284784,worker-recovery,2,200,0.009439 +1793284784,worker-recovery,2,200,0.016728 +1793284785,worker-recovery,2,200,0.016657 +1793284785,worker-recovery,2,200,0.007913 +1793284785,worker-recovery,2,200,0.014711 +1793284786,worker-recovery,2,200,0.011543 +1793284786,worker-recovery,2,200,0.008876 +1793284786,worker-recovery,2,200,0.030250 +1793284787,worker-recovery,2,200,0.011923 +1793284787,worker-recovery,2,200,0.006559 +1793284787,worker-recovery,2,200,0.009279 +1793284787,worker-recovery,2,200,0.029740 +1793284788,worker-recovery,2,200,0.014490 +1793284788,worker-recovery,2,200,0.017256 +1793284788,worker-recovery,2,200,0.015013 +1793284789,worker-recovery,2,200,0.010748 +1793284789,worker-recovery,2,200,0.011827 +1793284789,worker-recovery,2,200,0.009429 +1793284789,worker-recovery,2,200,0.008257 +1793284790,worker-recovery,2,200,0.007093 +1793284790,worker-recovery,2,200,0.018491 +1793284790,worker-recovery,2,200,0.014199 +1793284791,worker-recovery,2,200,0.022086 +1793284791,worker-recovery,2,200,0.018653 +1793284791,worker-recovery,2,200,0.016205 +1793284792,worker-recovery,2,200,0.011021 +1793284792,worker-recovery,2,200,0.011170 +1793284792,worker-recovery,2,200,0.011186 +1793284792,worker-recovery,2,200,0.006295 +1793284793,worker-recovery,2,200,0.010630 +1793284793,worker-recovery,2,200,0.012062 +1793284793,worker-recovery,2,200,0.026329 +1793284794,worker-recovery,2,200,0.021675 +1793284794,worker-recovery,2,200,0.006638 +1793284794,worker-recovery,2,200,0.016625 +1793284794,worker-recovery,2,200,0.021380 +1793284795,worker-recovery,2,200,0.008601 +1793284795,worker-recovery,2,200,0.021042 +1793284795,worker-recovery,2,200,0.010481 +1793284796,worker-recovery,2,200,0.015528 +1793284796,worker-recovery,2,200,0.014979 +1793284796,worker-recovery,2,200,0.009871 +1793284797,worker-recovery,2,200,0.013719 +1793284797,worker-recovery,2,200,0.014371 +1793284797,worker-recovery,2,200,0.007878 +1793284797,worker-recovery,2,200,0.008779 +1793284798,worker-recovery,2,200,0.006803 +1793284798,worker-recovery,2,200,0.007916 +1793284798,worker-recovery,2,200,0.009052 +1793284799,worker-recovery,2,200,0.031033 +1793284799,worker-recovery,2,200,0.009261 +1793284799,worker-recovery,2,200,0.010097 +1793284799,worker-recovery,2,200,0.016932 +1793284800,worker-recovery,2,200,0.010047 +1793284800,worker-recovery,2,200,0.008629 +1793284800,worker-recovery,2,200,0.017202 +1793284801,worker-recovery,2,200,0.009449 +1793284801,worker-recovery,2,200,0.007212 +1793284801,worker-recovery,2,200,0.007442 +1793284802,worker-recovery,2,200,0.006855 +1793284802,worker-recovery,2,200,0.006835 +1793284802,worker-recovery,2,200,0.007255 +1793284802,worker-recovery,2,200,0.009036 +1793284803,worker-recovery,2,200,0.017044 +1793284803,worker-recovery,2,200,0.009224 +1793284803,worker-recovery,2,200,0.009068 +1793284804,worker-recovery,2,200,0.012392 +1793284804,worker-recovery,2,200,0.008692 +1793284804,worker-recovery,2,200,0.017985 +1793284805,worker-recovery,2,200,0.009548 +1793284805,worker-recovery,2,200,0.014008 +1793284805,worker-recovery,2,200,0.006519 +1793284805,worker-recovery,2,200,0.016633 +1793284806,worker-recovery,2,200,0.013175 +1793284806,worker-recovery,2,200,0.007938 +1793284806,worker-recovery,2,200,0.030677 +1793284807,worker-recovery,2,200,0.007554 +1793284807,worker-recovery,2,200,0.006617 +1793284807,worker-recovery,2,200,0.016484 +1793284807,worker-recovery,2,200,0.010538 +1793284808,worker-recovery,2,200,0.008980 +1793284808,worker-recovery,2,200,0.010324 +1793284808,worker-recovery,2,200,0.010713 +1793284809,worker-recovery,2,200,0.009681 +1793284809,worker-recovery,2,200,0.006823 +1793284809,worker-recovery,2,200,0.007013 +1793284809,worker-recovery,2,200,0.010662 +1793284810,worker-recovery,2,200,0.006329 +1793284810,worker-recovery,2,200,0.007751 +1793284810,worker-recovery,2,200,0.009353 +1793284811,worker-recovery,2,200,0.012631 +1793284811,worker-recovery,2,200,0.014893 +1793284811,worker-recovery,2,200,0.010181 +1793284811,worker-recovery,2,200,0.014487 +1793284812,worker-recovery,2,200,0.007939 +1793284812,worker-recovery,2,200,0.009846 +1793284812,worker-recovery,2,200,0.007908 +1793284813,worker-recovery,2,200,0.008556 +1793284813,worker-recovery,2,200,0.005129 +1793284813,worker-recovery,2,200,0.007928 +1793284814,worker-recovery,2,200,0.012024 +1793284814,worker-recovery,2,200,0.009017 +1793284814,worker-recovery,2,200,0.008097 +1793284814,worker-recovery,2,200,0.006767 +1793284815,worker-recovery,2,200,0.006658 +1793284815,worker-recovery,2,200,0.007603 +1793284815,worker-recovery,2,200,0.006290 +1793284816,worker-recovery,2,200,0.016165 +1793284816,worker-recovery,2,200,0.008969 +1793284816,worker-recovery,2,200,0.010039 +1793284816,worker-recovery,2,200,0.009643 +1793284817,worker-recovery,2,200,0.007533 +1793284817,worker-recovery,2,200,0.008274 +1793284817,worker-recovery,2,200,0.008086 +1793284818,worker-recovery,2,200,0.015467 +1793284818,worker-recovery,2,200,0.004280 +1793284818,worker-recovery,2,200,0.011284 +1793284818,worker-recovery,2,200,0.009432 +1793284819,worker-recovery,2,200,0.008379 +1793284819,worker-recovery,2,200,0.005409 +1793284819,worker-recovery,2,200,0.009958 +1793284820,worker-recovery,2,200,0.012036 +1793284820,worker-recovery,2,200,0.009203 +1793284820,worker-recovery,2,200,0.010968 +1793284820,worker-recovery,2,200,0.012179 +1793284821,worker-recovery,2,200,0.015969 +1793284821,worker-recovery,2,200,0.018774 +1793284821,worker-recovery,2,200,0.008063 +1793284822,worker-recovery,2,200,0.010345 +1793284822,worker-recovery,2,200,0.013081 +1793284822,worker-recovery,2,200,0.009180 +1793284822,worker-recovery,2,200,0.009498 +1793284823,worker-recovery,2,200,0.009188 +1793284823,worker-recovery,2,200,0.006887 +1793284823,worker-recovery,2,200,0.014809 +1793284824,worker-recovery,2,200,0.008120 +1793284824,worker-recovery,2,200,0.008052 +1793284824,worker-recovery,2,200,0.014664 +1793284825,worker-recovery,2,200,0.008353 +1793284825,worker-recovery,2,200,0.006828 +1793284825,worker-recovery,2,200,0.010784 +1793284825,worker-recovery,2,200,0.010936 +1793284826,worker-recovery,2,200,0.014239 +1793284826,worker-recovery,2,200,0.009211 +1793284826,worker-recovery,2,200,0.006896 +1793284827,worker-recovery,2,200,0.006329 +1793284827,worker-recovery,2,200,0.012307 +1793284827,worker-recovery,2,200,0.009418 +1793284827,worker-recovery,2,200,0.007995 +1793284828,worker-recovery,2,200,0.010194 +1793284828,worker-recovery,2,200,0.010314 +1793284828,worker-recovery,2,200,0.009676 +1793284829,worker-recovery,2,200,0.013439 +1793284829,worker-recovery,2,200,0.008172 +1793284829,worker-recovery,2,200,0.018360 +1793284830,worker-recovery,2,200,0.016809 +1793284830,worker-recovery,2,200,0.014603 +1793284830,worker-recovery,2,200,0.012909 +1793284830,worker-recovery,2,200,0.006511 +1793284831,worker-recovery,2,200,0.011194 +1793284831,worker-recovery,2,200,0.009578 +1793284831,worker-recovery,2,200,0.007546 +1793284832,worker-recovery,2,200,0.022129 +1793284832,worker-recovery,2,200,0.010539 +1793284832,worker-recovery,2,200,0.014292 +1793284832,worker-recovery,2,200,0.020718 +1793284833,worker-recovery,2,200,0.009525 +1793284833,worker-recovery,2,200,0.009621 +1793284833,worker-recovery,2,200,0.010412 +1793284834,worker-recovery,2,200,0.010597 +1793284834,worker-recovery,2,200,0.022447 +1793284834,worker-recovery,2,200,0.016207 +1793284835,worker-recovery,2,200,0.012928 +1793284835,worker-recovery,2,200,0.018528 +1793284835,worker-recovery,2,200,0.011535 +1793284835,worker-recovery,2,200,0.006371 +1793284836,worker-recovery,2,200,0.007541 +1793284836,worker-recovery,2,200,0.007682 +1793284836,worker-recovery,2,200,0.006461 +1793284837,worker-recovery,2,200,0.015318 +1793284837,worker-recovery,2,200,0.015553 +1793284837,worker-recovery,2,200,0.042504 +1793284838,worker-recovery,2,200,0.008612 +1793284838,worker-recovery,2,200,0.007871 +1793284838,worker-recovery,2,200,0.009832 +1793284838,worker-recovery,2,200,0.007107 +1793284839,worker-recovery,2,200,0.019491 +1793284839,worker-recovery,2,200,0.008685 +1793284839,worker-recovery,2,200,0.006414 +1793284840,worker-recovery,2,200,0.014882 +1793284840,worker-recovery,2,200,0.008035 +1793284840,worker-recovery,2,200,0.006202 +1793284840,worker-recovery,2,200,0.007157 +1793284841,worker-recovery,2,200,0.022548 +1793284841,worker-recovery,2,200,0.007653 +1793284841,worker-recovery,2,200,0.007597 +1793284842,worker-recovery,2,200,0.018848 +1793284842,worker-recovery,2,200,0.012693 +1793284842,worker-recovery,2,200,0.010206 +1793284842,worker-recovery,2,200,0.009169 +1793284843,worker-recovery,2,200,0.009868 +1793284843,worker-recovery,2,200,0.008259 +1793284843,worker-recovery,2,200,0.007717 +1793284844,worker-recovery,2,200,0.007079 +1793284844,worker-recovery,2,200,0.034947 +1793284844,trace-backend-outage,2,200,0.019257 +1793284845,trace-backend-outage,2,200,0.015457 +1793284845,trace-backend-outage,2,200,0.007150 +1793284845,trace-backend-outage,2,200,0.008953 +1793284845,trace-backend-outage,2,200,0.007137 +1793284846,trace-backend-outage,2,200,0.019183 +1793284846,trace-backend-outage,2,200,0.007144 +1793284846,trace-backend-outage,2,200,0.014358 +1793284847,trace-backend-outage,2,200,0.022596 +1793284847,trace-backend-outage,2,200,0.010836 +1793284847,trace-backend-outage,2,200,0.023019 +1793284847,trace-backend-outage,2,200,0.027181 +1793284848,trace-backend-outage,2,200,0.008142 +1793284848,trace-backend-outage,2,200,0.012256 +1793284848,trace-backend-outage,2,200,0.011859 +1793284849,trace-backend-outage,2,200,0.009883 +1793284849,trace-backend-outage,2,200,0.020575 +1793284849,trace-backend-outage,2,200,0.009784 +1793284850,trace-backend-outage,2,200,0.011329 +1793284850,trace-backend-outage,2,200,0.012775 +1793284850,trace-backend-outage,2,200,0.021659 +1793284850,trace-backend-outage,2,200,0.010672 +1793284851,trace-backend-outage,2,200,0.014450 +1793284851,trace-backend-outage,2,200,0.043496 +1793284851,trace-backend-outage,2,200,0.013206 +1793284852,trace-backend-outage,2,200,0.009246 +1793284852,trace-backend-outage,2,200,0.007795 +1793284852,trace-backend-outage,2,200,0.007993 +1793284852,trace-backend-outage,2,200,0.013943 +1793284853,trace-backend-outage,2,200,0.025736 +1793284853,trace-backend-outage,2,200,0.015413 +1793284853,trace-backend-outage,2,200,0.011742 +1793284854,trace-backend-outage,2,200,0.007872 +1793284854,trace-backend-outage,2,200,0.011257 +1793284854,trace-backend-outage,2,200,0.015011 +1793284855,trace-backend-outage,2,200,0.004713 +1793284855,trace-backend-outage,2,200,0.006526 +1793284855,trace-backend-outage,2,200,0.012820 +1793284855,trace-backend-outage,2,200,0.008897 +1793284856,trace-backend-outage,2,200,0.012701 +1793284856,trace-backend-outage,2,200,0.010563 +1793284856,trace-backend-outage,2,200,0.009314 +1793284857,trace-backend-outage,2,200,0.007686 +1793284857,trace-backend-outage,2,200,0.009840 +1793284857,trace-backend-outage,2,200,0.008022 +1793284857,trace-backend-outage,2,200,0.007457 +1793284858,trace-backend-outage,2,200,0.017166 +1793284858,trace-backend-outage,2,200,0.011408 +1793284858,trace-backend-outage,2,200,0.009474 +1793284858,trace-backend-outage,2,200,0.010941 +1793284859,trace-backend-outage,2,200,0.013357 +1793284859,trace-backend-outage,2,200,0.006502 +1793284859,trace-backend-outage,2,200,0.008526 +1793284860,trace-backend-outage,2,200,0.007075 +1793284860,trace-backend-outage,2,200,0.004577 +1793284860,trace-backend-outage,2,200,0.008611 +1793284860,trace-backend-outage,2,200,0.006271 +1793284861,trace-backend-outage,2,200,0.005875 +1793284861,trace-backend-outage,2,200,0.010368 +1793284861,trace-backend-outage,2,200,0.004846 +1793284862,trace-backend-outage,2,200,0.017165 +1793284862,trace-backend-outage,2,200,0.006880 +1793284862,trace-backend-outage,2,200,0.006627 +1793284862,trace-backend-outage,2,200,0.008343 +1793284863,trace-backend-outage,2,200,0.008748 +1793284863,trace-backend-outage,2,200,0.008779 +1793284863,trace-backend-outage,2,200,0.009107 +1793284864,trace-backend-outage,2,200,0.010316 +1793284864,trace-backend-outage,2,200,0.008938 +1793284864,trace-backend-outage,2,200,0.012326 +1793284864,trace-backend-outage,2,200,0.007877 +1793284865,trace-backend-outage,2,200,0.015315 +1793284865,trace-backend-outage,2,200,0.017471 +1793284865,trace-backend-outage,2,200,0.014129 +1793284866,trace-backend-outage,2,200,0.021698 +1793284866,trace-backend-outage,2,200,0.011581 +1793284866,trace-backend-outage,2,200,0.009207 +1793284867,trace-backend-outage,2,200,0.015991 +1793284867,trace-backend-outage,2,200,0.016507 +1793284867,trace-backend-outage,2,200,0.013421 +1793284867,trace-backend-outage,2,200,0.009583 +1793284868,trace-backend-outage,2,200,0.020904 +1793284868,trace-backend-outage,2,200,0.016517 +1793284868,trace-backend-outage,2,200,0.009761 +1793284869,trace-backend-outage,2,200,0.016463 +1793284869,trace-backend-outage,2,200,0.013395 +1793284869,trace-backend-outage,2,200,0.010847 +1793284870,trace-backend-outage,2,200,0.019590 +1793284870,trace-backend-outage,2,200,0.013090 +1793284870,trace-backend-outage,2,200,0.013032 +1793284870,trace-backend-outage,2,200,0.010457 +1793284871,trace-backend-outage,2,200,0.014190 +1793284871,trace-backend-outage,2,200,0.017355 +1793284871,trace-backend-outage,2,200,0.009639 +1793284872,trace-backend-outage,2,200,0.018904 +1793284872,trace-backend-outage,2,200,0.010208 +1793284872,trace-backend-outage,2,200,0.023330 +1793284873,trace-backend-outage,2,200,0.019578 +1793284873,trace-backend-outage,2,200,0.019743 +1793284873,trace-backend-outage,2,200,0.014900 +1793284874,trace-backend-outage,2,200,0.009960 +1793284874,trace-backend-outage,2,200,0.018321 +1793284874,trace-backend-outage,2,200,0.017022 +1793284874,trace-backend-outage,2,200,0.008144 +1793284875,trace-backend-outage,2,200,0.024938 +1793284875,trace-backend-outage,2,200,0.024314 +1793284875,trace-backend-outage,2,200,0.010527 +1793284876,trace-backend-outage,2,200,0.015655 +1793284876,trace-backend-outage,2,200,0.014212 +1793284876,trace-backend-outage,2,200,0.017583 +1793284877,trace-backend-outage,2,200,0.016640 +1793284877,trace-backend-outage,2,200,0.017347 +1793284877,trace-backend-outage,2,200,0.018940 +1793284877,trace-backend-outage,2,200,0.015212 +1793284878,trace-backend-outage,2,200,0.015374 +1793284878,trace-backend-outage,2,200,0.019194 +1793284878,trace-backend-outage,2,200,0.010155 +1793284879,trace-backend-outage,2,200,0.017526 +1793284879,trace-backend-outage,2,200,0.016325 +1793284879,trace-backend-outage,2,200,0.021094 +1793284880,trace-backend-outage,2,200,0.017594 +1793284880,trace-backend-outage,2,200,0.017494 +1793284880,trace-backend-outage,2,200,0.018024 +1793284881,trace-backend-outage,2,200,0.019904 +1793284881,trace-backend-outage,2,200,0.018970 +1793284881,trace-backend-outage,2,200,0.014452 +1793284881,trace-backend-outage,2,200,0.010876 +1793284882,trace-backend-outage,2,200,0.016834 +1793284882,trace-backend-outage,2,200,0.012794 +1793284882,trace-backend-outage,2,200,0.021871 +1793284883,trace-backend-outage,2,200,0.018101 +1793284883,trace-backend-outage,2,200,0.016068 +1793284883,trace-backend-outage,2,200,0.010164 +1793284884,trace-backend-outage,2,200,0.008729 +1793284884,trace-backend-outage,2,200,0.025613 +1793284884,trace-backend-outage,2,200,0.014923 +1793284884,trace-backend-outage,2,200,0.008093 +1793284885,trace-backend-outage,2,200,0.013921 +1793284885,trace-backend-outage,2,200,0.018282 +1793284885,trace-backend-outage,2,200,0.020522 +1793284886,trace-backend-outage,2,200,0.021017 +1793284886,trace-backend-outage,2,200,0.020192 +1793284886,trace-backend-outage,2,200,0.020872 +1793284887,trace-backend-outage,2,200,0.020793 +1793284887,trace-backend-outage,2,200,0.022593 +1793284887,trace-backend-outage,2,200,0.016799 +1793284887,trace-backend-outage,2,200,0.009077 +1793284888,trace-backend-outage,2,200,0.011855 +1793284888,trace-backend-outage,2,200,0.010347 +1793284888,trace-backend-outage,2,200,0.012318 +1793284889,trace-backend-outage,2,200,0.008246 +1793284889,trace-backend-outage,2,200,0.021501 +1793284889,trace-backend-outage,2,200,0.014630 +1793284890,trace-backend-outage,2,200,0.020657 +1793284890,trace-backend-outage,2,200,0.014682 +1793284890,trace-backend-outage,2,200,0.015428 +1793284890,trace-backend-outage,2,200,0.012579 +1793284891,trace-backend-outage,2,200,0.010416 +1793284891,trace-backend-outage,2,200,0.011980 +1793284891,trace-backend-outage,2,200,0.009561 +1793284892,trace-backend-outage,2,200,0.009681 +1793284892,trace-backend-outage,2,200,0.007955 +1793284892,trace-backend-outage,2,200,0.022592 +1793284893,trace-backend-outage,2,200,0.049383 +1793284893,telemetry-recovery,2,200,0.018778 +1793284893,telemetry-recovery,2,200,0.010219 +1793284894,telemetry-recovery,2,200,0.012190 +1793284894,telemetry-recovery,2,200,0.015953 +1793284894,telemetry-recovery,2,200,0.011372 +1793284894,telemetry-recovery,2,200,0.009904 +1793284895,telemetry-recovery,2,200,0.009633 +1793284895,telemetry-recovery,2,200,0.008496 +1793284895,telemetry-recovery,2,200,0.008025 +1793284896,telemetry-recovery,2,200,0.014612 +1793284896,telemetry-recovery,2,200,0.016951 +1793284896,telemetry-recovery,2,200,0.014996 +1793284897,telemetry-recovery,2,200,0.014679 +1793284897,telemetry-recovery,2,200,0.015010 +1793284897,telemetry-recovery,2,200,0.016663 +1793284897,telemetry-recovery,2,200,0.009832 +1793284898,telemetry-recovery,2,200,0.017744 +1793284898,telemetry-recovery,2,200,0.022254 +1793284898,telemetry-recovery,2,200,0.017487 +1793284899,telemetry-recovery,2,200,0.014001 +1793284899,telemetry-recovery,2,200,0.011838 +1793284899,telemetry-recovery,2,200,0.012648 +1793284900,telemetry-recovery,2,200,0.021561 +1793284900,telemetry-recovery,2,200,0.014697 +1793284900,telemetry-recovery,2,200,0.015965 +1793284901,telemetry-recovery,2,200,0.018563 +1793284901,telemetry-recovery,2,200,0.011428 +1793284901,telemetry-recovery,2,200,0.007983 +1793284901,telemetry-recovery,2,200,0.010540 +1793284902,telemetry-recovery,2,200,0.014627 +1793284902,telemetry-recovery,2,200,0.013715 +1793284902,telemetry-recovery,2,200,0.013439 +1793284903,telemetry-recovery,2,200,0.014970 +1793284903,telemetry-recovery,2,200,0.010483 +1793284903,telemetry-recovery,2,200,0.014989 +1793284904,telemetry-recovery,2,200,0.012041 +1793284904,telemetry-recovery,2,200,0.016176 +1793284904,telemetry-recovery,2,200,0.016168 +1793284904,telemetry-recovery,2,200,0.013001 +1793284905,telemetry-recovery,2,200,0.028058 +1793284905,telemetry-recovery,2,200,0.008935 +1793284905,telemetry-recovery,2,200,0.015756 +1793284906,telemetry-recovery,2,200,0.009600 +1793284906,telemetry-recovery,2,200,1.019170 +1793284907,telemetry-recovery,2,200,0.012953 +1793284908,telemetry-recovery,2,200,0.009722 +1793284908,telemetry-recovery,2,200,0.010940 +1793284908,telemetry-recovery,2,200,0.019403 +1793284909,telemetry-recovery,2,200,0.014733 +1793284909,telemetry-recovery,2,200,0.010733 +1793284909,telemetry-recovery,2,200,0.018438 +1793284909,telemetry-recovery,2,200,0.013157 +1793284910,telemetry-recovery,2,200,0.012576 +1793284910,telemetry-recovery,2,200,0.014215 +1793284910,telemetry-recovery,2,200,0.014656 +1793284911,telemetry-recovery,2,200,0.017300 +1793284911,telemetry-recovery,2,200,0.008206 +1793284911,telemetry-recovery,2,200,0.011842 +1793284912,telemetry-recovery,2,200,0.014704 +1793284912,telemetry-recovery,2,200,0.014847 +1793284912,telemetry-recovery,2,200,0.014324 +1793284913,telemetry-recovery,2,200,0.017383 +1793284913,telemetry-recovery,2,200,0.017619 diff --git a/evidence/incidents/incident-timeline.png b/evidence/incidents/incident-timeline.png new file mode 100644 index 0000000..cfa9881 Binary files /dev/null and b/evidence/incidents/incident-timeline.png differ diff --git a/evidence/incidents/queue-size.json b/evidence/incidents/queue-size.json new file mode 100644 index 0000000..9455898 --- /dev/null +++ b/evidence/incidents/queue-size.json @@ -0,0 +1 @@ +{"status":"success","data":{"resultType":"vector","result":[{"metric":{},"value":[1793284947.202,"0"]}]}} \ No newline at end of file diff --git a/evidence/incidents/rss.txt b/evidence/incidents/rss.txt new file mode 100644 index 0000000..ee2b38f --- /dev/null +++ b/evidence/incidents/rss.txt @@ -0,0 +1,536 @@ +1793284209 baseline +35644 27968 0:00.09 +35645 30992 0:00.10 +35646 31632 0:00.10 +1793284214 baseline +35644 28784 0:00.10 +35645 32816 0:00.13 +35646 33248 0:00.13 +1793284219 baseline +35644 30736 0:00.12 +35645 33968 0:00.17 +35646 34320 0:00.16 +1793284224 baseline +35644 29136 0:00.13 +35645 34960 0:00.20 +35646 35472 0:00.20 +1793284229 baseline +35644 26512 0:00.14 +35645 29984 0:00.24 +35646 28928 0:00.23 +1793284234 baseline +35644 26864 0:00.15 +35645 30944 0:00.27 +35646 30240 0:00.27 +1793284239 baseline +35644 27104 0:00.16 +35645 30528 0:00.30 +35646 30240 0:00.30 +1793284244 baseline +35644 27296 0:00.18 +35645 31040 0:00.34 +35646 30816 0:00.33 +1793284250 baseline +35644 27024 0:00.19 +35645 31472 0:00.37 +35646 31056 0:00.37 +1793284255 baseline +35644 26944 0:00.20 +35645 32128 0:00.40 +35646 31664 0:00.40 +1793284260 baseline +35644 26720 0:00.21 +35645 31888 0:00.43 +35646 31040 0:00.43 +1793284265 baseline +35644 25648 0:00.22 +35645 28064 0:00.45 +35646 28768 0:00.46 +1793284270 baseline +35644 26048 0:00.22 +35645 29696 0:00.46 +35646 30976 0:00.49 +1793284275 baseline +35644 29136 0:00.24 +35645 30896 0:00.49 +35646 32032 0:00.52 +1793284280 baseline +35644 29312 0:00.25 +35645 32192 0:00.52 +35646 33280 0:00.55 +1793284285 baseline +35644 27568 0:00.27 +35645 32720 0:00.55 +35646 33840 0:00.59 +1793284290 baseline +35644 27408 0:00.28 +35645 32848 0:00.59 +35646 33872 0:00.62 +1793284295 baseline +35644 27648 0:00.29 +35645 45600 0:00.63 +35646 46240 0:00.67 +1793284300 baseline +35644 27456 0:00.30 +35645 36240 0:00.66 +35646 42048 0:00.71 +1793284305 baseline +35644 29552 0:00.32 +35645 37792 0:00.71 +35646 37504 0:00.75 +1793284310 baseline +35644 29728 0:00.34 +35645 38768 0:00.75 +35646 37920 0:00.78 +1793284315 baseline +35644 29968 0:00.35 +35645 38992 0:00.81 +35646 37920 0:00.79 +1793284320 baseline +35644 28784 0:00.37 +35645 38288 0:00.85 +35646 37408 0:00.82 +1793284326 baseline +35644 28960 0:00.38 +35645 38608 0:00.89 +35646 37680 0:00.86 +1793284331 short-db-spike +35644 29968 0:00.40 +35645 37792 0:00.93 +35646 37008 0:00.89 +1793284336 spike-recovery +35644 30096 0:00.42 +35645 37168 0:00.95 +35646 36240 0:00.89 +1793284341 spike-recovery +35644 25584 0:00.43 +35645 32640 0:00.99 +35646 31808 0:00.92 +1793284346 spike-recovery +35644 22800 0:00.45 +35645 30064 0:01.04 +35646 29328 0:00.97 +1793284351 spike-recovery +35644 23696 0:00.46 +35645 28176 0:01.07 +35646 27024 0:01.00 +1793284356 spike-recovery +35644 23088 0:00.48 +35645 28320 0:01.11 +35646 26272 0:01.04 +1793284361 spike-recovery +35644 20560 0:00.50 +35645 26912 0:01.17 +35646 24656 0:01.07 +1793284366 spike-recovery +35644 20688 0:00.51 +35645 27024 0:01.20 +35646 26560 0:01.12 +1793284371 spike-recovery +35644 20736 0:00.53 +35645 27008 0:01.25 +35646 23808 0:01.15 +1793284376 spike-recovery +35644 23136 0:00.55 +35645 24672 0:01.29 +35646 24512 0:01.20 +1793284382 spike-recovery +35644 23216 0:00.56 +35645 26608 0:01.33 +35646 25904 0:01.24 +1793284387 spike-recovery +35644 20896 0:00.58 +35645 25200 0:01.36 +35646 24784 0:01.27 +1793284392 spike-recovery +35644 20864 0:00.59 +35645 26800 0:01.41 +35646 26144 0:01.31 +1793284397 spike-recovery +35644 20960 0:00.60 +35645 25440 0:01.43 +35646 26288 0:01.35 +1793284402 spike-recovery +35644 23328 0:00.63 +35645 25872 0:01.48 +35646 25008 0:01.38 +1793284407 spike-recovery +35644 21200 0:00.64 +35645 25104 0:01.51 +35646 26112 0:01.42 +1793284412 spike-recovery +35644 21168 0:00.65 +35645 26896 0:01.55 +35646 26208 0:01.46 +1793284417 spike-recovery +35644 21184 0:00.66 +35645 27216 0:01.59 +35646 26400 0:01.50 +1793284422 spike-recovery +35644 21216 0:00.68 +35645 27168 0:01.63 +35646 26000 0:01.54 +1793284427 sustained-db-outage +35644 21248 0:00.69 +35645 27360 0:01.67 +35646 26496 0:01.59 +1793284432 sustained-db-outage +35644 23632 0:00.70 +35645 26656 0:01.68 +35646 25888 0:01.60 +1793284437 sustained-db-outage +35644 22320 0:00.71 +35645 25952 0:01.69 +35646 26480 0:01.61 +1793284442 sustained-db-outage +35644 22368 0:00.72 +35645 26128 0:01.70 +35646 26656 0:01.62 +1793284448 sustained-db-outage +35644 22416 0:00.73 +35645 27216 0:01.71 +35646 25824 0:01.62 +1793284453 sustained-db-outage +35644 22208 0:00.74 +35645 25744 0:01.72 +35646 25328 0:01.63 +1793284458 sustained-db-outage +35644 23920 0:00.75 +35645 25504 0:01.73 +35646 24880 0:01.64 +1793284463 sustained-db-outage +35644 22688 0:00.76 +35645 25664 0:01.74 +35646 25008 0:01.65 +1793284468 sustained-db-outage +35644 21520 0:00.77 +35645 25184 0:01.74 +35646 25952 0:01.66 +1793284473 sustained-db-outage +35644 21472 0:00.78 +35645 26672 0:01.75 +35646 25616 0:01.66 +1793284478 sustained-db-outage +35644 19888 0:00.78 +35645 22208 0:01.76 +35646 22256 0:01.67 +1793284483 sustained-db-outage +35644 20800 0:00.80 +35645 21408 0:01.77 +35646 21760 0:01.68 +1793284488 sustained-db-outage +35644 21168 0:00.81 +35645 22096 0:01.78 +35646 22336 0:01.69 +1793284493 sustained-db-outage +35644 19360 0:00.81 +35645 21312 0:01.79 +35646 22048 0:01.70 +1793284498 sustained-db-outage +35644 20048 0:00.82 +35645 21936 0:01.80 +35646 22336 0:01.71 +1793284503 sustained-db-outage +35644 19856 0:00.83 +35645 22576 0:01.81 +35646 21520 0:01.72 +1793284508 sustained-db-outage +35644 20880 0:00.85 +35645 22704 0:01.83 +35646 22096 0:01.73 +1793284513 sustained-db-outage +35644 21136 0:00.86 +35645 23168 0:01.84 +35646 22464 0:01.74 +1793284518 sustained-db-outage +35644 21312 0:00.88 +35645 23312 0:01.85 +35646 22912 0:01.75 +1793284523 sustained-db-outage +35644 21376 0:00.89 +35645 23600 0:01.86 +35646 25824 0:01.77 +1793284528 sustained-db-outage +35644 20928 0:00.90 +35645 25728 0:01.88 +35646 23376 0:01.78 +1793284533 sustained-db-outage +35644 23040 0:00.92 +35645 25632 0:01.89 +35646 23568 0:01.80 +1793284539 sustained-db-outage +35644 23120 0:00.94 +35645 25744 0:01.90 +35646 23616 0:01.81 +1793284544 sustained-db-outage +35644 21936 0:00.95 +35645 24176 0:01.91 +35646 23600 0:01.82 +1793284549 db-recovery +35644 21024 0:00.97 +35645 23904 0:01.92 +35646 23888 0:01.83 +1793284554 db-recovery +35644 21632 0:00.98 +35645 25824 0:01.96 +35646 24544 0:01.87 +1793284559 db-recovery +35644 23232 0:01.01 +35645 26192 0:02.01 +35646 25888 0:01.92 +1793284564 db-recovery +35644 23200 0:01.02 +35645 26240 0:02.05 +35646 25968 0:01.96 +1793284569 db-recovery +35644 22224 0:01.04 +35645 26304 0:02.09 +35646 26080 0:02.00 +1793284574 db-recovery +35644 22176 0:01.05 +35645 26288 0:02.13 +35646 26112 0:02.03 +1793284579 db-recovery +35644 22272 0:01.07 +35645 26320 0:02.17 +35646 26128 0:02.07 +1793284585 db-recovery +35644 23504 0:01.09 +35645 26544 0:02.21 +35646 25888 0:02.10 +1793284590 db-recovery +35644 23552 0:01.11 +35645 26752 0:02.25 +35646 26400 0:02.15 +1793284595 db-recovery +35644 23600 0:01.12 +35645 26800 0:02.29 +35646 26480 0:02.19 +1793284600 db-recovery +35644 22400 0:01.14 +35645 26736 0:02.33 +35646 26400 0:02.23 +1793284605 db-recovery +35644 22368 0:01.15 +35645 26768 0:02.38 +35646 26256 0:02.27 +1793284610 db-recovery +35644 22144 0:01.17 +35645 26720 0:02.41 +35646 26496 0:02.31 +1793284615 db-recovery +35644 23392 0:01.19 +35645 26784 0:02.45 +35646 26336 0:02.35 +1793284620 db-recovery +35644 22064 0:01.20 +35645 26160 0:02.49 +35646 26064 0:02.38 +1793284625 db-recovery +35644 21680 0:01.22 +35645 26112 0:02.55 +35646 24896 0:02.39 +1793284630 db-recovery +35644 20688 0:01.23 +35645 26144 0:02.58 +35646 26400 0:02.45 +1793284635 db-recovery +35644 20672 0:01.25 +35645 25568 0:02.59 +35646 26592 0:02.50 +1793284661 worker-stopped +35644 14656 0:01.27 +35645 29952 0:02.88 +35646 28960 0:02.67 +1793284667 worker-stopped +35644 14480 0:01.27 +35645 29984 0:02.92 +35646 29024 0:02.71 +1793284672 worker-stopped +35644 14480 0:01.27 +35645 30000 0:02.95 +35646 29136 0:02.74 +1793284677 worker-stopped +35644 14480 0:01.27 +35645 30048 0:02.99 +35646 29152 0:02.78 +1793284682 worker-stopped +35644 14480 0:01.27 +35645 29888 0:03.03 +35646 28640 0:02.82 +1793284687 worker-stopped +35644 14480 0:01.27 +35645 29952 0:03.07 +35646 28720 0:02.86 +1793284692 worker-stopped +35644 14464 0:01.27 +35645 29968 0:03.10 +35646 28720 0:02.90 +1793284697 worker-stopped +35644 14352 0:01.27 +35645 29248 0:03.14 +35646 27792 0:02.93 +1793284702 worker-stopped +35644 14272 0:01.27 +35645 26256 0:03.16 +35646 26880 0:02.94 +1793284707 worker-stopped +35644 14272 0:01.27 +35645 26560 0:03.20 +35646 26560 0:02.95 +1793284712 worker-stopped +35644 14272 0:01.27 +35645 27392 0:03.22 +35646 25856 0:02.97 +1793284717 worker-stopped +35644 14272 0:01.27 +35645 27520 0:03.25 +35646 26880 0:03.00 +1793284722 worker-stopped +35644 14272 0:01.27 +35645 27728 0:03.29 +35646 27024 0:03.04 +1793284727 worker-stopped +35644 14272 0:01.27 +35645 24976 0:03.31 +35646 25808 0:03.07 +1793284732 worker-stopped +35644 14272 0:01.27 +35645 27520 0:03.35 +35646 25088 0:03.10 +1793284737 worker-stopped +35644 14272 0:01.27 +35645 27840 0:03.39 +35646 27056 0:03.12 +1793284742 worker-stopped +35644 14272 0:01.27 +35645 27632 0:03.42 +35646 27104 0:03.16 +1793284747 worker-stopped +35644 14272 0:01.27 +35645 27792 0:03.45 +35646 27328 0:03.19 +1793284753 worker-recovery +35644 23280 0:01.28 +35645 27440 0:03.49 +35646 26560 0:03.22 +1793284758 worker-recovery +35644 28624 0:01.58 +35645 27744 0:03.52 +35646 27392 0:03.25 +1793284763 worker-recovery +35644 28640 0:01.59 +35645 27376 0:03.55 +35646 26320 0:03.28 +1793284768 worker-recovery +35644 28640 0:01.60 +35645 27776 0:03.59 +35646 27648 0:03.32 +1793284773 worker-recovery +35644 28640 0:01.61 +35645 27184 0:03.62 +35646 26032 0:03.33 +1793284778 worker-recovery +35644 26720 0:01.62 +35645 27376 0:03.66 +35646 25840 0:03.35 +1793284783 worker-recovery +35644 24976 0:01.63 +35645 26208 0:03.69 +35646 23392 0:03.38 +1793284788 worker-recovery +35644 20752 0:01.64 +35645 25712 0:03.73 +35646 24096 0:03.41 +1793284793 worker-recovery +35644 21152 0:01.65 +35645 26768 0:03.77 +35646 26784 0:03.45 +1793284798 worker-recovery +35644 21328 0:01.66 +35645 26144 0:03.80 +35646 25776 0:03.48 +1793284803 worker-recovery +35644 21568 0:01.67 +35645 27168 0:03.83 +35646 25744 0:03.51 +1793284808 worker-recovery +35644 24560 0:01.68 +35645 26384 0:03.87 +35646 25824 0:03.54 +1793284813 worker-recovery +35644 21888 0:01.69 +35645 27264 0:03.90 +35646 26944 0:03.58 +1793284818 worker-recovery +35644 22240 0:01.70 +35645 26640 0:03.93 +35646 25952 0:03.61 +1793284823 worker-recovery +35644 22128 0:01.71 +35645 27168 0:03.97 +35646 26848 0:03.64 +1793284828 worker-recovery +35644 20896 0:01.72 +35645 26896 0:04.01 +35646 24576 0:03.68 +1793284833 worker-recovery +35644 20128 0:01.73 +35645 23632 0:04.04 +35646 23184 0:03.71 +1793284838 worker-recovery +35644 24176 0:01.75 +35645 23760 0:04.07 +35646 23344 0:03.75 +1793284844 trace-backend-outage +35644 21792 0:01.76 +35645 24992 0:04.11 +35646 24704 0:03.78 +1793284849 trace-backend-outage +35644 20064 0:01.77 +35645 23616 0:04.16 +35646 26400 0:03.82 +1793284854 trace-backend-outage +35644 18752 0:01.78 +35645 22208 0:04.19 +35646 22528 0:03.86 +1793284859 trace-backend-outage +35644 19520 0:01.79 +35645 25376 0:04.22 +35646 25328 0:03.89 +1793284864 trace-backend-outage +35644 19184 0:01.80 +35645 22544 0:04.25 +35646 22864 0:03.92 +1793284869 trace-backend-outage +35644 22848 0:01.82 +35645 25472 0:04.29 +35646 25344 0:03.96 +1793284874 trace-backend-outage +35644 21952 0:01.83 +35645 25104 0:04.33 +35646 24192 0:03.99 +1793284879 trace-backend-outage +35644 20704 0:01.84 +35645 25632 0:04.37 +35646 25456 0:04.03 +1793284885 trace-backend-outage +35644 20736 0:01.85 +35645 24256 0:04.40 +35646 24000 0:04.06 +1793284893 telemetry-recovery +35644 21472 0:01.90 +35645 36160 0:04.61 +35646 43296 0:04.26 +1793284898 telemetry-recovery +35644 22368 0:01.92 +35645 36896 0:04.64 +35646 45392 0:04.31 +1793284903 telemetry-recovery +35644 24784 0:01.94 +35645 38912 0:04.68 +35646 45808 0:04.35 +1793284908 telemetry-recovery +35644 24992 0:01.95 +35645 39232 0:04.72 +35646 46016 0:04.38 diff --git a/evidence/incidents/smoke-with-tempo-down.txt b/evidence/incidents/smoke-with-tempo-down.txt new file mode 100644 index 0000000..dd4553e --- /dev/null +++ b/evidence/incidents/smoke-with-tempo-down.txt @@ -0,0 +1,2 @@ +python3 tests/smoke.py --base-url "http://127.0.0.1:55053" +PASS: lab6 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/incidents/summary.json b/evidence/incidents/summary.json new file mode 100644 index 0000000..fef08b1 --- /dev/null +++ b/evidence/incidents/summary.json @@ -0,0 +1,102 @@ +{ + "phases": { + "baseline": { + "requests": 789, + "errors": 2, + "p50_ms": 12.777999999999999, + "p95_ms": 21.69, + "p99_ms": 1009.4080000000001, + "duration_seconds": 122 + }, + "short-db-spike": { + "requests": 19, + "errors": 19, + "p50_ms": 2.2889999999999997, + "p95_ms": 1010.3819999999998, + "p99_ms": 1010.3819999999998, + "duration_seconds": 5 + }, + "spike-recovery": { + "requests": 591, + "errors": 4, + "p50_ms": 14.059000000000001, + "p95_ms": 22.608, + "p99_ms": 35.912, + "duration_seconds": 91 + }, + "sustained-db-outage": { + "requests": 552, + "errors": 552, + "p50_ms": 2.548, + "p95_ms": 1009.9240000000001, + "p99_ms": 1015.0600000000001, + "duration_seconds": 122 + }, + "db-recovery": { + "requests": 566, + "errors": 4, + "p50_ms": 16.916, + "p95_ms": 25.725, + "p99_ms": 1013.446, + "duration_seconds": 91 + }, + "worker-stopped": { + "requests": 734, + "errors": 0, + "p50_ms": 12.02, + "p95_ms": 19.837, + "p99_ms": 43.933, + "duration_seconds": 113 + }, + "worker-recovery": { + "requests": 619, + "errors": 0, + "p50_ms": 10.713000000000001, + "p95_ms": 22.551000000000002, + "p99_ms": 35.055, + "duration_seconds": 91 + }, + "trace-backend-outage": { + "requests": 328, + "errors": 0, + "p50_ms": 13.357000000000001, + "p95_ms": 23.36, + "p99_ms": 42.234, + "duration_seconds": 46 + }, + "telemetry-recovery": { + "requests": 126, + "errors": 0, + "p50_ms": 14.366999999999999, + "p95_ms": 22.253999999999998, + "p99_ms": 1011.9800000000001, + "duration_seconds": 20 + } + }, + "alerts": { + "MessengerAPIBurnRate": { + "first_firing": 1793284564, + "last_firing": 1793284595, + "first_resolved": 1793284600, + "firing_samples": 7 + }, + "MessengerReadyBurnRate": { + "first_firing": null, + "last_firing": null, + "first_resolved": null, + "firing_samples": 0 + }, + "MessengerQueueLag": { + "first_firing": 1793284727, + "last_firing": 1793284758, + "first_resolved": 1793284763, + "firing_samples": 7 + }, + "MessengerTelemetryDrops": { + "first_firing": 1793284885, + "last_firing": 1793284908, + "first_resolved": null, + "firing_samples": 5 + } + } +} diff --git a/evidence/local-correlation/correlated-logs.json b/evidence/local-correlation/correlated-logs.json new file mode 100644 index 0000000..4ba09c9 --- /dev/null +++ b/evidence/local-correlation/correlated-logs.json @@ -0,0 +1 @@ +{"status":"success","data":{"resultType":"streams","result":[{"stream":{"deployment_environment_name":"lab","detected_level":"info","event":"s3.put","flags":"1","observed_timestamp":"1793284207033795000","outcome":"ok","scope_name":"messenger","service_instance_id":"6cdf9035-fcaf-4543-8dd1-2263203ed2a8","service_name":"messenger-worker","service_version":"lab6","severity_number":"9","span_id":"2c92be6214d8cfbc","trace_id":"bfeacf9a52db37463ddefff2ed23b0cb"},"values":[["1793284207033794000","s3.put"]]},{"stream":{"deployment_environment_name":"lab","detected_level":"info","event":"s3.put","flags":"1","observed_timestamp":"1793284207030149000","outcome":"ok","scope_name":"messenger","service_instance_id":"6cdf9035-fcaf-4543-8dd1-2263203ed2a8","service_name":"messenger-worker","service_version":"lab6","severity_number":"9","span_id":"44a8831b72df12b8","trace_id":"bfeacf9a52db37463ddefff2ed23b0cb"},"values":[["1793284207030148000","s3.put"]]},{"stream":{"deployment_environment_name":"lab","detected_level":"info","event":"s3.put","flags":"1","observed_timestamp":"1793284207026879000","outcome":"ok","scope_name":"messenger","service_instance_id":"6cdf9035-fcaf-4543-8dd1-2263203ed2a8","service_name":"messenger-worker","service_version":"lab6","severity_number":"9","span_id":"57ef4c23890c827a","trace_id":"bfeacf9a52db37463ddefff2ed23b0cb"},"values":[["1793284207026874000","s3.put"]]},{"stream":{"deployment_environment_name":"lab","detected_level":"info","event":"s3.put","flags":"1","observed_timestamp":"1793284207023711000","outcome":"ok","scope_name":"messenger","service_instance_id":"6cdf9035-fcaf-4543-8dd1-2263203ed2a8","service_name":"messenger-worker","service_version":"lab6","severity_number":"9","span_id":"c4ad774acfa6e858","trace_id":"bfeacf9a52db37463ddefff2ed23b0cb"},"values":[["1793284207023690000","s3.put"]]},{"stream":{"deployment_environment_name":"lab","detected_level":"info","event":"s3.put","flags":"1","observed_timestamp":"1793284206318254000","outcome":"ok","scope_name":"messenger","service_instance_id":"cd2ca7bb-0626-4ccb-bdb4-6d4b50af696e","service_name":"messenger-api","service_version":"lab6","severity_number":"9","span_id":"2c2be991560d42af","trace_id":"bfeacf9a52db37463ddefff2ed23b0cb"},"values":[["1793284206318253000","s3.put"]]},{"stream":{"deployment_environment_name":"lab","detected_level":"info","event":"s3.get","flags":"1","observed_timestamp":"1793284206986675000","outcome":"ok","scope_name":"messenger","service_instance_id":"6cdf9035-fcaf-4543-8dd1-2263203ed2a8","service_name":"messenger-worker","service_version":"lab6","severity_number":"9","span_id":"fa2aa0e9b724c58d","trace_id":"bfeacf9a52db37463ddefff2ed23b0cb"},"values":[["1793284206986673000","s3.get"]]},{"stream":{"deployment_environment_name":"lab","detected_level":"info","event":"postgres.query","flags":"1","observed_timestamp":"1793284207038568000","outcome":"ok","scope_name":"messenger","service_instance_id":"6cdf9035-fcaf-4543-8dd1-2263203ed2a8","service_name":"messenger-worker","service_version":"lab6","severity_number":"9","span_id":"4a100fa1e4c2b969","trace_id":"bfeacf9a52db37463ddefff2ed23b0cb"},"values":[["1793284207038566000","postgres.query"]]},{"stream":{"deployment_environment_name":"lab","detected_level":"info","event":"postgres.query","flags":"1","observed_timestamp":"1793284207037393000","outcome":"ok","scope_name":"messenger","service_instance_id":"6cdf9035-fcaf-4543-8dd1-2263203ed2a8","service_name":"messenger-worker","service_version":"lab6","severity_number":"9","span_id":"d1b6e398736a6bfe","trace_id":"bfeacf9a52db37463ddefff2ed23b0cb"},"values":[["1793284207037393000","postgres.query"]]},{"stream":{"deployment_environment_name":"lab","detected_level":"info","event":"postgres.query","flags":"1","observed_timestamp":"1793284207036624000","outcome":"ok","scope_name":"messenger","service_instance_id":"6cdf9035-fcaf-4543-8dd1-2263203ed2a8","service_name":"messenger-worker","service_version":"lab6","severity_number":"9","span_id":"46dcce036653b92b","trace_id":"bfeacf9a52db37463ddefff2ed23b0cb"},"values":[["1793284207036623000","postgres.query"]]},{"stream":{"deployment_environment_name":"lab","detected_level":"info","event":"postgres.query","flags":"1","observed_timestamp":"1793284207035928000","outcome":"ok","scope_name":"messenger","service_instance_id":"6cdf9035-fcaf-4543-8dd1-2263203ed2a8","service_name":"messenger-worker","service_version":"lab6","severity_number":"9","span_id":"ebfecba049c1a20e","trace_id":"bfeacf9a52db37463ddefff2ed23b0cb"},"values":[["1793284207035927000","postgres.query"]]},{"stream":{"deployment_environment_name":"lab","detected_level":"info","event":"postgres.query","flags":"1","observed_timestamp":"1793284207034903000","outcome":"ok","scope_name":"messenger","service_instance_id":"6cdf9035-fcaf-4543-8dd1-2263203ed2a8","service_name":"messenger-worker","service_version":"lab6","severity_number":"9","span_id":"47c929ea4e19e23e","trace_id":"bfeacf9a52db37463ddefff2ed23b0cb"},"values":[["1793284207034901000","postgres.query"]]},{"stream":{"deployment_environment_name":"lab","detected_level":"info","event":"postgres.query","flags":"1","observed_timestamp":"1793284206981364000","outcome":"ok","scope_name":"messenger","service_instance_id":"6cdf9035-fcaf-4543-8dd1-2263203ed2a8","service_name":"messenger-worker","service_version":"lab6","severity_number":"9","span_id":"95cfec678c9926f5","trace_id":"bfeacf9a52db37463ddefff2ed23b0cb"},"values":[["1793284206981362000","postgres.query"]]},{"stream":{"deployment_environment_name":"lab","detected_level":"info","event":"postgres.query","flags":"1","observed_timestamp":"1793284206980261000","outcome":"ok","scope_name":"messenger","service_instance_id":"6cdf9035-fcaf-4543-8dd1-2263203ed2a8","service_name":"messenger-worker","service_version":"lab6","severity_number":"9","span_id":"313bca245b1b5cf1","trace_id":"bfeacf9a52db37463ddefff2ed23b0cb"},"values":[["1793284206980260000","postgres.query"]]},{"stream":{"deployment_environment_name":"lab","detected_level":"info","event":"postgres.query","flags":"1","observed_timestamp":"1793284206979430000","outcome":"ok","scope_name":"messenger","service_instance_id":"6cdf9035-fcaf-4543-8dd1-2263203ed2a8","service_name":"messenger-worker","service_version":"lab6","severity_number":"9","span_id":"7ca5756de05d503d","trace_id":"bfeacf9a52db37463ddefff2ed23b0cb"},"values":[["1793284206979429000","postgres.query"]]},{"stream":{"deployment_environment_name":"lab","detected_level":"info","event":"postgres.query","flags":"1","observed_timestamp":"1793284206978364000","outcome":"ok","scope_name":"messenger","service_instance_id":"6cdf9035-fcaf-4543-8dd1-2263203ed2a8","service_name":"messenger-worker","service_version":"lab6","severity_number":"9","span_id":"38817ec57d0810a3","trace_id":"bfeacf9a52db37463ddefff2ed23b0cb"},"values":[["1793284206978363000","postgres.query"]]},{"stream":{"deployment_environment_name":"lab","detected_level":"info","event":"postgres.query","flags":"1","observed_timestamp":"1793284206977397000","outcome":"ok","scope_name":"messenger","service_instance_id":"6cdf9035-fcaf-4543-8dd1-2263203ed2a8","service_name":"messenger-worker","service_version":"lab6","severity_number":"9","span_id":"b88a93de1e683897","trace_id":"bfeacf9a52db37463ddefff2ed23b0cb"},"values":[["1793284206977395000","postgres.query"]]},{"stream":{"deployment_environment_name":"lab","detected_level":"info","event":"postgres.query","flags":"1","observed_timestamp":"1793284206321023000","outcome":"ok","scope_name":"messenger","service_instance_id":"cd2ca7bb-0626-4ccb-bdb4-6d4b50af696e","service_name":"messenger-api","service_version":"lab6","severity_number":"9","span_id":"def488ac53c3cc1b","trace_id":"bfeacf9a52db37463ddefff2ed23b0cb"},"values":[["1793284206321022000","postgres.query"]]},{"stream":{"deployment_environment_name":"lab","detected_level":"info","event":"postgres.query","flags":"1","observed_timestamp":"1793284206320404000","outcome":"ok","scope_name":"messenger","service_instance_id":"cd2ca7bb-0626-4ccb-bdb4-6d4b50af696e","service_name":"messenger-api","service_version":"lab6","severity_number":"9","span_id":"0ff5f2630dbf18c0","trace_id":"bfeacf9a52db37463ddefff2ed23b0cb"},"values":[["1793284206320399000","postgres.query"]]},{"stream":{"deployment_environment_name":"lab","detected_level":"info","event":"postgres.query","flags":"1","observed_timestamp":"1793284206319560000","outcome":"ok","scope_name":"messenger","service_instance_id":"cd2ca7bb-0626-4ccb-bdb4-6d4b50af696e","service_name":"messenger-api","service_version":"lab6","severity_number":"9","span_id":"5e45760a1206d2af","trace_id":"bfeacf9a52db37463ddefff2ed23b0cb"},"values":[["1793284206319559000","postgres.query"]]},{"stream":{"deployment_environment_name":"lab","detected_level":"info","event":"postgres.query","flags":"1","observed_timestamp":"1793284206318507000","outcome":"ok","scope_name":"messenger","service_instance_id":"cd2ca7bb-0626-4ccb-bdb4-6d4b50af696e","service_name":"messenger-api","service_version":"lab6","severity_number":"9","span_id":"d54ed8aa0beece48","trace_id":"bfeacf9a52db37463ddefff2ed23b0cb"},"values":[["1793284206318506000","postgres.query"]]},{"stream":{"deployment_environment_name":"lab","detected_level":"info","event":"postgres.query","flags":"1","observed_timestamp":"1793284206313405000","outcome":"ok","scope_name":"messenger","service_instance_id":"cd2ca7bb-0626-4ccb-bdb4-6d4b50af696e","service_name":"messenger-api","service_version":"lab6","severity_number":"9","span_id":"8bf510c4eb77ebb0","trace_id":"bfeacf9a52db37463ddefff2ed23b0cb"},"values":[["1793284206313405000","postgres.query"]]},{"stream":{"deployment_environment_name":"lab","detected_level":"info","event":"postgres.query","flags":"1","observed_timestamp":"1793284206310750000","outcome":"ok","scope_name":"messenger","service_instance_id":"cd2ca7bb-0626-4ccb-bdb4-6d4b50af696e","service_name":"messenger-api","service_version":"lab6","severity_number":"9","span_id":"d07fc3adb57ada2f","trace_id":"bfeacf9a52db37463ddefff2ed23b0cb"},"values":[["1793284206310749000","postgres.query"]]},{"stream":{"deployment_environment_name":"lab","detected_level":"info","event":"postgres.query","flags":"1","observed_timestamp":"1793284206310374000","outcome":"ok","scope_name":"messenger","service_instance_id":"cd2ca7bb-0626-4ccb-bdb4-6d4b50af696e","service_name":"messenger-api","service_version":"lab6","severity_number":"9","span_id":"ae372acb877cfaae","trace_id":"bfeacf9a52db37463ddefff2ed23b0cb"},"values":[["1793284206310373000","postgres.query"]]},{"stream":{"deployment_environment_name":"lab","detected_level":"info","event":"postgres.query","flags":"1","observed_timestamp":"1793284206309486000","outcome":"ok","scope_name":"messenger","service_instance_id":"cd2ca7bb-0626-4ccb-bdb4-6d4b50af696e","service_name":"messenger-api","service_version":"lab6","severity_number":"9","span_id":"ed76c7624f715dbf","trace_id":"bfeacf9a52db37463ddefff2ed23b0cb"},"values":[["1793284206309485000","postgres.query"]]},{"stream":{"deployment_environment_name":"lab","detected_level":"info","event":"postgres.acquire","flags":"1","observed_timestamp":"1793284207033812000","outcome":"ok","scope_name":"messenger","service_instance_id":"6cdf9035-fcaf-4543-8dd1-2263203ed2a8","service_name":"messenger-worker","service_version":"lab6","severity_number":"9","span_id":"cfdbab4953dbd19b","trace_id":"bfeacf9a52db37463ddefff2ed23b0cb"},"values":[["1793284207033812000","postgres.acquire"]]},{"stream":{"deployment_environment_name":"lab","detected_level":"info","event":"postgres.acquire","flags":"1","observed_timestamp":"1793284206976338000","outcome":"ok","scope_name":"messenger","service_instance_id":"6cdf9035-fcaf-4543-8dd1-2263203ed2a8","service_name":"messenger-worker","service_version":"lab6","severity_number":"9","span_id":"2185027c7f3694f2","trace_id":"bfeacf9a52db37463ddefff2ed23b0cb"},"values":[["1793284206976337000","postgres.acquire"]]},{"stream":{"deployment_environment_name":"lab","detected_level":"info","event":"postgres.acquire","flags":"1","observed_timestamp":"1793284206318269000","outcome":"ok","scope_name":"messenger","service_instance_id":"cd2ca7bb-0626-4ccb-bdb4-6d4b50af696e","service_name":"messenger-api","service_version":"lab6","severity_number":"9","span_id":"2d64eaa92b0d634e","trace_id":"bfeacf9a52db37463ddefff2ed23b0cb"},"values":[["1793284206318269000","postgres.acquire"]]},{"stream":{"deployment_environment_name":"lab","detected_level":"info","event":"postgres.acquire","flags":"1","observed_timestamp":"1793284206312038000","outcome":"ok","scope_name":"messenger","service_instance_id":"cd2ca7bb-0626-4ccb-bdb4-6d4b50af696e","service_name":"messenger-api","service_version":"lab6","severity_number":"9","span_id":"b3bee743caff4a6e","trace_id":"bfeacf9a52db37463ddefff2ed23b0cb"},"values":[["1793284206312038000","postgres.acquire"]]},{"stream":{"deployment_environment_name":"lab","detected_level":"info","event":"postgres.acquire","flags":"1","observed_timestamp":"1793284206310385000","outcome":"ok","scope_name":"messenger","service_instance_id":"cd2ca7bb-0626-4ccb-bdb4-6d4b50af696e","service_name":"messenger-api","service_version":"lab6","severity_number":"9","span_id":"1b7a85eed610c24d","trace_id":"bfeacf9a52db37463ddefff2ed23b0cb"},"values":[["1793284206310385000","postgres.acquire"]]},{"stream":{"deployment_environment_name":"lab","detected_level":"info","event":"postgres.acquire","flags":"1","observed_timestamp":"1793284206309892000","outcome":"ok","scope_name":"messenger","service_instance_id":"cd2ca7bb-0626-4ccb-bdb4-6d4b50af696e","service_name":"messenger-api","service_version":"lab6","severity_number":"9","span_id":"3bdcbbbc167e0299","trace_id":"bfeacf9a52db37463ddefff2ed23b0cb"},"values":[["1793284206309891000","postgres.acquire"]]},{"stream":{"deployment_environment_name":"lab","detected_level":"info","event":"postgres.acquire","flags":"1","observed_timestamp":"1793284206308902000","outcome":"ok","scope_name":"messenger","service_instance_id":"cd2ca7bb-0626-4ccb-bdb4-6d4b50af696e","service_name":"messenger-api","service_version":"lab6","severity_number":"9","span_id":"ef09fffdc2fb210b","trace_id":"bfeacf9a52db37463ddefff2ed23b0cb"},"values":[["1793284206308900000","postgres.acquire"]]},{"stream":{"deployment_environment_name":"lab","detected_level":"info","event":"http.request","flags":"1","observed_timestamp":"1793284206321632000","outcome":"ok","scope_name":"messenger","service_instance_id":"cd2ca7bb-0626-4ccb-bdb4-6d4b50af696e","service_name":"messenger-api","service_version":"lab6","severity_number":"9","span_id":"c9e71db22335ec7e","trace_id":"bfeacf9a52db37463ddefff2ed23b0cb"},"values":[["1793284206321632000","http.request"]]},{"stream":{"deployment_environment_name":"lab","detected_level":"info","event":"http.request","flags":"1","observed_timestamp":"1793284206321630000","outcome":"2xx","request_id":"df097fd9-6840-44d5-afcd-54bcb2f6ec98","scope_name":"messenger","service_instance_id":"cd2ca7bb-0626-4ccb-bdb4-6d4b50af696e","service_name":"messenger-api","service_version":"lab6","severity_number":"9","span_id":"c9e71db22335ec7e","trace_id":"bfeacf9a52db37463ddefff2ed23b0cb"},"values":[["1793284206321630000","http.request"]]},{"stream":{"deployment_environment_name":"lab","detected_level":"info","event":"attachment.publish","flags":"1","observed_timestamp":"1793284206970442000","outcome":"ok","scope_name":"messenger","service_instance_id":"6cdf9035-fcaf-4543-8dd1-2263203ed2a8","service_name":"messenger-worker","service_version":"lab6","severity_number":"9","span_id":"b0c8c9c7237fcbac","trace_id":"bfeacf9a52db37463ddefff2ed23b0cb"},"values":[["1793284206970438000","attachment.publish"]]},{"stream":{"deployment_environment_name":"lab","detected_level":"info","event":"attachment.process","flags":"1","observed_timestamp":"1793284207043186000","outcome":"ok","scope_name":"messenger","service_instance_id":"6cdf9035-fcaf-4543-8dd1-2263203ed2a8","service_name":"messenger-worker","service_version":"lab6","severity_number":"9","span_id":"5945c74f298a3433","trace_id":"bfeacf9a52db37463ddefff2ed23b0cb"},"values":[["1793284207043185000","attachment.process"]]}],"stats":{"summary":{"bytesProcessedPerSecond":20484888,"linesProcessedPerSecond":102400,"totalBytesProcessed":854796,"totalLinesProcessed":4273,"execTime":0.041728,"queueTime":0.001535,"subqueries":0,"totalEntriesReturned":35,"splits":2,"shards":2,"totalPostFilterLines":35,"totalStructuredMetadataBytesProcessed":743818},"querier":{"store":{"totalChunksRef":0,"totalChunksDownloaded":0,"chunksDownloadTime":0,"queryReferencedStructuredMetadata":false,"chunk":{"headChunkBytes":0,"headChunkLines":0,"decompressedBytes":0,"decompressedLines":0,"compressedBytes":0,"totalDuplicates":0,"postFilterLines":0,"headChunkStructuredMetadataBytes":0,"decompressedStructuredMetadataBytes":0},"chunkRefsFetchTime":0,"congestionControlLatency":0,"pipelineWrapperFilteredLines":0}},"ingester":{"totalReached":2,"totalChunksMatched":2,"totalBatches":3,"totalLinesSent":35,"store":{"totalChunksRef":0,"totalChunksDownloaded":0,"chunksDownloadTime":0,"queryReferencedStructuredMetadata":true,"chunk":{"headChunkBytes":176484,"headChunkLines":1722,"decompressedBytes":678312,"decompressedLines":2551,"compressedBytes":26781,"totalDuplicates":0,"postFilterLines":35,"headChunkStructuredMetadataBytes":152368,"decompressedStructuredMetadataBytes":591450},"chunkRefsFetchTime":812126,"congestionControlLatency":0,"pipelineWrapperFilteredLines":0}},"cache":{"chunk":{"entriesFound":0,"entriesRequested":0,"entriesStored":0,"bytesReceived":0,"bytesSent":0,"requests":0,"downloadTime":0,"queryLengthServed":0},"index":{"entriesFound":0,"entriesRequested":0,"entriesStored":0,"bytesReceived":0,"bytesSent":0,"requests":0,"downloadTime":0,"queryLengthServed":0},"result":{"entriesFound":0,"entriesRequested":0,"entriesStored":0,"bytesReceived":0,"bytesSent":0,"requests":0,"downloadTime":0,"queryLengthServed":0},"statsResult":{"entriesFound":1,"entriesRequested":1,"entriesStored":0,"bytesReceived":261,"bytesSent":0,"requests":1,"downloadTime":569625,"queryLengthServed":1821000000000},"volumeResult":{"entriesFound":0,"entriesRequested":0,"entriesStored":0,"bytesReceived":0,"bytesSent":0,"requests":0,"downloadTime":0,"queryLengthServed":0},"seriesResult":{"entriesFound":0,"entriesRequested":0,"entriesStored":0,"bytesReceived":0,"bytesSent":0,"requests":0,"downloadTime":0,"queryLengthServed":0},"labelResult":{"entriesFound":0,"entriesRequested":0,"entriesStored":0,"bytesReceived":0,"bytesSent":0,"requests":0,"downloadTime":0,"queryLengthServed":0},"instantMetricResult":{"entriesFound":0,"entriesRequested":0,"entriesStored":0,"bytesReceived":0,"bytesSent":0,"requests":0,"downloadTime":0,"queryLengthServed":0}},"index":{"totalChunks":0,"postFilterChunks":0,"shardsDuration":0,"usedBloomFilters":false}}}} diff --git a/evidence/local-correlation/correlated-trace.json b/evidence/local-correlation/correlated-trace.json new file mode 100644 index 0000000..91b2a8d --- /dev/null +++ b/evidence/local-correlation/correlated-trace.json @@ -0,0 +1 @@ +{"batches":[{"resource":{"attributes":[{"key":"deployment.environment.name","value":{"stringValue":"lab"}},{"key":"service.instance.id","value":{"stringValue":"cd2ca7bb-0626-4ccb-bdb4-6d4b50af696e"}},{"key":"service.name","value":{"stringValue":"messenger-api"}},{"key":"service.version","value":{"stringValue":"lab6"}}]},"scopeSpans":[{"scope":{"name":"messenger"},"spans":[{"traceId":"v+rPmlLbN0Y93v/y7SOwyw==","spanId":"yecdsiM17H4=","parentSpanId":"YDfYgNXDI9g=","flags":768,"name":"POST /api/v1/chats/:chat_id/attachments","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1793284206308303000","endTimeUnixNano":"1793284206321632583","attributes":[{"key":"http.route","value":{"stringValue":"/api/v1/chats/:chat_id/attachments"}},{"key":"http.request.method","value":{"stringValue":"POST"}},{"key":"http.response.status_code","value":{"intValue":"202"}}],"status":{}},{"traceId":"v+rPmlLbN0Y93v/y7SOwyw==","spanId":"7wn//cL7IQs=","parentSpanId":"yecdsiM17H4=","flags":256,"name":"postgres.acquire","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793284206308896000","endTimeUnixNano":"1793284206308908250","status":{}},{"traceId":"v+rPmlLbN0Y93v/y7SOwyw==","spanId":"7XbHYk9xXb8=","parentSpanId":"yecdsiM17H4=","flags":256,"name":"postgres.query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793284206308913000","endTimeUnixNano":"1793284206309489500","status":{}},{"traceId":"v+rPmlLbN0Y93v/y7SOwyw==","spanId":"O9y7vBZ+Apk=","parentSpanId":"yecdsiM17H4=","flags":256,"name":"postgres.acquire","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793284206309889000","endTimeUnixNano":"1793284206309894542","status":{}},{"traceId":"v+rPmlLbN0Y93v/y7SOwyw==","spanId":"rjcqy4d8+q4=","parentSpanId":"yecdsiM17H4=","flags":256,"name":"postgres.query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793284206309897000","endTimeUnixNano":"1793284206310376291","status":{}},{"traceId":"v+rPmlLbN0Y93v/y7SOwyw==","spanId":"G3qF7tYQwk0=","parentSpanId":"yecdsiM17H4=","flags":256,"name":"postgres.acquire","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793284206310383000","endTimeUnixNano":"1793284206310385458","status":{}},{"traceId":"v+rPmlLbN0Y93v/y7SOwyw==","spanId":"0H/DrbV62i8=","parentSpanId":"yecdsiM17H4=","flags":256,"name":"postgres.query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793284206310386000","endTimeUnixNano":"1793284206310750583","status":{}},{"traceId":"v+rPmlLbN0Y93v/y7SOwyw==","spanId":"s77nQ8r/Sm4=","parentSpanId":"yecdsiM17H4=","flags":256,"name":"postgres.acquire","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793284206312038000","endTimeUnixNano":"1793284206312039250","status":{}},{"traceId":"v+rPmlLbN0Y93v/y7SOwyw==","spanId":"i/UQxOt367A=","parentSpanId":"yecdsiM17H4=","flags":256,"name":"postgres.query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793284206312041000","endTimeUnixNano":"1793284206313406292","status":{}},{"traceId":"v+rPmlLbN0Y93v/y7SOwyw==","spanId":"LCvpkVYNQq8=","parentSpanId":"yecdsiM17H4=","flags":256,"name":"s3.put","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793284206313411000","endTimeUnixNano":"1793284206318256708","status":{}},{"traceId":"v+rPmlLbN0Y93v/y7SOwyw==","spanId":"LWTqqSsNY04=","parentSpanId":"yecdsiM17H4=","flags":256,"name":"postgres.acquire","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793284206318267000","endTimeUnixNano":"1793284206318269833","status":{}},{"traceId":"v+rPmlLbN0Y93v/y7SOwyw==","spanId":"1U7Yqgvuzkg=","parentSpanId":"yecdsiM17H4=","flags":256,"name":"postgres.query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793284206318271000","endTimeUnixNano":"1793284206318508291","status":{}},{"traceId":"v+rPmlLbN0Y93v/y7SOwyw==","spanId":"XkV2ChIG0q8=","parentSpanId":"yecdsiM17H4=","flags":256,"name":"postgres.query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793284206318512000","endTimeUnixNano":"1793284206319562542","status":{}},{"traceId":"v+rPmlLbN0Y93v/y7SOwyw==","spanId":"D/XyYw2/GMA=","parentSpanId":"yecdsiM17H4=","flags":256,"name":"postgres.query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793284206319575000","endTimeUnixNano":"1793284206320407000","status":{}},{"traceId":"v+rPmlLbN0Y93v/y7SOwyw==","spanId":"3vSIrFPDzBs=","parentSpanId":"yecdsiM17H4=","flags":256,"name":"postgres.query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793284206320410000","endTimeUnixNano":"1793284206321023416","status":{}}]}]},{"resource":{"attributes":[{"key":"deployment.environment.name","value":{"stringValue":"lab"}},{"key":"service.instance.id","value":{"stringValue":"6cdf9035-fcaf-4543-8dd1-2263203ed2a8"}},{"key":"service.name","value":{"stringValue":"messenger-worker"}},{"key":"service.version","value":{"stringValue":"lab6"}}]},"scopeSpans":[{"scope":{"name":"messenger"},"spans":[{"traceId":"v+rPmlLbN0Y93v/y7SOwyw==","spanId":"sMjJxyN/y6w=","parentSpanId":"yecdsiM17H4=","flags":768,"name":"attachment.publish","kind":"SPAN_KIND_PRODUCER","startTimeUnixNano":"1793284206967017000","endTimeUnixNano":"1793284206970450917","status":{}},{"traceId":"v+rPmlLbN0Y93v/y7SOwyw==","spanId":"WUXHTymKNDM=","parentSpanId":"sMjJxyN/y6w=","flags":768,"name":"attachment.process","kind":"SPAN_KIND_CONSUMER","startTimeUnixNano":"1793284206976324000","endTimeUnixNano":"1793284207043187500","status":{}},{"traceId":"v+rPmlLbN0Y93v/y7SOwyw==","spanId":"IYUCfH82lPI=","parentSpanId":"WUXHTymKNDM=","flags":256,"name":"postgres.acquire","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793284206976335000","endTimeUnixNano":"1793284206976340666","status":{}},{"traceId":"v+rPmlLbN0Y93v/y7SOwyw==","spanId":"uIqT3h5oOJc=","parentSpanId":"WUXHTymKNDM=","flags":256,"name":"postgres.query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793284206976344000","endTimeUnixNano":"1793284206977400625","status":{}},{"traceId":"v+rPmlLbN0Y93v/y7SOwyw==","spanId":"OIF+xX0IEKM=","parentSpanId":"WUXHTymKNDM=","flags":256,"name":"postgres.query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793284206977410000","endTimeUnixNano":"1793284206978366125","status":{}},{"traceId":"v+rPmlLbN0Y93v/y7SOwyw==","spanId":"fKV1beBdUD0=","parentSpanId":"WUXHTymKNDM=","flags":256,"name":"postgres.query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793284206978401000","endTimeUnixNano":"1793284206979432041","status":{}},{"traceId":"v+rPmlLbN0Y93v/y7SOwyw==","spanId":"MTvKJFsbXPE=","parentSpanId":"WUXHTymKNDM=","flags":256,"name":"postgres.query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793284206979438000","endTimeUnixNano":"1793284206980264458","status":{}},{"traceId":"v+rPmlLbN0Y93v/y7SOwyw==","spanId":"lc/sZ4yZJvU=","parentSpanId":"WUXHTymKNDM=","flags":256,"name":"postgres.query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793284206980269000","endTimeUnixNano":"1793284206981367792","status":{}},{"traceId":"v+rPmlLbN0Y93v/y7SOwyw==","spanId":"+iqg6bckxY0=","parentSpanId":"WUXHTymKNDM=","flags":256,"name":"s3.get","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793284206981381000","endTimeUnixNano":"1793284206986681083","status":{}},{"traceId":"v+rPmlLbN0Y93v/y7SOwyw==","spanId":"xK13Ss+m6Fg=","parentSpanId":"WUXHTymKNDM=","flags":256,"name":"s3.put","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793284207015523000","endTimeUnixNano":"1793284207023735917","status":{}},{"traceId":"v+rPmlLbN0Y93v/y7SOwyw==","spanId":"V+9MI4kMgno=","parentSpanId":"WUXHTymKNDM=","flags":256,"name":"s3.put","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793284207023765000","endTimeUnixNano":"1793284207026886042","status":{}},{"traceId":"v+rPmlLbN0Y93v/y7SOwyw==","spanId":"RKiDG3LfErg=","parentSpanId":"WUXHTymKNDM=","flags":256,"name":"s3.put","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793284207026899000","endTimeUnixNano":"1793284207030156667","status":{}},{"traceId":"v+rPmlLbN0Y93v/y7SOwyw==","spanId":"LJK+YhTYz7w=","parentSpanId":"WUXHTymKNDM=","flags":256,"name":"s3.put","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793284207030167000","endTimeUnixNano":"1793284207033798042","status":{}},{"traceId":"v+rPmlLbN0Y93v/y7SOwyw==","spanId":"z9urSVPb0Zs=","parentSpanId":"WUXHTymKNDM=","flags":256,"name":"postgres.acquire","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793284207033807000","endTimeUnixNano":"1793284207033812959","status":{}},{"traceId":"v+rPmlLbN0Y93v/y7SOwyw==","spanId":"R8kp6k4Z4j4=","parentSpanId":"WUXHTymKNDM=","flags":256,"name":"postgres.query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793284207033816000","endTimeUnixNano":"1793284207034905458","status":{}},{"traceId":"v+rPmlLbN0Y93v/y7SOwyw==","spanId":"6/7LoEnBog4=","parentSpanId":"WUXHTymKNDM=","flags":256,"name":"postgres.query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793284207034918000","endTimeUnixNano":"1793284207035929000","status":{}},{"traceId":"v+rPmlLbN0Y93v/y7SOwyw==","spanId":"RtzOA2ZTuSs=","parentSpanId":"WUXHTymKNDM=","flags":256,"name":"postgres.query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793284207035935000","endTimeUnixNano":"1793284207036626083","status":{}},{"traceId":"v+rPmlLbN0Y93v/y7SOwyw==","spanId":"0bbjmHNqa/4=","parentSpanId":"WUXHTymKNDM=","flags":256,"name":"postgres.query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793284207036631000","endTimeUnixNano":"1793284207037395250","status":{}},{"traceId":"v+rPmlLbN0Y93v/y7SOwyw==","spanId":"ShAPoeTCuWk=","parentSpanId":"WUXHTymKNDM=","flags":256,"name":"postgres.query","kind":"SPAN_KIND_CLIENT","startTimeUnixNano":"1793284207037399000","endTimeUnixNano":"1793284207038568958","status":{}}]}]}]} \ No newline at end of file diff --git a/evidence/local-correlation/correlation.json b/evidence/local-correlation/correlation.json new file mode 100644 index 0000000..e74366e --- /dev/null +++ b/evidence/local-correlation/correlation.json @@ -0,0 +1,5 @@ +{ + "trace_id": "bfeacf9a52db37463ddefff2ed23b0cb", + "attachment_id": "1824ea29-d35f-4949-944e-b976a222768c", + "ready": true +} diff --git a/evidence/local-correlation/correlation.txt b/evidence/local-correlation/correlation.txt new file mode 100644 index 0000000..e14848a --- /dev/null +++ b/evidence/local-correlation/correlation.txt @@ -0,0 +1,2 @@ +curl: (22) The requested URL returned error: 404 +PASS: upload and worker share trace bfeacf9a52db37463ddefff2ed23b0cb; correlated structured logs found diff --git a/evidence/local/actions-errors.txt b/evidence/local/actions-errors.txt new file mode 100644 index 0000000..e69de29 diff --git a/evidence/local/actions.txt b/evidence/local/actions.txt new file mode 100644 index 0000000..b1c2758 --- /dev/null +++ b/evidence/local/actions.txt @@ -0,0 +1,8 @@ +2026-10-29T14:28:32Z target=local-b action=stop +2026-10-29T14:28:49Z action=restart-api-worker-postgres-valkey-s3-broker retained-container-filesystems +2026-10-29T14:29:29Z action=stop-worker +2026-10-29T14:29:29Z action=restart-api-worker-postgres-valkey-s3-broker retained-container-filesystems +2026-10-29T14:43:21Z target=local-b action=stop +2026-10-29T14:43:40Z action=restart-api-worker-postgres-valkey-s3-broker retained-container-filesystems +2026-10-29T14:44:19Z action=stop-worker +2026-10-29T14:44:20Z action=restart-api-worker-postgres-valkey-s3-broker retained-container-filesystems diff --git a/evidence/local/durable.txt b/evidence/local/durable.txt new file mode 100644 index 0000000..e4749d5 --- /dev/null +++ b/evidence/local/durable.txt @@ -0,0 +1,2 @@ +worker_stopped=true upload_status=202 snapshot=queued message_created=true content_status=409 +PASS: accepted intent survived API, worker, PostgreSQL, S3 and AOF broker restart; ready=1 variants=3 original_unchanged=true diff --git a/evidence/local/failover.txt b/evidence/local/failover.txt new file mode 100644 index 0000000..a6e8cf6 --- /dev/null +++ b/evidence/local/failover.txt @@ -0,0 +1,4 @@ +python3 tests/resilience.py --mode failover --action-script "/Users/marlo/go/src/gitflick.ru/vlad_marlo/gitea/mtusi2026lab6/scripts/local-action.sh" --base-url "http://127.0.0.1:49688" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 99, "failed_attempts": 0, "first_recovery_seconds": 0.287} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/local/local-a.txt b/evidence/local/local-a.txt new file mode 100644 index 0000000..e69de29 diff --git a/evidence/local/local-b.txt b/evidence/local/local-b.txt new file mode 100644 index 0000000..e69de29 diff --git a/evidence/local/persistence.txt b/evidence/local/persistence.txt new file mode 100644 index 0000000..5d0fd84 --- /dev/null +++ b/evidence/local/persistence.txt @@ -0,0 +1,3 @@ +python3 tests/resilience.py --mode persistence --action-script "/Users/marlo/go/src/gitflick.ru/vlad_marlo/gitea/mtusi2026lab6/scripts/local-action.sh" --base-url "http://127.0.0.1:49688" +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/local/smoke-after.txt b/evidence/local/smoke-after.txt new file mode 100644 index 0000000..f7e2100 --- /dev/null +++ b/evidence/local/smoke-after.txt @@ -0,0 +1,2 @@ +python3 tests/smoke.py --base-url "http://127.0.0.1:49688" +PASS: lab6 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/local/smoke.txt b/evidence/local/smoke.txt new file mode 100644 index 0000000..f7e2100 --- /dev/null +++ b/evidence/local/smoke.txt @@ -0,0 +1,2 @@ +python3 tests/smoke.py --base-url "http://127.0.0.1:49688" +PASS: lab6 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/local/worker.txt b/evidence/local/worker.txt new file mode 100644 index 0000000..e69de29 diff --git a/evidence/ready-slo-result.txt b/evidence/ready-slo-result.txt new file mode 100644 index 0000000..3fa396d --- /dev/null +++ b/evidence/ready-slo-result.txt @@ -0,0 +1 @@ +PASS: ready burn and queue lag resolved after real worker recovery diff --git a/evidence/ready-slo/alerts.jsonl b/evidence/ready-slo/alerts.jsonl new file mode 100644 index 0000000..190ae20 --- /dev/null +++ b/evidence/ready-slo/alerts.jsonl @@ -0,0 +1,42 @@ +{"at":1793285189,"phase":"worker-stopped","alerts":[]} +{"at":1793285194,"phase":"worker-stopped","alerts":[]} +{"at":1793285199,"phase":"worker-stopped","alerts":[]} +{"at":1793285204,"phase":"worker-stopped","alerts":[]} +{"at":1793285209,"phase":"worker-stopped","alerts":[]} +{"at":1793285214,"phase":"worker-stopped","alerts":[]} +{"at":1793285219,"phase":"worker-stopped","alerts":[]} +{"at":1793285224,"phase":"worker-stopped","alerts":[]} +{"at":1793285229,"phase":"worker-stopped","alerts":[]} +{"at":1793285234,"phase":"worker-stopped","alerts":[]} +{"at":1793285239,"phase":"worker-stopped","alerts":[]} +{"at":1793285244,"phase":"worker-stopped","alerts":[]} +{"at":1793285249,"phase":"worker-stopped","alerts":[]} +{"at":1793285254,"phase":"worker-stopped","alerts":[]} +{"at":1793285259,"phase":"worker-stopped","alerts":[{"labels":{"alertname":"MessengerQueueLag","severity":"warning"},"annotations":{"runbook":"REPORT.md#диагностика","summary":"Accepted work has been waiting for over 60 seconds"},"state":"pending","activeAt":"2026-10-29T14:47:36.842599208Z","value":"6.425125e+01"}]} +{"at":1793285264,"phase":"worker-stopped","alerts":[{"labels":{"alertname":"MessengerQueueLag","severity":"warning"},"annotations":{"runbook":"REPORT.md#диагностика","summary":"Accepted work has been waiting for over 60 seconds"},"state":"pending","activeAt":"2026-10-29T14:47:36.842599208Z","value":"6.9249197e+01"}]} +{"at":1793285269,"phase":"worker-stopped","alerts":[{"labels":{"alertname":"MessengerQueueLag","severity":"warning"},"annotations":{"runbook":"REPORT.md#диагностика","summary":"Accepted work has been waiting for over 60 seconds"},"state":"pending","activeAt":"2026-10-29T14:47:36.842599208Z","value":"7.4250454e+01"}]} +{"at":1793285274,"phase":"worker-stopped","alerts":[{"labels":{"alertname":"MessengerQueueLag","severity":"warning"},"annotations":{"runbook":"REPORT.md#диагностика","summary":"Accepted work has been waiting for over 60 seconds"},"state":"firing","activeAt":"2026-10-29T14:47:36.842599208Z","value":"7.9248933e+01"}]} +{"at":1793285279,"phase":"worker-recovery","alerts":[{"labels":{"alertname":"MessengerQueueLag","severity":"warning"},"annotations":{"runbook":"REPORT.md#диагностика","summary":"Accepted work has been waiting for over 60 seconds"},"state":"firing","activeAt":"2026-10-29T14:47:36.842599208Z","value":"8.4248099e+01"}]} +{"at":1793285284,"phase":"worker-recovery","alerts":[{"labels":{"alertname":"MessengerQueueLag","severity":"warning"},"annotations":{"runbook":"REPORT.md#диагностика","summary":"Accepted work has been waiting for over 60 seconds"},"state":"firing","activeAt":"2026-10-29T14:47:36.842599208Z","value":"8.9246841e+01"}]} +{"at":1793285289,"phase":"worker-recovery","alerts":[{"labels":{"alertname":"MessengerReadyBurnRate","severity":"warning"},"annotations":{"runbook":"REPORT.md#диагностика","summary":"More than 25% of completed jobs missed 60s in both windows (5x burn of 95% SLO)"},"state":"pending","activeAt":"2026-10-29T14:48:06.842599208Z","value":"1e+00"}]} +{"at":1793285294,"phase":"worker-recovery","alerts":[{"labels":{"alertname":"MessengerReadyBurnRate","severity":"warning"},"annotations":{"runbook":"REPORT.md#диагностика","summary":"More than 25% of completed jobs missed 60s in both windows (5x burn of 95% SLO)"},"state":"pending","activeAt":"2026-10-29T14:48:06.842599208Z","value":"1e+00"}]} +{"at":1793285299,"phase":"worker-recovery","alerts":[{"labels":{"alertname":"MessengerReadyBurnRate","severity":"warning"},"annotations":{"runbook":"REPORT.md#диагностика","summary":"More than 25% of completed jobs missed 60s in both windows (5x burn of 95% SLO)"},"state":"pending","activeAt":"2026-10-29T14:48:06.842599208Z","value":"1e+00"}]} +{"at":1793285305,"phase":"worker-recovery","alerts":[{"labels":{"alertname":"MessengerReadyBurnRate","severity":"warning"},"annotations":{"runbook":"REPORT.md#диагностика","summary":"More than 25% of completed jobs missed 60s in both windows (5x burn of 95% SLO)"},"state":"pending","activeAt":"2026-10-29T14:48:06.842599208Z","value":"1e+00"}]} +{"at":1793285310,"phase":"worker-recovery","alerts":[{"labels":{"alertname":"MessengerReadyBurnRate","severity":"warning"},"annotations":{"runbook":"REPORT.md#диагностика","summary":"More than 25% of completed jobs missed 60s in both windows (5x burn of 95% SLO)"},"state":"pending","activeAt":"2026-10-29T14:48:06.842599208Z","value":"1e+00"}]} +{"at":1793285315,"phase":"worker-recovery","alerts":[{"labels":{"alertname":"MessengerReadyBurnRate","severity":"warning"},"annotations":{"runbook":"REPORT.md#диагностика","summary":"More than 25% of completed jobs missed 60s in both windows (5x burn of 95% SLO)"},"state":"pending","activeAt":"2026-10-29T14:48:06.842599208Z","value":"1e+00"}]} +{"at":1793285320,"phase":"worker-recovery","alerts":[{"labels":{"alertname":"MessengerReadyBurnRate","severity":"warning"},"annotations":{"runbook":"REPORT.md#диагностика","summary":"More than 25% of completed jobs missed 60s in both windows (5x burn of 95% SLO)"},"state":"firing","activeAt":"2026-10-29T14:48:06.842599208Z","value":"1e+00"}]} +{"at":1793285325,"phase":"worker-recovery","alerts":[{"labels":{"alertname":"MessengerReadyBurnRate","severity":"warning"},"annotations":{"runbook":"REPORT.md#диагностика","summary":"More than 25% of completed jobs missed 60s in both windows (5x burn of 95% SLO)"},"state":"firing","activeAt":"2026-10-29T14:48:06.842599208Z","value":"1e+00"}]} +{"at":1793285330,"phase":"worker-recovery","alerts":[{"labels":{"alertname":"MessengerReadyBurnRate","severity":"warning"},"annotations":{"runbook":"REPORT.md#диагностика","summary":"More than 25% of completed jobs missed 60s in both windows (5x burn of 95% SLO)"},"state":"firing","activeAt":"2026-10-29T14:48:06.842599208Z","value":"1e+00"}]} +{"at":1793285335,"phase":"worker-recovery","alerts":[{"labels":{"alertname":"MessengerReadyBurnRate","severity":"warning"},"annotations":{"runbook":"REPORT.md#диагностика","summary":"More than 25% of completed jobs missed 60s in both windows (5x burn of 95% SLO)"},"state":"firing","activeAt":"2026-10-29T14:48:06.842599208Z","value":"1e+00"}]} +{"at":1793285340,"phase":"worker-recovery","alerts":[{"labels":{"alertname":"MessengerReadyBurnRate","severity":"warning"},"annotations":{"runbook":"REPORT.md#диагностика","summary":"More than 25% of completed jobs missed 60s in both windows (5x burn of 95% SLO)"},"state":"firing","activeAt":"2026-10-29T14:48:06.842599208Z","value":"1e+00"}]} +{"at":1793285345,"phase":"worker-recovery","alerts":[]} +{"at":1793285350,"phase":"worker-recovery","alerts":[]} +{"at":1793285355,"phase":"worker-recovery","alerts":[]} +{"at":1793285360,"phase":"worker-recovery","alerts":[]} +{"at":1793285365,"phase":"worker-recovery","alerts":[]} +{"at":1793285370,"phase":"worker-recovery","alerts":[]} +{"at":1793285375,"phase":"worker-recovery","alerts":[]} +{"at":1793285380,"phase":"worker-recovery","alerts":[]} +{"at":1793285385,"phase":"worker-recovery","alerts":[]} +{"at":1793285390,"phase":"worker-recovery","alerts":[]} +{"at":1793285395,"phase":"worker-recovery","alerts":[]} diff --git a/evidence/ready-slo/events.csv b/evidence/ready-slo/events.csv new file mode 100644 index 0000000..3e8d511 --- /dev/null +++ b/evidence/ready-slo/events.csv @@ -0,0 +1,2 @@ +1793285188,worker-stopped +1793285279,worker-resumed diff --git a/evidence/ready-slo/final-backlog.json b/evidence/ready-slo/final-backlog.json new file mode 100644 index 0000000..958a328 --- /dev/null +++ b/evidence/ready-slo/final-backlog.json @@ -0,0 +1 @@ +{"status":"success","data":{"resultType":"vector","result":[{"metric":{},"value":[1793285400.731,"1"]}]}} \ No newline at end of file diff --git a/evidence/ready-slo/summary.json b/evidence/ready-slo/summary.json new file mode 100644 index 0000000..65151a7 --- /dev/null +++ b/evidence/ready-slo/summary.json @@ -0,0 +1,12 @@ +{ + "MessengerReadyBurnRate": { + "first_firing": 1793285320, + "last_firing": 1793285340, + "first_resolved": 1793285345 + }, + "MessengerQueueLag": { + "first_firing": 1793285274, + "last_firing": 1793285284, + "first_resolved": 1793285289 + } +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..474fc77 --- /dev/null +++ b/go.mod @@ -0,0 +1,77 @@ +module gitea.privcloud.tech/vlad_marlo/mtusi2026lab6 + +go 1.26.1 + +require ( + github.com/disintegration/imaging v1.6.2 + github.com/gofiber/fiber/v2 v2.52.12 + github.com/google/uuid v1.6.0 + github.com/jackc/pgx/v5 v5.9.1 + github.com/minio/minio-go/v7 v7.0.95 + github.com/prometheus/client_golang v1.23.2 + github.com/redis/go-redis/v9 v9.17.2 + github.com/valyala/fasthttp v1.51.0 + go.lab260.ru/fielderr v0.0.0-20260609150058-2f9540b99e41 + go.opentelemetry.io/otel v1.38.0 + go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.14.0 + go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.38.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0 + go.opentelemetry.io/otel/log v0.14.0 + go.opentelemetry.io/otel/metric v1.38.0 + go.opentelemetry.io/otel/sdk v1.38.0 + go.opentelemetry.io/otel/sdk/log v0.14.0 + go.opentelemetry.io/otel/sdk/metric v1.38.0 + go.opentelemetry.io/otel/trace v1.38.0 + go.opentelemetry.io/proto/otlp v1.7.1 + go.uber.org/fx v1.24.0 + go.uber.org/zap v1.27.1 + golang.org/x/crypto v0.49.0 + golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8 + google.golang.org/protobuf v1.36.8 +) + +require ( + github.com/andybalholm/brotli v1.1.0 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/cenkalti/backoff/v5 v5.0.3 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/go-ini/ini v1.67.0 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/goccy/go-json v0.10.5 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect + github.com/klauspost/compress v1.18.0 // indirect + github.com/klauspost/cpuid/v2 v2.2.11 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mattn/go-runewidth v0.0.16 // indirect + github.com/minio/crc64nvme v1.0.2 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/philhofer/fwd v1.2.0 // indirect + github.com/prometheus/client_model v0.6.2 // indirect + github.com/prometheus/common v0.66.1 // indirect + github.com/prometheus/procfs v0.16.1 // indirect + github.com/rivo/uniseg v0.2.0 // indirect + github.com/rs/xid v1.6.0 // indirect + github.com/tinylib/msgp v1.3.0 // indirect + github.com/valyala/bytebufferpool v1.0.0 // indirect + github.com/valyala/tcplisten v1.0.0 // indirect + go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 // indirect + go.uber.org/dig v1.19.0 // indirect + go.uber.org/multierr v1.10.0 // indirect + go.yaml.in/yaml/v2 v2.4.2 // indirect + golang.org/x/net v0.51.0 // indirect + golang.org/x/sync v0.20.0 // indirect + golang.org/x/sys v0.42.0 // indirect + golang.org/x/text v0.35.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5 // indirect + google.golang.org/grpc v1.75.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..f45af2e --- /dev/null +++ b/go.sum @@ -0,0 +1,180 @@ +github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= +github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= +github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= +github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= +github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= +github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1ei82L+c= +github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= +github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= +github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/gofiber/fiber/v2 v2.52.12 h1:0LdToKclcPOj8PktUdIKo9BUohjjwfnQl42Dhw8/WUw= +github.com/gofiber/fiber/v2 v2.52.12/go.mod h1:YEcBbO/FB+5M1IZNBP9FO3J9281zgPAreiI1oqg8nDw= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 h1:8Tjv8EJ+pM1xP8mK6egEbD1OgnVTyacbefKhmbLhIhU= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2/go.mod h1:pkJQ2tZHJ0aFOVEEot6oZmaVEZcRme73eIFmhiVuRWs= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.9.1 h1:uwrxJXBnx76nyISkhr33kQLlUqjv7et7b9FjCen/tdc= +github.com/jackc/pgx/v5 v5.9.1/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU= +github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= +github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/minio/crc64nvme v1.0.2 h1:6uO1UxGAD+kwqWWp7mBFsi5gAse66C4NXO8cmcVculg= +github.com/minio/crc64nvme v1.0.2/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.95 h1:ywOUPg+PebTMTzn9VDsoFJy32ZuARN9zhB+K3IYEvYU= +github.com/minio/minio-go/v7 v7.0.95/go.mod h1:wOOX3uxS334vImCNRVyIDdXX9OsXDm89ToynKgqUKlo= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= +github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= +github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs= +github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA= +github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= +github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= +github.com/redis/go-redis/v9 v9.17.2 h1:P2EGsA4qVIM3Pp+aPocCJ7DguDHhqrXNhVcEp4ViluI= +github.com/redis/go-redis/v9 v9.17.2/go.mod h1:u410H11HMLoB+TP67dz8rL9s6QW2j76l0//kSOd3370= +github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/tinylib/msgp v1.3.0 h1:ULuf7GPooDaIlbyvgAxBV/FI7ynli6LZ1/nVUNu+0ww= +github.com/tinylib/msgp v1.3.0/go.mod h1:ykjzy2wzgrlvpDCRc4LA8UXy6D8bzMSuAF3WD57Gok0= +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasthttp v1.51.0 h1:8b30A5JlZ6C7AS81RsWjYMQmrZG6feChmgAolCl1SqA= +github.com/valyala/fasthttp v1.51.0/go.mod h1:oI2XroL+lI7vdXyYoQk03bXBThfFl2cVdIA3Xl7cH8g= +github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8= +github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= +go.lab260.ru/fielderr v0.0.0-20260609150058-2f9540b99e41 h1:kHFUC/1OnQ05ShJW1wjCAbrSa/XxylAgEnZ75BholE4= +go.lab260.ru/fielderr v0.0.0-20260609150058-2f9540b99e41/go.mod h1:OQKs2tIkkTxUjzewIMsqChKuHPWDjIgUz+SYqdhR9rM= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8= +go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.14.0 h1:QQqYw3lkrzwVsoEX0w//EhH/TCnpRdEenKBOOEIMjWc= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.14.0/go.mod h1:gSVQcr17jk2ig4jqJ2DX30IdWH251JcNAecvrqTxH1s= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.38.0 h1:Oe2z/BCg5q7k4iXC3cqJxKYg0ieRiOqF0cecFYdPTwk= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.38.0/go.mod h1:ZQM5lAJpOsKnYagGg/zV2krVqTtaVdYdDkhMoX6Oalg= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 h1:GqRJVj7UmLjCVyVJ3ZFLdPRmhDUp2zFmQe3RHIOsw24= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0/go.mod h1:ri3aaHSmCTVYu2AWv44YMauwAQc0aqI9gHKIcSbI1pU= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0 h1:aTL7F04bJHUlztTsNGJ2l+6he8c+y/b//eR0jjjemT4= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0/go.mod h1:kldtb7jDTeol0l3ewcmd8SDvx3EmIE7lyvqbasU3QC4= +go.opentelemetry.io/otel/log v0.14.0 h1:2rzJ+pOAZ8qmZ3DDHg73NEKzSZkhkGIua9gXtxNGgrM= +go.opentelemetry.io/otel/log v0.14.0/go.mod h1:5jRG92fEAgx0SU/vFPxmJvhIuDU9E1SUnEQrMlJpOno= +go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA= +go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI= +go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E= +go.opentelemetry.io/otel/sdk v1.38.0/go.mod h1:ghmNdGlVemJI3+ZB5iDEuk4bWA3GkTpW+DOoZMYBVVg= +go.opentelemetry.io/otel/sdk/log v0.14.0 h1:JU/U3O7N6fsAXj0+CXz21Czg532dW2V4gG1HE/e8Zrg= +go.opentelemetry.io/otel/sdk/log v0.14.0/go.mod h1:imQvII+0ZylXfKU7/wtOND8Hn4OpT3YUoIgqJVksUkM= +go.opentelemetry.io/otel/sdk/log/logtest v0.14.0 h1:Ijbtz+JKXl8T2MngiwqBlPaHqc4YCaP/i13Qrow6gAM= +go.opentelemetry.io/otel/sdk/log/logtest v0.14.0/go.mod h1:dCU8aEL6q+L9cYTqcVOk8rM9Tp8WdnHOPLiBgp0SGOA= +go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6qT5wthqPoM= +go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA= +go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE= +go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs= +go.opentelemetry.io/proto/otlp v1.7.1 h1:gTOMpGDb0WTBOP8JaO72iL3auEZhVmAQg4ipjOVAtj4= +go.opentelemetry.io/proto/otlp v1.7.1/go.mod h1:b2rVh6rfI/s2pHWNlB7ILJcRALpcNDzKhACevjI+ZnE= +go.uber.org/dig v1.19.0 h1:BACLhebsYdpQ7IROQ1AGPjrXcP5dF80U3gKoFzbaq/4= +go.uber.org/dig v1.19.0/go.mod h1:Us0rSJiThwCv2GteUN0Q7OKvU7n5J4dxZ9JKUXozFdE= +go.uber.org/fx v1.24.0 h1:wE8mruvpg2kiiL1Vqd0CC+tr0/24XIB10Iwp2lLWzkg= +go.uber.org/fx v1.24.0/go.mod h1:AmDeGyS+ZARGKM4tlH4FY2Jr63VjbEDJHtqXTGP5hbo= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= +go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc= +go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= +go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= +golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4= +golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA= +golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8 h1:hVwzHzIUGRjiF7EcUjqNxk3NCfkPxbDKRdnNE1Rpg0U= +golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo= +golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y= +golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= +golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= +golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8= +golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA= +gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= +gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5 h1:BIRfGDEjiHRrk0QKZe3Xv2ieMhtgRGeLcZQ0mIVn4EY= +google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5/go.mod h1:j3QtIyytwqGr1JUDtYXwtMXWPKsEa5LtzIFN1Wn5WvE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5 h1:eaY8u2EuxbRv7c3NiGK0/NedzVsCcV6hDuU5qPX5EGE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5/go.mod h1:M4/wBTSeyLxupu3W3tJtOgB14jILAS/XWPSSa3TAlJc= +google.golang.org/grpc v1.75.0 h1:+TW+dqTd2Biwe6KKfhE5JpiYIBWq865PhKGSXiivqt4= +google.golang.org/grpc v1.75.0/go.mod h1:JtPAzKiq4v1xcAB2hydNlWI2RnF85XXcV0mhKXr2ecQ= +google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc= +google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/admission/distributed.go b/internal/admission/distributed.go new file mode 100644 index 0000000..c268f97 --- /dev/null +++ b/internal/admission/distributed.go @@ -0,0 +1,18 @@ +package admission + +import ( + "context" + "errors" +) + +// ErrRate means the request exhausted a shared rate budget. +var ErrRate = errors.New("shared request rate exceeded") + +// ErrSlots means the request exhausted a shared concurrency budget. +var ErrSlots = errors.New("shared execution slots exhausted") + +// Distributed defines request admission independently of the backing database. +type Distributed interface { + // Enter reserves global/group or user/group budgets and returns a release operation. + Enter(ctx context.Context, group, user string) (func(), error) +} diff --git a/internal/admission/limiter.go b/internal/admission/limiter.go new file mode 100644 index 0000000..97e7f2a --- /dev/null +++ b/internal/admission/limiter.go @@ -0,0 +1,73 @@ +// Package admission bounds request rate and concurrent execution independently of transport. +package admission + +import ( + "fmt" + "sync" + "time" +) + +// Options defines the rate and execution bounds for one limiter. +type Options struct { + // MaxInFlight limits concurrent executions without an admission queue. + MaxInFlight int + // RequestsPerSecond is the token refill rate. + RequestsPerSecond int + // Burst is the maximum number of immediately available tokens. + Burst int +} + +// Limiter owns synchronization for the token bucket and execution slots. +// It knows neither HTTP response codes nor application shutdown state. +type Limiter struct { + mu sync.Mutex + rate float64 + burst float64 + tokens float64 + lastRefill time.Time + slots chan struct{} +} + +// New validates limits and creates a full token bucket with bounded execution slots. +func New(cfg Options) (*Limiter, error) { + if cfg.MaxInFlight <= 0 || cfg.RequestsPerSecond <= 0 || cfg.Burst <= 0 { + return nil, fmt.Errorf("admission limits must be positive") + } + return &Limiter{ + rate: float64(cfg.RequestsPerSecond), + burst: float64(cfg.Burst), + tokens: float64(cfg.Burst), + lastRefill: time.Now(), + slots: make(chan struct{}, cfg.MaxInFlight), + }, nil +} + +// Allow refills the token bucket and consumes a token when one is available. +func (limiter *Limiter) Allow(now time.Time) bool { + limiter.mu.Lock() + defer limiter.mu.Unlock() + elapsed := max(0, now.Sub(limiter.lastRefill).Seconds()) + limiter.tokens = min(limiter.burst, limiter.tokens+elapsed*limiter.rate) + limiter.lastRefill = now + if limiter.tokens < 1 { + return false + } + limiter.tokens-- + return true +} + +// Acquire reserves an execution slot without waiting. +func (limiter *Limiter) Acquire() bool { + select { + case limiter.slots <- struct{}{}: + return true + default: + return false + } +} + +// Release returns an execution slot previously reserved by Acquire. +// Each successful Acquire must be paired with exactly one Release. +func (limiter *Limiter) Release() { + <-limiter.slots +} diff --git a/internal/admission/limiter_test.go b/internal/admission/limiter_test.go new file mode 100644 index 0000000..9fcb7e4 --- /dev/null +++ b/internal/admission/limiter_test.go @@ -0,0 +1,148 @@ +package admission + +import ( + "testing" + "testing/synctest" + "time" +) + +// TestLimiter_Allow checks token exhaustion and refill with deterministic time. +func TestLimiter_Allow(t *testing.T) { + cases := []struct { + name string + elapsed time.Duration + want bool + }{ + {"Negative: exhausted bucket", 0, false}, + {"Negative: partial token", time.Millisecond * 500, false}, + {"Positive: refilled token", time.Second, true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + // Arrange. + limiter := newTestLimiter(t, Options{MaxInFlight: 1, RequestsPerSecond: 1, Burst: 1}) + if !limiter.Allow(time.Now()) { + t.Fatal("fixture token missing") + } + time.Sleep(tc.elapsed) + + // Act. + got := limiter.Allow(time.Now()) + + // Assert. + if got != tc.want { + t.Fatalf("Allow after %s=%v, want=%v", tc.elapsed, got, tc.want) + } + }) + }) + } +} + +// TestLimiter_Acquire checks available, occupied and released execution slots independently. +func TestLimiter_Acquire(t *testing.T) { + t.Run("Positive: free slot", func(t *testing.T) { + // Arrange. + limiter := newTestLimiter(t, Options{MaxInFlight: 1, RequestsPerSecond: 1, Burst: 1}) + + want := true + + // Act. + got := limiter.Acquire() + if got { + t.Cleanup(limiter.Release) + } + + // Assert. + if got != want { + t.Fatalf("Acquire=%v, want=%v", got, want) + } + }) + + t.Run("Negative: occupied slot", func(t *testing.T) { + // Arrange. + limiter := newTestLimiter(t, Options{MaxInFlight: 1, RequestsPerSecond: 1, Burst: 1}) + + if !limiter.Acquire() { + t.Fatal("fixture slot unavailable") + } + + t.Cleanup(limiter.Release) + + want := false + + // Act. + got := limiter.Acquire() + if got { + t.Cleanup(limiter.Release) + } + + // Assert. + if got != want { + t.Fatalf("Acquire=%v, want=%v", got, want) + } + }) + + t.Run("Positive: released slot", func(t *testing.T) { + // Arrange. + limiter := newTestLimiter(t, Options{MaxInFlight: 1, RequestsPerSecond: 1, Burst: 1}) + + if !limiter.Acquire() { + t.Fatal("fixture slot unavailable") + } + + limiter.Release() + + want := true + + // Act. + got := limiter.Acquire() + if got { + t.Cleanup(limiter.Release) + } + + // Assert. + if got != want { + t.Fatalf("Acquire=%v, want=%v", got, want) + } + }) +} + +// newTestLimiter creates a limiter or fails the test on invalid settings. +func newTestLimiter(t *testing.T, options Options) *Limiter { + t.Helper() + limiter, err := New(options) + if err != nil { + t.Fatal(err) + } + return limiter +} + +// TestLimiter_New rejects each nonpositive limit independently. +func TestLimiter_New(t *testing.T) { + cases := []struct { + name string + options Options + }{ + {"Negative: zero concurrency", Options{MaxInFlight: 0, RequestsPerSecond: 1, Burst: 1}}, + {"Negative: negative concurrency", Options{MaxInFlight: -1, RequestsPerSecond: 1, Burst: 1}}, + {"Negative: zero refill rate", Options{MaxInFlight: 1, RequestsPerSecond: 0, Burst: 1}}, + {"Negative: negative refill rate", Options{MaxInFlight: 1, RequestsPerSecond: -1, Burst: 1}}, + {"Negative: zero burst", Options{MaxInFlight: 1, RequestsPerSecond: 1, Burst: 0}}, + {"Negative: negative burst", Options{MaxInFlight: 1, RequestsPerSecond: 1, Burst: -1}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + // Arrange. + options := tc.options + + // Act. + limiter, err := New(options) + + // Assert. + if err == nil || limiter != nil { + t.Fatalf("invalid options accepted: %v, %v", limiter, err) + } + }) + } +} diff --git a/internal/config/config.go b/internal/config/config.go new file mode 100644 index 0000000..339faac --- /dev/null +++ b/internal/config/config.go @@ -0,0 +1,53 @@ +// Package config loads and validates the environment configuration of the application. +package config + +import "fmt" + +// Config groups settings by the component that consumes them. +type Config struct { + // HTTP configures the listener, request limits and shutdown deadline. + HTTP *Controller + // Memory configures the capacity of the process-local store. + Memory *Memory + // Logger configures structured process logging. + Logger *Logger +} + +// Default returns independent configuration sections with the documented defaults. +func Default() *Config { + return &Config{ + HTTP: defaultController(), + Memory: defaultMemory(), + Logger: &Logger{Level: "info"}, + } +} + +// New loads environment overrides and rejects invalid settings before startup. +func New() (*Config, error) { + cfg := Default() + if err := cfg.HTTP.loadEnvironment(); err != nil { + return nil, err + } + if err := cfg.Memory.loadEnvironment(); err != nil { + return nil, err + } + cfg.Logger.loadEnvironment() + if err := cfg.Validate(); err != nil { + return nil, err + } + return cfg, nil +} + +// Validate verifies the presence and invariants of each configuration section. +func (cfg *Config) Validate() error { + if cfg == nil || cfg.HTTP == nil || cfg.Memory == nil || cfg.Logger == nil { + return fmt.Errorf("HTTP, Memory and Logger configuration sections are required") + } + if err := cfg.HTTP.Validate(); err != nil { + return err + } + if err := cfg.Memory.Validate(); err != nil { + return err + } + return cfg.Logger.Validate() +} diff --git a/internal/config/config_test.go b/internal/config/config_test.go new file mode 100644 index 0000000..8396991 --- /dev/null +++ b/internal/config/config_test.go @@ -0,0 +1,241 @@ +package config + +import "testing" + +// TestConfig_New checks individual environment overrides and invalid settings. +func TestConfig_New(t *testing.T) { + t.Run("Positive: APP_PORT override", func(t *testing.T) { + // Arrange. + t.Setenv("APP_PORT", "18080") + + // Act. + cfg, err := New() + + // Assert. + if err != nil { + t.Fatal(err) + } + if cfg.HTTP.Port != 18080 { + t.Fatalf("APP_PORT=%v, want=%v", cfg.HTTP.Port, 18080) + } + }) + t.Run("Positive: REQUEST_TIMEOUT_SECONDS override", func(t *testing.T) { + // Arrange. + t.Setenv("REQUEST_TIMEOUT_SECONDS", "4") + + // Act. + cfg, err := New() + + // Assert. + if err != nil { + t.Fatal(err) + } + if cfg.HTTP.TimeoutSeconds != 4 { + t.Fatalf("REQUEST_TIMEOUT_SECONDS=%v, want=%v", cfg.HTTP.TimeoutSeconds, 4) + } + }) + t.Run("Positive: MAX_MESSAGES override", func(t *testing.T) { + // Arrange. + t.Setenv("MAX_MESSAGES", "25") + + // Act. + cfg, err := New() + + // Assert. + if err != nil { + t.Fatal(err) + } + if cfg.Memory.MaxMessages != 25 { + t.Fatalf("MAX_MESSAGES=%v, want=%v", cfg.Memory.MaxMessages, 25) + } + }) + t.Run("Positive: REQUESTS_PER_SECOND override", func(t *testing.T) { + // Arrange. + t.Setenv("REQUESTS_PER_SECOND", "12") + + // Act. + cfg, err := New() + + // Assert. + if err != nil { + t.Fatal(err) + } + if cfg.HTTP.Admission.RequestsPerSecond != 12 { + t.Fatalf("REQUESTS_PER_SECOND=%v, want=%v", cfg.HTTP.Admission.RequestsPerSecond, 12) + } + }) + t.Run("Positive: LOG_LEVEL override", func(t *testing.T) { + // Arrange. + t.Setenv("LOG_LEVEL", "debug") + + // Act. + cfg, err := New() + + // Assert. + if err != nil { + t.Fatal(err) + } + if cfg.Logger.Level != "debug" { + t.Fatalf("LOG_LEVEL=%v, want=%v", cfg.Logger.Level, "debug") + } + }) + cases := []struct { + name string + key string + value string + }{ + {"port exceeds TCP range", "APP_PORT", "65536"}, + {"zero message capacity", "MAX_MESSAGES", "0"}, + {"nonnumeric body limit", "MAX_BODY_BYTES", "bad"}, + {"shutdown deadline exceeds bound", "SHUTDOWN_TIMEOUT_SECONDS", "10"}, + {"hostname instead of bind IP", "APP_HOST", "hostname"}, + } + for _, tc := range cases { + t.Run("Negative: "+tc.name, func(t *testing.T) { + // Arrange. + t.Setenv(tc.key, tc.value) + + // Act. + cfg, err := New() + + // Assert. + if err == nil || cfg != nil { + t.Fatalf("invalid %s=%q returned config=%+v, error=%v", tc.key, tc.value, cfg, err) + } + }) + } +} + +// TestConfig_Validate rejects one missing section or invalid component per scenario. +func TestConfig_Validate(t *testing.T) { + t.Run("Negative: nil config", func(t *testing.T) { + // Arrange. + var cfg *Config + + // Act. + err := cfg.Validate() + + // Assert. + if err == nil { + t.Fatalf("accepted %s", "nil config") + } + }) + + t.Run("Negative: missing HTTP", func(t *testing.T) { + // Arrange. + cfg := Default() + + cfg.HTTP = nil + + // Act. + err := cfg.Validate() + + // Assert. + if err == nil { + t.Fatalf("accepted %s", "missing HTTP") + } + }) + + t.Run("Negative: missing Memory", func(t *testing.T) { + // Arrange. + cfg := Default() + + cfg.Memory = nil + + // Act. + err := cfg.Validate() + + // Assert. + if err == nil { + t.Fatalf("accepted %s", "missing Memory") + } + }) + + t.Run("Negative: missing Logger", func(t *testing.T) { + // Arrange. + cfg := Default() + + cfg.Logger = nil + + // Act. + err := cfg.Validate() + + // Assert. + if err == nil { + t.Fatalf("accepted %s", "missing Logger") + } + }) + + t.Run("Negative: unknown log level", func(t *testing.T) { + // Arrange. + cfg := Default() + + cfg.Logger.Level = "unknown" + + // Act. + err := cfg.Validate() + + // Assert. + if err == nil { + t.Fatalf("accepted %s", "unknown log level") + } + }) + + t.Run("Negative: zero storage capacity", func(t *testing.T) { + // Arrange. + cfg := Default() + + cfg.Memory.MaxUsers = 0 + + // Act. + err := cfg.Validate() + + // Assert. + if err == nil { + t.Fatalf("accepted %s", "zero storage capacity") + } + }) + + t.Run("Negative: zero concurrency", func(t *testing.T) { + // Arrange. + cfg := Default() + + cfg.HTTP.Admission.MaxInFlight = 0 + + // Act. + err := cfg.Validate() + + // Assert. + if err == nil { + t.Fatalf("accepted %s", "zero concurrency") + } + }) +} + +// TestLogger_Validate rejects an absent logging section without panicking. +func TestLogger_Validate(t *testing.T) { + // Arrange. + var cfg *Logger + + // Act. + err := cfg.Validate() + + // Assert. + if err == nil { + t.Fatal("nil logger accepted") + } +} + +// TestMemory_Validate rejects absent storage settings without panicking. +func TestMemory_Validate(t *testing.T) { + // Arrange. + var cfg *Memory + + // Act. + err := cfg.Validate() + + // Assert. + if err == nil { + t.Fatal("nil memory configuration accepted") + } +} diff --git a/internal/config/controller.go b/internal/config/controller.go new file mode 100644 index 0000000..d1538dd --- /dev/null +++ b/internal/config/controller.go @@ -0,0 +1,160 @@ +package config + +import ( + "fmt" + "net" + "os" + "strconv" + "time" +) + +const ( + maxTCPPort = 65535 + maxShutdownTimeoutSeconds = 9 + maxRequestTimeoutSeconds = 15 +) + +// Controller contains settings owned exclusively by the HTTP transport. +type Controller struct { + // InstanceID identifies this API process to the load balancer diagnostics. + InstanceID string + // Host is the IP address on which the listener binds. + Host string + // Port is the TCP port, in the range 1..65535. + Port int + // TimeoutSeconds limits request reads, response writes and idle connections. + TimeoutSeconds int + // ShutdownTimeoutSeconds bounds draining of active requests; it must not exceed nine seconds. + ShutdownTimeoutSeconds int + // MaxBodyBytes is the maximum accepted request body size before decoding. + MaxBodyBytes int + // MaxConnections bounds simultaneous connections handled by fasthttp. + MaxConnections int + // Admission controls business request throughput and execution concurrency. + Admission Admission +} + +// Admission defines shared request policies; health endpoints bypass them. +type Admission struct { + // MaxInFlight is the maximum number of executing requests, with no waiting queue. + MaxInFlight int + // RequestsPerSecond is the token refill rate for the global admission bucket. + RequestsPerSecond int + // Burst is the bucket capacity and maximum immediate request burst. + Burst int + // AuthRate limits public registration/login attempts per second. + AuthRate int + // AuthSlots bounds concurrent password KDF work globally. + AuthSlots int + // UserRate limits each authenticated account across all groups. + UserRate int + // UserSlots bounds concurrency across one account's groups. + UserSlots int + // UserWriteRate limits one account's write attempts per second. + UserWriteRate int + // UserWriteSlots bounds one account's concurrent writes. + UserWriteSlots int + // UserAuthRate limits session-management operations per account. + UserAuthRate int + // UserAuthSlots bounds one account's session-management concurrency. + UserAuthSlots int +} + +// defaultController returns the default listener, timeout and admission settings. +func defaultController() *Controller { + return &Controller{ + Host: "0.0.0.0", + Port: 8080, + TimeoutSeconds: 3, + ShutdownTimeoutSeconds: 5, + MaxBodyBytes: 10 << 20, + MaxConnections: 16, + Admission: Admission{ + MaxInFlight: 32, + RequestsPerSecond: 1000, + Burst: 100, + AuthRate: 100, + AuthSlots: 4, + UserRate: 100, + UserSlots: 8, + UserWriteRate: 50, + UserWriteSlots: 4, + UserAuthRate: 20, + UserAuthSlots: 2, + }, + } +} + +// loadEnvironment applies environment overrides to this configuration section. +func (cfg *Controller) loadEnvironment() error { + cfg.InstanceID = os.Getenv("INSTANCE_ID") + if host, exists := os.LookupEnv("APP_HOST"); exists { + cfg.Host = host + } + return loadIntegers( + integerSetting{"APP_PORT", &cfg.Port}, + integerSetting{"REQUEST_TIMEOUT_SECONDS", &cfg.TimeoutSeconds}, + integerSetting{"SHUTDOWN_TIMEOUT_SECONDS", &cfg.ShutdownTimeoutSeconds}, + integerSetting{"MAX_BODY_BYTES", &cfg.MaxBodyBytes}, + integerSetting{"MAX_CONNECTIONS", &cfg.MaxConnections}, + integerSetting{"MAX_IN_FLIGHT", &cfg.Admission.MaxInFlight}, + integerSetting{"REQUESTS_PER_SECOND", &cfg.Admission.RequestsPerSecond}, + integerSetting{"REQUEST_BURST", &cfg.Admission.Burst}, + integerSetting{"AUTH_RATE", &cfg.Admission.AuthRate}, + integerSetting{"AUTH_SLOTS", &cfg.Admission.AuthSlots}, + integerSetting{"USER_RATE", &cfg.Admission.UserRate}, + integerSetting{"USER_SLOTS", &cfg.Admission.UserSlots}, + integerSetting{"USER_WRITE_RATE", &cfg.Admission.UserWriteRate}, + integerSetting{"USER_WRITE_SLOTS", &cfg.Admission.UserWriteSlots}, + integerSetting{"USER_AUTH_RATE", &cfg.Admission.UserAuthRate}, + integerSetting{"USER_AUTH_SLOTS", &cfg.Admission.UserAuthSlots}, + ) +} + +// Validate checks listener addresses, timeouts and admission bounds. +func (cfg *Controller) Validate() error { + if cfg == nil { + return fmt.Errorf("HTTP configuration is required") + } + if cfg.TimeoutSeconds > maxRequestTimeoutSeconds { + return fmt.Errorf("REQUEST_TIMEOUT_SECONDS must not exceed %d", maxRequestTimeoutSeconds) + } + if net.ParseIP(cfg.Host) == nil { + return fmt.Errorf("APP_HOST must be an IP address") + } + if cfg.Port < 1 || cfg.Port > maxTCPPort { + return fmt.Errorf("APP_PORT must be 1..%d", maxTCPPort) + } + if cfg.ShutdownTimeoutSeconds < 1 || cfg.ShutdownTimeoutSeconds > maxShutdownTimeoutSeconds { + return fmt.Errorf("SHUTDOWN_TIMEOUT_SECONDS must be 1..%d", maxShutdownTimeoutSeconds) + } + settings := []integerSetting{ + {"REQUEST_TIMEOUT_SECONDS", &cfg.TimeoutSeconds}, + {"MAX_BODY_BYTES", &cfg.MaxBodyBytes}, + {"MAX_CONNECTIONS", &cfg.MaxConnections}, + {"MAX_IN_FLIGHT", &cfg.Admission.MaxInFlight}, + {"REQUESTS_PER_SECOND", &cfg.Admission.RequestsPerSecond}, + {"REQUEST_BURST", &cfg.Admission.Burst}, + } + for _, setting := range settings { + if err := validatePositive(setting.name, *setting.target); err != nil { + return err + } + } + return nil +} + +// Bind returns the listener address, including brackets for IPv6 hosts. +func (cfg *Controller) Bind() string { + return net.JoinHostPort(cfg.Host, strconv.Itoa(cfg.Port)) +} + +// Timeout returns the configured read, write and idle timeout. +func (cfg *Controller) Timeout() time.Duration { + return time.Duration(cfg.TimeoutSeconds) * time.Second +} + +// ShutdownTimeout returns the maximum time allowed to drain active requests. +func (cfg *Controller) ShutdownTimeout() time.Duration { + return time.Duration(cfg.ShutdownTimeoutSeconds) * time.Second +} diff --git a/internal/config/environment.go b/internal/config/environment.go new file mode 100644 index 0000000..db5dfd8 --- /dev/null +++ b/internal/config/environment.go @@ -0,0 +1,41 @@ +package config + +import ( + "fmt" + "os" + "strconv" +) + +const maxConfiguredInteger = 1 << 30 + +type integerSetting struct { + name string + target *int +} + +// loadIntegers parses configured integer overrides and reports invalid values. +func loadIntegers(settings ...integerSetting) error { + for _, setting := range settings { + raw, exists := os.LookupEnv(setting.name) + if !exists { + continue + } + value, err := strconv.Atoi(raw) + if err != nil { + return fmt.Errorf("%s must be an integer", setting.name) + } + if err := validatePositive(setting.name, value); err != nil { + return err + } + *setting.target = value + } + return nil +} + +// validatePositive checks that a setting is positive and within the supported range. +func validatePositive(name string, value int) error { + if value < 1 || value > maxConfiguredInteger { + return fmt.Errorf("%s must be between 1 and %d", name, maxConfiguredInteger) + } + return nil +} diff --git a/internal/config/logger.go b/internal/config/logger.go new file mode 100644 index 0000000..58f2450 --- /dev/null +++ b/internal/config/logger.go @@ -0,0 +1,32 @@ +package config + +import ( + "fmt" + "os" +) + +// Logger configures the minimum severity written as structured JSON to stderr. +type Logger struct { + // Level is one of debug, info, warn or error; the default is info. + Level string +} + +// loadEnvironment applies environment overrides to this configuration section. +func (cfg *Logger) loadEnvironment() { + if level, exists := os.LookupEnv("LOG_LEVEL"); exists { + cfg.Level = level + } +} + +// Validate rejects unsupported levels rather than silently changing verbosity. +func (cfg *Logger) Validate() error { + if cfg == nil { + return fmt.Errorf("Logger configuration is required") + } + switch cfg.Level { + case "debug", "info", "warn", "error": + return nil + default: + return fmt.Errorf("LOG_LEVEL must be debug, info, warn or error") + } +} diff --git a/internal/config/memory.go b/internal/config/memory.go new file mode 100644 index 0000000..401e67d --- /dev/null +++ b/internal/config/memory.go @@ -0,0 +1,54 @@ +package config + +import "fmt" + +// Memory defines logical capacity limits; MaxStateBytes is not a measurement of RSS. +type Memory struct { + // MaxUsers limits the number of registered users. + MaxUsers int + // MaxChats limits the number of immutable chats. + MaxChats int + // MaxMessages limits the total number of messages across all chats. + MaxMessages int + // MaxStateBytes limits the logical storage budget, including estimated record overhead. + MaxStateBytes int +} + +// defaultMemory returns the default storage capacity limits. +func defaultMemory() *Memory { + return &Memory{ + MaxUsers: 10000, + MaxChats: 10000, + MaxMessages: 100000, + MaxStateBytes: 32 << 20, + } +} + +// loadEnvironment applies environment overrides to this configuration section. +func (cfg *Memory) loadEnvironment() error { + return loadIntegers( + integerSetting{"MAX_USERS", &cfg.MaxUsers}, + integerSetting{"MAX_CHATS", &cfg.MaxChats}, + integerSetting{"MAX_MESSAGES", &cfg.MaxMessages}, + integerSetting{"MAX_STATE_BYTES", &cfg.MaxStateBytes}, + ) +} + +// Validate rejects disabled, negative and excessively large storage limits. +func (cfg *Memory) Validate() error { + if cfg == nil { + return fmt.Errorf("Memory configuration is required") + } + settings := []integerSetting{ + {"MAX_USERS", &cfg.MaxUsers}, + {"MAX_CHATS", &cfg.MaxChats}, + {"MAX_MESSAGES", &cfg.MaxMessages}, + {"MAX_STATE_BYTES", &cfg.MaxStateBytes}, + } + for _, setting := range settings { + if err := validatePositive(setting.name, *setting.target); err != nil { + return err + } + } + return nil +} diff --git a/internal/config/objectstore.go b/internal/config/objectstore.go new file mode 100644 index 0000000..169d9cd --- /dev/null +++ b/internal/config/objectstore.go @@ -0,0 +1,35 @@ +package config + +import ( + "fmt" + "os" +) + +// ObjectStore contains only the S3 adapter's connection settings. +type ObjectStore struct { + // Endpoint is host:port, without a URL scheme. + Endpoint string + // Bucket is the private application bucket. + Bucket string + // AccessKey is the restricted service identity. + AccessKey string + // SecretKey is supplied by the runtime secret provider. + SecretKey string + // Secure enables certificate-verified TLS. + Secure bool +} + +// NewObjectStore validates S3 settings independently of their use. +func NewObjectStore() (*ObjectStore, error) { + cfg := &ObjectStore{ + Endpoint: os.Getenv("S3_ENDPOINT"), + Bucket: os.Getenv("S3_BUCKET"), + AccessKey: os.Getenv("S3_ACCESS_KEY"), + SecretKey: os.Getenv("S3_SECRET_KEY"), + Secure: os.Getenv("S3_TLS") != "false", + } + if cfg.Endpoint == "" || cfg.Bucket == "" || cfg.AccessKey == "" || cfg.SecretKey == "" { + return nil, fmt.Errorf("S3 endpoint, bucket and scoped credentials required") + } + return cfg, nil +} diff --git a/internal/config/objectstore_test.go b/internal/config/objectstore_test.go new file mode 100644 index 0000000..8836738 --- /dev/null +++ b/internal/config/objectstore_test.go @@ -0,0 +1,24 @@ +package config + +import "testing" + +// TestObjectStoreConfiguration verifies mandatory credentials and opt-in local plaintext. +func TestObjectStoreConfiguration(t *testing.T) { + t.Run("missing credentials", func(t *testing.T) { + t.Setenv("S3_ENDPOINT", "") + if _, err := NewObjectStore(); err == nil { + t.Fatal("missing endpoint accepted") + } + }) + t.Run("local configuration", func(t *testing.T) { + t.Setenv("S3_ENDPOINT", "localhost:9000") + t.Setenv("S3_BUCKET", "attachments") + t.Setenv("S3_ACCESS_KEY", "test") + t.Setenv("S3_SECRET_KEY", "synthetic-only") + t.Setenv("S3_TLS", "false") + cfg, err := NewObjectStore() + if err != nil || cfg.Secure || cfg.Bucket != "attachments" { + t.Fatalf("invalid result: %v", err) + } + }) +} diff --git a/internal/config/persistence.go b/internal/config/persistence.go new file mode 100644 index 0000000..6950d4d --- /dev/null +++ b/internal/config/persistence.go @@ -0,0 +1,55 @@ +package config + +import ( + "fmt" + "os" + "strings" +) + +// Persistence contains database, cache and session settings. +type Persistence struct { + // CursorKey is the shared base64 signing key supplied through a secret. + CursorKey string + // DatabaseURL identifies the PostgreSQL database. + DatabaseURL string + // Shards maps stable identities to Valkey addresses. + Shards map[string]string + // Coordinator names the stable global admission shard. + Coordinator string + // PoolSize bounds connections per dependency. + PoolSize int + // TimeoutMilliseconds bounds each dependency operation. + TimeoutMilliseconds int + // SessionTTLSeconds sets the absolute session lifetime. + SessionTTLSeconds int +} + +// NewPersistence loads dependency settings separately from HTTP configuration. +func NewPersistence() (*Persistence, error) { + cfg := &Persistence{ + CursorKey: os.Getenv("CURSOR_SIGNING_KEY"), + DatabaseURL: os.Getenv("DATABASE_URL"), + Coordinator: os.Getenv("VALKEY_COORDINATOR"), + Shards: make(map[string]string), + PoolSize: 8, + TimeoutMilliseconds: 1000, + SessionTTLSeconds: 3600, + } + if cfg.Coordinator == "" { + cfg.Coordinator = "shard0" + } + for _, entry := range strings.Split(os.Getenv("VALKEY_SHARDS"), ",") { + id, address, ok := strings.Cut(entry, "=") + if !ok || id == "" || address == "" || cfg.Shards[id] != "" { + return nil, fmt.Errorf("VALKEY_SHARDS requires unique id=address entries") + } + cfg.Shards[id] = address + } + if cfg.DatabaseURL == "" || cfg.Shards[cfg.Coordinator] == "" { + return nil, fmt.Errorf("DATABASE_URL and a configured VALKEY_COORDINATOR are required") + } + if err := loadIntegers(integerSetting{"DEPENDENCY_POOL_SIZE", &cfg.PoolSize}, integerSetting{"DEPENDENCY_TIMEOUT_MS", &cfg.TimeoutMilliseconds}, integerSetting{"SESSION_TTL_SECONDS", &cfg.SessionTTLSeconds}); err != nil { + return nil, err + } + return cfg, nil +} diff --git a/internal/config/persistence_test.go b/internal/config/persistence_test.go new file mode 100644 index 0000000..6f9b63f --- /dev/null +++ b/internal/config/persistence_test.go @@ -0,0 +1,34 @@ +package config + +import "testing" + +// TestPersistenceConfiguration validates independent environment configurations. +func TestPersistenceConfiguration(t *testing.T) { + cases := []struct { + name string + database string + shards string + coordinator string + ttl string + valid bool + }{ + {"defaults", "postgres://localhost/test", "shard0=localhost:6379", "", "3600", true}, + {"missing database", "", "shard0=localhost:6379", "", "3600", false}, + {"missing coordinator", "postgres://localhost/test", "shard1=localhost:6379", "shard0", "3600", false}, + {"duplicate shard", "postgres://localhost/test", "shard0=a,shard0=b", "", "3600", false}, + {"invalid topology", "postgres://localhost/test", "wrong", "", "3600", false}, + {"invalid ttl", "postgres://localhost/test", "shard0=a", "", "no", false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Setenv("DATABASE_URL", tc.database) + t.Setenv("VALKEY_SHARDS", tc.shards) + t.Setenv("VALKEY_COORDINATOR", tc.coordinator) + t.Setenv("SESSION_TTL_SECONDS", tc.ttl) + _, err := NewPersistence() + if (err == nil) != tc.valid { + t.Fatalf("unexpected validation result: %v", err) + } + }) + } +} diff --git a/internal/config/telemetry.go b/internal/config/telemetry.go new file mode 100644 index 0000000..0c0dc41 --- /dev/null +++ b/internal/config/telemetry.go @@ -0,0 +1,47 @@ +package config + +import ( + "fmt" + "math" + "net/url" + "os" + "strconv" + + "github.com/google/uuid" +) + +// Telemetry holds validated SDK settings; absence of Endpoint disables export. +type Telemetry struct { + // Endpoint is the private Collector OTLP HTTP base URL. + Endpoint string + // Service distinguishes API and worker resource identity. + Service string + // Version identifies the application build. + Version string + // Instance is stable for one process lifetime. + Instance string + // SampleRatio is bounded to [0,1]. + SampleRatio float64 +} + +// NewTelemetry validates explicit export settings without contacting the backend. +func NewTelemetry(service string) (*Telemetry, error) { + cfg := &Telemetry{Endpoint: os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT"), Service: service, Version: os.Getenv("APP_VERSION"), Instance: uuid.NewString(), SampleRatio: 1} + if cfg.Version == "" { + cfg.Version = "lab6" + } + if raw := os.Getenv("OTEL_SAMPLE_RATIO"); raw != "" { + value, err := strconv.ParseFloat(raw, 64) + if err != nil || math.IsNaN(value) || value < 0 || value > 1 { + return nil, fmt.Errorf("OTEL_SAMPLE_RATIO must be in [0,1]") + } + cfg.SampleRatio = value + } + if cfg.Endpoint != "" { + u, err := url.Parse(cfg.Endpoint) + if err != nil || u.Host == "" || (u.Scheme != "http" && u.Scheme != "https") || u.User != nil || u.RawQuery != "" { + return nil, fmt.Errorf("OTLP endpoint must be an HTTP(S) base URL without credentials") + } + } + return cfg, nil +} diff --git a/internal/config/telemetry_test.go b/internal/config/telemetry_test.go new file mode 100644 index 0000000..451385c --- /dev/null +++ b/internal/config/telemetry_test.go @@ -0,0 +1,36 @@ +package config + +import "testing" + +// TestTelemetryConfiguration validates bounded sampling and credential-free Collector addresses. +func TestTelemetryConfiguration(t *testing.T) { + for _, tc := range []struct { + name string + endpoint string + ratio string + wantError bool + }{ + {"disabled", "", "", false}, + {"enabled", "http://collector:4318", "0.5", false}, + {"bad ratio", "", "bad", true}, + {"NaN", "", "NaN", true}, + {"negative", "", "-0.1", true}, + {"over one", "", "1.1", true}, + {"credentials", "http://user:pass@collector:4318", "", true}, + {"query", "http://collector:4318?token=x", "", true}, + {"bad scheme", "ftp://collector", "", true}, + {"relative", "collector:4318", "", true}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Setenv("OTEL_EXPORTER_OTLP_ENDPOINT", tc.endpoint) + t.Setenv("OTEL_SAMPLE_RATIO", tc.ratio) + cfg, err := NewTelemetry("messenger-api") + if (err != nil) != tc.wantError { + t.Fatalf("configuration: %v", err) + } + if err == nil && (cfg.Instance == "" || cfg.Version == "" || cfg.Service != "messenger-api") { + t.Fatal("resource identity missing") + } + }) + } +} diff --git a/internal/config/worker.go b/internal/config/worker.go new file mode 100644 index 0000000..a1b6407 --- /dev/null +++ b/internal/config/worker.go @@ -0,0 +1,25 @@ +package config + +import ( + "fmt" + "os" + + "github.com/google/uuid" +) + +// Worker contains the broker address and a process-unique consumer identity. +type Worker struct { + // BrokerAddress is a private Valkey Streams endpoint. + BrokerAddress string + // Consumer is never used as an unbounded metric label. + Consumer string +} + +// NewWorker requires an explicit broker separate from the admission shards. +func NewWorker() (*Worker, error) { + address := os.Getenv("BROKER_ADDRESS") + if address == "" { + return nil, fmt.Errorf("BROKER_ADDRESS is required") + } + return &Worker{BrokerAddress: address, Consumer: uuid.NewString()}, nil +} diff --git a/internal/config/worker_test.go b/internal/config/worker_test.go new file mode 100644 index 0000000..018584e --- /dev/null +++ b/internal/config/worker_test.go @@ -0,0 +1,24 @@ +package config + +import "testing" + +// TestWorkerConfiguration requires a broker and gives each process an independent consumer ID. +func TestWorkerConfiguration(t *testing.T) { + t.Run("missing", func(t *testing.T) { + t.Setenv("BROKER_ADDRESS", "") + if _, err := NewWorker(); err == nil { + t.Fatal("missing broker accepted") + } + }) + t.Run("configured", func(t *testing.T) { + t.Setenv("BROKER_ADDRESS", "broker:6379") + first, err := NewWorker() + if err != nil { + t.Fatal(err) + } + second, err := NewWorker() + if err != nil || first.Consumer == second.Consumer || first.BrokerAddress != "broker:6379" { + t.Fatal("invalid consumer configuration") + } + }) +} diff --git a/internal/controller/fx.go b/internal/controller/fx.go new file mode 100644 index 0000000..e99b226 --- /dev/null +++ b/internal/controller/fx.go @@ -0,0 +1,8 @@ +package controller + +import "go.uber.org/fx" + +// RunControllerFx lets Fx own startup, signal handling and graceful shutdown. +func RunControllerFx(lc fx.Lifecycle, ctrl Controller) { + lc.Append(fx.Hook{OnStart: ctrl.Start, OnStop: ctrl.Stop}) +} diff --git a/internal/controller/http/fiber/attachments.go b/internal/controller/http/fiber/attachments.go new file mode 100644 index 0000000..b34d2bb --- /dev/null +++ b/internal/controller/http/fiber/attachments.go @@ -0,0 +1,123 @@ +package fiber + +import ( + "context" + "io" + "net/http" + "time" + + "github.com/gofiber/fiber/v2" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/model/dto" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/service" +) + +// Attachments defines file operations required by the HTTP boundary. +type Attachments interface { + // Upload validates and publishes an original. + Upload(context.Context, string, string, string, string, []byte) (dto.Attachment, error) + // Get authorizes access to attachment metadata. + Get(context.Context, string, string) (dto.Attachment, error) + // Download returns an authorized original stream. + Download(context.Context, string, string, string) (dto.Attachment, io.ReadCloser, error) +} + +// WithAttachments binds the file service during composition before server startup. +func (ctrl *Controller) WithAttachments(attachments Attachments) *Controller { + ctrl.attachments = attachments + return ctrl +} + +type attachmentResponse struct { + // ID identifies the uploaded file. + ID string `json:"id"` + // ChatID defines the chat authorization scope. + ChatID string `json:"chat_id"` + // OwnerID identifies the original uploader. + OwnerID string `json:"owner_id"` + // Filename is the validated display name. + Filename string `json:"filename"` + // ContentType describes the verified original format. + ContentType string `json:"content_type"` + // SizeBytes is the encoded original length. + SizeBytes int64 `json:"size_bytes"` + // SHA256 is the original lowercase checksum. + SHA256 string `json:"sha256"` + // Status is the durable publication state. + Status string `json:"status"` + // Variants lists generated variants, empty in lab4. + Variants []dto.Variant `json:"variants"` + // ErrorCode is null for a ready original. + ErrorCode *string `json:"error_code"` + // CreatedAt is the reservation time. + CreatedAt time.Time `json:"created_at"` + // UpdatedAt is the publication time. + UpdatedAt time.Time `json:"updated_at"` +} + +// attachmentResponseOf deliberately omits private S3 coordinates. +func attachmentResponseOf(a dto.Attachment) attachmentResponse { + return attachmentResponse{ + ID: a.ID.String(), + ChatID: a.ChatID.String(), + OwnerID: a.OwnerID.String(), + Filename: a.Filename, + ContentType: a.ContentType, + SizeBytes: a.SizeBytes, + SHA256: a.SHA256, + Status: a.Status, + Variants: publicVariants(a.Variants), + ErrorCode: a.ErrorCode, + CreatedAt: a.CreatedAt, + UpdatedAt: a.UpdatedAt, + } +} + +// handleUpload passes raw bounded bytes to application validation. +func (ctrl *Controller) handleUpload(ctx *fiber.Ctx) error { + if ctrl.attachments == nil { + return service.ErrUnavailable + } + a, err := ctrl.attachments.Upload(ctx.UserContext(), identity(ctx), ctx.Params("chat_id"), ctx.Query("filename"), ctx.Get("Content-Type"), ctx.Body()) + if err != nil { + return err + } + return ctx.Status(http.StatusAccepted).JSON(attachmentResponseOf(a)) +} + +// handleAttachment returns only the public metadata of an accessible file. +func (ctrl *Controller) handleAttachment(ctx *fiber.Ctx) error { + if ctrl.attachments == nil { + return service.ErrUnavailable + } + a, err := ctrl.attachments.Get(ctx.UserContext(), identity(ctx), ctx.Params("id")) + if err != nil { + return err + } + return ctx.JSON(attachmentResponseOf(a)) +} + +// handleContent streams original bytes without redirecting around authorization. +func (ctrl *Controller) handleContent(ctx *fiber.Ctx) error { + if ctrl.attachments == nil { + return service.ErrUnavailable + } + a, reader, err := ctrl.attachments.Download(ctx.UserContext(), identity(ctx), ctx.Params("id"), ctx.Query("variant")) + if err != nil { + return err + } + ctx.Set("Content-Type", a.ContentType) + ctx.Set("X-Content-Type-Options", "nosniff") + return ctx.SendStream(&responseStream{Reader: reader, cleanup: func() { + }}, int(a.SizeBytes)) +} + +// publicVariants copies generated metadata without exposing storage keys. +func publicVariants(variants []dto.Variant) []dto.Variant { + result := make([]dto.Variant, 0, len(variants)) + for _, variant := range variants { + variant.ObjectKey = "" + result = append(result, variant) + } + return result +} diff --git a/internal/controller/http/fiber/attachments_test.go b/internal/controller/http/fiber/attachments_test.go new file mode 100644 index 0000000..7b993f4 --- /dev/null +++ b/internal/controller/http/fiber/attachments_test.go @@ -0,0 +1,102 @@ +package fiber + +import ( + "bytes" + "context" + "io" + "net/http" + "net/http/httptest" + "testing" + + "github.com/google/uuid" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/model/dto" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/service" +) + +type attachmentFixture struct { + attachment dto.Attachment + body []byte + err error +} + +// Upload returns the configured application outcome for an HTTP mapping scenario. +func (f attachmentFixture) Upload(context.Context, string, string, string, string, []byte) (dto.Attachment, error) { + return f.attachment, f.err +} + +// Get returns the configured metadata outcome for an HTTP mapping scenario. +func (f attachmentFixture) Get(context.Context, string, string) (dto.Attachment, error) { + return f.attachment, f.err +} + +// Download returns a stream that fails if the request context is canceled before transmission. +func (f attachmentFixture) Download(ctx context.Context, _, _, _ string) (dto.Attachment, io.ReadCloser, error) { + return f.attachment, &contextReader{Reader: bytes.NewReader(f.body), ctx: ctx}, f.err +} + +type contextReader struct { + *bytes.Reader + ctx context.Context +} + +// Read verifies the admission context survives until the response is actually read. +func (r *contextReader) Read(p []byte) (int, error) { + if err := r.ctx.Err(); err != nil { + return 0, err + } + return r.Reader.Read(p) +} + +// Close makes the synthetic response stream an owned closer. +func (r *contextReader) Close() error { + return nil +} + +// TestAttachmentHTTPMapping exercises early errors, successful metadata and byte-exact streaming. +func TestAttachmentHTTPMapping(t *testing.T) { + cases := []struct { + name string + method string + path string + fixture *attachmentFixture + status int + }{ + {"upload unavailable", "POST", "/api/v1/chats/test/attachments", nil, http.StatusServiceUnavailable}, + {"metadata unavailable", "GET", "/api/v1/attachments/test", nil, http.StatusServiceUnavailable}, + {"download unavailable", "GET", "/api/v1/attachments/test/content", nil, http.StatusServiceUnavailable}, + {"upload too large", "POST", "/api/v1/chats/test/attachments", &attachmentFixture{err: service.ErrTooLarge}, http.StatusRequestEntityTooLarge}, + {"upload unsupported", "POST", "/api/v1/chats/test/attachments", &attachmentFixture{err: service.ErrMedia}, http.StatusUnsupportedMediaType}, + {"metadata missing", "GET", "/api/v1/attachments/test", &attachmentFixture{err: service.ErrNotFound}, http.StatusNotFound}, + {"download missing", "GET", "/api/v1/attachments/test/content", &attachmentFixture{err: service.ErrNotFound}, http.StatusNotFound}, + {"upload queued", "POST", "/api/v1/chats/test/attachments", &attachmentFixture{attachment: dto.Attachment{ID: uuid.New(), Status: "queued"}}, http.StatusAccepted}, + {"metadata ready", "GET", "/api/v1/attachments/test", &attachmentFixture{attachment: dto.Attachment{ID: uuid.New(), Status: "ready"}}, http.StatusOK}, + {"download stream", "GET", "/api/v1/attachments/test/content", &attachmentFixture{attachment: dto.Attachment{SizeBytes: 4, ContentType: "image/png"}, body: []byte{0, 1, 2, 3}}, http.StatusOK}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + ctrl := testController(t, nil) + user := call(t, ctrl, http.MethodPost, "/api/v1/users", "", map[string]any{"username": "test_user", "display_name": "Test", "password": "synthetic-password"}, http.StatusCreated) + if tc.fixture != nil { + ctrl.WithAttachments(*tc.fixture) + } + req := httptest.NewRequest(tc.method, tc.path, bytes.NewReader([]byte("fixture"))) + req.Header.Set("Authorization", "Bearer "+user["id"].(string)) + req.Header.Set("Content-Type", "image/png") + response, err := ctrl.server.Test(req, 10000) + if err != nil { + t.Fatal(err) + } + body, err := io.ReadAll(response.Body) + if closeErr := response.Body.Close(); closeErr != nil { + t.Fatal(closeErr) + } + if err != nil || response.StatusCode != tc.status { + t.Fatalf("response status=%d err=%v", response.StatusCode, err) + } + if tc.fixture != nil && tc.fixture.body != nil && !bytes.Equal(body, tc.fixture.body) { + t.Fatalf("stream changed: %v", body) + } + }) + } +} diff --git a/internal/controller/http/fiber/auth.go b/internal/controller/http/fiber/auth.go new file mode 100644 index 0000000..1805c0b --- /dev/null +++ b/internal/controller/http/fiber/auth.go @@ -0,0 +1,76 @@ +package fiber + +import ( + "encoding/base64" + "net/http" + "strings" + + "github.com/gofiber/fiber/v2" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/service" +) + +const identityKey = "authenticated-user" + +// HandleLogin structurally decodes Basic credentials and delegates authentication. +func (ctrl *Controller) HandleLogin(ctx *fiber.Ctx) error { + scheme, encoded, ok := strings.Cut(ctx.Get(fiber.HeaderAuthorization), " ") + if !ok || !strings.EqualFold(scheme, "Basic") { + return service.ErrUnauthorized + } + decoded, err := base64.StdEncoding.DecodeString(encoded) + if err != nil { + return service.ErrUnauthorized + } + username, password, ok := strings.Cut(string(decoded), ":") + if !ok { + return service.ErrUnauthorized + } + if len(ctx.Body()) != 0 { + return service.ErrInvalid + } + session, err := ctrl.auth.Login(ctx.UserContext(), username, password) + if err != nil { + return err + } + return ctx.Status(http.StatusCreated).JSON(fiber.Map{ + "token": session.Token, + "token_type": "Bearer", + "expires_at": session.ExpiresAt, + "user": userResponse(session.User), + }) +} + +// HandleLogout revokes the current authenticated token before returning no content. +func (ctrl *Controller) HandleLogout(ctx *fiber.Ctx) error { + if err := ctrl.auth.Logout(ctx.UserContext(), bearer(ctx)); err != nil { + return err + } + return ctx.SendStatus(http.StatusNoContent) +} + +// bearer extracts a Bearer token without interpreting its identity or validity. +func bearer(ctx *fiber.Ctx) string { + scheme, token, ok := strings.Cut(ctx.Get(fiber.HeaderAuthorization), " ") + if !ok || !strings.EqualFold(scheme, "Bearer") { + return "" + } + return token +} + +// identity reads only the identity produced by successful session authentication. +func identity(ctx *fiber.Ctx) string { + value, _ := ctx.Locals(identityKey).(string) + return value +} + +// requestGroup selects a bounded policy group from the HTTP operation. +func requestGroup(ctx *fiber.Ctx) string { + if ctx.Path() == "/api/v1/users" || strings.HasPrefix(ctx.Path(), "/api/v1/auth/") { + return "auth" + } + if ctx.Method() == http.MethodGet { + return "read" + } + return "write" +} diff --git a/internal/controller/http/fiber/auth_test.go b/internal/controller/http/fiber/auth_test.go new file mode 100644 index 0000000..bca8e32 --- /dev/null +++ b/internal/controller/http/fiber/auth_test.go @@ -0,0 +1,161 @@ +package fiber + +import ( + "context" + "encoding/base64" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/google/uuid" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/model/dto" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/service" +) + +type sessionFixture struct { + fixtureAuthentication + login func(string, string) (dto.SessionDTO, error) + logoutError error +} + +// Login delegates the credentials decoded by the controller to the arranged scenario. +func (f sessionFixture) Login(_ context.Context, username, password string) (dto.SessionDTO, error) { + return f.login(username, password) +} + +// Logout returns the prescribed revocation outcome. +func (f sessionFixture) Logout(context.Context, string) error { + return f.logoutError +} + +// TestLoginHTTPChecksBasicEncoding verifies independent wire-level login scenarios. +func TestLoginHTTPChecksBasicEncoding(t *testing.T) { + cases := []struct { + name string + header string + body string + status int + }{ + {"missing header", "", "", http.StatusUnauthorized}, + {"wrong scheme", "Bearer abc", "", http.StatusUnauthorized}, + {"bad base64", "Basic %%%", "", http.StatusUnauthorized}, + {"no delimiter", "Basic " + base64.StdEncoding.EncodeToString([]byte("alice")), "", http.StatusUnauthorized}, + {"unexpected body", "Basic " + base64.StdEncoding.EncodeToString([]byte("alice:long-password:ok")), "{}", http.StatusBadRequest}, + {"valid colon in password", "Basic " + base64.StdEncoding.EncodeToString([]byte("alice:long-password:ok")), "", http.StatusCreated}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + ctrl := testController(t, nil) + ctrl.auth = sessionFixture{login: func(username, password string) (dto.SessionDTO, error) { + if username != "alice" || password != "long-password:ok" { + t.Fatal("credentials altered") + } + return dto.SessionDTO{Token: "opaque", ExpiresAt: time.Now().UTC().Add(time.Hour), User: dto.UserDTO{ID: uuid.New(), Username: "alice"}}, nil + }} + req := httptest.NewRequest(http.MethodPost, "/api/v1/auth/sessions", strings.NewReader(tc.body)) + req.Header.Set("Authorization", tc.header) + response, err := ctrl.server.Test(req) + if err != nil { + t.Fatal(err) + } + defer func() { + _ = response.Body.Close() + }() + if response.StatusCode != tc.status { + t.Fatalf("got %d want %d", response.StatusCode, tc.status) + } + if tc.status == http.StatusUnauthorized && response.Header.Get("WWW-Authenticate") != `Basic realm="messenger", charset="UTF-8"` { + t.Fatal("missing Basic challenge") + } + }) + } +} + +// TestLoginInvalidCredentialsUsesBasicChallenge checks the service rejection mapping. +func TestLoginInvalidCredentialsUsesBasicChallenge(t *testing.T) { + ctrl := testController(t, nil) + ctrl.auth = sessionFixture{login: func(string, string) (dto.SessionDTO, error) { + return dto.SessionDTO{}, service.ErrUnauthorized + }} + req := httptest.NewRequest(http.MethodPost, "/api/v1/auth/sessions", nil) + req.SetBasicAuth("alice", "invalid-password") + response, err := ctrl.server.Test(req) + if err != nil { + t.Fatal(err) + } + defer func() { + _ = response.Body.Close() + }() + if response.StatusCode != http.StatusUnauthorized || response.Header.Get("WWW-Authenticate") != `Basic realm="messenger", charset="UTF-8"` { + t.Fatal("incorrect login rejection") + } +} + +// TestLogoutHTTPReturnsNoContent checks the authenticated revocation response. +func TestLogoutHTTPReturnsNoContent(t *testing.T) { + ctrl := testController(t, nil) + actor := user(t, ctrl) + req := httptest.NewRequest(http.MethodDelete, "/api/v1/auth/sessions/current", nil) + req.Header.Set("Authorization", "Bearer "+actor) + response, err := ctrl.server.Test(req) + if err != nil { + t.Fatal(err) + } + defer func() { + _ = response.Body.Close() + }() + body, err := io.ReadAll(response.Body) + if err != nil || response.StatusCode != http.StatusNoContent || len(body) != 0 { + t.Fatalf("invalid logout response: %v", err) + } +} + +// TestLogoutHTTPPreservesFailure ensures an unsuccessful revoke does not claim success. +func TestLogoutHTTPPreservesFailure(t *testing.T) { + ctrl := testController(t, nil) + actor := user(t, ctrl) + ctrl.auth = sessionFixture{fixtureAuthentication: fixtureAuthentication{ctrl.service}, logoutError: context.DeadlineExceeded} + status, _, err := request(ctrl, http.MethodDelete, "/api/v1/auth/sessions/current", actor, nil) + if err != nil || status != http.StatusInternalServerError { + t.Fatalf("revocation failure hidden: %d %v", status, err) + } +} + +// TestLegacyIdentityDoesNotAuthenticate verifies the lab1 header cannot grant access. +func TestLegacyIdentityDoesNotAuthenticate(t *testing.T) { + ctrl := testController(t, nil) + actor := user(t, ctrl) + req := httptest.NewRequest(http.MethodGet, "/api/v1/users/me", nil) + req.Header.Set("X-User-Id", actor) + response, err := ctrl.server.Test(req) + if err != nil { + t.Fatal(err) + } + defer func() { + _ = response.Body.Close() + }() + if response.StatusCode != http.StatusUnauthorized || response.Header.Get("WWW-Authenticate") != `Bearer realm="messenger"` { + t.Fatal("legacy identity granted access or missing challenge") + } +} + +type unavailableReadiness struct{} + +// Ping simulates an unavailable mandatory dependency for readiness mapping. +func (unavailableReadiness) Ping(context.Context) error { + return context.DeadlineExceeded +} + +// TestReadinessRejectsDependencyFailure verifies readiness reflects mandatory infrastructure. +func TestReadinessRejectsDependencyFailure(t *testing.T) { + ctrl := testController(t, nil) + ctrl.readiness = unavailableReadiness{} + status, _, err := request(ctrl, http.MethodGet, "/health/ready", "", nil) + if err != nil || status != http.StatusServiceUnavailable { + t.Fatalf("incorrect readiness: %d %v", status, err) + } +} diff --git a/internal/controller/http/fiber/body_limit_test.go b/internal/controller/http/fiber/body_limit_test.go new file mode 100644 index 0000000..4561077 --- /dev/null +++ b/internal/controller/http/fiber/body_limit_test.go @@ -0,0 +1,69 @@ +package fiber + +import ( + "context" + "encoding/json" + "net" + "net/http" + "strings" + "testing" + "time" + + "github.com/gofiber/fiber/v2" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/config" + httpmodel "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/model/http" +) + +// TestBodyLimitInstanceHeader verifies parser-generated 413 responses over a real TCP listener. +func TestBodyLimitInstanceHeader(t *testing.T) { + // Arrange. + ctrl := testController(t, func(cfg *config.Config) { + cfg.HTTP.MaxBodyBytes = 64 << 10 + cfg.HTTP.InstanceID = "body-limit-test-instance" + }) + ctrl.cfg.Host = "127.0.0.1" + ctrl.cfg.Port = 0 + address := make(chan string, 1) + ctrl.server.Hooks().OnListen(func(data fiber.ListenData) error { + address <- net.JoinHostPort(data.Host, data.Port) + return nil + }) + if err := ctrl.Start(t.Context()); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + if err := ctrl.Stop(ctx); err != nil { + t.Error(err) + } + }) + client := &http.Client{Timeout: 3 * time.Second} + + // Act. + response, err := client.Post("http://"+<-address+"/api/v1/users", "application/json", strings.NewReader(strings.Repeat("x", ctrl.cfg.MaxBodyBytes+1))) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := response.Body.Close(); err != nil { + t.Error(err) + } + }() + + // Assert. + if response.StatusCode != http.StatusRequestEntityTooLarge { + t.Fatalf("status=%d", response.StatusCode) + } + if instance := response.Header.Get("X-Instance-Id"); instance != ctrl.instanceID { + t.Fatalf("instance=%q, want %q", instance, ctrl.instanceID) + } + var body httpmodel.ErrorResponse + if err := json.NewDecoder(response.Body).Decode(&body); err != nil { + t.Fatal(err) + } + if body.Error.Code != "payload_too_large" || body.Error.RequestID == "" || body.Error.RequestID != response.Header.Get(headerRequestID) { + t.Fatalf("invalid error envelope: %+v", body) + } +} diff --git a/internal/controller/http/fiber/chats_test.go b/internal/controller/http/fiber/chats_test.go new file mode 100644 index 0000000..d1660ba --- /dev/null +++ b/internal/controller/http/fiber/chats_test.go @@ -0,0 +1,312 @@ +package fiber + +import ( + "net/http" + "net/url" + "reflect" + "testing" + + "github.com/google/uuid" +) + +// TestController_ChatsCreate checks membership and request validation in isolated chats. +func TestController_ChatsCreate(t *testing.T) { + t.Run("Positive: caller included", func(t *testing.T) { + // Arrange. + ctrl := testController(t, nil) + actor := user(t, ctrl) + payload := mustJSON(t, map[string]any{"title": "Team", "member_ids": []string{actor}}) + + // Act. + status, body, err := request(ctrl, http.MethodPost, "/api/v1/chats", actor, payload) + + // Assert. + if err != nil { + t.Fatal(err) + } + assertHTTPStatus(t, status, http.StatusCreated, body) + if body["title"] != "Team" || !reflect.DeepEqual(body["member_ids"], []any{actor}) { + t.Fatalf("chat fields changed: %v", body) + } + }) + + t.Run("Negative: empty members", func(t *testing.T) { + // Arrange. + ctrl := testController(t, nil) + actor := user(t, ctrl) + payload := mustJSON(t, map[string]any{"title": "Team", "member_ids": []string{}}) + + // Act. + status, body, err := request(ctrl, http.MethodPost, "/api/v1/chats", actor, payload) + + // Assert. + if err != nil { + t.Fatal(err) + } + assertHTTPStatus(t, status, http.StatusBadRequest, body) + assertErrorCode(t, body, "invalid_request") + }) + + t.Run("Negative: duplicate members", func(t *testing.T) { + // Arrange. + ctrl := testController(t, nil) + actor := user(t, ctrl) + payload := mustJSON(t, map[string]any{"title": "Team", "member_ids": []string{actor, actor}}) + + // Act. + status, body, err := request(ctrl, http.MethodPost, "/api/v1/chats", actor, payload) + + // Assert. + if err != nil { + t.Fatal(err) + } + assertHTTPStatus(t, status, http.StatusBadRequest, body) + assertErrorCode(t, body, "invalid_request") + }) + + t.Run("Negative: caller omitted", func(t *testing.T) { + // Arrange. + ctrl := testController(t, nil) + actor := user(t, ctrl) + outsider := user(t, ctrl) + payload := mustJSON(t, map[string]any{"title": "Team", "member_ids": []string{outsider}}) + + // Act. + status, body, err := request(ctrl, http.MethodPost, "/api/v1/chats", actor, payload) + + // Assert. + if err != nil { + t.Fatal(err) + } + assertHTTPStatus(t, status, http.StatusBadRequest, body) + assertErrorCode(t, body, "invalid_request") + }) + + t.Run("Negative: unknown member", func(t *testing.T) { + // Arrange. + ctrl := testController(t, nil) + actor := user(t, ctrl) + payload := mustJSON(t, map[string]any{"title": "Team", "member_ids": []string{actor, uuid.NewString()}}) + + // Act. + status, body, err := request(ctrl, http.MethodPost, "/api/v1/chats", actor, payload) + + // Assert. + if err != nil { + t.Fatal(err) + } + assertHTTPStatus(t, status, http.StatusBadRequest, body) + assertErrorCode(t, body, "invalid_request") + }) + + t.Run("Negative: malformed member ID", func(t *testing.T) { + // Arrange. + ctrl := testController(t, nil) + actor := user(t, ctrl) + payload := mustJSON(t, map[string]any{"title": "Team", "member_ids": []string{actor, "invalid"}}) + + // Act. + status, body, err := request(ctrl, http.MethodPost, "/api/v1/chats", actor, payload) + + // Assert. + if err != nil { + t.Fatal(err) + } + assertHTTPStatus(t, status, http.StatusBadRequest, body) + assertErrorCode(t, body, "invalid_request") + }) + + t.Run("Negative: blank title", func(t *testing.T) { + // Arrange. + ctrl := testController(t, nil) + actor := user(t, ctrl) + payload := mustJSON(t, map[string]any{"title": " ", "member_ids": []string{actor}}) + + // Act. + status, body, err := request(ctrl, http.MethodPost, "/api/v1/chats", actor, payload) + + // Assert. + if err != nil { + t.Fatal(err) + } + assertHTTPStatus(t, status, http.StatusBadRequest, body) + assertErrorCode(t, body, "invalid_request") + }) + + t.Run("Negative: malformed JSON", func(t *testing.T) { + // Arrange. + ctrl := testController(t, nil) + actor := user(t, ctrl) + payload := []byte(`{`) + + // Act. + status, body, err := request(ctrl, http.MethodPost, "/api/v1/chats", actor, payload) + + // Assert. + if err != nil { + t.Fatal(err) + } + assertHTTPStatus(t, status, http.StatusBadRequest, body) + assertErrorCode(t, body, "invalid_request") + }) +} + +// TestController_ChatsGetByID checks access and the returned chat identity. +func TestController_ChatsGetByID(t *testing.T) { + t.Run("Positive: member reads chat", func(t *testing.T) { + // Arrange. + ctrl := testController(t, nil) + owner := user(t, ctrl) + id := chat(t, ctrl, owner) + actor, target := owner, id + + // Act. + status, body, err := request(ctrl, http.MethodGet, "/api/v1/chats/"+target, actor, nil) + + // Assert. + if err != nil { + t.Fatal(err) + } + assertHTTPStatus(t, status, http.StatusOK, body) + if body["id"] != id || body["title"] != "Чат" { + t.Fatalf("wrong chat: %v", body) + } + }) + + t.Run("Negative: outsider", func(t *testing.T) { + // Arrange. + ctrl := testController(t, nil) + owner := user(t, ctrl) + id := chat(t, ctrl, owner) + actor, target := user(t, ctrl), id + + // Act. + status, body, err := request(ctrl, http.MethodGet, "/api/v1/chats/"+target, actor, nil) + + // Assert. + if err != nil { + t.Fatal(err) + } + assertHTTPStatus(t, status, http.StatusNotFound, body) + assertErrorCode(t, body, "not_found") + }) + + t.Run("Negative: unknown chat", func(t *testing.T) { + // Arrange. + ctrl := testController(t, nil) + owner := user(t, ctrl) + actor, target := owner, uuid.NewString() + + // Act. + status, body, err := request(ctrl, http.MethodGet, "/api/v1/chats/"+target, actor, nil) + + // Assert. + if err != nil { + t.Fatal(err) + } + assertHTTPStatus(t, status, http.StatusNotFound, body) + assertErrorCode(t, body, "not_found") + }) + + t.Run("Negative: malformed chat ID", func(t *testing.T) { + // Arrange. + ctrl := testController(t, nil) + owner := user(t, ctrl) + actor, target := owner, "invalid" + + // Act. + status, body, err := request(ctrl, http.MethodGet, "/api/v1/chats/"+target, actor, nil) + + // Assert. + if err != nil { + t.Fatal(err) + } + assertHTTPStatus(t, status, http.StatusBadRequest, body) + assertErrorCode(t, body, "invalid_request") + }) + + t.Run("Negative: authentication precedes ID parsing", func(t *testing.T) { + // Arrange. + ctrl := testController(t, nil) + actor, target := "", "invalid" + + // Act. + status, body, err := request(ctrl, http.MethodGet, "/api/v1/chats/"+target, actor, nil) + + // Assert. + if err != nil { + t.Fatal(err) + } + assertHTTPStatus(t, status, http.StatusUnauthorized, body) + assertErrorCode(t, body, "unauthorized") + }) +} + +// TestController_ChatsGetPage checks that foreign memberships remain absent from lists. +func TestController_ChatsGetPage(t *testing.T) { + // Arrange. + ctrl := testController(t, nil) + owner := user(t, ctrl) + chat(t, ctrl, owner) + outsider := user(t, ctrl) + + // Act. + status, body, err := request(ctrl, http.MethodGet, "/api/v1/chats", outsider, nil) + + // Assert. + if err != nil { + t.Fatal(err) + } + assertHTTPStatus(t, status, http.StatusOK, body) + if len(body["items"].([]any)) != 0 || body["next_cursor"] != nil { + t.Fatalf("foreign chat leaked: %v", body) + } +} + +// TestController_ChatPagination recovers every chat exactly once across page boundaries. +func TestController_ChatPagination(t *testing.T) { + // Arrange. + ctrl := testController(t, nil) + actor := user(t, ctrl) + expected := make(map[string]bool) + for range 15 { + expected[chat(t, ctrl, actor)] = true + } + + // Act. + items := history(t, ctrl, actor, "/api/v1/chats") + + // Assert. + if len(items) != len(expected) { + t.Fatalf("got %d chats, want %d", len(items), len(expected)) + } + for _, item := range items { + id := item.(map[string]any)["id"].(string) + if !expected[id] { + t.Fatalf("duplicate or unexpected chat %s", id) + } + delete(expected, id) + } +} + +// TestController_ChatCursorScope rejects a continuation issued for another user's list. +func TestController_ChatCursorScope(t *testing.T) { + // Arrange. + ctrl := testController(t, nil) + owner := user(t, ctrl) + outsider := user(t, ctrl) + chat(t, ctrl, owner) + chat(t, ctrl, owner) + page := call(t, ctrl, http.MethodGet, "/api/v1/chats?limit=1", owner, nil, http.StatusOK) + token := page["next_cursor"].(string) + + // Act. + status, body, err := request(ctrl, http.MethodGet, "/api/v1/chats?cursor="+url.QueryEscape(token), outsider, nil) + + // Assert. + if err != nil { + t.Fatal(err) + } + assertHTTPStatus(t, status, http.StatusBadRequest, body) + assertErrorCode(t, body, "invalid_request") +} diff --git a/internal/controller/http/fiber/configure.go b/internal/controller/http/fiber/configure.go new file mode 100644 index 0000000..1f3da39 --- /dev/null +++ b/internal/controller/http/fiber/configure.go @@ -0,0 +1,37 @@ +package fiber + +import ( + "github.com/gofiber/fiber/v2" + "github.com/gofiber/fiber/v2/middleware/recover" +) + +// configure initializes the HTTP server, middleware and routes. +func (ctrl *Controller) configure() { + ctrl.createServer() + ctrl.initMiddlewares() + ctrl.initRoutes() +} + +// createServer creates Fiber with the configured transport limits and error handler. +func (ctrl *Controller) createServer() { + timeout := ctrl.cfg.Timeout() + ctrl.server = fiber.New(fiber.Config{ + AppName: "messenger-lab1", + DisableStartupMessage: true, + Immutable: true, + CaseSensitive: true, + StrictRouting: true, + BodyLimit: ctrl.cfg.MaxBodyBytes, + Concurrency: ctrl.cfg.MaxConnections, + ReadTimeout: timeout, + WriteTimeout: timeout, + IdleTimeout: timeout, + ErrorHandler: ctrl.handleErr, + }) +} + +// initMiddlewares registers request logging and panic recovery middleware. +func (ctrl *Controller) initMiddlewares() { + ctrl.server.Use(ctrl.request) + ctrl.server.Use(recover.New()) +} diff --git a/internal/controller/http/fiber/controller.go b/internal/controller/http/fiber/controller.go new file mode 100644 index 0000000..2681c53 --- /dev/null +++ b/internal/controller/http/fiber/controller.go @@ -0,0 +1,67 @@ +// Package fiber implements the messenger HTTP transport using Fiber. +package fiber + +import ( + "fmt" + "sync/atomic" + + "github.com/gofiber/fiber/v2" + "github.com/google/uuid" + "go.lab260.ru/fielderr" + "go.uber.org/zap" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/admission" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/config" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/controller" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/service" +) + +// Controller owns the HTTP server and delegates business operations to Service. +// Its configuration is copied at construction and must not change during serving. +// Fx manages its listener through Start and Stop. +type Controller struct { + observer HTTPObserver + attachments Attachments + metrics *httpMetrics + log *zap.Logger + cfg config.Controller + service service.Service + server *fiber.App + errorRegistry *fielderr.Registry + admission admission.Distributed + auth service.Authentication + readiness Readiness + stopping atomic.Bool + instanceID string + serveDone chan error +} + +var _ controller.Controller = (*Controller)(nil) + +// New validates dependencies and prepares the server without opening a listener. +// Routing, middleware and Fiber configuration are installed by configure. +func New(log *zap.Logger, cfg *config.Controller, svc service.Service, registry *fielderr.Registry, limiter admission.Distributed, auth service.Authentication, readiness Readiness) (*Controller, error) { + if err := cfg.Validate(); err != nil { + return nil, err + } + if log == nil || svc == nil || registry == nil || limiter == nil || auth == nil || readiness == nil { + return nil, fmt.Errorf("HTTP logger, service, error registry and admission limiter are required") + } + ctrl := &Controller{ + metrics: newHTTPMetrics(), + log: log.Named("controller.http.fiber"), + cfg: *cfg, + service: svc, + errorRegistry: registry, + admission: limiter, + auth: auth, + readiness: readiness, + serveDone: make(chan error, 1), + } + ctrl.instanceID = cfg.InstanceID + if ctrl.instanceID == "" { + ctrl.instanceID = uuid.NewString() + } + ctrl.configure() + return ctrl, nil +} diff --git a/internal/controller/http/fiber/controller_test.go b/internal/controller/http/fiber/controller_test.go new file mode 100644 index 0000000..a07fc87 --- /dev/null +++ b/internal/controller/http/fiber/controller_test.go @@ -0,0 +1,321 @@ +package fiber + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "net/url" + "reflect" + "strings" + "sync" + "testing" + + "github.com/google/uuid" + "go.uber.org/zap" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/admission" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/config" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/cursor" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/service/production" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/store/memory" +) + +// testController creates an integration fixture with the real controller, service and store. +func testController(t *testing.T, change func(*config.Config)) *Controller { + t.Helper() + cfg := config.Default() + cfg.HTTP.MaxConnections = 256 + cfg.HTTP.Admission.RequestsPerSecond = 100000 + cfg.HTTP.Admission.Burst = 100000 + if change != nil { + change(cfg) + } + repo, err := memory.New(cfg.Memory) + if err != nil { + t.Fatal(err) + } + codec, err := cursor.New() + if err != nil { + t.Fatal(err) + } + svc, err := production.New(repo, codec) + if err != nil { + t.Fatal(err) + } + registry, err := NewErrorRegistry() + if err != nil { + t.Fatal(err) + } + limiter, err := admission.New(admission.Options{ + MaxInFlight: cfg.HTTP.Admission.MaxInFlight, + RequestsPerSecond: cfg.HTTP.Admission.RequestsPerSecond, + Burst: cfg.HTTP.Admission.Burst, + }) + if err != nil { + t.Fatal(err) + } + ctrl, err := New(zap.NewNop(), cfg.HTTP, svc, registry, &fixtureAdmission{limiter}, fixtureAuthentication{svc}, fixtureReadiness{}) + if err != nil { + t.Fatal(err) + } + return ctrl +} + +// request sends a raw test request and decodes its response. +func request(ctrl *Controller, method, path, actor string, body []byte) (int, map[string]any, error) { + req := httptest.NewRequest(method, path, bytes.NewReader(body)) + req.Header.Set("Idempotency-Key", uuid.NewString()) + if body != nil { + req.Header.Set("Content-Type", "application/json") + } + if actor != "" { + req.Header.Set("Authorization", "Bearer "+actor) + } + resp, err := ctrl.server.Test(req, 10000) + if err != nil { + return 0, nil, err + } + defer func() { + _ = resp.Body.Close() + }() + b, err := io.ReadAll(resp.Body) + if err != nil { + return 0, nil, err + } + var data map[string]any + if err = json.Unmarshal(b, &data); err != nil { + return resp.StatusCode, nil, fmt.Errorf("invalid JSON response: %w", err) + } + if resp.Header.Get("X-Request-Id") == "" { + return resp.StatusCode, data, fmt.Errorf("missing request ID") + } + if resp.StatusCode >= http.StatusBadRequest { + envelope, ok := data["error"].(map[string]any) + if !ok || envelope["request_id"] != resp.Header.Get("X-Request-Id") { + return resp.StatusCode, data, fmt.Errorf("invalid error envelope") + } + } + return resp.StatusCode, data, nil +} + +// call sends a JSON test request and checks its expected HTTP status. +func call(t *testing.T, ctrl *Controller, method, path, actor string, body any, want int) map[string]any { + t.Helper() + var b []byte + var err error + if raw, ok := body.([]byte); ok { + b = raw + } else if body != nil { + b, err = json.Marshal(body) + if err != nil { + t.Fatal(err) + } + } + status, data, err := request(ctrl, method, path, actor, b) + if err != nil { + t.Fatal(err) + } + if status != want { + t.Fatalf("%s %s: got %d want %d: %v", method, path, status, want, data) + } + return data +} + +// user creates a test user and returns its identifier. +func user(t *testing.T, c *Controller) string { + t.Helper() + r := call(t, c, http.MethodPost, "/api/v1/users", "", map[string]any{"username": "u_" + uuid.New().String()[:8], "display_name": "Алиса 👋", "password": "fixture-password-123"}, http.StatusCreated) + return r["id"].(string) +} + +// chat creates a test chat containing the supplied user. +func chat(t *testing.T, c *Controller, u string) string { + t.Helper() + r := call(t, c, http.MethodPost, "/api/v1/chats", u, map[string]any{"title": "Чат", "member_ids": []string{u}}, http.StatusCreated) + return r["id"].(string) +} + +// TestDuplicateUsernameRace checks username uniqueness under concurrent requests. +func TestDuplicateUsernameRace(t *testing.T) { + // Arrange. + c := testController(t, nil) + results := make(chan int, 10) + failures := make(chan error, 10) + var wg sync.WaitGroup + // Act. + for range 10 { + wg.Go(func() { + status, _, err := request(c, http.MethodPost, "/api/v1/users", "", []byte(`{"username":"same","display_name":"User","password":"fixture-password-123"}`)) + if err != nil { + failures <- err + } + results <- status + }) + } + wg.Wait() + close(results) + close(failures) + // Assert. + for err := range failures { + t.Error(err) + } + created, conflicts := 0, 0 + for s := range results { + switch s { + case http.StatusCreated: + created++ + case http.StatusConflict: + conflicts++ + default: + t.Errorf("unexpected %d", s) + } + } + if created != 1 || conflicts != 9 { + t.Fatalf("created=%d conflicts=%d", created, conflicts) + } +} + +// history collects all test messages by following pagination cursors. +func history(t *testing.T, c *Controller, u, path string) []any { + t.Helper() + var items []any + next := path + "?limit=7" + for range 100 { + r := call(t, c, http.MethodGet, next, u, nil, http.StatusOK) + items = append(items, r["items"].([]any)...) + if r["next_cursor"] == nil { + return items + } + next = path + "?limit=7&cursor=" + url.QueryEscape(r["next_cursor"].(string)) + } + t.Fatal("pagination never finished") + return nil +} + +// TestConcurrent100Messages10Clients recovers the exact acknowledged records after concurrent writes. +func TestConcurrent100Messages10Clients(t *testing.T) { + // Arrange. + const clients = 10 + const messagesPerClient = 10 + const total = clients * messagesPerClient + ctrl := testController(t, nil) + actor := user(t, ctrl) + chatID := chat(t, ctrl, actor) + path := "/api/v1/chats/" + chatID + "/messages" + acknowledgements := make(chan map[string]any, total) + failures := make(chan error, total) + var writers sync.WaitGroup + + // Act. + for client := range clients { + writers.Go(func() { + for index := range messagesPerClient { + text := fmt.Sprintf("client %d message %d", client, index) + payload, err := json.Marshal(map[string]string{"text": text}) + if err != nil { + failures <- err + continue + } + status, body, err := request(ctrl, http.MethodPost, path, actor, payload) + if err != nil { + failures <- err + continue + } + if status != http.StatusCreated || body["text"] != text || body["sender_id"] != actor || body["chat_id"] != chatID { + failures <- fmt.Errorf("incorrect acknowledgement: status=%d, body=%v", status, body) + continue + } + acknowledgements <- body + } + }) + } + writers.Wait() + close(failures) + close(acknowledgements) + + // Assert. + for err := range failures { + t.Error(err) + } + if t.Failed() { + t.FailNow() + } + expected := make(map[string]map[string]any, total) + for message := range acknowledgements { + id, ok := message["id"].(string) + if !ok || id == "" { + t.Fatalf("invalid message ID: %v", message) + } + if _, duplicate := expected[id]; duplicate { + t.Fatalf("duplicate acknowledgement ID %s", id) + } + expected[id] = message + } + if len(expected) != total { + t.Fatalf("acknowledged=%d, want=%d", len(expected), total) + } + items := history(t, ctrl, actor, path) + if len(items) != total { + t.Fatalf("recovered=%d, want=%d", len(items), total) + } + for _, item := range items { + message := item.(map[string]any) + id := message["id"].(string) + if !reflect.DeepEqual(message, expected[id]) { + t.Fatalf("stored record differs from acknowledgement: got=%v, want=%v", message, expected[id]) + } + delete(expected, id) + } +} + +// TestUnsupportedMediaType checks rejection of unsupported request content types. +func TestUnsupportedMediaType(t *testing.T) { + // Arrange. + c := testController(t, nil) + req := httptest.NewRequest(http.MethodPost, "/api/v1/users", strings.NewReader(`{}`)) + req.Header.Set("Content-Type", "text/plain") + // Act. + r, e := c.server.Test(req) + + // Assert. + if e != nil { + t.Fatal(e) + } + defer func() { + _ = r.Body.Close() + }() + if r.StatusCode != http.StatusUnsupportedMediaType { + t.Fatalf("status %d", r.StatusCode) + } +} + +// TestConfigIndependentState prevents one application instance from accepting another's identity. +func TestConfigIndependentState(t *testing.T) { + // Arrange. + first := testController(t, nil) + actor := user(t, first) + second := testController(t, nil) + + // Act. + status, body, err := request(second, http.MethodGet, "/api/v1/users/me", actor, nil) + + // Assert. + if err != nil { + t.Fatal(err) + } + assertHTTPStatus(t, status, http.StatusUnauthorized, body) + assertErrorCode(t, body, "unauthorized") +} + +// mustJSON encodes fixture payloads and reports preparation failures at the caller. +func mustJSON(t *testing.T, value any) []byte { + t.Helper() + payload, err := json.Marshal(value) + if err != nil { + t.Fatal(err) + } + return payload +} diff --git a/internal/controller/http/fiber/decode.go b/internal/controller/http/fiber/decode.go new file mode 100644 index 0000000..83cf16c --- /dev/null +++ b/internal/controller/http/fiber/decode.go @@ -0,0 +1,34 @@ +package fiber + +import ( + "mime" + + "github.com/gofiber/fiber/v2" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/service" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/strictjson" +) + +// decode checks HTTP media type and maps strict JSON decoding errors to API input errors. +func decode(ctx *fiber.Ctx, target any, requiredFields ...string) error { + mediaType, _, err := mime.ParseMediaType(ctx.Get(fiber.HeaderContentType)) + if err != nil || mediaType != fiber.MIMEApplicationJSON { + return errMedia + } + if err := strictjson.DecodeObject(ctx.Body(), target, requiredFields...); err != nil { + return service.ErrInvalid + } + return nil +} + +// decodeMessage accepts the optional attachment list without weakening strict JSON validation. +func decodeMessage(ctx *fiber.Ctx, target any) error { + mediaType, _, err := mime.ParseMediaType(ctx.Get(fiber.HeaderContentType)) + if err != nil || mediaType != fiber.MIMEApplicationJSON { + return errMedia + } + if err := strictjson.DecodeOptional(ctx.Body(), target, []string{"text"}, []string{"attachment_ids"}); err != nil { + return service.ErrInvalid + } + return nil +} diff --git a/internal/controller/http/fiber/dependencies.go b/internal/controller/http/fiber/dependencies.go new file mode 100644 index 0000000..58a3a74 --- /dev/null +++ b/internal/controller/http/fiber/dependencies.go @@ -0,0 +1,9 @@ +package fiber + +import "context" + +// Readiness checks the dependencies required by the synchronous request path. +type Readiness interface { + // Ping reports whether requests can be safely served. + Ping(context.Context) error +} diff --git a/internal/controller/http/fiber/error_codes.go b/internal/controller/http/fiber/error_codes.go new file mode 100644 index 0000000..64decfa --- /dev/null +++ b/internal/controller/http/fiber/error_codes.go @@ -0,0 +1,22 @@ +package fiber + +const ( + // codeInvalid identifies malformed or invalid request input. + codeInvalid = "invalid_request" + // codeUnauthorized identifies an unresolved user identity. + codeUnauthorized = "unauthorized" + // codeNotFound deliberately covers both absent and inaccessible resources. + codeNotFound = "not_found" + // codeConflict identifies a violated uniqueness constraint. + codeConflict = "conflict" + // codeUnavailable identifies temporary admission or persistent capacity exhaustion. + codeUnavailable = "unavailable" + // codeRateLimited identifies exhaustion of the request token bucket. + codeRateLimited = "rate_limited" + // codeTooLarge identifies a request exceeding the transport byte budget. + codeTooLarge = "payload_too_large" + // codeMedia identifies an unsupported request media type. + codeMedia = "unsupported_media_type" + // codeInternal identifies a hidden unexpected failure. + codeInternal = "internal_error" +) diff --git a/internal/controller/http/fiber/error_registry.go b/internal/controller/http/fiber/error_registry.go new file mode 100644 index 0000000..d5e1007 --- /dev/null +++ b/internal/controller/http/fiber/error_registry.go @@ -0,0 +1,33 @@ +package fiber + +import ( + "net/http" + + "go.lab260.ru/fielderr" +) + +// NewErrorRegistry returns an immutable mapping of application statuses to HTTP codes. +// The registry is injected rather than installed in process-global state, allowing +// independent application instances in integration tests. +func NewErrorRegistry() (*fielderr.Registry, error) { + return fielderr.NewRegistry( + statusDefinition(codeInvalid, http.StatusBadRequest), + statusDefinition(codeUnauthorized, http.StatusUnauthorized), + statusDefinition(codeNotFound, http.StatusNotFound), + statusDefinition(codeConflict, http.StatusConflict), + statusDefinition(codeTooLarge, http.StatusRequestEntityTooLarge), + statusDefinition(codeMedia, http.StatusUnsupportedMediaType), + statusDefinition(codeRateLimited, http.StatusTooManyRequests), + statusDefinition(codeUnavailable, http.StatusServiceUnavailable), + statusDefinition(codeInternal, http.StatusInternalServerError), + ) +} + +// statusDefinition associates an application error code with its HTTP status. +func statusDefinition(name string, httpStatus int) fielderr.StatusDefinition { + return fielderr.StatusDefinition{ + Code: fielderr.NewStatusCode(name), + Name: name, + HTTPStatus: httpStatus, + } +} diff --git a/internal/controller/http/fiber/errors.go b/internal/controller/http/fiber/errors.go new file mode 100644 index 0000000..6839f7b --- /dev/null +++ b/internal/controller/http/fiber/errors.go @@ -0,0 +1,73 @@ +package fiber + +import ( + "errors" + "net/http" + + "github.com/gofiber/fiber/v2" + "github.com/google/uuid" + "go.lab260.ru/fielderr" + "go.uber.org/zap" + + httpmodel "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/model/http" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/service" +) + +// handleErr normalizes a failure and writes the public error envelope. +func (ctrl *Controller) handleErr(ctx *fiber.Ctx, err error) error { + // Parser failures such as BodyLimit rejection bypass request middleware. + ctx.Set("X-Instance-Id", ctrl.instanceID) + requestID := ctx.GetRespHeader(headerRequestID) + if requestID == "" { + requestID = uuid.NewString() + ctx.Set(headerRequestID, requestID) + } + err = publicError(normalizeTransportError(err)) + status, ok := fielderr.HTTPStatusFrom(ctrl.errorRegistry, err) + if !ok { + err = errInternal + status = http.StatusInternalServerError + } + if status == http.StatusUnauthorized { + challenge := `Bearer realm="messenger"` + if ctx.Path() == "/api/v1/auth/sessions" && ctx.Method() == http.MethodPost { + challenge = `Basic realm="messenger", charset="UTF-8"` + } + ctx.Set(fiber.HeaderWWWAuthenticate, challenge) + } + if status == http.StatusInternalServerError { + ctrl.log.Error("unexpected request failure", zap.String("request_id", requestID)) + } + return ctx.Status(status).JSON(errorResponse(err, requestID)) +} + +// normalizeTransportError maps Fiber failures to transport-independent application errors. +func normalizeTransportError(err error) error { + transportError, ok := errors.AsType[*fiber.Error](err) + if !ok { + return err + } + switch transportError.Code { + case http.StatusRequestEntityTooLarge: + return errTooLarge + case http.StatusNotFound, http.StatusMethodNotAllowed: + return service.ErrNotFound + case http.StatusBadRequest, http.StatusRequestTimeout, http.StatusRequestHeaderFieldsTooLarge: + return service.ErrInvalid + default: + return errInternal + } +} + +// errorResponse builds the public error envelope with its request identifier. +func errorResponse(err error, requestID string) httpmodel.ErrorResponse { + status, _ := fielderr.Status(err) + message, _ := fielderr.PublicMessage(err) + return httpmodel.ErrorResponse{ + Error: httpmodel.ErrorDetail{ + Code: status.String(), + Message: message, + RequestID: requestID, + }, + } +} diff --git a/internal/controller/http/fiber/errors_test.go b/internal/controller/http/fiber/errors_test.go new file mode 100644 index 0000000..79cc2aa --- /dev/null +++ b/internal/controller/http/fiber/errors_test.go @@ -0,0 +1,107 @@ +package fiber + +import ( + "encoding/json" + "errors" + "net/http" + "testing" + + "github.com/gofiber/fiber/v2" + "github.com/valyala/fasthttp" + "go.lab260.ru/fielderr" + + httpmodel "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/model/http" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/service" +) + +// TestTransportErrorMapping checks safe API errors for framework-generated failures. +func TestTransportErrorMapping(t *testing.T) { + t.Run("large body", func(t *testing.T) { + if normalizeTransportError(fiber.ErrRequestEntityTooLarge) != errTooLarge { + t.Fatal("incorrect body-limit error") + } + }) + t.Run("method not allowed", func(t *testing.T) { + if normalizeTransportError(fiber.ErrMethodNotAllowed) != service.ErrNotFound { + t.Fatal("unexpected method error") + } + }) + t.Run("invalid request", func(t *testing.T) { + if normalizeTransportError(fiber.ErrBadRequest) != service.ErrInvalid { + t.Fatal("unexpected input error") + } + }) + t.Run("unexpected framework failure", func(t *testing.T) { + if normalizeTransportError(fiber.ErrInternalServerError) != errInternal { + t.Fatal("unexpected failure escaped normalization") + } + }) +} + +// TestController_handleErr writes a safe, correlated fallback response. +func TestController_handleErr(t *testing.T) { + t.Run("Negative: unknown internal error", func(t *testing.T) { + // Arrange. + ctrl := testController(t, nil) + raw := &fasthttp.RequestCtx{} + ctx := ctrl.server.AcquireCtx(raw) + defer ctrl.server.ReleaseCtx(ctx) + + // Act. + err := ctrl.handleErr(ctx, errors.New("private storage secret")) + + // Assert. + if err != nil { + t.Fatal(err) + } + if raw.Response.StatusCode() != http.StatusInternalServerError { + t.Fatalf("status=%d", raw.Response.StatusCode()) + } + var body httpmodel.ErrorResponse + if err := json.Unmarshal(raw.Response.Body(), &body); err != nil { + t.Fatal(err) + } + requestID := string(raw.Response.Header.Peek(headerRequestID)) + if requestID == "" || body.Error.RequestID != requestID { + t.Fatalf("uncorrelated error: %+v, header=%q", body, requestID) + } + if body.Error.Code != "internal_error" || body.Error.Message != "internal server error" { + t.Fatalf("unsafe fallback: %+v", body) + } + }) + + t.Run("Negative: missing status mapping", func(t *testing.T) { + // Arrange. + ctrl := testController(t, nil) + registry, err := fielderr.NewRegistry() + if err != nil { + t.Fatal(err) + } + ctrl.errorRegistry = registry + raw := &fasthttp.RequestCtx{} + ctx := ctrl.server.AcquireCtx(raw) + defer ctrl.server.ReleaseCtx(ctx) + + // Act. + err = ctrl.handleErr(ctx, service.ErrInvalid) + + // Assert. + if err != nil { + t.Fatal(err) + } + if raw.Response.StatusCode() != http.StatusInternalServerError { + t.Fatalf("status=%d", raw.Response.StatusCode()) + } + var body httpmodel.ErrorResponse + if err := json.Unmarshal(raw.Response.Body(), &body); err != nil { + t.Fatal(err) + } + requestID := string(raw.Response.Header.Peek(headerRequestID)) + if requestID == "" || body.Error.RequestID != requestID { + t.Fatalf("uncorrelated error: %+v, header=%q", body, requestID) + } + if body.Error.Code != "internal_error" || body.Error.Message != "internal server error" { + t.Fatalf("unsafe fallback: %+v", body) + } + }) +} diff --git a/internal/controller/http/fiber/fixtures_auth_test.go b/internal/controller/http/fiber/fixtures_auth_test.go new file mode 100644 index 0000000..c20c685 --- /dev/null +++ b/internal/controller/http/fiber/fixtures_auth_test.go @@ -0,0 +1,61 @@ +package fiber + +import ( + "context" + "time" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/admission" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/model/dto" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/service" +) + +// fixtureAuthentication isolates HTTP decoding and business authorization from session persistence. +type fixtureAuthentication struct { + service service.Service +} + +// Register delegates profile validation to the real business service. +func (f fixtureAuthentication) Register(ctx context.Context, request dto.UsersCreateRequestDTO, _ string) (dto.UserDTO, error) { + return f.service.UsersCreate(ctx, request) +} + +// Authenticate uses fixture tokens equal to user IDs exclusively in controller tests. +func (f fixtureAuthentication) Authenticate(ctx context.Context, token string) (string, error) { + return token, f.service.UsersCheckIdentity(ctx, token) +} + +// Login is deliberately unsupported by this fixture; real login has separate integration coverage. +func (f fixtureAuthentication) Login(context.Context, string, string) (dto.SessionDTO, error) { + return dto.SessionDTO{}, service.ErrUnauthorized +} + +// Logout accepts the fixture session without persistent side effects. +func (f fixtureAuthentication) Logout(context.Context, string) error { + return nil +} + +type fixtureReadiness struct{} + +// Ping marks isolated controller fixtures ready without external resources. +func (fixtureReadiness) Ping(context.Context) error { + return nil +} + +type fixtureAdmission struct { + *admission.Limiter +} + +// Enter adapts the deterministic local policy fixture to the distributed boundary. +func (f *fixtureAdmission) Enter(_ context.Context, _, user string) (func(), error) { + if user != "" { + return func() { + }, nil + } + if !f.Allow(time.Now()) { + return nil, admission.ErrRate + } + if !f.Acquire() { + return nil, admission.ErrSlots + } + return f.Release, nil +} diff --git a/internal/controller/http/fiber/handlers.go b/internal/controller/http/fiber/handlers.go new file mode 100644 index 0000000..260e604 --- /dev/null +++ b/internal/controller/http/fiber/handlers.go @@ -0,0 +1,98 @@ +package fiber + +import ( + "net/http" + + "github.com/gofiber/fiber/v2" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/model/dto" + httpmodel "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/model/http" +) + +// HandleUsersCreate decodes registration input and returns the created user. +func (ctrl *Controller) HandleUsersCreate(ctx *fiber.Ctx) error { + var request httpmodel.UsersCreateRequest + if err := decode(ctx, &request, "username", "display_name", "password"); err != nil { + return err + } + user, err := ctrl.auth.Register(ctx.UserContext(), dto.UsersCreateRequestDTO{ + Username: request.Username, + DisplayName: request.DisplayName, + }, request.Password) + if err != nil { + return err + } + return ctx.Status(http.StatusCreated).JSON(userResponse(user)) +} + +// HandleUsersGetMe returns the account selected by the validated session. +func (ctrl *Controller) HandleUsersGetMe(ctx *fiber.Ctx) error { + user, err := ctrl.service.UsersGetMe(ctx.UserContext(), identity(ctx)) + if err != nil { + return err + } + return ctx.Status(http.StatusOK).JSON(userResponse(user)) +} + +// HandleChatsCreate decodes the chat payload and returns the new chat. +func (ctrl *Controller) HandleChatsCreate(ctx *fiber.Ctx) error { + var request httpmodel.ChatsCreateRequest + if err := decode(ctx, &request, "title", "member_ids"); err != nil { + return err + } + chat, err := ctrl.service.ChatsCreate(ctx.UserContext(), dto.ChatsCreateRequestDTO{ + UserID: identity(ctx), + Title: request.Title, + MemberIDs: request.MemberIDs, + }) + if err != nil { + return err + } + return ctx.Status(http.StatusCreated).JSON(chatResponse(chat)) +} + +// HandleChatsGetByID delegates chat authorization and maps the accessible chat. +func (ctrl *Controller) HandleChatsGetByID(ctx *fiber.Ctx) error { + chat, err := ctrl.service.ChatsGetByID(ctx.UserContext(), identity(ctx), ctx.Params("chat_id")) + if err != nil { + return err + } + return ctx.Status(http.StatusOK).JSON(chatResponse(chat)) +} + +// HandleChatsGetPage returns the service's page of visible chats. +func (ctrl *Controller) HandleChatsGetPage(ctx *fiber.Ctx) error { + page, err := ctrl.service.ChatsGetPage(ctx.UserContext(), pageRequest(ctx)) + if err != nil { + return err + } + return ctx.Status(http.StatusOK).JSON(chatPageResponse(page)) +} + +// HandleMessagesCreate decodes text; the service derives and authorizes the sender. +func (ctrl *Controller) HandleMessagesCreate(ctx *fiber.Ctx) error { + var request httpmodel.MessagesCreateRequest + if err := decodeMessage(ctx, &request); err != nil { + return err + } + message, err := ctrl.service.MessagesCreate(ctx.UserContext(), dto.MessagesCreateRequestDTO{ + AttachmentIDs: request.AttachmentIDs, + IdempotencyKey: ctx.Get("Idempotency-Key"), + UserID: identity(ctx), + ChatID: ctx.Params("chat_id"), + Text: request.Text, + }) + if err != nil { + return err + } + return ctx.Status(http.StatusCreated).JSON(messageResponse(message)) +} + +// HandleMessagesGetPage returns the service's ordered history page. +func (ctrl *Controller) HandleMessagesGetPage(ctx *fiber.Ctx) error { + page, err := ctrl.service.MessagesGetPage(ctx.UserContext(), pageRequest(ctx)) + if err != nil { + return err + } + return ctx.Status(http.StatusOK).JSON(messagePageResponse(page)) +} diff --git a/internal/controller/http/fiber/headers.go b/internal/controller/http/fiber/headers.go new file mode 100644 index 0000000..d699168 --- /dev/null +++ b/internal/controller/http/fiber/headers.go @@ -0,0 +1,7 @@ +package fiber + +const ( + headerUserID = "X-User-Id" + headerRequestID = "X-Request-Id" + retryAfterSeconds = 1 +) diff --git a/internal/controller/http/fiber/health.go b/internal/controller/http/fiber/health.go new file mode 100644 index 0000000..5e37850 --- /dev/null +++ b/internal/controller/http/fiber/health.go @@ -0,0 +1,28 @@ +package fiber + +import ( + "github.com/gofiber/fiber/v2" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/service" +) + +// handleLive reports that the server process is running. +func (ctrl *Controller) handleLive(ctx *fiber.Ctx) error { + return ctx.JSON(fiber.Map{"status": "ok"}) +} + +// handleReady reports readiness unless shutdown has started. +func (ctrl *Controller) handleReady(ctx *fiber.Ctx) error { + if ctrl.stopping.Load() { + return errBusy + } + if err := ctrl.readiness.Ping(ctx.UserContext()); err != nil { + return errBusy + } + return ctrl.handleLive(ctx) +} + +// handleNotFound returns the application error for an unmatched route. +func (ctrl *Controller) handleNotFound(_ *fiber.Ctx) error { + return service.ErrNotFound +} diff --git a/internal/controller/http/fiber/lifecycle.go b/internal/controller/http/fiber/lifecycle.go new file mode 100644 index 0000000..455e213 --- /dev/null +++ b/internal/controller/http/fiber/lifecycle.go @@ -0,0 +1,70 @@ +package fiber + +import ( + "context" + "net" + "sync" + + "go.uber.org/zap" +) + +// Start opens the listener synchronously so Fx observes bind errors immediately. +// It must be called once per Controller; Stop terminates the serving goroutine. +func (ctrl *Controller) Start(ctx context.Context) error { + listener, err := net.Listen("tcp", ctrl.cfg.Bind()) + if err != nil { + return err + } + ready := make(chan struct{}) + go ctrl.serve(&acceptingListener{Listener: listener, ready: ready}) + select { + case <-ready: + ctrl.log.Info("http listener started", zap.String("address", listener.Addr().String())) + return nil + case err := <-ctrl.serveDone: + return err + case <-ctx.Done(): + _ = listener.Close() + return ctx.Err() + } +} + +// acceptingListener signals after fasthttp registers the listener and enters its +// accept loop. An OnListen hook runs earlier and cannot guarantee safe immediate Stop. +type acceptingListener struct { + // Listener is the bound socket owned by the HTTP lifecycle. + net.Listener + ready chan struct{} + once sync.Once +} + +// Accept marks the server as started before waiting for the next connection. +func (listener *acceptingListener) Accept() (net.Conn, error) { + listener.once.Do(func() { + close(listener.ready) + }) + return listener.Listener.Accept() +} + +// serve serves the listener and publishes its terminal result to the lifecycle. +func (ctrl *Controller) serve(listener net.Listener) { + ctrl.serveDone <- ctrl.server.Listener(listener) +} + +// Stop disables admission, closes the listener and drains active requests. +// The configured timeout is bounded by the caller's Fx shutdown context. +func (ctrl *Controller) Stop(ctx context.Context) error { + ctrl.stopping.Store(true) + ctx, cancel := context.WithTimeout(ctx, ctrl.cfg.ShutdownTimeout()) + defer cancel() + if err := ctrl.server.ShutdownWithContext(ctx); err != nil { + return err + } + select { + case err := <-ctrl.serveDone: + ctrl.log.Info("http listener stopped") + return err + case <-ctx.Done(): + return ctx.Err() + } +} diff --git a/internal/controller/http/fiber/lifecycle_test.go b/internal/controller/http/fiber/lifecycle_test.go new file mode 100644 index 0000000..ed9e1b4 --- /dev/null +++ b/internal/controller/http/fiber/lifecycle_test.go @@ -0,0 +1,132 @@ +package fiber + +import ( + "context" + "errors" + "net" + "net/http" + "testing" + "time" + + "github.com/gofiber/fiber/v2" +) + +// TestImmediateShutdown checks the startup/stop race without waiting for a first request. +func TestImmediateShutdown(t *testing.T) { + const repetitions = 10 + const stopDeadline = time.Second + for range repetitions { + // Arrange. + ctrl := testController(t, nil) + // Select an ephemeral test port after validating the normal application config. + ctrl.cfg.Host = "127.0.0.1" + ctrl.cfg.Port = 0 + if err := ctrl.Start(t.Context()); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(t.Context(), stopDeadline) + // Act. + err := ctrl.Stop(ctx) + cancel() + // Assert. + if err != nil { + t.Fatalf("immediate shutdown leaked the listener or serve goroutine: %v", err) + } + } +} + +// TestController_Stop_ActiveRequestDeadline bounds draining while a real handler is still running. +func TestController_Stop_ActiveRequestDeadline(t *testing.T) { + // Arrange. + const deadline = 3 * time.Second + ctrl := testController(t, nil) + ctrl.cfg.Host = "127.0.0.1" + ctrl.cfg.Port = 0 + // Install one controlled handler to test lifecycle independently of business routes. + ctrl.createServer() + entered := make(chan struct{}) + release := make(chan struct{}) + address := make(chan string, 1) + ctrl.server.Hooks().OnListen(func(data fiber.ListenData) error { + address <- net.JoinHostPort(data.Host, data.Port) + return nil + }) + ctrl.server.Get("/blocked", func(ctx *fiber.Ctx) error { + close(entered) + <-release + return ctx.SendStatus(http.StatusOK) + }) + if err := ctrl.Start(t.Context()); err != nil { + t.Fatal(err) + } + client := &http.Client{Timeout: deadline} + completed := make(chan error, 1) + go func() { + response, err := client.Get("http://" + <-address + "/blocked") + if err == nil { + err = response.Body.Close() + } + completed <- err + }() + t.Cleanup(func() { + close(release) + select { + case err := <-completed: + if err != nil { + t.Error(err) + } + case <-time.After(deadline): + t.Error("blocked request did not terminate after release") + } + stopCtx, cancel := context.WithTimeout(context.Background(), deadline) + defer cancel() + if err := ctrl.Stop(stopCtx); err != nil { + t.Error(err) + } + }) + select { + case <-entered: + case <-time.After(deadline): + t.Fatal("request did not reach handler") + } + ctx, cancel := context.WithCancel(t.Context()) + cancel() + + // Act. + err := ctrl.Stop(ctx) + + // Assert. + if !errors.Is(err, context.Canceled) { + t.Fatalf("expected canceled drain, got %v", err) + } + if !ctrl.stopping.Load() { + t.Fatal("shutdown did not disable admission") + } +} + +// TestCanceledStartup closes its socket while server initialization is still blocked. +func TestCanceledStartup(t *testing.T) { + // Arrange. + ctrl := testController(t, nil) + ctrl.cfg.Host = "127.0.0.1" + ctrl.cfg.Port = 0 + release := make(chan struct{}) + ctrl.server.Hooks().OnListen(func(_ fiber.ListenData) error { + <-release + return nil + }) + ctx, cancel := context.WithCancel(t.Context()) + cancel() + // Act. + err := ctrl.Start(ctx) + close(release) + // Assert. + if !errors.Is(err, context.Canceled) { + t.Fatalf("expected canceled startup, got %v", err) + } + select { + case <-ctrl.serveDone: + case <-time.After(time.Second): + t.Fatal("serving goroutine leaked after canceled startup") + } +} diff --git a/internal/controller/http/fiber/mapping.go b/internal/controller/http/fiber/mapping.go new file mode 100644 index 0000000..b0448e4 --- /dev/null +++ b/internal/controller/http/fiber/mapping.go @@ -0,0 +1,79 @@ +package fiber + +import ( + "github.com/gofiber/fiber/v2" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/model/dto" + httpmodel "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/model/http" +) + +// userResponse converts a user DTO into its HTTP representation. +func userResponse(user dto.UserDTO) httpmodel.UserResponse { + return httpmodel.UserResponse{ + ID: user.ID.String(), + Username: user.Username, + DisplayName: user.DisplayName, + CreatedAt: user.CreatedAt, + } +} + +// chatResponse converts a chat DTO and its member identifiers into an HTTP response. +func chatResponse(chat dto.ChatDTO) httpmodel.ChatResponse { + members := make([]string, len(chat.MemberIDs)) + for index, id := range chat.MemberIDs { + members[index] = id.String() + } + return httpmodel.ChatResponse{ + ID: chat.ID.String(), + Title: chat.Title, + MemberIDs: members, + CreatedAt: chat.CreatedAt, + } +} + +// messageResponse converts a message DTO into its HTTP representation. +func messageResponse(message dto.MessageDTO) httpmodel.MessageResponse { + ids := append([]string{}, message.AttachmentIDs...) + return httpmodel.MessageResponse{ + AttachmentIDs: ids, + ID: message.ID.String(), + ChatID: message.ChatID.String(), + SenderID: message.SenderID.String(), + Text: message.Text, + CreatedAt: message.CreatedAt, + } +} + +// pageRequest extracts identity, resource and pagination parameters from the request. +func pageRequest(ctx *fiber.Ctx) dto.PageRequestDTO { + return dto.PageRequestDTO{ + UserID: identity(ctx), + ChatID: ctx.Params("chat_id"), + Limit: ctx.Query("limit"), + Cursor: ctx.Query("cursor"), + } +} + +// chatPageResponse converts a page of chat DTOs into an HTTP response. +func chatPageResponse(page dto.ChatPageDTO) httpmodel.ChatPageResponse { + response := httpmodel.ChatPageResponse{ + Items: make([]httpmodel.ChatResponse, len(page.Items)), + NextCursor: page.NextCursor, + } + for index, chat := range page.Items { + response.Items[index] = chatResponse(chat) + } + return response +} + +// messagePageResponse converts a page of message DTOs into an HTTP response. +func messagePageResponse(page dto.MessagePageDTO) httpmodel.MessagePageResponse { + response := httpmodel.MessagePageResponse{ + Items: make([]httpmodel.MessageResponse, len(page.Items)), + NextCursor: page.NextCursor, + } + for index, message := range page.Items { + response.Items[index] = messageResponse(message) + } + return response +} diff --git a/internal/controller/http/fiber/messages_test.go b/internal/controller/http/fiber/messages_test.go new file mode 100644 index 0000000..c0dd806 --- /dev/null +++ b/internal/controller/http/fiber/messages_test.go @@ -0,0 +1,270 @@ +package fiber + +import ( + "net/http" + "net/url" + "strings" + "testing" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/config" +) + +// TestController_MessagesCreate checks message validation and identity-derived sender fields. +func TestController_MessagesCreate(t *testing.T) { + t.Run("Positive: Unicode at rune limit", func(t *testing.T) { + // Arrange. + ctrl := testController(t, nil) + owner := user(t, ctrl) + id := chat(t, ctrl, owner) + actor := owner + text := strings.Repeat("界", 4000) + payload := mustJSON(t, map[string]string{"text": text}) + + // Act. + status, body, err := request(ctrl, http.MethodPost, "/api/v1/chats/"+id+"/messages", actor, payload) + + // Assert. + if err != nil { + t.Fatal(err) + } + assertHTTPStatus(t, status, http.StatusCreated, body) + if body["sender_id"] != owner || body["chat_id"] != id || body["text"] != text { + t.Fatalf("message fields changed: %v", body) + } + }) + + t.Run("Negative: empty text", func(t *testing.T) { + // Arrange. + ctrl := testController(t, nil) + owner := user(t, ctrl) + id := chat(t, ctrl, owner) + actor := owner + payload := mustJSON(t, map[string]string{"text": ""}) + + // Act. + status, body, err := request(ctrl, http.MethodPost, "/api/v1/chats/"+id+"/messages", actor, payload) + + // Assert. + if err != nil { + t.Fatal(err) + } + assertHTTPStatus(t, status, http.StatusBadRequest, body) + assertErrorCode(t, body, "invalid_request") + }) + + t.Run("Negative: whitespace text", func(t *testing.T) { + // Arrange. + ctrl := testController(t, nil) + owner := user(t, ctrl) + id := chat(t, ctrl, owner) + actor := owner + payload := mustJSON(t, map[string]string{"text": " \n\t"}) + + // Act. + status, body, err := request(ctrl, http.MethodPost, "/api/v1/chats/"+id+"/messages", actor, payload) + + // Assert. + if err != nil { + t.Fatal(err) + } + assertHTTPStatus(t, status, http.StatusBadRequest, body) + assertErrorCode(t, body, "invalid_request") + }) + + t.Run("Negative: text exceeds rune limit", func(t *testing.T) { + // Arrange. + ctrl := testController(t, nil) + owner := user(t, ctrl) + id := chat(t, ctrl, owner) + actor := owner + payload := mustJSON(t, map[string]string{"text": strings.Repeat("界", 4001)}) + + // Act. + status, body, err := request(ctrl, http.MethodPost, "/api/v1/chats/"+id+"/messages", actor, payload) + + // Assert. + if err != nil { + t.Fatal(err) + } + assertHTTPStatus(t, status, http.StatusBadRequest, body) + assertErrorCode(t, body, "invalid_request") + }) + + t.Run("Negative: sender spoof", func(t *testing.T) { + // Arrange. + ctrl := testController(t, nil) + owner := user(t, ctrl) + id := chat(t, ctrl, owner) + actor := owner + outsider := user(t, ctrl) + payload := mustJSON(t, map[string]string{"text": "hello", "sender_id": outsider}) + + // Act. + status, body, err := request(ctrl, http.MethodPost, "/api/v1/chats/"+id+"/messages", actor, payload) + + // Assert. + if err != nil { + t.Fatal(err) + } + assertHTTPStatus(t, status, http.StatusBadRequest, body) + assertErrorCode(t, body, "invalid_request") + }) + + t.Run("Negative: outsider", func(t *testing.T) { + // Arrange. + ctrl := testController(t, nil) + owner := user(t, ctrl) + id := chat(t, ctrl, owner) + actor := user(t, ctrl) + payload := mustJSON(t, map[string]string{"text": "hello"}) + + // Act. + status, body, err := request(ctrl, http.MethodPost, "/api/v1/chats/"+id+"/messages", actor, payload) + + // Assert. + if err != nil { + t.Fatal(err) + } + assertHTTPStatus(t, status, http.StatusNotFound, body) + assertErrorCode(t, body, "not_found") + }) + + t.Run("Negative: authentication precedes JSON decoding", func(t *testing.T) { + // Arrange. + ctrl := testController(t, nil) + owner := user(t, ctrl) + id := chat(t, ctrl, owner) + actor := "" + payload := []byte(`{`) + + // Act. + status, body, err := request(ctrl, http.MethodPost, "/api/v1/chats/"+id+"/messages", actor, payload) + + // Assert. + if err != nil { + t.Fatal(err) + } + assertHTTPStatus(t, status, http.StatusUnauthorized, body) + assertErrorCode(t, body, "unauthorized") + }) +} + +// TestController_MessagesGetPage rejects invalid pagination separately from access failures. +func TestController_MessagesGetPage(t *testing.T) { + cases := []struct { + name string + query string + outsider bool + status int + code string + }{ + {"Negative: zero limit", "limit=0", false, http.StatusBadRequest, "invalid_request"}, + {"Negative: limit above maximum", "limit=101", false, http.StatusBadRequest, "invalid_request"}, + {"Negative: negative limit", "limit=-1", false, http.StatusBadRequest, "invalid_request"}, + {"Negative: nonnumeric limit", "limit=x", false, http.StatusBadRequest, "invalid_request"}, + {"Negative: fractional limit", "limit=1.0", false, http.StatusBadRequest, "invalid_request"}, + {"Negative: malformed cursor", "cursor=invalid", false, http.StatusBadRequest, "invalid_request"}, + {"Negative: outsider", "", true, http.StatusNotFound, "not_found"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + // Arrange. + ctrl := testController(t, nil) + actor := user(t, ctrl) + id := chat(t, ctrl, actor) + if tc.outsider { + actor = user(t, ctrl) + } + + // Act. + status, body, err := request(ctrl, http.MethodGet, "/api/v1/chats/"+id+"/messages?"+tc.query, actor, nil) + + // Assert. + if err != nil { + t.Fatal(err) + } + assertHTTPStatus(t, status, tc.status, body) + assertErrorCode(t, body, tc.code) + }) + } +} + +// TestController_MessageCursorScope rejects a continuation from another chat's history. +func TestController_MessageCursorScope(t *testing.T) { + // Arrange. + ctrl := testController(t, nil) + actor := user(t, ctrl) + first := chat(t, ctrl, actor) + second := chat(t, ctrl, actor) + path := "/api/v1/chats/" + first + "/messages" + for _, text := range []string{"first", "second"} { + call(t, ctrl, http.MethodPost, path, actor, map[string]string{"text": text}, http.StatusCreated) + } + page := call(t, ctrl, http.MethodGet, path+"?limit=1", actor, nil, http.StatusOK) + token := page["next_cursor"].(string) + + // Act. + status, body, err := request(ctrl, http.MethodGet, "/api/v1/chats/"+second+"/messages?cursor="+url.QueryEscape(token), actor, nil) + + // Assert. + if err != nil { + t.Fatal(err) + } + assertHTTPStatus(t, status, http.StatusBadRequest, body) + assertErrorCode(t, body, "invalid_request") +} + +// TestController_MessagePageLimitChange preserves continuation when the client changes page size. +func TestController_MessagePageLimitChange(t *testing.T) { + // Arrange. + ctrl := testController(t, nil) + actor := user(t, ctrl) + path := "/api/v1/chats/" + chat(t, ctrl, actor) + "/messages" + for _, text := range []string{"first", "second", "third"} { + call(t, ctrl, http.MethodPost, path, actor, map[string]string{"text": text}, http.StatusCreated) + } + first := call(t, ctrl, http.MethodGet, path+"?limit=1", actor, nil, http.StatusOK) + token := first["next_cursor"].(string) + + // Act. + status, body, err := request(ctrl, http.MethodGet, path+"?limit=100&cursor="+url.QueryEscape(token), actor, nil) + + // Assert. + if err != nil { + t.Fatal(err) + } + assertHTTPStatus(t, status, http.StatusOK, body) + items := body["items"].([]any) + if len(items) != 2 || body["next_cursor"] != nil { + t.Fatalf("unexpected final page: %v", body) + } + if items[0].(map[string]any)["text"] != "second" || items[1].(map[string]any)["text"] != "third" { + t.Fatalf("lost order or skipped messages: %v", items) + } +} + +// TestController_MessageCapacity preserves acknowledged data after rejecting an overflow write. +func TestController_MessageCapacity(t *testing.T) { + // Arrange. + ctrl := testController(t, func(cfg *config.Config) { + cfg.Memory.MaxMessages = 2 + }) + actor := user(t, ctrl) + path := "/api/v1/chats/" + chat(t, ctrl, actor) + "/messages" + first := call(t, ctrl, http.MethodPost, path, actor, map[string]string{"text": "first"}, http.StatusCreated) + second := call(t, ctrl, http.MethodPost, path, actor, map[string]string{"text": "second"}, http.StatusCreated) + + // Act. + status, body, err := request(ctrl, http.MethodPost, path, actor, []byte(`{"text":"rejected"}`)) + + // Assert. + if err != nil { + t.Fatal(err) + } + assertHTTPStatus(t, status, http.StatusServiceUnavailable, body) + assertErrorCode(t, body, "unavailable") + items := history(t, ctrl, actor, path) + if len(items) != 2 || items[0].(map[string]any)["id"] != first["id"] || items[1].(map[string]any)["id"] != second["id"] { + t.Fatalf("overflow changed acknowledged messages: %v", items) + } +} diff --git a/internal/controller/http/fiber/metrics.go b/internal/controller/http/fiber/metrics.go new file mode 100644 index 0000000..448b7f7 --- /dev/null +++ b/internal/controller/http/fiber/metrics.go @@ -0,0 +1,61 @@ +package fiber + +import ( + "net/http" + "time" + + "github.com/gofiber/fiber/v2" + "github.com/gofiber/fiber/v2/middleware/adaptor" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" +) + +// httpMetrics owns a per-server registry with bounded labels and no user identifiers. +type httpMetrics struct { + registry *prometheus.Registry + requests *prometheus.CounterVec + duration *prometheus.HistogramVec + active prometheus.Gauge +} + +// newHTTPMetrics creates independent collectors so multiple test servers do not share state. +func newHTTPMetrics() *httpMetrics { + metrics := &httpMetrics{ + registry: prometheus.NewRegistry(), + requests: prometheus.NewCounterVec(prometheus.CounterOpts{Name: "messenger_http_requests_total", Help: "Completed requests by route, method and status class."}, []string{"route", "method", "status_class"}), + duration: prometheus.NewHistogramVec(prometheus.HistogramOpts{Name: "messenger_http_duration_seconds", Help: "Completed request latency.", Buckets: prometheus.DefBuckets}, []string{"route", "method"}), + active: prometheus.NewGauge(prometheus.GaugeOpts{Name: "messenger_http_in_flight", Help: "Currently executing HTTP requests."}), + } + metrics.registry.MustRegister(metrics.requests, metrics.duration, metrics.active) + return metrics +} + +// observe records bounded route patterns, methods and status classes after error handling. +func (m *httpMetrics) observe(ctx *fiber.Ctx, elapsed time.Duration) { + route := "unmatched" + if ctx.Route() != nil && ctx.Route().Path != "/" { + route = ctx.Route().Path + } + method := ctx.Method() + switch method { + case http.MethodGet, http.MethodPost, http.MethodDelete, http.MethodHead, http.MethodOptions: + default: + method = "other" + } + status := "5xx" + switch ctx.Response().StatusCode() / 100 { + case 2: + status = "2xx" + case 3: + status = "3xx" + case 4: + status = "4xx" + } + m.requests.WithLabelValues(route, method, status).Inc() + m.duration.WithLabelValues(route, method).Observe(elapsed.Seconds()) +} + +// handleMetrics exposes this process's Prometheus registry without external backend calls. +func (ctrl *Controller) handleMetrics(ctx *fiber.Ctx) error { + return adaptor.HTTPHandler(promhttp.HandlerFor(ctrl.metrics.registry, promhttp.HandlerOpts{}))(ctx) +} diff --git a/internal/controller/http/fiber/metrics_labels_test.go b/internal/controller/http/fiber/metrics_labels_test.go new file mode 100644 index 0000000..73cc983 --- /dev/null +++ b/internal/controller/http/fiber/metrics_labels_test.go @@ -0,0 +1,38 @@ +package fiber + +import ( + "net/http" + "testing" + "time" + + "github.com/valyala/fasthttp" +) + +// TestMetricsBoundUnknownMethods keeps arbitrary client methods out of metric labels. +func TestMetricsBoundUnknownMethods(t *testing.T) { + ctrl := testController(t, nil) + raw := &fasthttp.RequestCtx{} + raw.Request.Header.SetMethod("CUSTOM_RANDOM_METHOD") + raw.Response.SetStatusCode(http.StatusTemporaryRedirect) + ctx := ctrl.server.AcquireCtx(raw) + defer ctrl.server.ReleaseCtx(ctx) + ctrl.metrics.observe(ctx, time.Millisecond) + families, err := ctrl.metrics.registry.Gather() + if err != nil { + t.Fatal(err) + } + found := false + for _, family := range families { + for _, metric := range family.Metric { + for _, label := range metric.Label { + if label.GetValue() == "CUSTOM_RANDOM_METHOD" { + t.Fatal("unbounded method label") + } + found = found || label.GetValue() == "other" + } + } + } + if !found { + t.Fatal("unknown method was not grouped") + } +} diff --git a/internal/controller/http/fiber/metrics_test.go b/internal/controller/http/fiber/metrics_test.go new file mode 100644 index 0000000..9f2d4e9 --- /dev/null +++ b/internal/controller/http/fiber/metrics_test.go @@ -0,0 +1,73 @@ +package fiber + +import ( + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +// TestMetricsEndpoint exposes completed requests without token or user labels. +func TestMetricsEndpoint(t *testing.T) { + ctrl := testController(t, nil) + call(t, ctrl, http.MethodGet, "/health/live", "", nil, http.StatusOK) + resp, err := ctrl.server.Test(httptest.NewRequest(http.MethodGet, "/metrics", nil)) + if err != nil { + t.Fatal(err) + } + defer func() { + _ = resp.Body.Close() + }() + body, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatal(err) + } + if resp.StatusCode != http.StatusOK || !strings.Contains(string(body), "messenger_http_requests_total") || !strings.Contains(string(body), "messenger_http_duration_seconds_bucket") { + t.Fatalf("invalid metrics response: status %d", resp.StatusCode) + } + if resp.Header.Get("X-Instance-Id") == "" { + t.Fatal("metrics response lacks instance ID") + } +} + +// TestInstanceIdentityRemainsStable verifies errors and successes identify the same process. +func TestInstanceIdentityRemainsStable(t *testing.T) { + ctrl := testController(t, nil) + var instance string + for _, path := range []string{"/health/live", "/unknown"} { + resp, err := ctrl.server.Test(httptest.NewRequest(http.MethodGet, path, nil)) + if err != nil { + t.Fatal(err) + } + if instance == "" { + instance = resp.Header.Get("X-Instance-Id") + } + if instance == "" || resp.Header.Get("X-Instance-Id") != instance { + t.Fatal("unstable instance identity") + } + if err := resp.Body.Close(); err != nil { + t.Fatal(err) + } + } +} + +// TestMissingIdempotencyHeader rejects an otherwise valid authenticated write. +func TestMissingIdempotencyHeader(t *testing.T) { + ctrl := testController(t, nil) + caller := user(t, ctrl) + room := chat(t, ctrl, caller) + req := httptest.NewRequest(http.MethodPost, "/api/v1/chats/"+room+"/messages", strings.NewReader(`{"text":"valid"}`)) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer "+caller) + resp, err := ctrl.server.Test(req) + if err != nil { + t.Fatal(err) + } + defer func() { + _ = resp.Body.Close() + }() + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("missing key accepted: %d", resp.StatusCode) + } +} diff --git a/internal/controller/http/fiber/middleware.go b/internal/controller/http/fiber/middleware.go new file mode 100644 index 0000000..4dc28a5 --- /dev/null +++ b/internal/controller/http/fiber/middleware.go @@ -0,0 +1,104 @@ +package fiber + +import ( + "context" + "errors" + "strconv" + "time" + + "github.com/gofiber/fiber/v2" + "github.com/google/uuid" + "go.uber.org/zap" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/admission" +) + +// request assigns a request identifier, handles errors and logs the completed request. +func (ctrl *Controller) request(ctx *fiber.Ctx) error { + ctx.Set(headerRequestID, uuid.NewString()) + ctx.Set("X-Instance-Id", ctrl.instanceID) + started := time.Now() + var finish func(error) + if ctrl.observer != nil { + parent := ctrl.observer.Extract(ctx.UserContext(), ctx.Get("traceparent"), ctx.Get("tracestate")) + observed, end := ctrl.observer.Start(parent, "http.request", "server") + ctx.SetUserContext(observed) + finish = end + } + ctrl.metrics.active.Inc() + defer ctrl.metrics.active.Dec() + err := ctx.Next() + if err != nil { + err = ctrl.handleErr(ctx, err) + } + ctrl.logRequest(ctx, time.Since(started)) + if ctrl.observer != nil { + route := "unmatched" + if ctx.Route() != nil { + route = ctx.Route().Path + } + ctrl.observer.HTTP(ctx.UserContext(), ctx.Method(), route, ctx.Response().StatusCode(), time.Since(started), ctx.GetRespHeader(headerRequestID)) + finish(err) + } else if ctx.Path() != "/metrics" { + ctrl.metrics.observe(ctx, time.Since(started)) + } + return err +} + +// logRequest logs request metadata without including message contents. +func (ctrl *Controller) logRequest(ctx *fiber.Ctx, elapsed time.Duration) { + route := "unmatched" + if ctx.Route() != nil { + route = ctx.Route().Path + } + ctrl.log.Info("http request", + zap.String("method", ctx.Method()), + zap.String("route", route), + zap.Int("status", ctx.Response().StatusCode()), + zap.String("request_id", ctx.GetRespHeader(headerRequestID)), + zap.Duration("duration", elapsed), + ) +} + +// admit reserves global budgets and bounds downstream request work. +func (ctrl *Controller) admit(ctx *fiber.Ctx) error { + if ctrl.stopping.Load() { + return errBusy + } + requestCtx, cancel := context.WithTimeout(ctx.UserContext(), ctrl.cfg.Timeout()) + ctx.SetUserContext(requestCtx) + release, err := ctrl.admission.Enter(requestCtx, requestGroup(ctx), "") + if err != nil { + cancel() + return admissionError(ctx, err) + } + defer finishRequest(ctx, func() { + release() + cancel() + }) + return ctx.Next() +} + +// requireIdentity resolves the session before reserving per-user budgets. +func (ctrl *Controller) requireIdentity(ctx *fiber.Ctx) error { + user, err := ctrl.auth.Authenticate(ctx.UserContext(), bearer(ctx)) + if err != nil { + return err + } + ctx.Locals(identityKey, user) + release, err := ctrl.admission.Enter(ctx.UserContext(), requestGroup(ctx), user) + if err != nil { + return admissionError(ctx, err) + } + defer finishRequest(ctx, release) + return ctx.Next() +} + +// admissionError maps policy rejection and backend failure into HTTP-owned errors. +func admissionError(ctx *fiber.Ctx, err error) error { + ctx.Set(fiber.HeaderRetryAfter, strconv.Itoa(retryAfterSeconds)) + if errors.Is(err, admission.ErrRate) { + return errRateLimited + } + return errBusy +} diff --git a/internal/controller/http/fiber/middleware_test.go b/internal/controller/http/fiber/middleware_test.go new file mode 100644 index 0000000..50e57d7 --- /dev/null +++ b/internal/controller/http/fiber/middleware_test.go @@ -0,0 +1,211 @@ +package fiber + +import ( + "net/http" + "testing" + "testing/synctest" + "time" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/config" +) + +// TestController_InFlightAdmission checks rejection, recovery and health exemption independently. +func TestController_InFlightAdmission(t *testing.T) { + cases := []struct { + name string + release bool + path string + status int + }{ + {"Negative: occupied execution slot", false, "/api/v1/users/me", http.StatusServiceUnavailable}, + {"Positive: released execution slot", true, "/api/v1/users/me", http.StatusUnauthorized}, + {"Positive: health bypasses occupied slot", false, "/health/live", http.StatusOK}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + // Arrange. + ctrl := testController(t, func(cfg *config.Config) { + cfg.HTTP.Admission.MaxInFlight = 1 + }) + if !ctrl.admission.(*fixtureAdmission).Acquire() { + t.Fatal("fixture could not reserve slot") + } + if tc.release { + ctrl.admission.(*fixtureAdmission).Release() + } else { + t.Cleanup(ctrl.admission.(*fixtureAdmission).Release) + } + + // Act. + status, body, err := request(ctrl, http.MethodGet, tc.path, "", nil) + + // Assert. + if err != nil { + t.Fatal(err) + } + assertHTTPStatus(t, status, tc.status, body) + }) + } +} + +// TestController_RateAdmission checks exhausted tokens without timing-dependent HTTP setup. +func TestController_RateAdmission(t *testing.T) { + t.Run("/api/v1/users/me", func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + // Arrange. + ctrl := testController(t, func(cfg *config.Config) { + cfg.HTTP.Admission.RequestsPerSecond = 1 + cfg.HTTP.Admission.Burst = 1 + }) + if !ctrl.admission.(*fixtureAdmission).Allow(time.Now()) { + t.Fatal("fixture token missing") + } + expected := http.StatusTooManyRequests + + // Act. + status, body, err := request(ctrl, http.MethodGet, "/api/v1/users/me", "", nil) + + // Assert. + if err != nil { + t.Fatal(err) + } + assertHTTPStatus(t, status, expected, body) + }) + }) + + t.Run("/health/ready", func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + // Arrange. + ctrl := testController(t, func(cfg *config.Config) { + cfg.HTTP.Admission.RequestsPerSecond = 1 + cfg.HTTP.Admission.Burst = 1 + }) + if !ctrl.admission.(*fixtureAdmission).Allow(time.Now()) { + t.Fatal("fixture token missing") + } + expected := http.StatusOK + + // Act. + status, body, err := request(ctrl, http.MethodGet, "/health/ready", "", nil) + + // Assert. + if err != nil { + t.Fatal(err) + } + assertHTTPStatus(t, status, expected, body) + }) + }) +} + +// TestController_Draining checks each endpoint's admission policy during shutdown. +func TestController_Draining(t *testing.T) { + cases := []struct { + name string + path string + status int + }{ + {"Negative: readiness disabled", "/health/ready", http.StatusServiceUnavailable}, + {"Positive: liveness remains available", "/health/live", http.StatusOK}, + {"Negative: business requests rejected", "/api/v1/users/me", http.StatusServiceUnavailable}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + // Arrange. + ctrl := testController(t, nil) + ctrl.stopping.Store(true) + + // Act. + status, body, err := request(ctrl, http.MethodGet, tc.path, "", nil) + + // Assert. + if err != nil { + t.Fatal(err) + } + assertHTTPStatus(t, status, tc.status, body) + }) + } +} + +// TestController_UnknownRoute checks the public fallback for an unmatched path. +func TestController_UnknownRoute(t *testing.T) { + // Arrange. + ctrl := testController(t, nil) + + // Act. + status, body, err := request(ctrl, http.MethodGet, "/missing", "", nil) + + // Assert. + if err != nil { + t.Fatal(err) + } + assertHTTPStatus(t, status, http.StatusNotFound, body) + assertErrorCode(t, body, "not_found") +} + +// TestController_New rejects each missing dependency independently. +func TestController_New(t *testing.T) { + t.Run("Negative: missing configuration", func(t *testing.T) { + // Arrange. + fixture := testController(t, nil) + + // Act. + ctrl, err := New(fixture.log, nil, fixture.service, fixture.errorRegistry, fixture.admission, fixture.auth, fixture.readiness) + + // Assert. + if err == nil || ctrl != nil { + t.Fatalf("incomplete controller accepted: %v, %v", ctrl, err) + } + }) + + t.Run("Negative: missing logger", func(t *testing.T) { + // Arrange. + fixture := testController(t, nil) + + // Act. + ctrl, err := New(nil, &fixture.cfg, fixture.service, fixture.errorRegistry, fixture.admission, fixture.auth, fixture.readiness) + + // Assert. + if err == nil || ctrl != nil { + t.Fatalf("incomplete controller accepted: %v, %v", ctrl, err) + } + }) + + t.Run("Negative: missing service", func(t *testing.T) { + // Arrange. + fixture := testController(t, nil) + + // Act. + ctrl, err := New(fixture.log, &fixture.cfg, nil, fixture.errorRegistry, fixture.admission, fixture.auth, fixture.readiness) + + // Assert. + if err == nil || ctrl != nil { + t.Fatalf("incomplete controller accepted: %v, %v", ctrl, err) + } + }) + + t.Run("Negative: missing registry", func(t *testing.T) { + // Arrange. + fixture := testController(t, nil) + + // Act. + ctrl, err := New(fixture.log, &fixture.cfg, fixture.service, nil, fixture.admission, fixture.auth, fixture.readiness) + + // Assert. + if err == nil || ctrl != nil { + t.Fatalf("incomplete controller accepted: %v, %v", ctrl, err) + } + }) + + t.Run("Negative: missing limiter", func(t *testing.T) { + // Arrange. + fixture := testController(t, nil) + + // Act. + ctrl, err := New(fixture.log, &fixture.cfg, fixture.service, fixture.errorRegistry, nil, fixture.auth, fixture.readiness) + + // Assert. + if err == nil || ctrl != nil { + t.Fatalf("incomplete controller accepted: %v, %v", ctrl, err) + } + }) +} diff --git a/internal/controller/http/fiber/observation.go b/internal/controller/http/fiber/observation.go new file mode 100644 index 0000000..ca1fb27 --- /dev/null +++ b/internal/controller/http/fiber/observation.go @@ -0,0 +1,22 @@ +package fiber + +import ( + "context" + "time" +) + +// HTTPObserver consumes only safe transport metadata and W3C context. +type HTTPObserver interface { + // Start begins a server operation and returns its completion callback. + Start(context.Context, string, string) (context.Context, func(error)) + // Extract restores the upstream W3C parent without baggage. + Extract(context.Context, string, string) context.Context + // HTTP observes the completed response using a route template. + HTTP(context.Context, string, string, int, time.Duration, string) +} + +// WithObserver installs optional instrumentation before the HTTP listener starts. +func (ctrl *Controller) WithObserver(observer HTTPObserver) *Controller { + ctrl.observer = observer + return ctrl +} diff --git a/internal/controller/http/fiber/public_errors.go b/internal/controller/http/fiber/public_errors.go new file mode 100644 index 0000000..e52b86d --- /dev/null +++ b/internal/controller/http/fiber/public_errors.go @@ -0,0 +1,55 @@ +package fiber + +import ( + "errors" + + "go.lab260.ru/fielderr" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/service" +) + +var ( + errBusy = publicFailure(codeUnavailable, "server is busy") + errRateLimited = publicFailure(codeRateLimited, "request rate exceeded") + errTooLarge = publicFailure(codeTooLarge, "request body is too large") + errMedia = publicFailure(codeMedia, "expected application/json") + errInternal = publicFailure(codeInternal, "internal server error") +) + +// publicFailure creates a transport-owned error with a safe API message. +func publicFailure(code, message string) error { + return fielderr.WithStatus(errors.New(message), fielderr.NewStatusCode(code), message) +} + +// publicError maps application failures into the public API envelope. +// Unknown errors receive a fixed response without exposing their details. +func publicError(err error) error { + switch { + case errors.Is(err, service.ErrTooLarge): + return errTooLarge + case errors.Is(err, service.ErrMedia): + return errMedia + case errors.Is(err, service.ErrUnavailable): + return publicFailure(codeUnavailable, "dependency unavailable") + case errors.Is(err, service.ErrInvalid): + return fielderr.WithStatus(err, fielderr.NewStatusCode(codeInvalid), "invalid request") + case errors.Is(err, service.ErrUnauthorized): + return fielderr.WithStatus(err, fielderr.NewStatusCode(codeUnauthorized), "unknown user identity") + case errors.Is(err, service.ErrNotFound): + return fielderr.WithStatus(err, fielderr.NewStatusCode(codeNotFound), "resource not found") + case errors.Is(err, service.ErrConflict): + return fielderr.WithStatus(err, fielderr.NewStatusCode(codeConflict), "username already exists") + case errors.Is(err, service.ErrCapacity): + return fielderr.WithStatus(err, fielderr.NewStatusCode(codeUnavailable), "in-memory storage capacity reached") + case errors.Is(err, errBusy): + return errBusy + case errors.Is(err, errRateLimited): + return errRateLimited + case errors.Is(err, errTooLarge): + return errTooLarge + case errors.Is(err, errMedia): + return errMedia + default: + return errInternal + } +} diff --git a/internal/controller/http/fiber/public_errors_test.go b/internal/controller/http/fiber/public_errors_test.go new file mode 100644 index 0000000..13a8fc5 --- /dev/null +++ b/internal/controller/http/fiber/public_errors_test.go @@ -0,0 +1,43 @@ +package fiber + +import ( + "errors" + "fmt" + "testing" + + "go.lab260.ru/fielderr" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/service" +) + +// TestPublicErrorSafety maps wrapped application failures without exposing internal details. +func TestPublicErrorSafety(t *testing.T) { + t.Run("wrapped application error", func(t *testing.T) { + err := publicError(fmt.Errorf("private database detail: %w", service.ErrConflict)) + response := errorResponse(err, "request") + if response.Error.Code != codeConflict || response.Error.Message != "username already exists" { + t.Fatalf("unexpected response: %+v", response) + } + }) + t.Run("unknown error", func(t *testing.T) { + err := publicError(errors.New("private database detail")) + if err != errInternal { + t.Fatalf("unexpected public error: %v", err) + } + }) + t.Run("foreign public status", func(t *testing.T) { + err := fielderr.WithStatus(errors.New("private detail"), fielderr.NewStatusCode(codeInvalid), "private detail") + if publicError(err) != errInternal { + t.Fatal("untrusted error metadata reached the public response") + } + }) +} + +// TestPublicErrorUnavailable keeps dependency details out of the public envelope. +func TestPublicErrorUnavailable(t *testing.T) { + err := publicError(fmt.Errorf("private database detail: %w", service.ErrUnavailable)) + response := errorResponse(err, "request") + if response.Error.Code != codeUnavailable || response.Error.Message != "dependency unavailable" { + t.Fatalf("unexpected response: %+v", response) + } +} diff --git a/internal/controller/http/fiber/routes.go b/internal/controller/http/fiber/routes.go new file mode 100644 index 0000000..31f17de --- /dev/null +++ b/internal/controller/http/fiber/routes.go @@ -0,0 +1,26 @@ +package fiber + +// initRoutes registers health endpoints and authenticated application routes. +func (ctrl *Controller) initRoutes() { + ctrl.server.Get("/metrics", ctrl.handleMetrics) + ctrl.server.Get("/health/live", ctrl.handleLive) + ctrl.server.Get("/health/ready", ctrl.handleReady) + ctrl.server.Use(ctrl.admit) + ctrl.server.Post("/api/v1/users", ctrl.HandleUsersCreate) + + ctrl.server.Post("/api/v1/auth/sessions", ctrl.HandleLogin) + + api := ctrl.server.Group("/api/v1", ctrl.requireIdentity) + api.Delete("/auth/sessions/current", ctrl.HandleLogout) + api.Get("/users/me", ctrl.HandleUsersGetMe) + api.Post("/chats", ctrl.HandleChatsCreate) + api.Get("/chats", ctrl.HandleChatsGetPage) + api.Get("/chats/:chat_id", ctrl.HandleChatsGetByID) + api.Post("/chats/:chat_id/attachments", ctrl.handleUpload) + api.Get("/attachments/:id", ctrl.handleAttachment) + api.Get("/attachments/:id/content", ctrl.handleContent) + api.Post("/chats/:chat_id/messages", ctrl.HandleMessagesCreate) + api.Get("/chats/:chat_id/messages", ctrl.HandleMessagesGetPage) + + ctrl.server.Use(ctrl.handleNotFound) +} diff --git a/internal/controller/http/fiber/stream.go b/internal/controller/http/fiber/stream.go new file mode 100644 index 0000000..4fef6d8 --- /dev/null +++ b/internal/controller/http/fiber/stream.go @@ -0,0 +1,40 @@ +package fiber + +import ( + "io" + "sync" + + "github.com/gofiber/fiber/v2" +) + +type responseStream struct { + io.Reader + cleanup func() + once sync.Once +} + +// Close releases request budgets after fasthttp finishes transmitting the stream. +func (s *responseStream) Close() error { + var err error + s.once.Do(func() { + if closer, ok := s.Reader.(io.Closer); ok { + err = closer.Close() + } + s.cleanup() + }) + return err +} + +// finishRequest keeps deadlines and admission leases alive until streamed responses close. +func finishRequest(ctx *fiber.Ctx, cleanup func()) { + stream, ok := ctx.Response().BodyStream().(*responseStream) + if !ok { + cleanup() + return + } + previous := stream.cleanup + stream.cleanup = func() { + previous() + cleanup() + } +} diff --git a/internal/controller/http/fiber/users_test.go b/internal/controller/http/fiber/users_test.go new file mode 100644 index 0000000..61d7e35 --- /dev/null +++ b/internal/controller/http/fiber/users_test.go @@ -0,0 +1,130 @@ +package fiber + +import ( + "net/http" + "strings" + "testing" + + "github.com/google/uuid" +) + +// TestController_UsersCreate checks one registration contract scenario per subtest. +func TestController_UsersCreate(t *testing.T) { + cases := []struct { + name string + body string + status int + }{ + {"Positive: valid Unicode display name", `{"password":"fixture-password-123","username":"alice","display_name":"Алиса 👋"}`, http.StatusCreated}, + {"Positive: display name at rune limit", `{"password":"fixture-password-123","username":"alice","display_name":"` + strings.Repeat("👋", 100) + `"}`, http.StatusCreated}, + {"Negative: display name exceeds rune limit", `{"password":"fixture-password-123","username":"alice","display_name":"` + strings.Repeat("👋", 101) + `"}`, http.StatusBadRequest}, + {"Negative: uppercase username", `{"password":"fixture-password-123","username":"Alice","display_name":"Alice"}`, http.StatusBadRequest}, + {"Negative: malformed JSON", `{`, http.StatusBadRequest}, + {"Negative: null root", `null`, http.StatusBadRequest}, + {"Negative: array root", `[]`, http.StatusBadRequest}, + {"Negative: missing fields", `{}`, http.StatusBadRequest}, + {"Negative: null field", `{"password":"fixture-password-123","username":"alice","display_name":null}`, http.StatusBadRequest}, + {"Negative: wrong field type", `{"password":"fixture-password-123","username":5,"display_name":"Alice"}`, http.StatusBadRequest}, + {"Negative: unknown field", `{"password":"fixture-password-123","username":"alice","display_name":"Alice","unknown":"secret"}`, http.StatusBadRequest}, + {"Negative: field name case", `{"password":"fixture-password-123","Username":"alice","display_name":"Alice"}`, http.StatusBadRequest}, + {"Negative: duplicate field", `{"password":"fixture-password-123","username":"alice","username":"bob","display_name":"Alice"}`, http.StatusBadRequest}, + {"Negative: trailing object", `{"password":"fixture-password-123","username":"alice","display_name":"Alice"} {}`, http.StatusBadRequest}, + {"Negative: invalid UTF-8", "{\"username\":\"alice\",\"display_name\":\"\xff\"}", http.StatusBadRequest}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + // Arrange. + ctrl := testController(t, nil) + + // Act. + status, body, err := request(ctrl, http.MethodPost, "/api/v1/users", "", []byte(tc.body)) + + // Assert. + if err != nil { + t.Fatal(err) + } + assertHTTPStatus(t, status, tc.status, body) + if tc.status == http.StatusBadRequest { + assertErrorCode(t, body, "invalid_request") + return + } + if body["username"] != "alice" || body["display_name"] == "" { + t.Fatalf("registration fields changed: %v", body) + } + if _, err := uuid.Parse(body["id"].(string)); err != nil { + t.Fatalf("invalid assigned ID: %v", err) + } + }) + } +} + +// TestController_UsersGetMe verifies profile selection and authentication failures. +func TestController_UsersGetMe(t *testing.T) { + t.Run("Positive: own profile", func(t *testing.T) { + // Arrange. + ctrl := testController(t, nil) + actor := user(t, ctrl) + identity := actor + + // Act. + status, body, err := request(ctrl, http.MethodGet, "/api/v1/users/me", identity, nil) + + // Assert. + if err != nil { + t.Fatal(err) + } + assertHTTPStatus(t, status, http.StatusOK, body) + if body["id"] != actor || body["display_name"] != "Алиса 👋" { + t.Fatalf("wrong profile: %v", body) + } + }) + + t.Run("Negative: missing identity", func(t *testing.T) { + // Arrange. + ctrl := testController(t, nil) + identity := "" + + // Act. + status, body, err := request(ctrl, http.MethodGet, "/api/v1/users/me", identity, nil) + + // Assert. + if err != nil { + t.Fatal(err) + } + assertHTTPStatus(t, status, http.StatusUnauthorized, body) + assertErrorCode(t, body, "unauthorized") + }) + + t.Run("Negative: unknown identity", func(t *testing.T) { + // Arrange. + ctrl := testController(t, nil) + identity := uuid.NewString() + + // Act. + status, body, err := request(ctrl, http.MethodGet, "/api/v1/users/me", identity, nil) + + // Assert. + if err != nil { + t.Fatal(err) + } + assertHTTPStatus(t, status, http.StatusUnauthorized, body) + assertErrorCode(t, body, "unauthorized") + }) +} + +// assertHTTPStatus reports the response body alongside a mismatched status. +func assertHTTPStatus(t *testing.T, got, want int, body map[string]any) { + t.Helper() + if got != want { + t.Fatalf("status=%d, want=%d, body=%v", got, want, body) + } +} + +// assertErrorCode checks the documented error classification independently of status. +func assertErrorCode(t *testing.T, body map[string]any, want string) { + t.Helper() + envelope, ok := body["error"].(map[string]any) + if !ok || envelope["code"] != want { + t.Fatalf("error=%v, want code %q", body, want) + } +} diff --git a/internal/controller/interface.go b/internal/controller/interface.go new file mode 100644 index 0000000..46e9af5 --- /dev/null +++ b/internal/controller/interface.go @@ -0,0 +1,12 @@ +// Package controller connects transport lifecycles to Fx. +package controller + +import "context" + +// Controller is a process-owned transport server. +type Controller interface { + // Start acquires the listener and reports startup errors to the lifecycle owner. + Start(context.Context) error + // Stop stops accepting work and drains active requests within the supplied deadline. + Stop(context.Context) error +} diff --git a/internal/cursor/codec.go b/internal/cursor/codec.go new file mode 100644 index 0000000..6f4cd94 --- /dev/null +++ b/internal/cursor/codec.go @@ -0,0 +1,119 @@ +// Package cursor signs and verifies resource-bound keyset positions. +package cursor + +import ( + "crypto/hmac" + "crypto/rand" + "crypto/sha256" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "time" + + "github.com/google/uuid" +) + +const ( + cursorSigningKeyBytes = 32 + maxEncodedCursorBytes = 1024 +) + +// ErrInvalid means a cursor is malformed, tampered with or bound to another resource. +var ErrInvalid = errors.New("invalid cursor") + +// Position identifies the last record already returned by keyset pagination. +type Position struct { + // CreatedAt is the creation timestamp of the last returned record. + CreatedAt time.Time + // ID breaks ties between records sharing a timestamp. + ID uuid.UUID +} + +// Codec authenticates pagination positions with an immutable process-local key. +type Codec struct { + key [cursorSigningKeyBytes]byte +} + +type cursorPayload struct { + // Resource binds the position to a chat history or one user's chat list. + Resource string `json:"r"` + // Time is the creation timestamp of the last record in the previous page. + Time time.Time `json:"t"` + // ID breaks ties between records with the same creation timestamp. + ID uuid.UUID `json:"i"` +} + +// New creates a cursor codec with a random process-local signing key. +func New() (*Codec, error) { + codec := &Codec{} + if _, err := rand.Read(codec.key[:]); err != nil { + return nil, err + } + return codec, nil +} + +// Encode serializes and signs a pagination position bound to a resource. +func (codec *Codec) Encode(resource string, position Position) (string, error) { + payload, err := json.Marshal(cursorPayload{ + Resource: resource, + Time: position.CreatedAt, + ID: position.ID, + }) + if err != nil { + return "", fmt.Errorf("encode cursor: %w", err) + } + payload = append(payload, codec.signature(payload)...) + return base64.RawURLEncoding.EncodeToString(payload), nil +} + +// Decode verifies the cursor signature and extracts its resource-bound position. +func (codec *Codec) Decode(encoded, resource string) (Position, error) { + if encoded == "" { + return Position{}, nil + } + if len(encoded) > maxEncodedCursorBytes { + return Position{}, ErrInvalid + } + signed, err := base64.RawURLEncoding.DecodeString(encoded) + if err != nil || len(signed) <= sha256.Size { + return Position{}, ErrInvalid + } + boundary := len(signed) - sha256.Size + payload, signature := signed[:boundary], signed[boundary:] + if !hmac.Equal(signature, codec.signature(payload)) { + return Position{}, ErrInvalid + } + return decodeCursorPosition(payload, resource) +} + +// signature computes the HMAC-SHA256 authentication tag for a cursor payload. +func (codec *Codec) signature(payload []byte) []byte { + mac := hmac.New(sha256.New, codec.key[:]) + // hash.Hash.Write is documented to always return a nil error. + _, _ = mac.Write(payload) + return mac.Sum(nil) +} + +// decodeCursorPosition validates the decoded cursor resource and timestamp. +func decodeCursorPosition(payload []byte, resource string) (Position, error) { + var cursor cursorPayload + if err := json.Unmarshal(payload, &cursor); err != nil { + return Position{}, ErrInvalid + } + if cursor.Resource != resource || cursor.Time.IsZero() { + return Position{}, ErrInvalid + } + return Position{CreatedAt: cursor.Time, ID: cursor.ID}, nil +} + +// NewShared creates a codec from the common 256-bit key provisioned to every replica. +func NewShared(encoded string) (*Codec, error) { + key, err := base64.StdEncoding.DecodeString(encoded) + if err != nil || len(key) != cursorSigningKeyBytes { + return nil, fmt.Errorf("CURSOR_SIGNING_KEY must encode exactly 32 bytes as base64") + } + codec := &Codec{} + copy(codec.key[:], key) + return codec, nil +} diff --git a/internal/cursor/codec_test.go b/internal/cursor/codec_test.go new file mode 100644 index 0000000..441e530 --- /dev/null +++ b/internal/cursor/codec_test.go @@ -0,0 +1,212 @@ +package cursor + +import ( + "encoding/base64" + "errors" + "strings" + "testing" + "time" + + "github.com/google/uuid" +) + +// TestCodec_EncodeDecode preserves the complete position through an authenticated round trip. +func TestCodec_EncodeDecode(t *testing.T) { + // Arrange. + codec := newTestCodec(t) + position := Position{CreatedAt: time.Unix(1700000000, 123456789).UTC(), ID: uuid.New()} + + // Act. + encoded, err := codec.Encode("chats:alice", position) + + // Assert. + if err != nil { + t.Fatal(err) + } + if encoded == "" { + t.Fatal("encoded position is empty") + } + decoded, err := codec.Decode(encoded, "chats:alice") + if err != nil || decoded != position { + t.Fatalf("decoded=%v, want=%v, error=%v", decoded, position, err) + } +} + +// TestCodec_Decode checks first-page behavior and independent rejection conditions. +func TestCodec_Decode(t *testing.T) { + t.Run("Positive: empty token starts first page", func(t *testing.T) { + // Arrange. + codec := newTestCodec(t) + + // Act. + got, err := codec.Decode("", "chats:alice") + + // Assert. + if err != nil || got != (Position{}) { + t.Fatalf("first-page position=%v, error=%v", got, err) + } + }) + + t.Run("Negative: another resource", func(t *testing.T) { + // Arrange. + codec := newTestCodec(t) + encoded := encodeFixture(t, codec, "chats:alice") + + // Act. + got, err := codec.Decode(encoded, "chats:bob") + + // Assert. + if !errors.Is(err, ErrInvalid) || got != (Position{}) { + t.Fatalf("position=%v, error=%v", got, err) + } + }) + + t.Run("Negative: another process key", func(t *testing.T) { + // Arrange. + codec := newTestCodec(t) + encoded := encodeFixture(t, codec, "chats:alice") + codec = newTestCodec(t) + + // Act. + got, err := codec.Decode(encoded, "chats:alice") + + // Assert. + if !errors.Is(err, ErrInvalid) || got != (Position{}) { + t.Fatalf("position=%v, error=%v", got, err) + } + }) + + t.Run("Negative: tampered payload", func(t *testing.T) { + // Arrange. + codec := newTestCodec(t) + encoded := encodeFixture(t, codec, "chats:alice") + signed, err := base64.RawURLEncoding.DecodeString(encoded) + if err != nil { + t.Fatal(err) + } + signed[0] ^= 1 + encoded = base64.RawURLEncoding.EncodeToString(signed) + + // Act. + got, err := codec.Decode(encoded, "chats:alice") + + // Assert. + if !errors.Is(err, ErrInvalid) || got != (Position{}) { + t.Fatalf("position=%v, error=%v", got, err) + } + }) + + t.Run("Negative: oversized token", func(t *testing.T) { + // Arrange. + codec := newTestCodec(t) + encoded := strings.Repeat("a", 1025) + + // Act. + got, err := codec.Decode(encoded, "chats:alice") + + // Assert. + if !errors.Is(err, ErrInvalid) || got != (Position{}) { + t.Fatalf("position=%v, error=%v", got, err) + } + }) + + t.Run("Negative: malformed base64", func(t *testing.T) { + // Arrange. + codec := newTestCodec(t) + encoded := "!invalid!" + + // Act. + got, err := codec.Decode(encoded, "chats:alice") + + // Assert. + if !errors.Is(err, ErrInvalid) || got != (Position{}) { + t.Fatalf("position=%v, error=%v", got, err) + } + }) + + t.Run("Negative: signed malformed JSON", func(t *testing.T) { + // Arrange. + codec := newTestCodec(t) + encoded := signedPayload(codec, []byte("not JSON")) + + // Act. + got, err := codec.Decode(encoded, "chats:alice") + + // Assert. + if !errors.Is(err, ErrInvalid) || got != (Position{}) { + t.Fatalf("position=%v, error=%v", got, err) + } + }) + + t.Run("Negative: signed zero timestamp", func(t *testing.T) { + // Arrange. + codec := newTestCodec(t) + encoded := signedPayload(codec, []byte(`{"r":"chats:alice","t":"0001-01-01T00:00:00Z","i":"11111111-1111-1111-1111-111111111111"}`)) + + // Act. + got, err := codec.Decode(encoded, "chats:alice") + + // Assert. + if !errors.Is(err, ErrInvalid) || got != (Position{}) { + t.Fatalf("position=%v, error=%v", got, err) + } + }) +} + +// signedPayload creates authenticated malformed fixtures to isolate payload validation. +func signedPayload(codec *Codec, payload []byte) string { + return base64.RawURLEncoding.EncodeToString(append(payload, codec.signature(payload)...)) +} + +// newTestCodec creates an independent signing key for one scenario. +func newTestCodec(t *testing.T) *Codec { + t.Helper() + codec, err := New() + if err != nil { + t.Fatal(err) + } + return codec +} + +// encodeFixture prepares a valid token before testing a separate decoding condition. +func encodeFixture(t *testing.T, codec *Codec, resource string) string { + t.Helper() + token, err := codec.Encode(resource, Position{ID: uuid.New(), CreatedAt: time.Unix(1700000000, 0).UTC()}) + if err != nil { + t.Fatal(err) + } + return token +} + +// TestSharedCodec verifies a second process can verify the first process's cursor. +func TestSharedCodec(t *testing.T) { + key := base64.StdEncoding.EncodeToString(make([]byte, 32)) + first, err := NewShared(key) + if err != nil { + t.Fatal(err) + } + second, err := NewShared(key) + if err != nil { + t.Fatal(err) + } + position := Position{CreatedAt: time.Now().UTC(), ID: uuid.New()} + encoded, err := first.Encode("chat", position) + if err != nil { + t.Fatal(err) + } + result, err := second.Decode(encoded, "chat") + if err != nil || result != position { + t.Fatalf("cross-replica cursor: %v", err) + } +} + +// TestSharedCodecRejectsInvalidKey requires a full 256-bit decoded secret. +func TestSharedCodecRejectsInvalidKey(t *testing.T) { + for _, key := range []string{"", "broken!", base64.StdEncoding.EncodeToString([]byte("short"))} { + t.Run(key, func(t *testing.T) { + if _, err := NewShared(key); err == nil { + t.Fatal("invalid key accepted") + } + }) + } +} diff --git a/internal/hashring/ring.go b/internal/hashring/ring.go new file mode 100644 index 0000000..f8c0362 --- /dev/null +++ b/internal/hashring/ring.go @@ -0,0 +1,62 @@ +// Package hashring implements immutable consistent hashing with virtual nodes. +package hashring + +import ( + "cmp" + "crypto/sha256" + "encoding/binary" + "fmt" + "slices" + "sort" + "strconv" +) + +type point struct { + hash uint64 + node string +} + +// Ring maps arbitrary keys to stable node identifiers. +type Ring struct { + points []point +} + +// New creates an order-independent ring and rejects ambiguous node identities. +func New(nodes []string, replicas int) (*Ring, error) { + if len(nodes) == 0 || replicas < 1 { + return nil, fmt.Errorf("nodes and positive virtual node count are required") + } + ring := &Ring{} + seen := make(map[string]bool) + for _, node := range nodes { + if node == "" || seen[node] { + return nil, fmt.Errorf("node identifiers must be unique and nonempty") + } + seen[node] = true + for index := range replicas { + ring.points = append(ring.points, point{hash: hash(node + ":" + strconv.Itoa(index)), node: node}) + } + } + slices.SortFunc(ring.points, func(a, b point) int { + if order := cmp.Compare(a.hash, b.hash); order != 0 { + return order + } + return cmp.Compare(a.node, b.node) + }) + return ring, nil +} + +// Owner selects the first virtual node clockwise from the key hash. +func (r *Ring) Owner(key string) string { + value := hash(key) + index := sort.Search(len(r.points), func(index int) bool { + return r.points[index].hash >= value + }) + return r.points[index%len(r.points)].node +} + +// hash maps a stable string into the ring's 64-bit coordinate space. +func hash(value string) uint64 { + sum := sha256.Sum256([]byte(value)) + return binary.BigEndian.Uint64(sum[:8]) +} diff --git a/internal/hashring/ring_test.go b/internal/hashring/ring_test.go new file mode 100644 index 0000000..56bd762 --- /dev/null +++ b/internal/hashring/ring_test.go @@ -0,0 +1,73 @@ +package hashring + +import ( + "fmt" + "testing" +) + +// TestExpansionMovesOnlyKeysOwnedByNewNode verifies the consistent hashing invariant. +func TestExpansionMovesOnlyKeysOwnedByNewNode(t *testing.T) { + before, err := New([]string{"a", "b"}, 256) + if err != nil { + t.Fatal(err) + } + after, err := New([]string{"c", "b", "a"}, 256) + if err != nil { + t.Fatal(err) + } + moved := 0 + for index := range 100000 { + key := fmt.Sprint(index) + oldOwner, newOwner := before.Owner(key), after.Owner(key) + if oldOwner != newOwner { + moved++ + if newOwner != "c" { + t.Fatalf("existing nodes exchanged key %s", key) + } + } + } + fraction := float64(moved) / 100000 + if fraction < 0.2 || fraction > 0.45 { + t.Fatalf("unexpected moved fraction: %f", fraction) + } + t.Logf("100000 keys: moved=%d fraction=%.5f", moved, fraction) +} + +// TestRingRejectsAmbiguousConfiguration checks independent malformed rings. +func TestRingRejectsAmbiguousConfiguration(t *testing.T) { + cases := []struct { + name string + nodes []string + replicas int + }{ + {"empty", nil, 1}, + {"zero replicas", []string{"a"}, 0}, + {"empty node", []string{""}, 1}, + {"duplicate", []string{"a", "a"}, 1}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if _, err := New(tc.nodes, tc.replicas); err == nil { + t.Fatal("invalid ring accepted") + } + }) + } +} + +// TestRingIgnoresInputOrder verifies that replicas agree despite configuration ordering. +func TestRingIgnoresInputOrder(t *testing.T) { + a, err := New([]string{"a", "b"}, 64) + if err != nil { + t.Fatal(err) + } + b, err := New([]string{"b", "a"}, 64) + if err != nil { + t.Fatal(err) + } + for index := range 1000 { + key := fmt.Sprint(index) + if a.Owner(key) != b.Owner(key) { + t.Fatal("node order changes ownership") + } + } +} diff --git a/internal/imageprocessor/processor.go b/internal/imageprocessor/processor.go new file mode 100644 index 0000000..4a107e2 --- /dev/null +++ b/internal/imageprocessor/processor.go @@ -0,0 +1,98 @@ +// Package imageprocessor creates bounded contract variants with no client-controlled paths. +package imageprocessor + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "image" + "image/color" + "image/draw" + "image/jpeg" + "image/png" + "os/exec" + + "github.com/disintegration/imaging" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/model/dto" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/pipeline" +) + +// Processor generates JPEGs in-process and WebP through a cancellable bounded encoder. +type Processor struct { + webp string +} + +// New resolves the required encoder before accepting work. +func New() (*Processor, error) { + path, err := exec.LookPath("cwebp") + if err != nil { + return nil, fmt.Errorf("cwebp encoder is required") + } + return &Processor{webp: path}, nil +} + +// Process verifies dimensions before decode, applies EXIF orientation and strips metadata. +func (p *Processor) Process(ctx context.Context, source []byte) ([]pipeline.Output, error) { + cfg, format, err := image.DecodeConfig(bytes.NewReader(source)) + if err != nil || (format != "png" && format != "jpeg") || len(source) > 10*1024*1024 || int64(cfg.Width)*int64(cfg.Height) > 40_000_000 { + return nil, pipeline.ErrInvalidImage + } + if err := ctx.Err(); err != nil { + return nil, err + } + decoded, err := imaging.Decode(bytes.NewReader(source), imaging.AutoOrientation(true)) + if err != nil { + return nil, pipeline.ErrInvalidImage + } + bounds := decoded.Bounds() + white := image.NewNRGBA(image.Rect(0, 0, bounds.Dx(), bounds.Dy())) + draw.Draw(white, white.Bounds(), image.NewUniform(color.White), image.Point{}, draw.Src) + draw.Draw(white, white.Bounds(), decoded, bounds.Min, draw.Over) + thumbnail := fit(white, 256) + crop := imaging.Fill(white, 128, 128, imaging.Center, imaging.Lanczos) + optimized := fit(white, 1280) + outputs := make([]pipeline.Output, 0, 3) + for _, item := range []struct { + name string + picture image.Image + }{{"thumbnail", thumbnail}, {"crop", crop}} { + if err := ctx.Err(); err != nil { + return nil, err + } + var encoded bytes.Buffer + if err := jpeg.Encode(&encoded, item.picture, &jpeg.Options{Quality: 85}); err != nil { + return nil, err + } + outputs = append(outputs, output(item.name, "image/jpeg", item.picture.Bounds(), encoded.Bytes())) + } + var input bytes.Buffer + if err := png.Encode(&input, optimized); err != nil { + return nil, err + } + command := exec.CommandContext(ctx, p.webp, "-quiet", "-q", "80", "-metadata", "none", "-o", "-", "--", "-") + command.Stdin = &input + encoded, err := command.Output() + if err != nil { + return nil, fmt.Errorf("webp encoding failed: %w", err) + } + return append(outputs, output("optimized", "image/webp", optimized.Bounds(), encoded)), nil +} + +// fit floors dimensions and never enlarges a small source. +func fit(source image.Image, limit int) image.Image { + width, height := source.Bounds().Dx(), source.Bounds().Dy() + largest := max(width, height) + if largest <= limit { + return source + } + return imaging.Resize(source, max(1, width*limit/largest), max(1, height*limit/largest), imaging.Lanczos) +} + +// output derives immutable variant metadata from the actual encoded result. +func output(name, mime string, bounds image.Rectangle, data []byte) pipeline.Output { + sum := sha256.Sum256(data) + return pipeline.Output{Metadata: dto.Variant{Name: name, ContentType: mime, Width: bounds.Dx(), Height: bounds.Dy(), SizeBytes: int64(len(data)), SHA256: hex.EncodeToString(sum[:])}, Bytes: data} +} diff --git a/internal/imageprocessor/processor_test.go b/internal/imageprocessor/processor_test.go new file mode 100644 index 0000000..ba61582 --- /dev/null +++ b/internal/imageprocessor/processor_test.go @@ -0,0 +1,107 @@ +package imageprocessor + +import ( + "bytes" + "context" + "errors" + "fmt" + + "image" + "image/color" + "image/png" + "testing" + + _ "golang.org/x/image/webp" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/pipeline" +) + +// sourcePNG creates a transparent synthetic source with controlled dimensions. +func sourcePNG(t *testing.T, width, height int) []byte { + t.Helper() + pic := image.NewNRGBA(image.Rect(0, 0, width, height)) + pic.Set(0, 0, color.NRGBA{R: 255, A: 128}) + var data bytes.Buffer + if err := png.Encode(&data, pic); err != nil { + t.Fatal(err) + } + return data.Bytes() +} + +// TestVariantsDimensions verifies real JPEG/WebP output and white transparency composition. +func TestVariantsDimensions(t *testing.T) { + for _, size := range []struct { + width int + height int + thumbWidth int + thumbHeight int + }{{320, 200, 256, 160}, {16, 8, 16, 8}, {200, 320, 160, 256}, {1600, 800, 256, 128}} { + t.Run(fmt.Sprintf("%dx%d", size.width, size.height), func(t *testing.T) { + processor, err := New() + if err != nil { + t.Fatal(err) + } + outputs, err := processor.Process(t.Context(), sourcePNG(t, size.width, size.height)) + if err != nil || len(outputs) != 3 { + t.Fatalf("variants: %v", err) + } + for _, out := range outputs { + decoded, format, err := image.Decode(bytes.NewReader(out.Bytes)) + if err != nil { + t.Fatal(err) + } + if decoded.Bounds().Dx() != out.Metadata.Width || decoded.Bounds().Dy() != out.Metadata.Height { + t.Fatal("metadata dimensions differ") + } + if out.Metadata.Name == "thumbnail" && (out.Metadata.Width != size.thumbWidth || out.Metadata.Height != size.thumbHeight || format != "jpeg") { + t.Fatal("thumbnail contract") + } + if out.Metadata.Name == "crop" && (out.Metadata.Width != 128 || out.Metadata.Height != 128 || format != "jpeg") { + t.Fatal("crop contract") + } + if out.Metadata.Name == "optimized" && (max(out.Metadata.Width, out.Metadata.Height) > 1280 || format != "webp") { + t.Fatal("optimized contract") + } + r, g, b, _ := decoded.At(decoded.Bounds().Max.X-1, decoded.Bounds().Max.Y-1).RGBA() + if min(r, g, b) < 60000 { + t.Fatal("alpha not composed on white") + } + } + }) + } +} + +// TestProcessorInvalidImage verifies malformed bytes are a permanent safe error. +func TestProcessorInvalidImage(t *testing.T) { + processor := &Processor{webp: "unused"} + _, err := processor.Process(t.Context(), []byte("not an image")) + if !errors.Is(err, pipeline.ErrInvalidImage) { + t.Fatal(err) + } +} + +// TestProcessorCancellation prevents decode after an expired job deadline. +func TestProcessorCancellation(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + cancel() + _, err := (&Processor{}).Process(ctx, sourcePNG(t, 10, 10)) + if !errors.Is(err, context.Canceled) { + t.Fatal(err) + } +} + +// TestProcessorEncoderFailure verifies a missing encoder is not a successful variant set. +func TestProcessorEncoderFailure(t *testing.T) { + _, err := (&Processor{webp: "/missing-encoder"}).Process(t.Context(), sourcePNG(t, 10, 10)) + if err == nil { + t.Fatal("missing encoder accepted") + } +} + +// TestProcessorRequiresEncoder validates the deployment dependency at startup. +func TestProcessorRequiresEncoder(t *testing.T) { + t.Setenv("PATH", t.TempDir()) + if _, err := New(); err == nil { + t.Fatal("missing cwebp accepted") + } +} diff --git a/internal/media/orphans.go b/internal/media/orphans.go new file mode 100644 index 0000000..e199786 --- /dev/null +++ b/internal/media/orphans.go @@ -0,0 +1,76 @@ +package media + +import ( + "context" + "fmt" + "time" +) + +// ObjectInfo describes an immutable object without its contents. +type ObjectInfo struct { + // Key is the server-assigned private coordinate. + Key string + // ModifiedAt determines eligibility for a grace-protected scan. + ModifiedAt time.Time +} + +// OrphanMetadata checks all reservations, not only published files. +type OrphanMetadata interface { + // HasObject reports whether any state references the key. + HasObject(context.Context, string) (bool, error) +} + +// OrphanObjects supports resumable listing and single-object cleanup. +type OrphanObjects interface { + // List returns a bounded lexicographic page after the given key. + List(context.Context, string) ([]ObjectInfo, error) + // Remove deletes an explicitly identified unreferenced key. + Remove(context.Context, string) error +} + +// OrphanResult reports one page of a safe S3-to-database reconciliation. +type OrphanResult struct { + // Scanned counts objects examined. + Scanned int `json:"scanned"` + // Orphans counts old objects with no metadata reservation. + Orphans int `json:"orphans"` + // Removed counts applied removals. + Removed int `json:"removed"` + // Next resumes the lexicographic scan. + Next string `json:"next"` +} + +// ReconcileOrphans protects fresh and referenced objects, and defaults to dry-run at the CLI. +func ReconcileOrphans(ctx context.Context, metadata OrphanMetadata, objects OrphanObjects, after string, grace time.Duration, apply bool) (OrphanResult, error) { + var result OrphanResult + if grace < time.Hour { + return result, fmt.Errorf("reconciliation grace must be at least one hour") + } + entries, err := objects.List(ctx, after) + if err != nil { + return result, err + } + cutoff := time.Now().UTC().Add(-grace) + for _, entry := range entries { + result.Scanned++ + result.Next = entry.Key + if entry.ModifiedAt.IsZero() || entry.ModifiedAt.After(cutoff) { + continue + } + referenced, err := metadata.HasObject(ctx, entry.Key) + if err != nil { + return result, err + } + if referenced { + continue + } + result.Orphans++ + if apply { + if err := objects.Remove(ctx, entry.Key); err != nil { + return result, err + } + result.Removed++ + } + } + return result, nil +} diff --git a/internal/media/reconcile.go b/internal/media/reconcile.go new file mode 100644 index 0000000..383e0b3 --- /dev/null +++ b/internal/media/reconcile.go @@ -0,0 +1,102 @@ +package media + +import ( + "context" + "fmt" + "time" + + "github.com/google/uuid" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/model/dto" +) + +// Inventory supports bounded, resumable scans and guarded state transitions. +type Inventory interface { + // Scan returns the next at most limit metadata records by ID. + Scan(context.Context, uuid.UUID, int) ([]dto.Attachment, error) + // Reclaim deletes only an old unfinished unreferenced reservation after claiming it. + Reclaim(context.Context, uuid.UUID, time.Time, bool) (bool, error) + // MarkMissing preserves metadata and references when an original disappears. + MarkMissing(context.Context, uuid.UUID, bool) error +} + +// ObjectInventory exposes bounded inspection and explicit removal. +type ObjectInventory interface { + // Exists distinguishes absence from infrastructure failure. + Exists(context.Context, string) (bool, error) + // Remove deletes a single immutable object key. + Remove(context.Context, string) error +} + +// ReconcileResult records observations and the continuation cursor without file contents. +type ReconcileResult struct { + // Scanned counts inspected metadata rows. + Scanned int `json:"scanned"` + // Missing counts published originals absent from S3. + Missing int `json:"missing"` + // Abandoned counts old incomplete reservations eligible for cleanup. + Abandoned int `json:"abandoned"` + // Changed counts successful applied repairs, zero for dry run. + Changed int `json:"changed"` + // Next continues scanning without retaining the full database in RAM. + Next uuid.UUID `json:"next"` +} + +// Reconcile inspects at most 1000 records and protects active uploads with a one-hour minimum grace. +func Reconcile(ctx context.Context, metadata Inventory, objects ObjectInventory, after uuid.UUID, grace time.Duration, apply bool) (ReconcileResult, error) { + var result ReconcileResult + if grace < time.Hour { + return result, fmt.Errorf("reconciliation grace must be at least one hour") + } + records, err := metadata.Scan(ctx, after, 1000) + if err != nil { + return result, err + } + cutoff := time.Now().UTC().Add(-grace) + for _, a := range records { + result.Scanned++ + result.Next = a.ID + if a.Status == "queued" || a.Status == "processing" || a.Status == "failed" || a.CreatedAt.After(cutoff) { + continue + } + exists, err := objects.Exists(ctx, a.ObjectKey) + if err != nil { + return result, err + } + switch { + case a.Status == "uploading" || a.Status == "deleting": + result.Abandoned++ + if !apply { + continue + } + claimed, err := metadata.Reclaim(ctx, a.ID, cutoff, false) + if err != nil { + return result, err + } + if !claimed { + continue + } + if err := objects.Remove(ctx, a.ObjectKey); err != nil { + return result, err + } + if _, err := metadata.Reclaim(ctx, a.ID, cutoff, true); err != nil { + return result, err + } + result.Changed++ + case !exists: + result.Missing++ + if apply && a.Status != "missing" { + if err := metadata.MarkMissing(ctx, a.ID, true); err != nil { + return result, err + } + result.Changed++ + } + case a.Status == "missing" && apply: + if err := metadata.MarkMissing(ctx, a.ID, false); err != nil { + return result, err + } + result.Changed++ + } + } + return result, nil +} diff --git a/internal/media/reconcile_test.go b/internal/media/reconcile_test.go new file mode 100644 index 0000000..b5271dc --- /dev/null +++ b/internal/media/reconcile_test.go @@ -0,0 +1,159 @@ +package media + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/google/uuid" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/model/dto" +) + +type reconciliationFixture struct { + records []dto.Attachment + entries []ObjectInfo + exists bool + referenced bool + claim bool + scanErr error + existsErr error + claimErr error + finishErr error + removeErr error + markErr error + listErr error + referenceErr error + removed int +} + +// Scan returns an independent bounded fixture page. +func (f *reconciliationFixture) Scan(context.Context, uuid.UUID, int) ([]dto.Attachment, error) { + return f.records, f.scanErr +} + +// Reclaim injects failures at the claim or finish boundary independently. +func (f *reconciliationFixture) Reclaim(_ context.Context, _ uuid.UUID, _ time.Time, finish bool) (bool, error) { + if finish { + return f.claim, f.finishErr + } + return f.claim, f.claimErr +} + +// MarkMissing models availability-state persistence without deleting metadata. +func (f *reconciliationFixture) MarkMissing(context.Context, uuid.UUID, bool) error { + return f.markErr +} + +// Exists returns an explicit existence or infrastructure-failure outcome. +func (f *reconciliationFixture) Exists(context.Context, string) (bool, error) { + return f.exists, f.existsErr +} + +// Remove counts applied side effects for dry-run and referenced-object assertions. +func (f *reconciliationFixture) Remove(context.Context, string) error { + f.removed++ + return f.removeErr +} + +// List supplies object timestamps independently of database state. +func (f *reconciliationFixture) List(context.Context, string) ([]ObjectInfo, error) { + return f.entries, f.listErr +} + +// HasObject includes reservations so active uploads remain protected. +func (f *reconciliationFixture) HasObject(context.Context, string) (bool, error) { + return f.referenced, f.referenceErr +} + +// TestReconciliationSafety exercises interrupted cleanup, age protection and missing-object repair. +func TestReconciliationSafety(t *testing.T) { + fault := errors.New("dependency failure") + old := time.Now().Add(-2 * time.Hour) + cases := []struct { + name string + fixture reconciliationFixture + status string + created time.Time + apply bool + wantErr bool + changed int + }{ + {name: "fresh", created: time.Now(), apply: true}, + {name: "scan failure", fixture: reconciliationFixture{scanErr: fault}, wantErr: true}, + {name: "head failure", fixture: reconciliationFixture{existsErr: fault}, wantErr: true}, + {name: "dry abandoned", status: "uploading"}, + {name: "abandoned applied", status: "uploading", apply: true, fixture: reconciliationFixture{claim: true}, changed: 1}, + {name: "claimed by another", status: "uploading", apply: true}, + {name: "claim failure", status: "uploading", apply: true, fixture: reconciliationFixture{claimErr: fault}, wantErr: true}, + {name: "delete failure", status: "deleting", apply: true, fixture: reconciliationFixture{claim: true, removeErr: fault}, wantErr: true}, + {name: "finish failure", status: "deleting", apply: true, fixture: reconciliationFixture{claim: true, finishErr: fault}, wantErr: true}, + {name: "missing dry", status: "ready"}, + {name: "missing applied", status: "ready", apply: true, changed: 1}, + {name: "mark failure", status: "ready", apply: true, fixture: reconciliationFixture{markErr: fault}, wantErr: true}, + {name: "missing repeat", status: "missing", apply: true}, + {name: "restored", status: "missing", apply: true, fixture: reconciliationFixture{exists: true}, changed: 1}, + {name: "restore failure", status: "missing", apply: true, fixture: reconciliationFixture{exists: true, markErr: fault}, wantErr: true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + f := tc.fixture + created := tc.created + if created.IsZero() { + created = old + } + f.records = []dto.Attachment{{ID: uuid.New(), Status: tc.status, CreatedAt: created, ObjectKey: "original/test"}} + result, err := Reconcile(t.Context(), &f, &f, uuid.Nil, time.Hour, tc.apply) + if (err != nil) != tc.wantErr || result.Changed != tc.changed { + t.Fatalf("result=%+v error=%v", result, err) + } + }) + } +} + +// TestOrphanSafety rejects deleting fresh, reserved and unknown-age objects. +func TestOrphanSafety(t *testing.T) { + fault := errors.New("dependency failure") + cases := []struct { + name string + fixture reconciliationFixture + fresh bool + apply bool + wantErr bool + removed int + }{ + {name: "fresh", fresh: true, apply: true}, + {name: "referenced", fixture: reconciliationFixture{referenced: true}, apply: true}, + {name: "dry run"}, + {name: "remove", apply: true, removed: 1}, + {name: "list failure", fixture: reconciliationFixture{listErr: fault}, wantErr: true}, + {name: "metadata failure", fixture: reconciliationFixture{referenceErr: fault}, wantErr: true}, + {name: "remove failure", fixture: reconciliationFixture{removeErr: fault}, apply: true, wantErr: true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + f := tc.fixture + modified := time.Now().Add(-2 * time.Hour) + if tc.fresh { + modified = time.Now() + } + f.entries = []ObjectInfo{{Key: "original/test", ModifiedAt: modified}} + result, err := ReconcileOrphans(t.Context(), &f, &f, "", time.Hour, tc.apply) + if (err != nil) != tc.wantErr || result.Removed != tc.removed { + t.Fatalf("result=%+v error=%v", result, err) + } + }) + } +} + +// TestGraceCannotBeDisabled rejects unsafe cleanup windows at both scan entrypoints. +func TestGraceCannotBeDisabled(t *testing.T) { + f := &reconciliationFixture{} + if _, err := Reconcile(t.Context(), f, f, uuid.Nil, time.Second, true); err == nil { + t.Fatal("unsafe metadata grace accepted") + } + if _, err := ReconcileOrphans(t.Context(), f, f, "", time.Second, true); err == nil { + t.Fatal("unsafe object grace accepted") + } +} diff --git a/internal/media/run_reconcile.go b/internal/media/run_reconcile.go new file mode 100644 index 0000000..ae840b4 --- /dev/null +++ b/internal/media/run_reconcile.go @@ -0,0 +1,55 @@ +package media + +import ( + "context" + "encoding/json" + "io" + "time" + + "github.com/google/uuid" +) + +// ReconcileMetadata combines bounded database inventory operations for the operator workflow. +type ReconcileMetadata interface { + Inventory + OrphanMetadata +} + +// ReconcileObjects combines object inspection and listing for the operator workflow. +type ReconcileObjects interface { + ObjectInventory + OrphanObjects +} + +// RunReconciliation emits one bounded result per page and stops on any uncertain dependency outcome. +func RunReconciliation(ctx context.Context, metadata ReconcileMetadata, objects ReconcileObjects, output io.Writer, apply bool) error { + encoder := json.NewEncoder(output) + after := uuid.Nil + for { + result, err := Reconcile(ctx, metadata, objects, after, time.Hour, apply) + if err != nil { + return err + } + if err := encoder.Encode(result); err != nil { + return err + } + if result.Scanned < 1000 { + break + } + after = result.Next + } + key := "" + for { + result, err := ReconcileOrphans(ctx, metadata, objects, key, time.Hour, apply) + if err != nil { + return err + } + if err := encoder.Encode(result); err != nil { + return err + } + if result.Scanned < 1000 { + return nil + } + key = result.Next + } +} diff --git a/internal/media/run_reconcile_test.go b/internal/media/run_reconcile_test.go new file mode 100644 index 0000000..3c2c5ef --- /dev/null +++ b/internal/media/run_reconcile_test.go @@ -0,0 +1,92 @@ +package media + +import ( + "context" + "errors" + "io" + "testing" + "time" + + "github.com/google/uuid" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/model/dto" +) + +type pagedFixture struct { + reconciliationFixture + metadataPage int + objectPage int +} + +// Scan returns one full fresh page followed by an empty continuation. +func (f *pagedFixture) Scan(context.Context, uuid.UUID, int) ([]dto.Attachment, error) { + f.metadataPage++ + if f.metadataPage > 1 { + return nil, nil + } + records := make([]dto.Attachment, 1000) + for i := range records { + records[i] = dto.Attachment{ID: uuid.New(), CreatedAt: time.Now()} + } + return records, nil +} + +// List returns one full fresh page followed by an empty continuation. +func (f *pagedFixture) List(context.Context, string) ([]ObjectInfo, error) { + f.objectPage++ + if f.objectPage > 1 { + return nil, nil + } + records := make([]ObjectInfo, 1000) + for i := range records { + records[i] = ObjectInfo{Key: uuid.NewString(), ModifiedAt: time.Now()} + } + return records, nil +} + +type failingWriter struct { + successful int +} + +// Write injects a broken report sink after a fixed number of complete pages. +func (f *failingWriter) Write(data []byte) (int, error) { + if f.successful == 0 { + return 0, io.ErrClosedPipe + } + f.successful-- + return len(data), nil +} + +// TestReconciliationPagination proves both inventories advance beyond a full page. +func TestReconciliationPagination(t *testing.T) { + f := &pagedFixture{} + if err := RunReconciliation(t.Context(), f, f, io.Discard, false); err != nil { + t.Fatal(err) + } + if f.metadataPage != 2 || f.objectPage != 2 { + t.Fatalf("pages: metadata=%d objects=%d", f.metadataPage, f.objectPage) + } +} + +// TestReconciliationStopsOnFailures prevents misleading successful reports after lost evidence or dependencies. +func TestReconciliationStopsOnFailures(t *testing.T) { + fault := errors.New("dependency failure") + cases := []struct { + name string + fixture reconciliationFixture + writer io.Writer + }{ + {"metadata", reconciliationFixture{scanErr: fault}, io.Discard}, + {"objects", reconciliationFixture{listErr: fault}, io.Discard}, + {"first report", reconciliationFixture{}, &failingWriter{}}, + {"second report", reconciliationFixture{}, &failingWriter{successful: 1}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + f := tc.fixture + if err := RunReconciliation(t.Context(), &f, &f, tc.writer, false); err == nil { + t.Fatal("failed run reported success") + } + }) + } +} diff --git a/internal/media/service.go b/internal/media/service.go new file mode 100644 index 0000000..e56885d --- /dev/null +++ b/internal/media/service.go @@ -0,0 +1,157 @@ +// Package media implements attachment authorization and the S3 publication protocol. +package media + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "time" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/store" + + "github.com/google/uuid" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/model/dto" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/service" +) + +// Metadata persists reservations before object creation. +type Metadata interface { + // Reserve inserts an uploading reservation. + Reserve(context.Context, dto.Attachment) error + // QueueAttachment atomically exposes queued metadata and durable processing intent. + QueueAttachment(context.Context, uuid.UUID) (dto.Attachment, error) + // Attachment retrieves metadata by a validated ID. + Attachment(context.Context, uuid.UUID) (dto.Attachment, error) +} + +// Objects exposes private object operations without a concrete S3 client. +type Objects interface { + // Put uploads a known bounded length without a multipart buffer. + Put(context.Context, string, string, io.Reader, int64) error + // Get opens an original stream; the caller closes it. + Get(context.Context, string) (io.ReadCloser, error) +} + +// Chats authorizes access using the existing business service. +type Chats interface { + // ChatsGetByID checks the caller and membership. + ChatsGetByID(context.Context, string, string) (dto.ChatDTO, error) +} + +// Service coordinates bounded validation and durable publication. +type Service struct { + metadata Metadata + objects Objects + chats Chats + slots chan struct{} +} + +// New validates required dependencies and caps simultaneous image decoding. +func New(metadata Metadata, objects Objects, chats Chats) (*Service, error) { + if metadata == nil || objects == nil || chats == nil { + return nil, fmt.Errorf("attachment dependencies are required") + } + return &Service{metadata: metadata, objects: objects, chats: chats, slots: make(chan struct{}, 2)}, nil +} + +// Upload validates access and original bytes before reserving and publishing. +func (s *Service) Upload(ctx context.Context, user, chat, filename, contentType string, body []byte) (dto.Attachment, error) { + scope, err := s.chats.ChatsGetByID(ctx, user, chat) + if err != nil { + return dto.Attachment{}, err + } + select { + case s.slots <- struct{}{}: + defer func() { + <-s.slots + }() + default: + return dto.Attachment{}, service.ErrUnavailable + } + if err := validateImage(filename, contentType, body); err != nil { + return dto.Attachment{}, err + } + owner, err := uuid.Parse(user) + if err != nil { + return dto.Attachment{}, service.ErrUnauthorized + } + sum := sha256.Sum256(body) + id := uuid.New() + now := time.Now().UTC() + attachment := dto.Attachment{ + ID: id, + ChatID: scope.ID, + OwnerID: owner, + Filename: filename, + ContentType: contentType, + SizeBytes: int64(len(body)), + SHA256: hex.EncodeToString(sum[:]), + Status: "uploading", + ObjectKey: "staging/" + id.String(), + CreatedAt: now, + UpdatedAt: now, + } + if err := s.metadata.Reserve(ctx, attachment); err != nil { + return dto.Attachment{}, err + } + if err := s.objects.Put(ctx, attachment.ObjectKey, contentType, bytes.NewReader(body), attachment.SizeBytes); err != nil { + return dto.Attachment{}, service.ErrUnavailable + } + // A lost COMMIT acknowledgement must never trigger deletion of a possibly published object. + return s.metadata.QueueAttachment(ctx, id) +} + +// Get authorizes metadata access without exposing private object coordinates. +func (s *Service) Get(ctx context.Context, user, rawID string) (dto.Attachment, error) { + id, err := uuid.Parse(rawID) + if err != nil { + return dto.Attachment{}, service.ErrInvalid + } + attachment, err := s.metadata.Attachment(ctx, id) + if errors.Is(err, store.ErrNotFound) { + return dto.Attachment{}, service.ErrNotFound + } + if err != nil { + return dto.Attachment{}, err + } + if _, err := s.chats.ChatsGetByID(ctx, user, attachment.ChatID.String()); err != nil { + return dto.Attachment{}, err + } + return attachment, nil +} + +// Download validates access and variant before opening the original stream. +func (s *Service) Download(ctx context.Context, user, id, variant string) (dto.Attachment, io.ReadCloser, error) { + attachment, err := s.Get(ctx, user, id) + if err != nil { + return dto.Attachment{}, nil, err + } + if attachment.Status != "ready" { + return dto.Attachment{}, nil, service.ErrConflict + } + if variant != "" && variant != "original" { + found := false + for _, candidate := range attachment.Variants { + if candidate.Name == variant { + attachment.ObjectKey = candidate.ObjectKey + attachment.ContentType = candidate.ContentType + attachment.SizeBytes = candidate.SizeBytes + found = true + break + } + } + if !found { + return dto.Attachment{}, nil, service.ErrNotFound + } + } + reader, err := s.objects.Get(ctx, attachment.ObjectKey) + if err != nil { + return dto.Attachment{}, nil, service.ErrUnavailable + } + return attachment, reader, nil +} diff --git a/internal/media/service_test.go b/internal/media/service_test.go new file mode 100644 index 0000000..93d5d43 --- /dev/null +++ b/internal/media/service_test.go @@ -0,0 +1,225 @@ +package media + +import ( + "bytes" + "context" + "errors" + "io" + "testing" + + "github.com/google/uuid" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/model/dto" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/service" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/store" +) + +type mediaFixture struct { + attachment dto.Attachment + body []byte + authorizeErr error + reserveErr error + publishErr error + getErr error + objectErr error +} + +// ChatsGetByID models a single authorization outcome for one scenario. +func (f *mediaFixture) ChatsGetByID(_ context.Context, _, chat string) (dto.ChatDTO, error) { + id, _ := uuid.Parse(chat) + return dto.ChatDTO{ID: id}, f.authorizeErr +} + +// Reserve records durable intent before a test object can be written. +func (f *mediaFixture) Reserve(_ context.Context, a dto.Attachment) error { + f.attachment = a + return f.reserveErr +} + +// QueueAttachment models an acknowledged or uncertain database commit. +func (f *mediaFixture) QueueAttachment(_ context.Context, _ uuid.UUID) (dto.Attachment, error) { + f.attachment.Status = "queued" + return f.attachment, f.publishErr +} + +// Attachment returns fixture metadata without bypassing service authorization. +func (f *mediaFixture) Attachment(_ context.Context, _ uuid.UUID) (dto.Attachment, error) { + return f.attachment, f.getErr +} + +// Put captures exact original bytes or injects an object failure. +func (f *mediaFixture) Put(_ context.Context, _, _ string, r io.Reader, _ int64) error { + if f.objectErr != nil { + return f.objectErr + } + var err error + f.body, err = io.ReadAll(r) + return err +} + +// Get returns a separate read cursor over the captured original. +func (f *mediaFixture) Get(_ context.Context, _ string) (io.ReadCloser, error) { + return io.NopCloser(bytes.NewReader(f.body)), f.objectErr +} + +// mediaService constructs an independent service with explicit fixture dependencies. +func mediaService(t *testing.T, f *mediaFixture) *Service { + t.Helper() + s, err := New(f, f, f) + if err != nil { + t.Fatal(err) + } + return s +} + +// TestUploadPublication verifies exact bytes and durable publication ordering. +func TestUploadPublication(t *testing.T) { + f := &mediaFixture{} + s := mediaService(t, f) + original := imageBytes(t) + a, err := s.Upload(t.Context(), uuid.NewString(), uuid.NewString(), "a.png", "image/png", original) + if err != nil || a.Status != "queued" || a.SizeBytes != int64(len(original)) || len(a.SHA256) != 64 || !bytes.Equal(f.body, original) { + t.Fatalf("upload result: %+v %v", a, err) + } +} + +// TestUploadRejectsUnauthorized verifies authorization precedes object creation. +func TestUploadRejectsUnauthorized(t *testing.T) { + f := &mediaFixture{authorizeErr: service.ErrNotFound} + s := mediaService(t, f) + _, err := s.Upload(t.Context(), uuid.NewString(), uuid.NewString(), "a.png", "image/png", imageBytes(t)) + if !errors.Is(err, service.ErrNotFound) || f.body != nil { + t.Fatalf("unauthorized upload: %v", err) + } +} + +// TestUploadDependencyFailures verifies failed operations are not acknowledged as ready. +func TestUploadDependencyFailures(t *testing.T) { + cases := []struct { + name string + fixture mediaFixture + }{ + {name: "reserve", fixture: mediaFixture{reserveErr: service.ErrUnavailable}}, + {name: "put", fixture: mediaFixture{objectErr: service.ErrUnavailable}}, + {name: "commit acknowledgement", fixture: mediaFixture{publishErr: service.ErrUnavailable}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + f := tc.fixture + s := mediaService(t, &f) + _, err := s.Upload(t.Context(), uuid.NewString(), uuid.NewString(), "a.png", "image/png", imageBytes(t)) + if !errors.Is(err, service.ErrUnavailable) { + t.Fatalf("error=%v", err) + } + }) + } +} + +// TestUploadConcurrencyBound verifies decoding cannot create an unbounded queue. +func TestUploadConcurrencyBound(t *testing.T) { + s := mediaService(t, &mediaFixture{}) + s.slots <- struct{}{} + s.slots <- struct{}{} + _, err := s.Upload(t.Context(), uuid.NewString(), uuid.NewString(), "a.png", "image/png", imageBytes(t)) + if !errors.Is(err, service.ErrUnavailable) { + t.Fatal(err) + } +} + +// TestGetAttachment checks metadata authorization and intermediate state handling. +func TestGetAttachment(t *testing.T) { + cases := []struct { + name string + fixture mediaFixture + id string + want error + }{ + {name: "invalid id", id: "bad", want: service.ErrInvalid}, + {name: "missing", id: uuid.NewString(), fixture: mediaFixture{getErr: store.ErrNotFound}, want: service.ErrNotFound}, + {name: "database unavailable", id: uuid.NewString(), fixture: mediaFixture{getErr: service.ErrUnavailable}, want: service.ErrUnavailable}, + {name: "foreign", id: uuid.NewString(), fixture: mediaFixture{authorizeErr: service.ErrNotFound}, want: service.ErrNotFound}, + {name: "unfinished", id: uuid.NewString(), fixture: mediaFixture{attachment: dto.Attachment{Status: "queued"}}}, + {name: "ready", id: uuid.NewString(), fixture: mediaFixture{attachment: dto.Attachment{Status: "ready"}}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + f := tc.fixture + s := mediaService(t, &f) + _, err := s.Get(t.Context(), uuid.NewString(), tc.id) + if !errors.Is(err, tc.want) { + t.Fatalf("got %v want %v", err, tc.want) + } + }) + } +} + +// TestDownloadOriginal checks byte identity through an independently authorized read. +func TestDownloadOriginal(t *testing.T) { + f := &mediaFixture{attachment: dto.Attachment{Status: "ready"}, body: imageBytes(t)} + s := mediaService(t, f) + _, reader, err := s.Download(t.Context(), uuid.NewString(), uuid.NewString(), "original") + if err != nil { + t.Fatal(err) + } + got, err := io.ReadAll(reader) + if closeErr := reader.Close(); closeErr != nil { + t.Fatal(closeErr) + } + if err != nil || !bytes.Equal(got, f.body) { + t.Fatalf("download: %v", err) + } +} + +// TestMediaRequiresDependencies rejects incomplete service composition. +func TestMediaRequiresDependencies(t *testing.T) { + if _, err := New(nil, nil, nil); err == nil { + t.Fatal("missing dependencies accepted") + } +} + +// TestUploadRejectsInvalidOriginal never reserves invalid file metadata. +func TestUploadRejectsInvalidOriginal(t *testing.T) { + f := &mediaFixture{} + s := mediaService(t, f) + _, err := s.Upload(t.Context(), uuid.NewString(), uuid.NewString(), "a.png", "image/png", []byte("invalid")) + if !errors.Is(err, service.ErrMedia) || f.attachment.ID != uuid.Nil { + t.Fatalf("invalid bytes persisted: %v", err) + } +} + +// TestUploadRejectsMalformedOwner does not persist an invalid authenticated identity. +func TestUploadRejectsMalformedOwner(t *testing.T) { + f := &mediaFixture{} + s := mediaService(t, f) + _, err := s.Upload(t.Context(), "bad", uuid.NewString(), "a.png", "image/png", imageBytes(t)) + if !errors.Is(err, service.ErrUnauthorized) { + t.Fatal(err) + } +} + +// TestDownloadFailures keeps authorization, variant and dependency failures distinct. +func TestDownloadFailures(t *testing.T) { + cases := []struct { + name string + fixture mediaFixture + variant string + want error + }{ + {name: "foreign", fixture: mediaFixture{authorizeErr: service.ErrNotFound}, want: service.ErrNotFound}, + {name: "variant missing", fixture: mediaFixture{attachment: dto.Attachment{Status: "ready"}}, variant: "thumbnail", want: service.ErrNotFound}, + {name: "S3 unavailable", fixture: mediaFixture{attachment: dto.Attachment{Status: "ready"}, objectErr: service.ErrUnavailable}, want: service.ErrUnavailable}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + f := tc.fixture + s := mediaService(t, &f) + _, reader, err := s.Download(t.Context(), uuid.NewString(), uuid.NewString(), tc.variant) + if reader != nil { + _ = reader.Close() + } + if !errors.Is(err, tc.want) { + t.Fatalf("got %v want %v", err, tc.want) + } + }) + } +} diff --git a/internal/media/validation.go b/internal/media/validation.go new file mode 100644 index 0000000..75a83f0 --- /dev/null +++ b/internal/media/validation.go @@ -0,0 +1,48 @@ +package media + +import ( + "bytes" + "image" + + // Register the only accepted original image decoders. + _ "image/jpeg" + // Register PNG for content validation before storage. + _ "image/png" + "strings" + "unicode" + "unicode/utf8" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/service" +) + +const maxImageBytes = 10 << 20 +const maxImagePixels = 40_000_000 + +// validateImage checks encoded size, display name, actual format and complete decoding. +func validateImage(filename, mime string, body []byte) error { + if len(body) > maxImageBytes { + return service.ErrTooLarge + } + if !utf8.ValidString(filename) || strings.TrimSpace(filename) == "" || utf8.RuneCountInString(filename) > 255 { + return service.ErrInvalid + } + for _, r := range filename { + if r == '/' || r == '\\' || unicode.IsControl(r) { + return service.ErrInvalid + } + } + if mime != "image/png" && mime != "image/jpeg" { + return service.ErrMedia + } + dimensions, format, err := image.DecodeConfig(bytes.NewReader(body)) + if err != nil || "image/"+format != mime { + return service.ErrMedia + } + if dimensions.Width < 1 || dimensions.Height < 1 || int64(dimensions.Width)*int64(dimensions.Height) > maxImagePixels { + return service.ErrTooLarge + } + if _, _, err := image.Decode(bytes.NewReader(body)); err != nil { + return service.ErrMedia + } + return nil +} diff --git a/internal/media/validation_test.go b/internal/media/validation_test.go new file mode 100644 index 0000000..9a5936d --- /dev/null +++ b/internal/media/validation_test.go @@ -0,0 +1,65 @@ +package media + +import ( + "bytes" + "encoding/binary" + "errors" + "hash/crc32" + "image" + "image/png" + "testing" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/service" +) + +// imageBytes creates a valid independent synthetic PNG fixture. +func imageBytes(t *testing.T) []byte { + t.Helper() + var output bytes.Buffer + if err := png.Encode(&output, image.NewRGBA(image.Rect(0, 0, 3, 2))); err != nil { + t.Fatal(err) + } + return output.Bytes() +} + +// TestValidateImage checks file-level rejection independently of persistence. +func TestValidateImage(t *testing.T) { + valid := imageBytes(t) + cases := []struct { + name string + filename string + mime string + body []byte + want error + }{ + {"valid", "test.png", "image/png", valid, nil}, + {"traversal", "../test.png", "image/png", valid, service.ErrInvalid}, + {"backslash", "a\\b", "image/png", valid, service.ErrInvalid}, + {"control", "a\n", "image/png", valid, service.ErrInvalid}, + {"empty name", "", "image/png", valid, service.ErrInvalid}, + {"mime mismatch", "a.jpg", "image/jpeg", valid, service.ErrMedia}, + {"unsupported", "a.png", "text/plain", valid, service.ErrMedia}, + {"corrupt header", "a.png", "image/png", []byte("bad"), service.ErrMedia}, + {"truncated payload", "a.png", "image/png", valid[:len(valid)-12], service.ErrMedia}, + {"encoded limit", "a.png", "image/png", make([]byte, maxImageBytes+1), service.ErrTooLarge}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + err := validateImage(tc.filename, tc.mime, tc.body) + if !errors.Is(err, tc.want) { + t.Fatalf("got %v want %v", err, tc.want) + } + }) + } +} + +// TestPixelLimitPrecedesDecode rejects a valid PNG header before allocating a huge pixel buffer. +func TestPixelLimitPrecedesDecode(t *testing.T) { + body := append([]byte{}, imageBytes(t)...) + binary.BigEndian.PutUint32(body[16:20], 10000) + binary.BigEndian.PutUint32(body[20:24], 10000) + binary.BigEndian.PutUint32(body[29:33], crc32.ChecksumIEEE(body[12:29])) + if err := validateImage("huge.png", "image/png", body); !errors.Is(err, service.ErrTooLarge) { + t.Fatalf("pixel limit: %v", err) + } +} diff --git a/internal/model/dto/attachment.go b/internal/model/dto/attachment.go new file mode 100644 index 0000000..cb71cb0 --- /dev/null +++ b/internal/model/dto/attachment.go @@ -0,0 +1,57 @@ +package dto + +import ( + "time" + + "github.com/google/uuid" +) + +// Attachment contains private persistence metadata and the public file description. +type Attachment struct { + // Variants contains generated immutable objects. + Variants []Variant + // ErrorCode is a safe terminal failure category. + ErrorCode *string + // PipelineVersion fences older processing jobs. + PipelineVersion int + // ID identifies the immutable upload. + ID uuid.UUID + // ChatID defines the authorization scope. + ChatID uuid.UUID + // OwnerID identifies the uploader. + OwnerID uuid.UUID + // Filename is display-only and never an object path. + Filename string + // ContentType is validated against decoded bytes. + ContentType string + // SizeBytes is the original encoded length. + SizeBytes int64 + // SHA256 is the lowercase original checksum. + SHA256 string + // Status is the durable upload state. + Status string + // ObjectKey is private and must not reach HTTP responses. + ObjectKey string + // CreatedAt is the reservation timestamp. + CreatedAt time.Time + // UpdatedAt is the latest durable transition. + UpdatedAt time.Time +} + +// Variant describes a generated object; ObjectKey is persistence-only. +type Variant struct { + // Name is the contract variant name. + Name string `json:"name"` + // ContentType identifies the actual encoding. + ContentType string `json:"content_type"` + // Width is the decoded pixel width. + Width int `json:"width"` + // Height is the decoded pixel height. + Height int `json:"height"` + // SizeBytes is the encoded length. + SizeBytes int64 `json:"size_bytes"` + // SHA256 is the encoded checksum. + SHA256 string `json:"sha256"` + // ObjectKey is not copied into HTTP responses. + ObjectKey string `json:"object_key,omitempty"` +} diff --git a/internal/model/dto/models.go b/internal/model/dto/models.go new file mode 100644 index 0000000..61227e1 --- /dev/null +++ b/internal/model/dto/models.go @@ -0,0 +1,118 @@ +// Package dto defines the transport-independent data shared by service and store. +package dto + +import ( + "time" + + "github.com/google/uuid" +) + +// UserDTO is the public account profile without credentials. +type UserDTO struct { + // ID is the server-generated user identifier. + ID uuid.UUID + // Username is the unique lowercase ASCII account name. + Username string + // DisplayName is the display label, limited to 100 Unicode code points. + DisplayName string + // CreatedAt is the server-assigned UTC creation time. + CreatedAt time.Time +} + +// ChatDTO is a chat with immutable membership. +type ChatDTO struct { + // ID is the server-generated chat identifier. + ID uuid.UUID + // Title is the nonblank title, limited to 100 Unicode code points. + Title string + // MemberIDs contains the fixed set of users allowed to read and write the chat. + MemberIDs []uuid.UUID + // CreatedAt is the server-assigned UTC creation time. + CreatedAt time.Time +} + +// MessageDTO is an immutable message stored in a chat. +type MessageDTO struct { + // AttachmentIDs references ordered immutable files in the same chat. + AttachmentIDs []string + // IdempotencyKey identifies a validated retry within the sender and chat scope. + IdempotencyKey string + // ID is the server-generated message identifier. + ID uuid.UUID + // ChatID identifies the owning chat. + ChatID uuid.UUID + // SenderID identifies the user resolved from the request identity. + SenderID uuid.UUID + // Text contains the original text, limited to 4000 Unicode code points. + Text string + // CreatedAt is the server-assigned UTC ordering timestamp. + CreatedAt time.Time +} + +// UsersCreateRequestDTO carries untrusted registration values into the service. +type UsersCreateRequestDTO struct { + // Username is the raw requested account name. + Username string + // DisplayName is the raw requested display label. + DisplayName string +} + +// ChatsCreateRequestDTO carries untrusted chat creation input. +type ChatsCreateRequestDTO struct { + // UserID is the caller identifier resolved from a validated session. + UserID string + // Title is the raw requested chat title. + Title string + // MemberIDs contains raw user UUIDs; the service checks uniqueness and membership. + MemberIDs []string +} + +// MessagesCreateRequestDTO carries untrusted message creation input. +type MessagesCreateRequestDTO struct { + // AttachmentIDs references ordered immutable files in the same chat. + AttachmentIDs []string + // IdempotencyKey identifies a validated retry within the sender and chat scope. + IdempotencyKey string + // UserID is the authenticated caller identifier, never a supplied sender field. + UserID string + // ChatID is the raw chat UUID from the route. + ChatID string + // Text is the unmodified message text to validate. + Text string +} + +// PageRequestDTO carries raw pagination values for service-level validation. +type PageRequestDTO struct { + // UserID is the caller identifier resolved from a validated session. + UserID string + // ChatID is the requested chat UUID; it is unused when listing chats. + ChatID string + // Limit is the raw page size; empty selects the contract default. + Limit string + // Cursor is the opaque continuation token; empty starts at the beginning. + Cursor string +} + +// PositionDTO is a validated keyset boundary; its zero value starts a history. +type PositionDTO struct { + // CreatedAt is the timestamp of the last record already returned. + CreatedAt time.Time + // ID is the tie-breaking identifier of the last record already returned. + ID uuid.UUID +} + +// ChatPageDTO contains an ordered page of accessible chats. +type ChatPageDTO struct { + // Items contains at most the requested number of chats. + Items []ChatDTO + // NextCursor continues the same user-specific list; nil marks its current end. + NextCursor *string +} + +// MessagePageDTO contains an ordered page of chat history. +type MessagePageDTO struct { + // Items contains at most the requested number of messages. + Items []MessageDTO + // NextCursor continues the same chat history; nil marks its current end. + NextCursor *string +} diff --git a/internal/model/dto/session.go b/internal/model/dto/session.go new file mode 100644 index 0000000..9d1193c --- /dev/null +++ b/internal/model/dto/session.go @@ -0,0 +1,13 @@ +package dto + +import "time" + +// SessionDTO is an issued session, exposing the raw token only at creation. +type SessionDTO struct { + // Token contains 256 random bits encoded for a Bearer header. + Token string + // ExpiresAt is an absolute UTC deadline which is not extended by reads. + ExpiresAt time.Time + // User is the authenticated account without credentials. + User UserDTO +} diff --git a/internal/model/http/models.go b/internal/model/http/models.go new file mode 100644 index 0000000..e9db290 --- /dev/null +++ b/internal/model/http/models.go @@ -0,0 +1,102 @@ +// Package http defines the public JSON boundary, separate from DTOs. +package http + +import "time" + +// UsersCreateRequest is the registration payload. +type UsersCreateRequest struct { + // Password is the unmodified Unicode registration secret. + Password string `json:"password"` + // Username is the unique lowercase ASCII username. + Username string `json:"username"` + // DisplayName is the user-visible name, limited to 100 Unicode code points. + DisplayName string `json:"display_name"` +} + +// ChatsCreateRequest is the chat payload. +type ChatsCreateRequest struct { + // Title is the nonblank chat title, limited to 100 Unicode code points. + Title string `json:"title"` + // MemberIDs contains unique user UUIDs, including the caller when creating a chat. + MemberIDs []string `json:"member_ids"` +} + +// MessagesCreateRequest is the message payload. +type MessagesCreateRequest struct { + // AttachmentIDs references up to ten files of this chat. + AttachmentIDs []string `json:"attachment_ids"` + // Text contains nonblank text, limited to 4000 Unicode code points. + Text string `json:"text"` +} + +// UserResponse is the public user model. +type UserResponse struct { + // ID is the canonical server-generated UUID. + ID string `json:"id"` + // Username is the unique lowercase ASCII username. + Username string `json:"username"` + // DisplayName is the user-visible name, limited to 100 Unicode code points. + DisplayName string `json:"display_name"` + // CreatedAt is the UTC creation time serialized as RFC 3339. + CreatedAt time.Time `json:"created_at"` +} + +// ChatResponse is the public chat model. +type ChatResponse struct { + // ID is the canonical server-generated UUID. + ID string `json:"id"` + // Title is the nonblank chat title, limited to 100 Unicode code points. + Title string `json:"title"` + // MemberIDs contains unique user UUIDs, including the caller when creating a chat. + MemberIDs []string `json:"member_ids"` + // CreatedAt is the UTC creation time serialized as RFC 3339. + CreatedAt time.Time `json:"created_at"` +} + +// MessageResponse is the public message model. +type MessageResponse struct { + // AttachmentIDs references up to ten files of this chat. + AttachmentIDs []string `json:"attachment_ids"` + // ID is the canonical server-generated UUID. + ID string `json:"id"` + // ChatID is the canonical UUID of the chat containing the message. + ChatID string `json:"chat_id"` + // SenderID is derived from the request identity, never from a client-supplied sender. + SenderID string `json:"sender_id"` + // Text contains nonblank text, limited to 4000 Unicode code points. + Text string `json:"text"` + // CreatedAt is the UTC creation time serialized as RFC 3339. + CreatedAt time.Time `json:"created_at"` +} + +// ChatPageResponse contains visible chats. +type ChatPageResponse struct { + // Items contains the ordered page records; an empty page is serialized as an empty array. + Items []ChatResponse `json:"items"` + // NextCursor is the opaque continuation token or null when the current page is last. + NextCursor *string `json:"next_cursor"` +} + +// MessagePageResponse contains the requested history page. +type MessagePageResponse struct { + // Items contains the ordered page records; an empty page is serialized as an empty array. + Items []MessageResponse `json:"items"` + // NextCursor is the opaque continuation token or null when the current page is last. + NextCursor *string `json:"next_cursor"` +} + +// ErrorResponse is the mandatory error envelope. +type ErrorResponse struct { + // Error contains the safe public error details. + Error ErrorDetail `json:"error"` +} + +// ErrorDetail never exposes internal errors or input contents. +type ErrorDetail struct { + // Code is one of the machine-readable error codes declared by the API contract. + Code string `json:"code"` + // Message is a safe public explanation without internal errors or input contents. + Message string `json:"message"` + // RequestID matches the X-Request-Id response header for log correlation. + RequestID string `json:"request_id"` +} diff --git a/internal/objectstore/client.go b/internal/objectstore/client.go new file mode 100644 index 0000000..2a1c596 --- /dev/null +++ b/internal/objectstore/client.go @@ -0,0 +1,122 @@ +// Package objectstore adapts the S3 protocol to bounded application operations. +package objectstore + +import ( + "context" + "fmt" + "io" + "time" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/config" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/observation" +) + +const operationTimeout = 15 * time.Second + +// Client owns a reusable S3 client and its private bucket scope. +type Client struct { + tracer observation.Tracer + client *minio.Client + bucket string +} + +// New configures a lazy client with explicit service credentials. +func New(cfg *config.ObjectStore) (*Client, error) { + client, err := minio.New(cfg.Endpoint, &minio.Options{ + Creds: credentials.NewStaticV4(cfg.AccessKey, cfg.SecretKey, ""), + Secure: cfg.Secure, + }) + if err != nil { + return nil, err + } + return &Client{client: client, bucket: cfg.Bucket}, nil +} + +// Put uses one bounded PUT for an already validated original. +func (c *Client) Put(ctx context.Context, key, mime string, reader io.Reader, size int64) (result error) { + ctx, end := c.start(ctx, "s3.put") + defer func() { end(result) }() + ctx, cancel := context.WithTimeout(ctx, operationTimeout) + defer cancel() + _, err := c.client.PutObject(ctx, c.bucket, key, reader, size, minio.PutObjectOptions{ + ContentType: mime, + DisableMultipart: true, + }) + return err +} + +// Get verifies object availability before handing its bounded stream to the caller. +func (c *Client) Get(ctx context.Context, key string) (io.ReadCloser, error) { + ctx, end := c.start(ctx, "s3.get") + ctx, cancel := context.WithTimeout(ctx, operationTimeout) + object, err := c.client.GetObject(ctx, c.bucket, key, minio.GetObjectOptions{}) + if err != nil { + end(err) + cancel() + return nil, err + } + if _, err := object.Stat(); err != nil { + end(err) + _ = object.Close() + cancel() + return nil, err + } + return &stream{ReadCloser: object, cancel: cancel, end: end}, nil +} + +type stream struct { + io.ReadCloser + cancel context.CancelFunc + end func(error) +} + +// Close releases the connection and its operation deadline. +func (s *stream) Close() error { + defer s.cancel() + err := s.ReadCloser.Close() + if s.end != nil { + s.end(err) + } + return err +} + +// Ping verifies the bucket is accessible with the configured scoped identity. +func (c *Client) Ping(ctx context.Context) error { + ctx, cancel := context.WithTimeout(ctx, operationTimeout) + defer cancel() + exists, err := c.client.BucketExists(ctx, c.bucket) + if err == nil && !exists { + return fmt.Errorf("private bucket missing") + } + return err +} + +// Exists distinguishes a missing object from a failed dependency. +func (c *Client) Exists(ctx context.Context, key string) (bool, error) { + ctx, cancel := context.WithTimeout(ctx, operationTimeout) + defer cancel() + _, err := c.client.StatObject(ctx, c.bucket, key, minio.StatObjectOptions{}) + if minio.ToErrorResponse(err).Code == "NoSuchKey" { + return false, nil + } + return err == nil, err +} + +// Remove deletes one explicitly claimed immutable object. +func (c *Client) Remove(ctx context.Context, key string) error { + ctx, cancel := context.WithTimeout(ctx, operationTimeout) + defer cancel() + return c.client.RemoveObject(ctx, c.bucket, key, minio.RemoveObjectOptions{}) +} + +// Bootstrap creates a bucket idempotently; no anonymous policy is installed. +func (c *Client) Bootstrap(ctx context.Context) error { + exists, err := c.client.BucketExists(ctx, c.bucket) + if err != nil || exists { + return err + } + return c.client.MakeBucket(ctx, c.bucket, minio.MakeBucketOptions{}) +} diff --git a/internal/objectstore/client_test.go b/internal/objectstore/client_test.go new file mode 100644 index 0000000..5b33887 --- /dev/null +++ b/internal/objectstore/client_test.go @@ -0,0 +1,193 @@ +package objectstore + +import ( + "bytes" + "context" + "io" + "net/http" + "os" + "testing" + "time" + + "github.com/google/uuid" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/config" +) + +// testObjects creates a scoped client against an explicitly isolated S3 server. +func testObjects(t *testing.T) *Client { + t.Helper() + if os.Getenv("S3_ENDPOINT") == "" { + t.Skip("isolated S3_ENDPOINT required") + } + cfg, err := config.NewObjectStore() + if err != nil { + t.Fatal(err) + } + client, err := New(cfg) + if err != nil { + t.Fatal(err) + } + return client +} + +// TestOriginalRoundTrip checks real S3 publication, streaming, listing and explicit deletion. +func TestOriginalRoundTrip(t *testing.T) { + client := testObjects(t) + key := "original/" + uuid.NewString() + original := []byte("synthetic original bytes") + if err := client.Put(t.Context(), key, "application/octet-stream", bytes.NewReader(original), int64(len(original))); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := client.Remove(context.Background(), key); err != nil { + t.Error(err) + } + }) + reader, err := client.Get(t.Context(), key) + if err != nil { + t.Fatal(err) + } + data, err := io.ReadAll(reader) + if closeErr := reader.Close(); closeErr != nil { + t.Fatal(closeErr) + } + if err != nil || !bytes.Equal(data, original) { + t.Fatalf("original changed: %v", err) + } + found, err := client.Exists(t.Context(), key) + if err != nil || !found { + t.Fatalf("existence: %v", err) + } + entries, err := client.List(t.Context(), "") + if err != nil { + t.Fatal(err) + } + found = false + for _, entry := range entries { + found = found || entry.Key == key + } + if !found { + t.Fatal("original absent from inventory") + } +} + +// TestMissingOriginal checks absence is distinct from unavailable S3. +func TestMissingOriginal(t *testing.T) { + client := testObjects(t) + key := "original/" + uuid.NewString() + exists, err := client.Exists(t.Context(), key) + if err != nil || exists { + t.Fatalf("missing: %v", err) + } + if reader, err := client.Get(t.Context(), key); err == nil { + _ = reader.Close() + t.Fatal("missing stream accepted") + } +} + +// TestUnavailableS3 returns bounded failures instead of converting them into missing objects. +func TestUnavailableS3(t *testing.T) { + client, err := New(&config.ObjectStore{Endpoint: "127.0.0.1:1", Bucket: "attachments", AccessKey: "test", SecretKey: "test"}) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(t.Context(), 50*time.Millisecond) + defer cancel() + if err := client.Put(ctx, "original/test", "image/png", bytes.NewReader(nil), 0); err == nil { + t.Fatal("unavailable PUT succeeded") + } + if _, err := client.Get(ctx, "original/test"); err == nil { + t.Fatal("unavailable GET succeeded") + } + if _, err := client.Exists(ctx, "original/test"); err == nil { + t.Fatal("unavailable HEAD classified as missing") + } + if _, err := client.List(ctx, ""); err == nil { + t.Fatal("unavailable LIST succeeded") + } + if err := client.Ping(ctx); err == nil { + t.Fatal("unavailable bucket ready") + } + if err := client.Bootstrap(ctx); err == nil { + t.Fatal("unavailable bootstrap succeeded") + } +} + +// TestInvalidEndpoint rejects malformed S3 configuration before a network request. +func TestInvalidEndpoint(t *testing.T) { + if _, err := New(&config.ObjectStore{Endpoint: "http://invalid/path"}); err == nil { + t.Fatal("invalid endpoint accepted") + } +} + +// TestBucketBootstrap creates an isolated private bucket twice and verifies scoped credentials cannot access it. +func TestBucketBootstrap(t *testing.T) { + scoped := testObjects(t) + if err := scoped.Ping(t.Context()); err != nil { + t.Fatal(err) + } + cfg, err := config.NewObjectStore() + if err != nil { + t.Fatal(err) + } + secret := os.Getenv("S3_ADMIN_SECRET") + if secret == "" { + t.Skip("isolated S3_ADMIN_SECRET required for bootstrap test") + } + cfg.AccessKey = "operator" + cfg.SecretKey = secret + cfg.Bucket = "test-" + uuid.NewString() + admin, err := New(cfg) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := admin.client.RemoveBucket(context.Background(), cfg.Bucket); err != nil { + t.Error(err) + } + }) + if err := admin.Bootstrap(t.Context()); err != nil { + t.Fatal(err) + } + if err := admin.Bootstrap(t.Context()); err != nil { + t.Fatal(err) + } + scoped.bucket = cfg.Bucket + if _, err := scoped.List(t.Context(), ""); err == nil { + t.Fatal("service identity accessed another bucket") + } +} + +// TestPrivateBucketRejectsAnonymous verifies object ACL independently of application authorization. +func TestPrivateBucketRejectsAnonymous(t *testing.T) { + client := testObjects(t) + key := "original/" + uuid.NewString() + if err := client.Put(t.Context(), key, "application/octet-stream", bytes.NewReader([]byte("private")), 7); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := client.Remove(context.Background(), key); err != nil { + t.Error(err) + } + }) + cfg, err := config.NewObjectStore() + if err != nil { + t.Fatal(err) + } + scheme := "https://" + if !cfg.Secure { + scheme = "http://" + } + httpClient := &http.Client{Timeout: time.Second} + response, err := httpClient.Get(scheme + cfg.Endpoint + "/" + cfg.Bucket + "/" + key) + if err != nil { + t.Fatal(err) + } + if err := response.Body.Close(); err != nil { + t.Fatal(err) + } + if response.StatusCode != http.StatusForbidden { + t.Fatalf("anonymous access status=%d", response.StatusCode) + } +} diff --git a/internal/objectstore/list.go b/internal/objectstore/list.go new file mode 100644 index 0000000..65d8f97 --- /dev/null +++ b/internal/objectstore/list.go @@ -0,0 +1,26 @@ +package objectstore + +import ( + "context" + + "github.com/minio/minio-go/v7" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/media" +) + +// List bounds each reconciliation page and cancels the SDK producer before returning. +func (c *Client) List(ctx context.Context, after string) ([]media.ObjectInfo, error) { + ctx, cancel := context.WithTimeout(ctx, operationTimeout) + defer cancel() + result := make([]media.ObjectInfo, 0, 1000) + for object := range c.client.ListObjects(ctx, c.bucket, minio.ListObjectsOptions{Prefix: "", StartAfter: after, Recursive: true, MaxKeys: 1000}) { + if object.Err != nil { + return nil, object.Err + } + result = append(result, media.ObjectInfo{Key: object.Key, ModifiedAt: object.LastModified}) + if len(result) == 1000 { + break + } + } + return result, nil +} diff --git a/internal/objectstore/observation.go b/internal/objectstore/observation.go new file mode 100644 index 0000000..630f94a --- /dev/null +++ b/internal/objectstore/observation.go @@ -0,0 +1,22 @@ +package objectstore + +import ( + "context" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/observation" +) + +// WithTracer binds optional operation tracing without exposing credentials or object keys. +func (c *Client) WithTracer(tracer observation.Tracer) *Client { + c.tracer = tracer + return c +} + +// start records only the stable S3 operation name. +func (c *Client) start(ctx context.Context, operation string) (context.Context, func(error)) { + if c.tracer != nil { + return c.tracer.Start(ctx, operation, "client") + } + return ctx, func(error) { + } +} diff --git a/internal/objectstore/observation_test.go b/internal/objectstore/observation_test.go new file mode 100644 index 0000000..0190276 --- /dev/null +++ b/internal/objectstore/observation_test.go @@ -0,0 +1,30 @@ +package objectstore + +import ( + "context" + "testing" +) + +type tracerFixture struct { + operation string + finished bool +} + +// Start records a stable operation name and closes it through the returned callback. +func (f *tracerFixture) Start(ctx context.Context, name, _ string) (context.Context, func(error)) { + f.operation = name + return ctx, func(error) { + f.finished = true + } +} + +// TestObjectObservationOmitsCoordinates keeps bucket keys and credentials outside the tracer boundary. +func TestObjectObservationOmitsCoordinates(t *testing.T) { + tracer := &tracerFixture{} + client := (&Client{}).WithTracer(tracer) + _, finish := client.start(t.Context(), "s3.put") + finish(nil) + if tracer.operation != "s3.put" || !tracer.finished { + t.Fatal("operation span missing") + } +} diff --git a/internal/observation/snapshot.go b/internal/observation/snapshot.go new file mode 100644 index 0000000..99269ac --- /dev/null +++ b/internal/observation/snapshot.go @@ -0,0 +1,11 @@ +package observation + +// Snapshot describes bounded persistence saturation without record identifiers. +type Snapshot struct { + // QueueDepth counts accepted unfinished jobs in the current pipeline version. + QueueDepth int64 + // OldestSeconds measures time since the oldest durable acceptance. + OldestSeconds float64 + // PoolUsed counts checked-out database connections. + PoolUsed int64 +} diff --git a/internal/observation/tracer.go b/internal/observation/tracer.go new file mode 100644 index 0000000..291b0a9 --- /dev/null +++ b/internal/observation/tracer.go @@ -0,0 +1,10 @@ +// Package observation defines optional consumer-owned instrumentation boundaries. +package observation + +import "context" + +// Tracer starts bounded spans without exposing an SDK provider to business logic. +type Tracer interface { + // Start creates an operation span and a completion callback accepting its result. + Start(context.Context, string, string) (context.Context, func(error)) +} diff --git a/internal/password/password.go b/internal/password/password.go new file mode 100644 index 0000000..630e160 --- /dev/null +++ b/internal/password/password.go @@ -0,0 +1,54 @@ +// Package password implements a bounded Argon2id password KDF. +package password + +import ( + "crypto/rand" + "crypto/subtle" + "encoding/base64" + "strings" + + "golang.org/x/crypto/argon2" +) + +const ( + memoryKiB = 19 * 1024 + iterations = 2 + parallelism = 1 + saltBytes = 16 + hashBytes = 32 + prefix = "$argon2id$v=19$m=19456,t=2,p=1$" +) + +// Hasher implements password hashing without transport or storage dependencies. +type Hasher struct{} + +// Hash generates a salted Argon2id encoding using fixed resource bounds. +func (Hasher) Hash(password string) (string, error) { + salt := make([]byte, saltBytes) + if _, err := rand.Read(salt); err != nil { + return "", err + } + hash := argon2.IDKey([]byte(password), salt, iterations, memoryKiB, parallelism, hashBytes) + return prefix + base64.RawStdEncoding.EncodeToString(salt) + "$" + base64.RawStdEncoding.EncodeToString(hash), nil +} + +// Verify accepts only this application's bounded KDF encoding and compares in constant time. +func (Hasher) Verify(encoded, password string) bool { + if !strings.HasPrefix(encoded, prefix) { + return false + } + parts := strings.Split(strings.TrimPrefix(encoded, prefix), "$") + if len(parts) != 2 { + return false + } + salt, err := base64.RawStdEncoding.DecodeString(parts[0]) + if err != nil || len(salt) != saltBytes { + return false + } + expected, err := base64.RawStdEncoding.DecodeString(parts[1]) + if err != nil || len(expected) != hashBytes { + return false + } + actual := argon2.IDKey([]byte(password), salt, iterations, memoryKiB, parallelism, hashBytes) + return subtle.ConstantTimeCompare(actual, expected) == 1 +} diff --git a/internal/password/password_test.go b/internal/password/password_test.go new file mode 100644 index 0000000..565df62 --- /dev/null +++ b/internal/password/password_test.go @@ -0,0 +1,37 @@ +package password + +import ( + "strings" + "testing" +) + +// TestPasswordRoundTrip verifies long Unicode input without bcrypt-style truncation. +func TestPasswordRoundTrip(t *testing.T) { + password := strings.Repeat("🔐", 128) + hasher := Hasher{} + encoded, err := hasher.Hash(password) + if err != nil { + t.Fatal(err) + } + if !hasher.Verify(encoded, password) || hasher.Verify(encoded, password+"x") { + t.Fatal("password verification mismatch") + } + second, err := hasher.Hash(password) + if err != nil { + t.Fatal(err) + } + if encoded == second { + t.Fatal("salt reused") + } +} + +// TestPasswordRejectsInvalidEncoding checks corrupt or unbounded KDF descriptions. +func TestPasswordRejectsInvalidEncoding(t *testing.T) { + for _, encoded := range []string{"", "$argon2id$v=19$m=999999999,t=2,p=1$a$b", prefix + "x", prefix + "%%%$abc", prefix + "YWJj$abc", prefix + "AAAAAAAAAAAAAAAAAAAAAA$%%%", prefix + "AAAAAAAAAAAAAAAAAAAAAA$YWJj"} { + t.Run(encoded, func(t *testing.T) { + if (Hasher{}).Verify(encoded, "password") { + t.Fatal("invalid encoding accepted") + } + }) + } +} diff --git a/internal/pipeline/cleanup.go b/internal/pipeline/cleanup.go new file mode 100644 index 0000000..8f84e0d --- /dev/null +++ b/internal/pipeline/cleanup.go @@ -0,0 +1,36 @@ +package pipeline + +import "context" + +// Retention owns replay fencing and deletion acknowledgements. +type Retention interface { + // ExpireStaging claims at most 100 old terminal source keys. + ExpireStaging(context.Context) ([]string, error) + // StagingRemoved acknowledges idempotent source removal. + StagingRemoved(context.Context, string) error +} + +// Remover deletes only a server-claimed immutable key. +type Remover interface { + // Remove is safe to repeat after an uncertain acknowledgement. + Remove(context.Context, string) error +} + +// Cleanup removes expired staging only after the database has excluded replay. +func Cleanup(ctx context.Context, store Retention, objects Remover) (int, error) { + keys, err := store.ExpireStaging(ctx) + if err != nil { + return 0, err + } + removed := 0 + for _, key := range keys { + if err := objects.Remove(ctx, key); err != nil { + return removed, err + } + if err := store.StagingRemoved(ctx, key); err != nil { + return removed, err + } + removed++ + } + return removed, nil +} diff --git a/internal/pipeline/cleanup_test.go b/internal/pipeline/cleanup_test.go new file mode 100644 index 0000000..2b69316 --- /dev/null +++ b/internal/pipeline/cleanup_test.go @@ -0,0 +1,49 @@ +package pipeline + +import ( + "context" + "testing" +) + +type cleanupFixture struct { + claimErr error + removeErr error + confirmErr error +} + +// ExpireStaging supplies one safely claimed source key. +func (f cleanupFixture) ExpireStaging(context.Context) ([]string, error) { + return []string{"staging/claimed"}, f.claimErr +} + +// Remove injects an independent deletion acknowledgement outcome. +func (f cleanupFixture) Remove(context.Context, string) error { + return f.removeErr +} + +// StagingRemoved injects acknowledgement persistence failure. +func (f cleanupFixture) StagingRemoved(context.Context, string) error { + return f.confirmErr +} + +// TestCleanupBoundaries verifies deletion is counted only after both acknowledgements. +func TestCleanupBoundaries(t *testing.T) { + for _, tc := range []struct { + name string + fixture cleanupFixture + want int + failed bool + }{ + {"claim failed", cleanupFixture{claimErr: errInjected}, 0, true}, + {"delete failed", cleanupFixture{removeErr: errInjected}, 0, true}, + {"confirm failed", cleanupFixture{confirmErr: errInjected}, 0, true}, + {"completed", cleanupFixture{}, 1, false}, + } { + t.Run(tc.name, func(t *testing.T) { + count, err := Cleanup(t.Context(), tc.fixture, tc.fixture) + if count != tc.want || (err != nil) != tc.failed { + t.Fatalf("cleanup: %d %v", count, err) + } + }) + } +} diff --git a/internal/pipeline/observation.go b/internal/pipeline/observation.go new file mode 100644 index 0000000..c8c78fe --- /dev/null +++ b/internal/pipeline/observation.go @@ -0,0 +1,42 @@ +package pipeline + +import ( + "context" + "time" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/observation" +) + +// Observer preserves job causality and records bounded processing measurements. +type Observer interface { + observation.Tracer + // Extract restores a durable parent context. + Extract(context.Context, string, string) context.Context + // Inject serializes the producer's W3C carrier. + Inject(context.Context) (string, string) + // JobStarted increments current worker occupancy. + JobStarted() + // JobFinished records a real attempt's outcome and duration. + JobFinished(context.Context, time.Duration, error) + // Ready records latency from durable acceptance to confirmed publication. + Ready(context.Context, time.Time) +} + +// WithObserver binds optional telemetry at the composition root. +func (w *Worker) WithObserver(observer Observer) *Worker { + w.observer = observer + return w +} + +// publish traces one durable handoff and propagates the producer context to its consumer. +func (w *Worker) publish(ctx context.Context, job Job) error { + if w.observer == nil { + return w.queue.Publish(ctx, job) + } + ctx = w.observer.Extract(ctx, job.Traceparent, job.Tracestate) + ctx, end := w.observer.Start(ctx, "attachment.publish", "producer") + job.Traceparent, job.Tracestate = w.observer.Inject(ctx) + err := w.queue.Publish(ctx, job) + end(err) + return err +} diff --git a/internal/pipeline/observation_test.go b/internal/pipeline/observation_test.go new file mode 100644 index 0000000..2d6da35 --- /dev/null +++ b/internal/pipeline/observation_test.go @@ -0,0 +1,92 @@ +package pipeline + +import ( + "context" + "errors" + "testing" + "time" +) + +type observerFixture struct { + extracted string + injected bool + started int + finished int + ready int + spans int +} + +// Start counts actual operation scopes without imposing an SDK on pipeline tests. +func (o *observerFixture) Start(ctx context.Context, _ string, _ string) (context.Context, func(error)) { + o.spans++ + return ctx, func(error) { + } +} + +// Extract captures only the intended durable carrier. +func (o *observerFixture) Extract(ctx context.Context, parent, _ string) context.Context { + o.extracted = parent + return ctx +} + +// Inject models the producer's outgoing parent context. +func (o *observerFixture) Inject(context.Context) (string, string) { + o.injected = true + return "producer-parent", "state" +} + +// JobStarted records one claimed attempt. +func (o *observerFixture) JobStarted() { + o.started++ +} + +// JobFinished releases one claimed attempt after its actual result. +func (o *observerFixture) JobFinished(context.Context, time.Duration, error) { + o.finished++ +} + +// Ready counts only confirmed durable results. +func (o *observerFixture) Ready(context.Context, time.Time) { + o.ready++ +} + +// TestObservedDispatchPropagatesProducer verifies producer spans surround the real queue call. +func TestObservedDispatchPropagatesProducer(t *testing.T) { + worker, _, _ := fixtureWorker(t) + observer := &observerFixture{} + worker.WithObserver(observer) + if err := worker.Dispatch(t.Context()); err != nil { + t.Fatal(err) + } + if !observer.injected || observer.spans != 1 { + t.Fatal("producer handoff not observed") + } +} + +// TestObservedAttemptMeasuresDurableCompletion ensures metrics do not count only notification receipt. +func TestObservedAttemptMeasuresDurableCompletion(t *testing.T) { + worker, _, delivery := fixtureWorker(t) + delivery.Job.Traceparent = "accepted-parent" + observer := &observerFixture{} + worker.WithObserver(observer) + if err := worker.Handle(t.Context(), delivery); err != nil { + t.Fatal(err) + } + if observer.extracted != "accepted-parent" || observer.started != 1 || observer.finished != 1 || observer.ready != 1 { + t.Fatal("incorrect attempt observation") + } +} + +// TestObservedFailedCommitDoesNotCountReady protects the end-to-end success histogram from failed publication. +func TestObservedFailedCommitDoesNotCountReady(t *testing.T) { + worker, fixture, delivery := fixtureWorker(t) + fixture.completeErr = errors.New("uncertain commit") + observer := &observerFixture{} + worker.WithObserver(observer) + if err := worker.Handle(t.Context(), delivery); err != nil { + t.Fatal(err) + } + if observer.ready != 0 || observer.finished != 1 { + t.Fatal("uncertain result counted ready") + } +} diff --git a/internal/pipeline/types.go b/internal/pipeline/types.go new file mode 100644 index 0000000..6467fa5 --- /dev/null +++ b/internal/pipeline/types.go @@ -0,0 +1,100 @@ +// Package pipeline coordinates durable jobs through consumer-owned interfaces. +package pipeline + +import ( + "context" + "io" + "time" + + "github.com/google/uuid" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/model/dto" +) + +// Job is the versioned broker envelope; storage paths never enter the broker. +type Job struct { + // SchemaVersion fixes the wire schema. + SchemaVersion int `json:"schema_version"` + // ID remains stable across retries. + ID uuid.UUID `json:"job_id"` + // Type identifies the processing operation. + Type string `json:"type"` + // AttachmentID identifies trusted metadata. + AttachmentID uuid.UUID `json:"attachment_id"` + // Version fences superseded pipelines. + Version int `json:"pipeline_version"` + // Attempt is a managed processing attempt, starting at one. + Attempt int `json:"attempt"` + // CreatedAt records durable intent. + CreatedAt time.Time `json:"created_at"` + // Traceparent propagates a W3C parent when telemetry is enabled. + Traceparent string `json:"traceparent,omitempty"` + // Tracestate carries optional vendor state. + Tracestate string `json:"tracestate,omitempty"` +} + +// Lease carries a fenced claim and trusted storage metadata. +type Lease struct { + // Job is the current durable attempt. + Job Job + // Token prevents a late attempt from publishing. + Token uuid.UUID + // Attachment describes the original. + Attachment dto.Attachment + // StagingKey is read only from the database. + StagingKey string +} + +// Delivery pairs the broker receipt with its envelope. +type Delivery struct { + // Receipt is used only for acknowledgement. + Receipt string + // Job is the validated notification. + Job Job +} + +// Store owns atomic job and metadata transitions. +type Store interface { + // Due returns bounded durable intents requiring publication. + Due(context.Context, int) ([]Job, error) + // Published records a successful broker confirmation. + Published(context.Context, Job) error + // Claim obtains a lease or returns nil for an ineligible notification. + Claim(context.Context, Job, time.Duration, int) (*Lease, error) + // Complete atomically publishes the current fenced result. + Complete(context.Context, Lease, string, []dto.Variant) (bool, error) + // Fail persists retry scheduling or a safe terminal failure. + Fail(context.Context, Lease, string, time.Duration, int) error +} + +// Queue transports durable notifications with explicit acknowledgements. +type Queue interface { + // Publish confirms acceptance by the persistent broker. + Publish(context.Context, Job) error + // Receive reads one notification, including abandoned deliveries. + Receive(context.Context) (*Delivery, error) + // Ack releases a delivery after its durable transition. + Ack(context.Context, string) error +} + +// Objects accesses private bytes; all keys come from trusted code. +type Objects interface { + // Get opens an object owned by the job. + Get(context.Context, string) (io.ReadCloser, error) + // Put persists an immutable attempt output. + Put(context.Context, string, string, io.Reader, int64) error +} + +// Output pairs public metadata with generated encoded bytes. +type Output struct { + // Metadata describes this variant. + Metadata dto.Variant + // Bytes contains a bounded encoded image. + Bytes []byte +} + +// Processor converts one validated source within the supplied deadline. +type Processor interface { + // Process returns all variants required before ready. + Process(context.Context, []byte) ([]Output, error) +} diff --git a/internal/pipeline/worker.go b/internal/pipeline/worker.go new file mode 100644 index 0000000..0d503f5 --- /dev/null +++ b/internal/pipeline/worker.go @@ -0,0 +1,177 @@ +package pipeline + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "math/rand/v2" + "time" + + "go.uber.org/zap" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/model/dto" +) + +// ErrInvalidImage identifies a permanent decode/checksum failure. +var ErrInvalidImage = errors.New("invalid image") + +const sourceLimit = 10 * 1024 * 1024 +const maxAttempts = 5 +const claimDuration = 2 * time.Minute +const processingTimeout = 45 * time.Second +const outboxBatch = 100 + +// Worker processes one job at a time, independently of API admission. +type Worker struct { + observer Observer + store Store + queue Queue + objects Objects + processor Processor + log *zap.Logger +} + +// New validates all side-effect boundaries at composition time. +func New(store Store, queue Queue, objects Objects, processor Processor, log *zap.Logger) (*Worker, error) { + if store == nil || queue == nil || objects == nil || processor == nil || log == nil { + return nil, fmt.Errorf("worker dependencies are required") + } + return &Worker{store: store, queue: queue, objects: objects, processor: processor, log: log}, nil +} + +// Dispatch republishes durable intents whose notification or consumer was lost. +func (w *Worker) Dispatch(ctx context.Context) error { + jobs, err := w.store.Due(ctx, outboxBatch) + if err != nil { + return err + } + for _, job := range jobs { + if err := w.publish(ctx, job); err != nil { + return err + } + if err := w.store.Published(ctx, job); err != nil { + return err + } + } + return nil +} + +// Run owns a bounded serial processing loop and observes cancellation between operations. +func (w *Worker) Run(ctx context.Context) { + for ctx.Err() == nil { + if err := w.Dispatch(ctx); err != nil { + w.log.Warn("job dispatch unavailable") + } + delivery, err := w.queue.Receive(ctx) + if err != nil { + w.log.Warn("job broker unavailable") + wait(ctx, time.Second) + continue + } + if delivery == nil { + continue + } + if err := w.Handle(ctx, *delivery); err != nil { + w.log.Warn("job transition unavailable") + } + } +} + +// Handle acknowledges only a durable completion, retry, terminal result or fenced no-op. +func (w *Worker) Handle(ctx context.Context, delivery Delivery) (result error) { + var processingError error + if w.observer != nil { + ctx = w.observer.Extract(ctx, delivery.Job.Traceparent, delivery.Job.Tracestate) + var finish func(error) + ctx, finish = w.observer.Start(ctx, "attachment.process", "consumer") + defer func() { finish(errors.Join(result, processingError)) }() + } + lease, err := w.store.Claim(ctx, delivery.Job, claimDuration, maxAttempts) + if err != nil { + return err + } + if lease == nil { + return w.queue.Ack(ctx, delivery.Receipt) + } + started := time.Now() + if w.observer != nil { + w.observer.JobStarted() + } + operation, cancel := context.WithTimeout(ctx, processingTimeout) + err = w.process(operation, *lease) + processingError = err + if w.observer != nil { + w.observer.JobFinished(ctx, time.Since(started), err) + } + cancel() + if err != nil { + code := "processing_failed" + if errors.Is(err, ErrInvalidImage) { + code = "invalid_image" + } + delay := time.Duration(1< sourceLimit || int64(len(source)) != lease.Attachment.SizeBytes || hex.EncodeToString(sum[:]) != lease.Attachment.SHA256 { + return ErrInvalidImage + } + outputs, err := w.processor.Process(ctx, source) + if err != nil { + return err + } + prefix := fmt.Sprintf("results/%s/v%d/%s/", lease.Job.AttachmentID, lease.Job.Version, lease.Token) + original := prefix + "original" + if err := w.objects.Put(ctx, original, lease.Attachment.ContentType, bytes.NewReader(source), int64(len(source))); err != nil { + return err + } + variants := make([]dto.Variant, 0, len(outputs)) + for _, output := range outputs { + if err := ctx.Err(); err != nil { + return err + } + output.Metadata.ObjectKey = prefix + output.Metadata.Name + if err := w.objects.Put(ctx, output.Metadata.ObjectKey, output.Metadata.ContentType, bytes.NewReader(output.Bytes), int64(len(output.Bytes))); err != nil { + return err + } + variants = append(variants, output.Metadata) + } + published, err := w.store.Complete(ctx, lease, original, variants) + if err == nil && published && w.observer != nil { + w.observer.Ready(ctx, lease.Job.CreatedAt) + } + return err +} + +// wait backs off broker failures without delaying shutdown. +func wait(ctx context.Context, duration time.Duration) { + timer := time.NewTimer(duration) + defer timer.Stop() + select { + case <-ctx.Done(): + case <-timer.C: + } +} diff --git a/internal/pipeline/worker_test.go b/internal/pipeline/worker_test.go new file mode 100644 index 0000000..bf6e580 --- /dev/null +++ b/internal/pipeline/worker_test.go @@ -0,0 +1,255 @@ +package pipeline + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "io" + "testing" + "time" + + "github.com/google/uuid" + "go.uber.org/zap" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/model/dto" +) + +var errInjected = errors.New("errInjected dependency failure") + +type workerFixture struct { + lease *Lease + source []byte + outputs []Output + dueErr error + publishErr error + confirmErr error + claimErr error + getErr error + readErr error + closeErr error + processErr error + completeErr error + failErr error + ackErr error + putAt int + puts int + code string + delay time.Duration + acked int + completed int + received bool + cancel context.CancelFunc +} + +// fixtureWorker provides independent dependencies for one delivery invariant. +func fixtureWorker(t *testing.T) (*Worker, *workerFixture, Delivery) { + t.Helper() + source := []byte("bounded synthetic original") + sum := sha256.Sum256(source) + job := Job{ID: uuid.New(), AttachmentID: uuid.New(), Version: 1, Attempt: 1} + f := &workerFixture{source: source, lease: &Lease{Job: job, Token: uuid.New(), Attachment: dto.Attachment{SHA256: hex.EncodeToString(sum[:]), SizeBytes: int64(len(source)), ContentType: "image/png"}, StagingKey: "staging/test"}, outputs: []Output{{Metadata: dto.Variant{Name: "thumbnail", ContentType: "image/jpeg"}, Bytes: []byte("variant")}}} + worker, err := New(f, f, f, f, zap.NewNop()) + if err != nil { + t.Fatal(err) + } + return worker, f, Delivery{Job: job, Receipt: "receipt"} +} + +// Due supplies one durable fixture intent or a database failure. +func (f *workerFixture) Due(context.Context, int) ([]Job, error) { + return []Job{f.lease.Job}, f.dueErr +} + +// Published models acknowledgement persistence after broker acceptance. +func (f *workerFixture) Published(context.Context, Job) error { + return f.confirmErr +} + +// Claim supplies a fenced claim or the configured database outcome. +func (f *workerFixture) Claim(context.Context, Job, time.Duration, int) (*Lease, error) { + return f.lease, f.claimErr +} + +// Complete records publication attempts without implementing a second SQL model. +func (f *workerFixture) Complete(context.Context, Lease, string, []dto.Variant) (bool, error) { + f.completed++ + return f.completeErr == nil, f.completeErr +} + +// Fail captures safe failure classification and bounded retry scheduling. +func (f *workerFixture) Fail(_ context.Context, _ Lease, code string, delay time.Duration, _ int) error { + f.code = code + f.delay = delay + return f.failErr +} + +// Publish supplies one broker acceptance outcome. +func (f *workerFixture) Publish(context.Context, Job) error { + return f.publishErr +} + +// Receive delivers once and cancels the fixture loop on its second read. +func (f *workerFixture) Receive(context.Context) (*Delivery, error) { + if f.received { + f.cancel() + return nil, errInjected + } + f.received = true + return &Delivery{Job: f.lease.Job, Receipt: "receipt"}, nil +} + +// Ack records whether the coordinator released the delivery. +func (f *workerFixture) Ack(context.Context, string) error { + f.acked++ + return f.ackErr +} + +// Get supplies a bounded reader with independent read and close failures. +func (f *workerFixture) Get(context.Context, string) (io.ReadCloser, error) { + return &faultReader{Reader: bytes.NewReader(f.source), readErr: f.readErr, closeErr: f.closeErr}, f.getErr +} + +// Put injects a failure at a configured original or variant write. +func (f *workerFixture) Put(context.Context, string, string, io.Reader, int64) error { + f.puts++ + if f.putAt == f.puts { + return errInjected + } + return nil +} + +// Process supplies one independent transformation outcome. +func (f *workerFixture) Process(context.Context, []byte) ([]Output, error) { + return f.outputs, f.processErr +} + +type faultReader struct { + *bytes.Reader + readErr error + closeErr error +} + +// Read injects a transport read failure before consuming source bytes. +func (r *faultReader) Read(p []byte) (int, error) { + if r.readErr != nil { + return 0, r.readErr + } + return r.Reader.Read(p) +} + +// Close exposes an independently failed stream finalization. +func (r *faultReader) Close() error { + return r.closeErr +} + +// TestHandlePublishesBeforeAck verifies all output writes precede the durable completion. +func TestHandlePublishesBeforeAck(t *testing.T) { + worker, f, delivery := fixtureWorker(t) + if err := worker.Handle(t.Context(), delivery); err != nil { + t.Fatal(err) + } + if f.puts != 2 || f.completed != 1 || f.acked != 1 || f.code != "" { + t.Fatal("incomplete publication acknowledged") + } +} + +// TestHandlePersistsRetryBeforeAck checks all transient I/O windows converge to durable retry. +func TestHandlePersistsRetryBeforeAck(t *testing.T) { + cases := []struct { + name string + setup func(*workerFixture) + code string + }{ + {"read source", func(f *workerFixture) { f.getErr = errInjected }, "processing_failed"}, + {"read bytes", func(f *workerFixture) { f.readErr = errInjected }, "processing_failed"}, + {"close source", func(f *workerFixture) { f.closeErr = errInjected }, "processing_failed"}, + {"decode", func(f *workerFixture) { f.processErr = ErrInvalidImage }, "invalid_image"}, + {"checksum", func(f *workerFixture) { f.source = []byte("tampered") }, "invalid_image"}, + {"original write", func(f *workerFixture) { f.putAt = 1 }, "processing_failed"}, + {"variant write", func(f *workerFixture) { f.putAt = 2 }, "processing_failed"}, + {"result commit", func(f *workerFixture) { f.completeErr = errInjected }, "processing_failed"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + worker, f, delivery := fixtureWorker(t) + tc.setup(f) + if err := worker.Handle(t.Context(), delivery); err != nil { + t.Fatal(err) + } + if f.code != tc.code || f.acked != 1 || f.delay < 2*time.Second || f.delay >= 3*time.Second { + t.Fatal("unsafe failure transition") + } + }) + } +} + +// TestHandleLeavesUncommittedFailurePending prevents acknowledgement when retry persistence fails. +func TestHandleLeavesUncommittedFailurePending(t *testing.T) { + worker, f, delivery := fixtureWorker(t) + f.getErr = errInjected + f.failErr = errInjected + if err := worker.Handle(t.Context(), delivery); !errors.Is(err, errInjected) || f.acked != 0 { + t.Fatal("uncommitted retry acknowledged") + } +} + +// TestHandleUnavailableClaimLeavesPending preserves delivery when the source of truth is unavailable. +func TestHandleUnavailableClaimLeavesPending(t *testing.T) { + worker, f, delivery := fixtureWorker(t) + f.claimErr = errInjected + if err := worker.Handle(t.Context(), delivery); !errors.Is(err, errInjected) || f.acked != 0 { + t.Fatal("failed claim acknowledged") + } +} + +// TestHandleDuplicateAcknowledgesWithoutEffects verifies fenced no-ops never touch S3. +func TestHandleDuplicateAcknowledgesWithoutEffects(t *testing.T) { + worker, f, delivery := fixtureWorker(t) + f.lease = nil + if err := worker.Handle(t.Context(), delivery); err != nil || f.acked != 1 || f.puts != 0 { + t.Fatal("duplicate caused effects") + } +} + +// TestDispatchFailureBoundaries checks uncertainty leaves durable outbox intent for another pass. +func TestDispatchFailureBoundaries(t *testing.T) { + for _, tc := range []struct { + name string + setup func(*workerFixture) + }{ + {"database read", func(f *workerFixture) { f.dueErr = errInjected }}, + {"broker publish", func(f *workerFixture) { f.publishErr = errInjected }}, + {"confirmation commit", func(f *workerFixture) { f.confirmErr = errInjected }}, + } { + t.Run(tc.name, func(t *testing.T) { + worker, f, _ := fixtureWorker(t) + tc.setup(f) + if err := worker.Dispatch(t.Context()); !errors.Is(err, errInjected) { + t.Fatal(err) + } + }) + } +} + +// TestWorkerLoopStops verifies the loop releases ownership on cancellation. +func TestWorkerLoopStops(t *testing.T) { + worker, f, _ := fixtureWorker(t) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + f.cancel = cancel + f.failErr = errInjected + f.getErr = errInjected + worker.Run(ctx) + if ctx.Err() == nil { + t.Fatal("worker did not stop") + } +} + +// TestWorkerRejectsMissingDependencies keeps invalid composition from starting a background loop. +func TestWorkerRejectsMissingDependencies(t *testing.T) { + if _, err := New(nil, nil, nil, nil, nil); err == nil { + t.Fatal("missing dependencies accepted") + } +} diff --git a/internal/service/auth.go b/internal/service/auth.go new file mode 100644 index 0000000..58eab95 --- /dev/null +++ b/internal/service/auth.go @@ -0,0 +1,44 @@ +package service + +import ( + "context" + "time" + + "github.com/google/uuid" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/model/dto" +) + +// Passwords is the password KDF boundary used by authentication. +type Passwords interface { + // Hash creates a salted password encoding. + Hash(password string) (string, error) + // Verify checks the supplied password against its encoding. + Verify(encoded, password string) bool +} + +// Accounts persists credentials and session validity independently of HTTP. +type Accounts interface { + // Register atomically stores a profile with credentials. + Register(context.Context, dto.UserDTO, string) (dto.UserDTO, error) + // Credentials looks up the password encoding and account profile. + Credentials(context.Context, string) (dto.UserDTO, string, error) + // SaveSession persists an opaque token digest and absolute expiry. + SaveSession(context.Context, string, uuid.UUID, time.Time) error + // SessionUser resolves an unexpired token digest. + SessionUser(context.Context, string) (uuid.UUID, error) + // RevokeSession invalidates a token digest. + RevokeSession(context.Context, string) error +} + +// Authentication defines registration and opaque session operations. +type Authentication interface { + // Register validates registration and persists a password-protected account. + Register(context.Context, dto.UsersCreateRequestDTO, string) (dto.UserDTO, error) + // Login verifies credentials and issues an opaque session. + Login(context.Context, string, string) (dto.SessionDTO, error) + // Authenticate resolves a valid token to its immutable owner. + Authenticate(context.Context, string) (string, error) + // Logout revokes the current token. + Logout(context.Context, string) error +} diff --git a/internal/service/authentication/service.go b/internal/service/authentication/service.go new file mode 100644 index 0000000..1df3a22 --- /dev/null +++ b/internal/service/authentication/service.go @@ -0,0 +1,123 @@ +// Package authentication validates credentials and owns the session lifecycle. +package authentication + +import ( + "context" + "crypto/rand" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "errors" + "fmt" + "time" + "unicode/utf8" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/model/dto" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/service" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/service/production" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/store" +) + +const ( + minimumPasswordRunes = 12 + maximumPasswordRunes = 128 + tokenBytes = 32 +) + +// Service performs password and session operations through explicit boundaries. +type Service struct { + accounts service.Accounts + passwords service.Passwords + ttl time.Duration + dummyHash string +} + +// New validates dependencies and prepares the unknown-account verification hash. +func New(accounts service.Accounts, passwords service.Passwords, ttl time.Duration) (*Service, error) { + if accounts == nil || passwords == nil || ttl <= 0 { + return nil, fmt.Errorf("accounts, passwords and positive session TTL are required") + } + dummyHash, err := passwords.Hash("unknown-account-dummy-password") + if err != nil { + return nil, err + } + return &Service{accounts: accounts, passwords: passwords, ttl: ttl, dummyHash: dummyHash}, nil +} + +// Register checks profile and Unicode password bounds before hashing and persistence. +func (s *Service) Register(ctx context.Context, request dto.UsersCreateRequestDTO, password string) (dto.UserDTO, error) { + user, err := production.PrepareUser(request) + if err != nil || !validPassword(password) { + return dto.UserDTO{}, service.ErrInvalid + } + hash, err := s.passwords.Hash(password) + if err != nil { + return dto.UserDTO{}, err + } + user, err = s.accounts.Register(ctx, user, hash) + if errors.Is(err, store.ErrConflict) { + return dto.UserDTO{}, service.ErrConflict + } + return user, err +} + +// Login verifies credentials uniformly and persists a new session before returning its token. +func (s *Service) Login(ctx context.Context, username, password string) (dto.SessionDTO, error) { + if !validPassword(password) { + return dto.SessionDTO{}, service.ErrUnauthorized + } + user, hash, err := s.accounts.Credentials(ctx, username) + if err != nil && !errors.Is(err, store.ErrNotFound) { + return dto.SessionDTO{}, err + } + missing := errors.Is(err, store.ErrNotFound) + if missing { + hash = s.dummyHash + } + if !s.passwords.Verify(hash, password) || missing { + return dto.SessionDTO{}, service.ErrUnauthorized + } + random := make([]byte, tokenBytes) + if _, err := rand.Read(random); err != nil { + return dto.SessionDTO{}, err + } + token := base64.RawURLEncoding.EncodeToString(random) + expiry := time.Now().UTC().Add(s.ttl) + if err := s.accounts.SaveSession(ctx, digest(token), user.ID, expiry); err != nil { + return dto.SessionDTO{}, err + } + return dto.SessionDTO{Token: token, ExpiresAt: expiry, User: user}, nil +} + +// Authenticate checks absolute expiry in the source of truth on every request. +func (s *Service) Authenticate(ctx context.Context, token string) (string, error) { + decoded, err := base64.RawURLEncoding.DecodeString(token) + if err != nil || len(decoded) != tokenBytes { + return "", service.ErrUnauthorized + } + id, err := s.accounts.SessionUser(ctx, digest(token)) + if errors.Is(err, store.ErrNotFound) { + return "", service.ErrUnauthorized + } + if err != nil { + return "", err + } + return id.String(), nil +} + +// Logout removes the current session from the authoritative store. +func (s *Service) Logout(ctx context.Context, token string) error { + return s.accounts.RevokeSession(ctx, digest(token)) +} + +// digest produces the storage key without persisting the bearer secret. +func digest(token string) string { + hash := sha256.Sum256([]byte(token)) + return hex.EncodeToString(hash[:]) +} + +// validPassword enforces the contract in Unicode code points without truncation. +func validPassword(password string) bool { + count := utf8.RuneCountInString(password) + return utf8.ValidString(password) && count >= minimumPasswordRunes && count <= maximumPasswordRunes +} diff --git a/internal/service/authentication/service_test.go b/internal/service/authentication/service_test.go new file mode 100644 index 0000000..e4d3a11 --- /dev/null +++ b/internal/service/authentication/service_test.go @@ -0,0 +1,232 @@ +package authentication + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + "github.com/google/uuid" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/model/dto" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/service" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/store" +) + +type accountFixture struct { + user dto.UserDTO + hash string + tokenHash string + expiry time.Time + failure error + registerFailure error + saveFailure error +} + +// Register captures one synthetic account or a prescribed persistence failure. +func (f *accountFixture) Register(_ context.Context, user dto.UserDTO, hash string) (dto.UserDTO, error) { + f.user = user + f.hash = hash + return user, f.registerFailure +} + +// Credentials returns the fixture account and its password encoding. +func (f *accountFixture) Credentials(context.Context, string) (dto.UserDTO, string, error) { + return f.user, f.hash, f.failure +} + +// SaveSession captures the digest and absolute expiry without storing the raw token. +func (f *accountFixture) SaveSession(_ context.Context, hash string, _ uuid.UUID, expiry time.Time) error { + f.tokenHash = hash + f.expiry = expiry + return f.saveFailure +} + +// SessionUser implements expiry and revocation for the service unit fixture. +func (f *accountFixture) SessionUser(_ context.Context, hash string) (uuid.UUID, error) { + if f.failure != nil { + return uuid.Nil, f.failure + } + if hash != f.tokenHash || !f.expiry.After(time.Now()) { + return uuid.Nil, store.ErrNotFound + } + return f.user.ID, nil +} + +// RevokeSession invalidates the fixture digest. +func (f *accountFixture) RevokeSession(context.Context, string) error { + f.tokenHash = "" + return f.failure +} + +type passwordFixture struct { + failure error +} + +// Hash creates an intentionally noncryptographic encoding for service-only unit tests. +func (f passwordFixture) Hash(password string) (string, error) { + return "fixture:" + password, f.failure +} + +// Verify compares only test fixture encodings; production uses Argon2id. +func (passwordFixture) Verify(encoded, password string) bool { + return encoded == "fixture:"+password +} + +// authFixture assembles deterministic service boundaries. +func authFixture(t *testing.T) (*Service, *accountFixture) { + t.Helper() + accounts := &accountFixture{user: dto.UserDTO{ID: uuid.New(), Username: "alice"}, hash: "fixture:synthetic-password"} + s, err := New(accounts, passwordFixture{}, time.Hour) + if err != nil { + t.Fatal(err) + } + return s, accounts +} + +// TestRegisterValidatesPassword checks contract boundaries without truncation. +func TestRegisterValidatesPassword(t *testing.T) { + cases := []struct { + name string + password string + valid bool + }{ + {"short", "short", false}, + {"minimum", strings.Repeat("a", 12), true}, + {"unicode maximum", strings.Repeat("🔐", 128), true}, + {"too long", strings.Repeat("a", 129), false}, + {"invalid UTF8", strings.Repeat("a", 12) + "\xff", false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + s, _ := authFixture(t) + _, err := s.Register(t.Context(), dto.UsersCreateRequestDTO{Username: "alice", DisplayName: "Alice"}, tc.password) + if (err == nil) != tc.valid { + t.Fatalf("unexpected registration result: %v", err) + } + }) + } +} + +// TestRegisterConflictMapsToApplicationError verifies uniqueness error translation. +func TestRegisterConflictMapsToApplicationError(t *testing.T) { + s, accounts := authFixture(t) + accounts.registerFailure = store.ErrConflict + _, err := s.Register(t.Context(), dto.UsersCreateRequestDTO{Username: "alice", DisplayName: "Alice"}, "synthetic-password") + if !errors.Is(err, service.ErrConflict) { + t.Fatal(err) + } +} + +// TestRegisterHashFailurePreservesCause checks KDF failures before persistence. +func TestRegisterHashFailurePreservesCause(t *testing.T) { + s, _ := authFixture(t) + failure := errors.New("KDF unavailable") + s.passwords = passwordFixture{failure: failure} + _, err := s.Register(t.Context(), dto.UsersCreateRequestDTO{Username: "alice", DisplayName: "Alice"}, "synthetic-password") + if !errors.Is(err, failure) { + t.Fatal(err) + } +} + +// TestLoginPersistsOnlyTokenDigest checks issued entropy, owner and deadline together. +func TestLoginPersistsOnlyTokenDigest(t *testing.T) { + s, accounts := authFixture(t) + session, err := s.Login(t.Context(), "alice", "synthetic-password") + if err != nil { + t.Fatal(err) + } + if session.Token == accounts.tokenHash || len(accounts.tokenHash) != 64 || session.User.ID != accounts.user.ID || !session.ExpiresAt.Equal(accounts.expiry) { + t.Fatal("invalid issued session") + } + id, err := s.Authenticate(t.Context(), session.Token) + if err != nil || id != accounts.user.ID.String() { + t.Fatalf("session does not authenticate its owner: %v", err) + } +} + +// TestLoginRejectsCredentials checks independent authentication rejection scenarios. +func TestLoginRejectsCredentials(t *testing.T) { + cases := []struct { + name string + password string + failure error + expected error + }{ + {"short password", "short", nil, service.ErrUnauthorized}, + {"wrong password", "incorrect-password", nil, service.ErrUnauthorized}, + {"missing account", "synthetic-password", store.ErrNotFound, service.ErrUnauthorized}, + {"database unavailable", "synthetic-password", context.DeadlineExceeded, context.DeadlineExceeded}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + s, accounts := authFixture(t) + accounts.failure = tc.failure + _, err := s.Login(t.Context(), "alice", tc.password) + if !errors.Is(err, tc.expected) { + t.Fatalf("expected %v, got %v", tc.expected, err) + } + }) + } +} + +// TestLoginDoesNotReturnUnpersistedSession checks failure after successful password verification. +func TestLoginDoesNotReturnUnpersistedSession(t *testing.T) { + s, accounts := authFixture(t) + accounts.saveFailure = context.DeadlineExceeded + session, err := s.Login(t.Context(), "alice", "synthetic-password") + if !errors.Is(err, context.DeadlineExceeded) || session.Token != "" { + t.Fatal("unpersisted session escaped") + } +} + +// TestAuthenticateRejectsUnknownToken checks malformed and correctly encoded unknown tokens. +func TestAuthenticateRejectsUnknownToken(t *testing.T) { + for _, token := range []string{"", "%%%", strings.Repeat("A", 43)} { + t.Run(token, func(t *testing.T) { + s, _ := authFixture(t) + if _, err := s.Authenticate(t.Context(), token); !errors.Is(err, service.ErrUnauthorized) { + t.Fatal(err) + } + }) + } +} + +// TestAuthenticatePropagatesDatabaseFailure prevents trusting a token during an outage. +func TestAuthenticatePropagatesDatabaseFailure(t *testing.T) { + s, accounts := authFixture(t) + accounts.failure = context.DeadlineExceeded + if _, err := s.Authenticate(t.Context(), strings.Repeat("A", 43)); !errors.Is(err, context.DeadlineExceeded) { + t.Fatal(err) + } +} + +// TestLogoutRevokesIssuedToken checks that the same token cannot be reused. +func TestLogoutRevokesIssuedToken(t *testing.T) { + s, _ := authFixture(t) + session, err := s.Login(t.Context(), "alice", "synthetic-password") + if err != nil { + t.Fatal(err) + } + if err := s.Logout(t.Context(), session.Token); err != nil { + t.Fatal(err) + } + if _, err := s.Authenticate(t.Context(), session.Token); !errors.Is(err, service.ErrUnauthorized) { + t.Fatal("revoked token authenticated") + } +} + +// TestNewRejectsIncompleteDependencies checks constructor validation and KDF startup failure. +func TestNewRejectsIncompleteDependencies(t *testing.T) { + t.Run("missing accounts", func(t *testing.T) { + if _, err := New(nil, passwordFixture{}, time.Hour); err == nil { + t.Fatal("missing accounts accepted") + } + }) + t.Run("hash failure", func(t *testing.T) { + if _, err := New(&accountFixture{}, passwordFixture{failure: context.Canceled}, time.Hour); !errors.Is(err, context.Canceled) { + t.Fatal(err) + } + }) +} diff --git a/internal/service/cache.go b/internal/service/cache.go new file mode 100644 index 0000000..46c2610 --- /dev/null +++ b/internal/service/cache.go @@ -0,0 +1,17 @@ +package service + +import ( + "context" + + "github.com/google/uuid" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/model/dto" +) + +// ProfileCache stores immutable public account profiles; misses fall back to PostgreSQL. +type ProfileCache interface { + // GetProfile returns a cached profile or a miss, including on backend failure. + GetProfile(context.Context, uuid.UUID) (dto.UserDTO, bool) + // PutProfile stores a bounded-lifetime profile on a best-effort basis. + PutProfile(context.Context, dto.UserDTO) +} diff --git a/internal/service/cursor.go b/internal/service/cursor.go new file mode 100644 index 0000000..1eaa399 --- /dev/null +++ b/internal/service/cursor.go @@ -0,0 +1,12 @@ +package service + +import "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/cursor" + +// CursorCodec defines the pagination operations required by application services. +type CursorCodec interface { + // Encode signs a position for the specified resource. + Encode(resource string, position cursor.Position) (string, error) + + // Decode verifies a token and returns its position; empty tokens start a page sequence. + Decode(encoded, resource string) (cursor.Position, error) +} diff --git a/internal/service/errors.go b/internal/service/errors.go new file mode 100644 index 0000000..1892e49 --- /dev/null +++ b/internal/service/errors.go @@ -0,0 +1,27 @@ +package service + +import ( + "errors" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/store" +) + +var ( + // ErrTooLarge rejects encoded or decoded images beyond policy. + ErrTooLarge = errors.New("image exceeds limit") + // ErrMedia rejects unsupported or corrupted image bytes. + ErrMedia = errors.New("invalid image") + // ErrUnavailable preserves the dependency failure category across service boundaries. + ErrUnavailable = store.ErrUnavailable + + // ErrInvalid indicates invalid operation input. + ErrInvalid = errors.New("invalid request") + // ErrUnauthorized indicates an unresolved current user. + ErrUnauthorized = errors.New("unknown user identity") + // ErrNotFound hides both absent and inaccessible resources. + ErrNotFound = errors.New("resource not found") + // ErrConflict indicates a duplicate username. + ErrConflict = errors.New("username already exists") + // ErrCapacity indicates exhausted storage capacity without eviction. + ErrCapacity = errors.New("in-memory storage capacity reached") +) diff --git a/internal/service/interface.go b/internal/service/interface.go new file mode 100644 index 0000000..4a27d47 --- /dev/null +++ b/internal/service/interface.go @@ -0,0 +1,36 @@ +// Package service defines the application operations consumed by HTTP controllers. +package service + +import ( + "context" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/model/dto" +) + +// Service validates raw input and enforces identity and membership requirements. +// Implementations must not expose transport-specific types or storage internals. +type Service interface { + // UsersCreate registers a user or reports invalid input, conflict or capacity exhaustion. + UsersCreate(ctx context.Context, request dto.UsersCreateRequestDTO) (dto.UserDTO, error) + + // UsersCheckIdentity verifies that an identity belongs to an existing user. + UsersCheckIdentity(ctx context.Context, userID string) error + + // UsersGetMe resolves the authenticated caller, rejecting malformed or unknown identifiers. + UsersGetMe(ctx context.Context, userID string) (dto.UserDTO, error) + + // ChatsCreate validates all members and creates one chat atomically. + ChatsCreate(ctx context.Context, request dto.ChatsCreateRequestDTO) (dto.ChatDTO, error) + + // ChatsGetByID hides missing and inaccessible chats behind the same not-found result. + ChatsGetByID(ctx context.Context, userID, chatID string) (dto.ChatDTO, error) + + // ChatsGetPage lists only chats visible to the caller, using resource-bound cursors. + ChatsGetPage(ctx context.Context, request dto.PageRequestDTO) (dto.ChatPageDTO, error) + + // MessagesCreate derives the sender from the identity and appends validated text. + MessagesCreate(ctx context.Context, request dto.MessagesCreateRequestDTO) (dto.MessageDTO, error) + + // MessagesGetPage authorizes the caller and returns history after a verified cursor. + MessagesGetPage(ctx context.Context, request dto.PageRequestDTO) (dto.MessagePageDTO, error) +} diff --git a/internal/service/production/access.go b/internal/service/production/access.go new file mode 100644 index 0000000..12b2c9e --- /dev/null +++ b/internal/service/production/access.go @@ -0,0 +1,62 @@ +package production + +import ( + "context" + "slices" + + "github.com/google/uuid" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/model/dto" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/service" +) + +// parseIdentity maps malformed identity values to the authentication error. +func parseIdentity(rawID string) (uuid.UUID, error) { + id, err := parseID(rawID) + if err != nil { + return uuid.Nil, service.ErrUnauthorized + } + return id, nil +} + +// requireUserID resolves an existing identity without loading a user profile. +func (s *Service) requireUserID(ctx context.Context, rawID string) (uuid.UUID, error) { + id, err := parseIdentity(rawID) + if err != nil { + return uuid.Nil, err + } + exists, err := s.repo.UsersExist(ctx, []uuid.UUID{id}) + if err != nil { + return uuid.Nil, normalizeStoreError(err) + } + if !exists { + return uuid.Nil, service.ErrUnauthorized + } + return id, nil +} + +// requireChatMember resolves an existing caller and a chat available to that caller. +func (s *Service) requireChatMember(ctx context.Context, userID, chatID string) (uuid.UUID, dto.ChatDTO, error) { + caller, err := s.requireUserID(ctx, userID) + if err != nil { + return uuid.Nil, dto.ChatDTO{}, err + } + chat, err := s.findChat(ctx, chatID) + if err != nil { + return uuid.Nil, dto.ChatDTO{}, err + } + if !slices.Contains(chat.MemberIDs, caller) { + return uuid.Nil, dto.ChatDTO{}, service.ErrNotFound + } + return caller, chat, nil +} + +// findChat parses a chat identifier and translates storage lookup errors. +func (s *Service) findChat(ctx context.Context, rawID string) (dto.ChatDTO, error) { + id, err := parseID(rawID) + if err != nil { + return dto.ChatDTO{}, err + } + chat, err := s.repo.Chat(ctx, id) + return chat, normalizeStoreError(err) +} diff --git a/internal/service/production/cache_test.go b/internal/service/production/cache_test.go new file mode 100644 index 0000000..8c08ab3 --- /dev/null +++ b/internal/service/production/cache_test.go @@ -0,0 +1,84 @@ +package production + +import ( + "context" + "testing" + + "github.com/google/uuid" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/cursor" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/model/dto" +) + +type cacheFixture struct { + profile dto.UserDTO + hit bool + writes int +} + +// GetProfile returns the arranged optional-cache lookup outcome. +func (f *cacheFixture) GetProfile(context.Context, uuid.UUID) (dto.UserDTO, bool) { + return f.profile, f.hit +} + +// PutProfile records successful cache population from authoritative data. +func (f *cacheFixture) PutProfile(_ context.Context, user dto.UserDTO) { + f.profile = user + f.writes++ +} + +// TestProfileCacheHitAvoidsProfileQuery verifies that the cache has a concrete role. +func TestProfileCacheHitAvoidsProfileQuery(t *testing.T) { + codec, err := cursor.New() + if err != nil { + t.Fatal(err) + } + cache := &cacheFixture{profile: dto.UserDTO{ID: uuid.New()}, hit: true} + svc, err := NewCached(&storeStub{t: t}, codec, cache) + if err != nil { + t.Fatal(err) + } + got, err := svc.UsersGetMe(t.Context(), cache.profile.ID.String()) + if err != nil || got.ID != cache.profile.ID { + t.Fatalf("cache hit ignored: %v", err) + } +} + +// TestProfileCacheMissPopulatesAfterDatabaseRead checks the authoritative fallback. +func TestProfileCacheMissPopulatesAfterDatabaseRead(t *testing.T) { + codec, err := cursor.New() + if err != nil { + t.Fatal(err) + } + id := uuid.New() + repo := &storeStub{t: t, user: func(context.Context, uuid.UUID) (dto.UserDTO, error) { + return dto.UserDTO{ID: id}, nil + }} + cache := &cacheFixture{} + svc, err := NewCached(repo, codec, cache) + if err != nil { + t.Fatal(err) + } + got, err := svc.UsersGetMe(t.Context(), id.String()) + if err != nil || got.ID != id || cache.writes != 1 { + t.Fatalf("fallback failed: %v", err) + } +} + +// TestCachedServiceRejectsMissingDependencies checks explicit service boundaries. +func TestCachedServiceRejectsMissingDependencies(t *testing.T) { + t.Run("store", func(t *testing.T) { + if _, err := NewCached(nil, nil, &cacheFixture{}); err == nil { + t.Fatal("missing store accepted") + } + }) + t.Run("cache", func(t *testing.T) { + codec, err := cursor.New() + if err != nil { + t.Fatal(err) + } + if _, err := NewCached(&storeStub{t: t}, codec, nil); err == nil { + t.Fatal("missing cache accepted") + } + }) +} diff --git a/internal/service/production/chats.go b/internal/service/production/chats.go new file mode 100644 index 0000000..7ce5682 --- /dev/null +++ b/internal/service/production/chats.go @@ -0,0 +1,47 @@ +package production + +import ( + "context" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/model/dto" +) + +// ChatsCreate resolves the caller, validates the proposed chat and persists it atomically. +func (s *Service) ChatsCreate(ctx context.Context, request dto.ChatsCreateRequestDTO) (dto.ChatDTO, error) { + caller, err := s.requireUserID(ctx, request.UserID) + if err != nil { + return dto.ChatDTO{}, err + } + chat, err := prepareChat(request, caller) + if err != nil { + return dto.ChatDTO{}, err + } + if err := s.requireExistingMembers(ctx, chat.MemberIDs); err != nil { + return dto.ChatDTO{}, err + } + created, err := s.repo.CreateChat(ctx, chat) + return created, normalizeStoreError(err) +} + +// ChatsGetByID returns a chat only to a current member, hiding its existence otherwise. +func (s *Service) ChatsGetByID(ctx context.Context, userID, chatID string) (dto.ChatDTO, error) { + _, chat, err := s.requireChatMember(ctx, userID, chatID) + return chat, err +} + +// ChatsGetPage authorizes the caller, loads a page and prepares its continuation. +func (s *Service) ChatsGetPage(ctx context.Context, request dto.PageRequestDTO) (dto.ChatPageDTO, error) { + caller, err := s.requireUserID(ctx, request.UserID) + if err != nil { + return dto.ChatPageDTO{}, err + } + page, err := s.parsePage(request, chatListResource(caller)) + if err != nil { + return dto.ChatPageDTO{}, err + } + items, err := s.repo.Chats(ctx, caller, page.position, page.fetchLimit()) + if err != nil { + return dto.ChatPageDTO{}, normalizeStoreError(err) + } + return s.chatPage(items, page) +} diff --git a/internal/service/production/chats_page_test.go b/internal/service/production/chats_page_test.go new file mode 100644 index 0000000..f000143 --- /dev/null +++ b/internal/service/production/chats_page_test.go @@ -0,0 +1,207 @@ +package production + +import ( + "context" + "errors" + "reflect" + "testing" + "time" + + "github.com/google/uuid" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/cursor" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/model/dto" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/service" +) + +// TestService_ChatsGetPage validates access and pagination before issuing a typed store query. +func TestService_ChatsGetPage(t *testing.T) { + t.Run("Positive: continuation uses scope, position and lookahead", func(t *testing.T) { + // Arrange. + repo := newStoreStub(t) + actor := uuid.New() + allowIdentity(repo, actor) + + previous := cursor.Position{ID: uuid.New(), CreatedAt: time.Unix(1700000000, 0).UTC()} + records := []dto.ChatDTO{ + {ID: uuid.New(), CreatedAt: time.Unix(1700000001, 0).UTC()}, + {ID: uuid.New(), CreatedAt: time.Unix(1700000002, 0).UTC()}, + {ID: uuid.New(), CreatedAt: time.Unix(1700000003, 0).UTC()}, + } + calls := 0 + repo.chats = func(_ context.Context, id uuid.UUID, position dto.PositionDTO, limit int) ([]dto.ChatDTO, error) { + calls++ + expected := dto.PositionDTO{ID: previous.ID, CreatedAt: previous.CreatedAt} + if id != actor || position != expected || limit != 3 { + t.Fatalf("wrong page query: id=%s, position=%v, limit=%d", id, position, limit) + } + return records, nil + } + svc := newStubService(t, repo) + resource := "chats:" + actor.String() + token, err := svc.cursors.Encode(resource, previous) + if err != nil { + t.Fatal(err) + } + input := dto.PageRequestDTO{UserID: actor.String(), Limit: "2", Cursor: token} + + // Act. + got, err := svc.ChatsGetPage(t.Context(), input) + + // Assert. + + if err != nil { + t.Fatal(err) + } + if calls != 1 || !reflect.DeepEqual(got.Items, records[:2]) || got.NextCursor == nil { + t.Fatalf("calls=%d, page=%+v", calls, got) + } + position, err := svc.cursors.Decode(*got.NextCursor, resource) + expected := cursor.Position{ID: records[1].ID, CreatedAt: records[1].CreatedAt} + if err != nil || position != expected { + t.Fatalf("next position=%v, want=%v, error=%v", position, expected, err) + } + + }) + + t.Run("Positive: empty history uses default limit and no continuation", func(t *testing.T) { + // Arrange. + repo := newStoreStub(t) + actor := uuid.New() + allowIdentity(repo, actor) + + calls := 0 + repo.chats = func(_ context.Context, id uuid.UUID, position dto.PositionDTO, limit int) ([]dto.ChatDTO, error) { + calls++ + if id != actor || position != (dto.PositionDTO{}) || limit != 51 { + t.Fatalf("wrong initial page query: id=%s, position=%v, limit=%d", id, position, limit) + } + return []dto.ChatDTO{}, nil + } + svc := newStubService(t, repo) + input := dto.PageRequestDTO{UserID: actor.String()} + + // Act. + got, err := svc.ChatsGetPage(t.Context(), input) + + // Assert. + if err != nil { + t.Fatal(err) + } + if calls != 1 || got.Items == nil || len(got.Items) != 0 || got.NextCursor != nil { + t.Fatalf("calls=%d, page=%+v", calls, got) + } + }) + + t.Run("Negative: cursor for another resource skips page lookup", func(t *testing.T) { + // Arrange. + repo := newStoreStub(t) + actor := uuid.New() + allowIdentity(repo, actor) + + svc := newStubService(t, repo) + token, err := svc.cursors.Encode("another-resource", cursor.Position{ID: uuid.New(), CreatedAt: time.Unix(1700000000, 0).UTC()}) + if err != nil { + t.Fatal(err) + } + input := dto.PageRequestDTO{UserID: actor.String(), Cursor: token} + + // Act. + got, err := svc.ChatsGetPage(t.Context(), input) + + // Assert. + if !errors.Is(err, service.ErrInvalid) || len(got.Items) != 0 || got.NextCursor != nil { + t.Fatalf("page=%+v, error=%v", got, err) + } + }) + + t.Run("Negative: codec failure skips page lookup", func(t *testing.T) { + // Arrange. + repo := newStoreStub(t) + actor := uuid.New() + allowIdentity(repo, actor) + + failure := errors.New("cursor unavailable") + svc := newStubService(t, repo) + svc.cursors = failingCursor{failure: failure} + input := dto.PageRequestDTO{UserID: actor.String(), Cursor: "token"} + + // Act. + got, err := svc.ChatsGetPage(t.Context(), input) + + // Assert. + if !errors.Is(err, failure) || len(got.Items) != 0 || got.NextCursor != nil { + t.Fatalf("page=%+v, error=%v", got, err) + } + }) + + t.Run("Negative: page storage fails", func(t *testing.T) { + // Arrange. + repo := newStoreStub(t) + actor := uuid.New() + allowIdentity(repo, actor) + + failure := errors.New("page unavailable") + repo.chats = func(_ context.Context, _ uuid.UUID, _ dto.PositionDTO, _ int) ([]dto.ChatDTO, error) { + return nil, failure + } + svc := newStubService(t, repo) + input := dto.PageRequestDTO{UserID: actor.String()} + + // Act. + got, err := svc.ChatsGetPage(t.Context(), input) + + // Assert. + if !errors.Is(err, failure) || len(got.Items) != 0 || got.NextCursor != nil { + t.Fatalf("page=%+v, error=%v", got, err) + } + }) + + t.Run("Negative: malformed identity skips all storage", func(t *testing.T) { + // Arrange. + svc := newStubService(t, newStoreStub(t)) + input := dto.PageRequestDTO{UserID: "invalid", ChatID: uuid.NewString()} + + // Act. + got, err := svc.ChatsGetPage(t.Context(), input) + + // Assert. + if !errors.Is(err, service.ErrUnauthorized) || len(got.Items) != 0 || got.NextCursor != nil { + t.Fatalf("page=%+v, error=%v", got, err) + } + }) +} + +// TestService_ChatsGetPage_InvalidLimit rejects invalid limits before reading stored pages. +func TestService_ChatsGetPage_InvalidLimit(t *testing.T) { + cases := []struct { + name string + limit string + }{ + {"Negative: zero limit", "0"}, + {"Negative: negative limit", "-1"}, + {"Negative: limit above maximum", "101"}, + {"Negative: nonnumeric limit", "text"}, + {"Negative: fractional limit", "1.5"}, + {"Negative: integer overflow", "99999999999999999999999999"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + // Arrange. + repo := newStoreStub(t) + actor := uuid.New() + allowIdentity(repo, actor) + + svc := newStubService(t, repo) + input := dto.PageRequestDTO{UserID: actor.String(), Limit: tc.limit} + + // Act. + got, err := svc.ChatsGetPage(t.Context(), input) + + // Assert. + if !errors.Is(err, service.ErrInvalid) || len(got.Items) != 0 || got.NextCursor != nil { + t.Fatalf("page=%+v, error=%v", got, err) + } + }) + } +} diff --git a/internal/service/production/chats_test.go b/internal/service/production/chats_test.go new file mode 100644 index 0000000..2a1382c --- /dev/null +++ b/internal/service/production/chats_test.go @@ -0,0 +1,261 @@ +package production + +import ( + "context" + "errors" + "reflect" + "slices" + "testing" + "time" + + "github.com/google/uuid" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/model/dto" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/service" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/store" +) + +// TestService_ChatsCreate checks membership in one batch and persists only validated chats. +func TestService_ChatsCreate(t *testing.T) { + t.Run("Positive: one membership query and original chat fields", func(t *testing.T) { + // Arrange. + repo := newStoreStub(t) + actor, other := uuid.New(), uuid.New() + var queries [][]uuid.UUID + repo.usersExist = func(_ context.Context, ids []uuid.UUID) (bool, error) { + queries = append(queries, slices.Clone(ids)) + return true, nil + } + var persisted dto.ChatDTO + repo.createChat = func(_ context.Context, input dto.ChatDTO) (dto.ChatDTO, error) { + persisted = input + input.CreatedAt = time.Unix(1700000000, 0).UTC() + return input, nil + } + svc := newStubService(t, repo) + input := dto.ChatsCreateRequestDTO{UserID: actor.String(), Title: " Team 👋 ", MemberIDs: []string{actor.String(), other.String()}} + + // Act. + got, err := svc.ChatsCreate(t.Context(), input) + + // Assert. + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(queries, [][]uuid.UUID{{actor}, {actor, other}}) { + t.Fatalf("existence queries=%v", queries) + } + if persisted.ID == uuid.Nil || persisted.Title != input.Title || !slices.Equal(persisted.MemberIDs, []uuid.UUID{actor, other}) { + t.Fatalf("wrong persistence input: %+v", persisted) + } + if got.ID != persisted.ID || got.CreatedAt.IsZero() || got.Title != input.Title || !slices.Equal(got.MemberIDs, persisted.MemberIDs) { + t.Fatalf("wrong returned chat: %+v", got) + } + }) + + t.Run("Negative: unknown member prevents persistence", func(t *testing.T) { + // Arrange. + repo := newStoreStub(t) + actor, missing := uuid.New(), uuid.New() + repo.usersExist = func(_ context.Context, ids []uuid.UUID) (bool, error) { + return !slices.Contains(ids, missing), nil + } + svc := newStubService(t, repo) + input := dto.ChatsCreateRequestDTO{UserID: actor.String(), Title: "Team", MemberIDs: []string{actor.String(), missing.String()}} + + // Act. + got, err := svc.ChatsCreate(t.Context(), input) + + // Assert. + if !errors.Is(err, service.ErrInvalid) || !reflect.DeepEqual(got, dto.ChatDTO{}) { + t.Fatalf("result=%+v, error=%v", got, err) + } + }) + + t.Run("Negative: membership lookup failure prevents persistence", func(t *testing.T) { + // Arrange. + repo := newStoreStub(t) + actor, other := uuid.New(), uuid.New() + failure := errors.New("membership unavailable") + repo.usersExist = func(_ context.Context, ids []uuid.UUID) (bool, error) { + if len(ids) == 1 { + return true, nil + } + return false, failure + } + svc := newStubService(t, repo) + input := dto.ChatsCreateRequestDTO{UserID: actor.String(), Title: "Team", MemberIDs: []string{actor.String(), other.String()}} + + // Act. + got, err := svc.ChatsCreate(t.Context(), input) + + // Assert. + if !errors.Is(err, failure) || !reflect.DeepEqual(got, dto.ChatDTO{}) { + t.Fatalf("result=%+v, error=%v", got, err) + } + }) + + t.Run("Negative: unknown creator prevents membership and persistence calls", func(t *testing.T) { + // Arrange. + repo := newStoreStub(t) + repo.usersExist = func(_ context.Context, _ []uuid.UUID) (bool, error) { + return false, nil + } + svc := newStubService(t, repo) + actor := uuid.New() + input := dto.ChatsCreateRequestDTO{UserID: actor.String(), Title: "Team", MemberIDs: []string{actor.String()}} + + // Act. + got, err := svc.ChatsCreate(t.Context(), input) + + // Assert. + if !errors.Is(err, service.ErrUnauthorized) || !reflect.DeepEqual(got, dto.ChatDTO{}) { + t.Fatalf("result=%+v, error=%v", got, err) + } + }) +} + +// TestService_ChatsCreate_InvalidInput rejects malformed chat fields before checking member existence. +func TestService_ChatsCreate_InvalidInput(t *testing.T) { + actor := uuid.New() + cases := []struct { + name string + title string + members []string + }{ + {"Negative: empty member set", "Team", nil}, + {"Negative: duplicate member", "Team", []string{actor.String(), actor.String()}}, + {"Negative: caller omitted", "Team", []string{uuid.NewString()}}, + {"Negative: malformed member ID", "Team", []string{actor.String(), "invalid"}}, + {"Negative: blank title", " ", []string{actor.String()}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + // Arrange. + repo := newStoreStub(t) + calls := 0 + repo.usersExist = func(_ context.Context, ids []uuid.UUID) (bool, error) { + calls++ + if calls != 1 || !slices.Equal(ids, []uuid.UUID{actor}) { + t.Fatalf("unexpected membership check: %v", ids) + } + return true, nil + } + svc := newStubService(t, repo) + input := dto.ChatsCreateRequestDTO{UserID: actor.String(), Title: tc.title, MemberIDs: tc.members} + + // Act. + got, err := svc.ChatsCreate(t.Context(), input) + + // Assert. + if !errors.Is(err, service.ErrInvalid) || !reflect.DeepEqual(got, dto.ChatDTO{}) { + t.Fatalf("result=%+v, error=%v", got, err) + } + if calls != 1 { + t.Fatalf("identity checks=%d, want=1", calls) + } + }) + } +} + +// TestService_ChatsGetByID authorizes the requested chat and conceals inaccessible records. +func TestService_ChatsGetByID(t *testing.T) { + t.Run("Positive: member receives the complete chat", func(t *testing.T) { + // Arrange. + repo := newStoreStub(t) + actor := uuid.New() + expected := dto.ChatDTO{ID: uuid.New(), Title: "Team", MemberIDs: []uuid.UUID{actor}, CreatedAt: time.Unix(1700000000, 0).UTC()} + allowChatAccess(repo, actor, expected) + svc := newStubService(t, repo) + + // Act. + got, err := svc.ChatsGetByID(t.Context(), actor.String(), expected.ID.String()) + + // Assert. + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, expected) { + t.Fatalf("chat=%+v, want=%+v", got, expected) + } + }) + + t.Run("Negative: outsider receives no chat details", func(t *testing.T) { + // Arrange. + repo := newStoreStub(t) + actor := uuid.New() + chat := dto.ChatDTO{ID: uuid.New(), Title: "Private", MemberIDs: []uuid.UUID{uuid.New()}} + allowChatAccess(repo, actor, chat) + svc := newStubService(t, repo) + + // Act. + got, err := svc.ChatsGetByID(t.Context(), actor.String(), chat.ID.String()) + + // Assert. + if !errors.Is(err, service.ErrNotFound) || !reflect.DeepEqual(got, dto.ChatDTO{}) { + t.Fatalf("private chat exposed: %+v, error=%v", got, err) + } + }) + + t.Run("Negative: malformed chat ID skips chat lookup", func(t *testing.T) { + // Arrange. + repo := newStoreStub(t) + actor := uuid.New() + allowIdentity(repo, actor) + svc := newStubService(t, repo) + + // Act. + got, err := svc.ChatsGetByID(t.Context(), actor.String(), "invalid") + + // Assert. + if !errors.Is(err, service.ErrInvalid) || !reflect.DeepEqual(got, dto.ChatDTO{}) { + t.Fatalf("result=%+v, error=%v", got, err) + } + }) + + t.Run("Negative: malformed identity skips storage", func(t *testing.T) { + // Arrange. + svc := newStubService(t, newStoreStub(t)) + + // Act. + got, err := svc.ChatsGetByID(t.Context(), "invalid", uuid.NewString()) + + // Assert. + if !errors.Is(err, service.ErrUnauthorized) || !reflect.DeepEqual(got, dto.ChatDTO{}) { + t.Fatalf("result=%+v, error=%v", got, err) + } + }) +} + +// TestService_ChatsGetByID_StoreErrors translates missing records and retains unexpected failures. +func TestService_ChatsGetByID_StoreErrors(t *testing.T) { + failure := errors.New("chat lookup unavailable") + cases := []struct { + name string + cause error + want error + }{ + {"Negative: absent chat", store.ErrNotFound, service.ErrNotFound}, + {"Negative: storage unavailable", failure, failure}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + // Arrange. + repo := newStoreStub(t) + actor := uuid.New() + allowIdentity(repo, actor) + repo.chat = func(_ context.Context, _ uuid.UUID) (dto.ChatDTO, error) { + return dto.ChatDTO{}, tc.cause + } + svc := newStubService(t, repo) + + // Act. + got, err := svc.ChatsGetByID(t.Context(), actor.String(), uuid.NewString()) + + // Assert. + if !errors.Is(err, tc.want) || !reflect.DeepEqual(got, dto.ChatDTO{}) { + t.Fatalf("result=%+v, error=%v, want=%v", got, err, tc.want) + } + }) + } +} diff --git a/internal/service/production/creation.go b/internal/service/production/creation.go new file mode 100644 index 0000000..86c00dd --- /dev/null +++ b/internal/service/production/creation.go @@ -0,0 +1,75 @@ +package production + +import ( + "strings" + "unicode/utf8" + + "github.com/google/uuid" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/model/dto" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/service" +) + +// PrepareUser validates registration fields and assigns the new user identifier. +func PrepareUser(request dto.UsersCreateRequestDTO) (dto.UserDTO, error) { + if !validUsername(request.Username) || !validText(request.DisplayName, maxDisplayNameRunes) { + return dto.UserDTO{}, service.ErrInvalid + } + return dto.UserDTO{ + ID: uuid.New(), + Username: request.Username, + DisplayName: request.DisplayName, + }, nil +} + +// prepareChat validates the title and member identifiers before storage access. +func prepareChat(request dto.ChatsCreateRequestDTO, caller uuid.UUID) (dto.ChatDTO, error) { + if !validText(request.Title, maxChatTitleRunes) { + return dto.ChatDTO{}, service.ErrInvalid + } + members, err := parseMembers(request.MemberIDs, caller) + if err != nil { + return dto.ChatDTO{}, err + } + return dto.ChatDTO{ + ID: uuid.New(), + Title: request.Title, + MemberIDs: members, + }, nil +} + +// prepareMessage validates text and creates a record for the authorized sender and chat. +func prepareMessage(request dto.MessagesCreateRequestDTO, sender, chatID uuid.UUID) (dto.MessageDTO, error) { + if (strings.TrimSpace(request.Text) == "" && len(request.AttachmentIDs) == 0) || !utf8.ValidString(request.Text) || utf8.RuneCountInString(request.Text) > maxMessageRunes || len(request.AttachmentIDs) > 10 || !validIdempotencyKey(request.IdempotencyKey) { + return dto.MessageDTO{}, service.ErrInvalid + } + seen := make(map[string]bool) + for _, id := range request.AttachmentIDs { + if _, err := parseID(id); err != nil || seen[id] { + return dto.MessageDTO{}, service.ErrInvalid + } + seen[id] = true + } + return dto.MessageDTO{ + ID: uuid.New(), + ChatID: chatID, + SenderID: sender, + Text: request.Text, + AttachmentIDs: append([]string{}, request.AttachmentIDs...), + IdempotencyKey: request.IdempotencyKey, + }, nil +} + +// validIdempotencyKey accepts only the bounded ASCII alphabet from the contract. +func validIdempotencyKey(key string) bool { + if len(key) < 1 || len(key) > 128 { + return false + } + for _, c := range key { + if c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' || c == '.' || c == '_' || c == ':' || c == '-' { + continue + } + return false + } + return true +} diff --git a/internal/service/production/errors.go b/internal/service/production/errors.go new file mode 100644 index 0000000..686bb8c --- /dev/null +++ b/internal/service/production/errors.go @@ -0,0 +1,23 @@ +package production + +import ( + "errors" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/service" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/store" +) + +// normalizeStoreError maps expected storage failures to application errors. +// Unexpected failures retain their cause for the transport's safe fallback. +func normalizeStoreError(err error) error { + switch { + case errors.Is(err, store.ErrNotFound): + return service.ErrNotFound + case errors.Is(err, store.ErrConflict): + return service.ErrConflict + case errors.Is(err, store.ErrCapacity): + return service.ErrCapacity + default: + return err + } +} diff --git a/internal/service/production/fixture_test.go b/internal/service/production/fixture_test.go new file mode 100644 index 0000000..569871a --- /dev/null +++ b/internal/service/production/fixture_test.go @@ -0,0 +1,147 @@ +package production + +import ( + "context" + "testing" + + "github.com/google/uuid" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/cursor" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/model/dto" +) + +// storeStub permits only the interactions explicitly arranged by each service scenario. +type storeStub struct { + t *testing.T + createUser func(ctx context.Context, user dto.UserDTO) (dto.UserDTO, error) + user func(ctx context.Context, id uuid.UUID) (dto.UserDTO, error) + usersExist func(ctx context.Context, ids []uuid.UUID) (bool, error) + createChat func(ctx context.Context, chat dto.ChatDTO) (dto.ChatDTO, error) + chat func(ctx context.Context, id uuid.UUID) (dto.ChatDTO, error) + chats func(ctx context.Context, id uuid.UUID, position dto.PositionDTO, limit int) ([]dto.ChatDTO, error) + createMessage func(ctx context.Context, message dto.MessageDTO) (dto.MessageDTO, error) + messages func(ctx context.Context, id uuid.UUID, position dto.PositionDTO, limit int) ([]dto.MessageDTO, error) +} + +// CreateUser delegates to the arranged behavior and rejects unexpected storage access. +func (repo *storeStub) CreateUser(ctx context.Context, user dto.UserDTO) (dto.UserDTO, error) { + repo.t.Helper() + if repo.createUser == nil { + repo.t.Fatalf("unexpected store.CreateUser call") + return dto.UserDTO{}, nil + } + return repo.createUser(ctx, user) +} + +// User delegates to the arranged behavior and rejects unexpected storage access. +func (repo *storeStub) User(ctx context.Context, id uuid.UUID) (dto.UserDTO, error) { + repo.t.Helper() + if repo.user == nil { + repo.t.Fatalf("unexpected store.User call") + return dto.UserDTO{}, nil + } + return repo.user(ctx, id) +} + +// UsersExist delegates to the arranged behavior and rejects unexpected storage access. +func (repo *storeStub) UsersExist(ctx context.Context, ids []uuid.UUID) (bool, error) { + repo.t.Helper() + if repo.usersExist == nil { + repo.t.Fatalf("unexpected store.UsersExist call") + return false, nil + } + return repo.usersExist(ctx, ids) +} + +// CreateChat delegates to the arranged behavior and rejects unexpected storage access. +func (repo *storeStub) CreateChat(ctx context.Context, chat dto.ChatDTO) (dto.ChatDTO, error) { + repo.t.Helper() + if repo.createChat == nil { + repo.t.Fatalf("unexpected store.CreateChat call") + return dto.ChatDTO{}, nil + } + return repo.createChat(ctx, chat) +} + +// Chat delegates to the arranged behavior and rejects unexpected storage access. +func (repo *storeStub) Chat(ctx context.Context, id uuid.UUID) (dto.ChatDTO, error) { + repo.t.Helper() + if repo.chat == nil { + repo.t.Fatalf("unexpected store.Chat call") + return dto.ChatDTO{}, nil + } + return repo.chat(ctx, id) +} + +// Chats delegates to the arranged behavior and rejects unexpected storage access. +func (repo *storeStub) Chats(ctx context.Context, id uuid.UUID, position dto.PositionDTO, limit int) ([]dto.ChatDTO, error) { + repo.t.Helper() + if repo.chats == nil { + repo.t.Fatalf("unexpected store.Chats call") + return nil, nil + } + return repo.chats(ctx, id, position, limit) +} + +// CreateMessage delegates to the arranged behavior and rejects unexpected storage access. +func (repo *storeStub) CreateMessage(ctx context.Context, message dto.MessageDTO) (dto.MessageDTO, error) { + repo.t.Helper() + if repo.createMessage == nil { + repo.t.Fatalf("unexpected store.CreateMessage call") + return dto.MessageDTO{}, nil + } + return repo.createMessage(ctx, message) +} + +// Messages delegates to the arranged behavior and rejects unexpected storage access. +func (repo *storeStub) Messages(ctx context.Context, id uuid.UUID, position dto.PositionDTO, limit int) ([]dto.MessageDTO, error) { + repo.t.Helper() + if repo.messages == nil { + repo.t.Fatalf("unexpected store.Messages call") + return nil, nil + } + return repo.messages(ctx, id, position, limit) +} + +// newStoreStub creates a repository that fails on every unarranged interaction. +func newStoreStub(t *testing.T) *storeStub { + t.Helper() + return &storeStub{t: t} +} + +// newStubService composes the service with controlled storage and a real cursor codec. +func newStubService(t *testing.T, repo *storeStub) *Service { + t.Helper() + codec, err := cursor.New() + if err != nil { + t.Fatal(err) + } + svc, err := New(repo, codec) + if err != nil { + t.Fatal(err) + } + return svc +} + +// allowIdentity permits exactly the arranged actor's existence lookup, never a profile read. +func allowIdentity(repo *storeStub, actor uuid.UUID) { + repo.t.Helper() + repo.usersExist = func(_ context.Context, ids []uuid.UUID) (bool, error) { + if len(ids) != 1 || ids[0] != actor { + repo.t.Fatalf("unexpected identity query: %v", ids) + } + return true, nil + } +} + +// allowChatAccess supplies a typed chat after checking the lookup key at the service boundary. +func allowChatAccess(repo *storeStub, actor uuid.UUID, chat dto.ChatDTO) { + repo.t.Helper() + allowIdentity(repo, actor) + repo.chat = func(_ context.Context, id uuid.UUID) (dto.ChatDTO, error) { + if id != chat.ID { + repo.t.Fatalf("chat queried for %s, want %s", id, chat.ID) + } + return chat, nil + } +} diff --git a/internal/service/production/members.go b/internal/service/production/members.go new file mode 100644 index 0000000..7800348 --- /dev/null +++ b/internal/service/production/members.go @@ -0,0 +1,42 @@ +package production + +import ( + "context" + + "github.com/google/uuid" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/service" +) + +// parseMembers validates unique member identifiers and requires the caller to be included. +func parseMembers(rawIDs []string, caller uuid.UUID) ([]uuid.UUID, error) { + if len(rawIDs) == 0 || len(rawIDs) > maxChatMembers { + return nil, service.ErrInvalid + } + members := make([]uuid.UUID, 0, len(rawIDs)) + seen := make(map[uuid.UUID]bool, len(rawIDs)) + for _, rawID := range rawIDs { + id, err := parseID(rawID) + if err != nil || seen[id] { + return nil, service.ErrInvalid + } + seen[id] = true + members = append(members, id) + } + if !seen[caller] { + return nil, service.ErrInvalid + } + return members, nil +} + +// requireExistingMembers validates the complete membership set with one storage operation. +func (s *Service) requireExistingMembers(ctx context.Context, members []uuid.UUID) error { + exists, err := s.repo.UsersExist(ctx, members) + if err != nil { + return normalizeStoreError(err) + } + if !exists { + return service.ErrInvalid + } + return nil +} diff --git a/internal/service/production/messages.go b/internal/service/production/messages.go new file mode 100644 index 0000000..e593c5b --- /dev/null +++ b/internal/service/production/messages.go @@ -0,0 +1,38 @@ +package production + +import ( + "context" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/model/dto" +) + +// MessagesCreate authorizes the sender, validates the message and stores it atomically. +func (s *Service) MessagesCreate(ctx context.Context, request dto.MessagesCreateRequestDTO) (dto.MessageDTO, error) { + caller, chat, err := s.requireChatMember(ctx, request.UserID, request.ChatID) + if err != nil { + return dto.MessageDTO{}, err + } + message, err := prepareMessage(request, caller, chat.ID) + if err != nil { + return dto.MessageDTO{}, err + } + created, err := s.repo.CreateMessage(ctx, message) + return created, normalizeStoreError(err) +} + +// MessagesGetPage authorizes access, loads history and prepares its continuation. +func (s *Service) MessagesGetPage(ctx context.Context, request dto.PageRequestDTO) (dto.MessagePageDTO, error) { + _, chat, err := s.requireChatMember(ctx, request.UserID, request.ChatID) + if err != nil { + return dto.MessagePageDTO{}, err + } + page, err := s.parsePage(request, messageHistoryResource(chat.ID)) + if err != nil { + return dto.MessagePageDTO{}, err + } + items, err := s.repo.Messages(ctx, chat.ID, page.position, page.fetchLimit()) + if err != nil { + return dto.MessagePageDTO{}, normalizeStoreError(err) + } + return s.messagePage(items, page) +} diff --git a/internal/service/production/messages_page_test.go b/internal/service/production/messages_page_test.go new file mode 100644 index 0000000..fc47bc5 --- /dev/null +++ b/internal/service/production/messages_page_test.go @@ -0,0 +1,231 @@ +package production + +import ( + "context" + "errors" + "reflect" + "testing" + "time" + + "github.com/google/uuid" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/cursor" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/model/dto" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/service" +) + +// TestService_MessagesGetPage validates access and pagination before issuing a typed store query. +func TestService_MessagesGetPage(t *testing.T) { + t.Run("Positive: continuation uses scope, position and lookahead", func(t *testing.T) { + // Arrange. + repo := newStoreStub(t) + actor := uuid.New() + chat := dto.ChatDTO{ID: uuid.New(), MemberIDs: []uuid.UUID{actor}} + allowChatAccess(repo, actor, chat) + + previous := cursor.Position{ID: uuid.New(), CreatedAt: time.Unix(1700000000, 0).UTC()} + records := []dto.MessageDTO{ + {ID: uuid.New(), CreatedAt: time.Unix(1700000001, 0).UTC()}, + {ID: uuid.New(), CreatedAt: time.Unix(1700000002, 0).UTC()}, + {ID: uuid.New(), CreatedAt: time.Unix(1700000003, 0).UTC()}, + } + calls := 0 + repo.messages = func(_ context.Context, id uuid.UUID, position dto.PositionDTO, limit int) ([]dto.MessageDTO, error) { + calls++ + expected := dto.PositionDTO{ID: previous.ID, CreatedAt: previous.CreatedAt} + if id != chat.ID || position != expected || limit != 3 { + t.Fatalf("wrong page query: id=%s, position=%v, limit=%d", id, position, limit) + } + return records, nil + } + svc := newStubService(t, repo) + resource := "messages:" + chat.ID.String() + token, err := svc.cursors.Encode(resource, previous) + if err != nil { + t.Fatal(err) + } + input := dto.PageRequestDTO{UserID: actor.String(), ChatID: chat.ID.String(), Limit: "2", Cursor: token} + + // Act. + got, err := svc.MessagesGetPage(t.Context(), input) + + // Assert. + + if err != nil { + t.Fatal(err) + } + if calls != 1 || !reflect.DeepEqual(got.Items, records[:2]) || got.NextCursor == nil { + t.Fatalf("calls=%d, page=%+v", calls, got) + } + position, err := svc.cursors.Decode(*got.NextCursor, resource) + expected := cursor.Position{ID: records[1].ID, CreatedAt: records[1].CreatedAt} + if err != nil || position != expected { + t.Fatalf("next position=%v, want=%v, error=%v", position, expected, err) + } + + }) + + t.Run("Positive: empty history uses default limit and no continuation", func(t *testing.T) { + // Arrange. + repo := newStoreStub(t) + actor := uuid.New() + chat := dto.ChatDTO{ID: uuid.New(), MemberIDs: []uuid.UUID{actor}} + allowChatAccess(repo, actor, chat) + + calls := 0 + repo.messages = func(_ context.Context, id uuid.UUID, position dto.PositionDTO, limit int) ([]dto.MessageDTO, error) { + calls++ + if id != chat.ID || position != (dto.PositionDTO{}) || limit != 51 { + t.Fatalf("wrong initial page query: id=%s, position=%v, limit=%d", id, position, limit) + } + return []dto.MessageDTO{}, nil + } + svc := newStubService(t, repo) + input := dto.PageRequestDTO{UserID: actor.String(), ChatID: chat.ID.String()} + + // Act. + got, err := svc.MessagesGetPage(t.Context(), input) + + // Assert. + if err != nil { + t.Fatal(err) + } + if calls != 1 || got.Items == nil || len(got.Items) != 0 || got.NextCursor != nil { + t.Fatalf("calls=%d, page=%+v", calls, got) + } + }) + + t.Run("Negative: cursor for another resource skips page lookup", func(t *testing.T) { + // Arrange. + repo := newStoreStub(t) + actor := uuid.New() + chat := dto.ChatDTO{ID: uuid.New(), MemberIDs: []uuid.UUID{actor}} + allowChatAccess(repo, actor, chat) + + svc := newStubService(t, repo) + token, err := svc.cursors.Encode("another-resource", cursor.Position{ID: uuid.New(), CreatedAt: time.Unix(1700000000, 0).UTC()}) + if err != nil { + t.Fatal(err) + } + input := dto.PageRequestDTO{UserID: actor.String(), ChatID: chat.ID.String(), Cursor: token} + + // Act. + got, err := svc.MessagesGetPage(t.Context(), input) + + // Assert. + if !errors.Is(err, service.ErrInvalid) || len(got.Items) != 0 || got.NextCursor != nil { + t.Fatalf("page=%+v, error=%v", got, err) + } + }) + + t.Run("Negative: codec failure skips page lookup", func(t *testing.T) { + // Arrange. + repo := newStoreStub(t) + actor := uuid.New() + chat := dto.ChatDTO{ID: uuid.New(), MemberIDs: []uuid.UUID{actor}} + allowChatAccess(repo, actor, chat) + + failure := errors.New("cursor unavailable") + svc := newStubService(t, repo) + svc.cursors = failingCursor{failure: failure} + input := dto.PageRequestDTO{UserID: actor.String(), ChatID: chat.ID.String(), Cursor: "token"} + + // Act. + got, err := svc.MessagesGetPage(t.Context(), input) + + // Assert. + if !errors.Is(err, failure) || len(got.Items) != 0 || got.NextCursor != nil { + t.Fatalf("page=%+v, error=%v", got, err) + } + }) + + t.Run("Negative: page storage fails", func(t *testing.T) { + // Arrange. + repo := newStoreStub(t) + actor := uuid.New() + chat := dto.ChatDTO{ID: uuid.New(), MemberIDs: []uuid.UUID{actor}} + allowChatAccess(repo, actor, chat) + + failure := errors.New("page unavailable") + repo.messages = func(_ context.Context, _ uuid.UUID, _ dto.PositionDTO, _ int) ([]dto.MessageDTO, error) { + return nil, failure + } + svc := newStubService(t, repo) + input := dto.PageRequestDTO{UserID: actor.String(), ChatID: chat.ID.String()} + + // Act. + got, err := svc.MessagesGetPage(t.Context(), input) + + // Assert. + if !errors.Is(err, failure) || len(got.Items) != 0 || got.NextCursor != nil { + t.Fatalf("page=%+v, error=%v", got, err) + } + }) + + t.Run("Negative: malformed identity skips all storage", func(t *testing.T) { + // Arrange. + svc := newStubService(t, newStoreStub(t)) + input := dto.PageRequestDTO{UserID: "invalid", ChatID: uuid.NewString()} + + // Act. + got, err := svc.MessagesGetPage(t.Context(), input) + + // Assert. + if !errors.Is(err, service.ErrUnauthorized) || len(got.Items) != 0 || got.NextCursor != nil { + t.Fatalf("page=%+v, error=%v", got, err) + } + }) + + t.Run("Negative: outsider cannot read message history", func(t *testing.T) { + // Arrange. + repo := newStoreStub(t) + actor := uuid.New() + chat := dto.ChatDTO{ID: uuid.New(), MemberIDs: []uuid.UUID{uuid.New()}} + allowChatAccess(repo, actor, chat) + svc := newStubService(t, repo) + input := dto.PageRequestDTO{UserID: actor.String(), ChatID: chat.ID.String()} + + // Act. + got, err := svc.MessagesGetPage(t.Context(), input) + + // Assert. + if !errors.Is(err, service.ErrNotFound) || len(got.Items) != 0 || got.NextCursor != nil { + t.Fatalf("page=%+v, error=%v", got, err) + } + }) +} + +// TestService_MessagesGetPage_InvalidLimit rejects invalid limits before reading stored pages. +func TestService_MessagesGetPage_InvalidLimit(t *testing.T) { + cases := []struct { + name string + limit string + }{ + {"Negative: zero limit", "0"}, + {"Negative: negative limit", "-1"}, + {"Negative: limit above maximum", "101"}, + {"Negative: nonnumeric limit", "text"}, + {"Negative: fractional limit", "1.5"}, + {"Negative: integer overflow", "99999999999999999999999999"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + // Arrange. + repo := newStoreStub(t) + actor := uuid.New() + chat := dto.ChatDTO{ID: uuid.New(), MemberIDs: []uuid.UUID{actor}} + allowChatAccess(repo, actor, chat) + + svc := newStubService(t, repo) + input := dto.PageRequestDTO{UserID: actor.String(), ChatID: chat.ID.String(), Limit: tc.limit} + + // Act. + got, err := svc.MessagesGetPage(t.Context(), input) + + // Assert. + if !errors.Is(err, service.ErrInvalid) || len(got.Items) != 0 || got.NextCursor != nil { + t.Fatalf("page=%+v, error=%v", got, err) + } + }) + } +} diff --git a/internal/service/production/messages_test.go b/internal/service/production/messages_test.go new file mode 100644 index 0000000..428c904 --- /dev/null +++ b/internal/service/production/messages_test.go @@ -0,0 +1,170 @@ +package production + +import ( + "context" + "errors" + "reflect" + "strings" + "testing" + "time" + + "github.com/google/uuid" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/model/dto" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/service" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/store" +) + +// TestService_MessagesCreate authorizes the sender before validating and persisting the message. +func TestService_MessagesCreate(t *testing.T) { + t.Run("Positive: authenticated sender and original text persisted", func(t *testing.T) { + // Arrange. + repo := newStoreStub(t) + actor := uuid.New() + chat := dto.ChatDTO{ID: uuid.New(), MemberIDs: []uuid.UUID{actor}} + allowChatAccess(repo, actor, chat) + var persisted dto.MessageDTO + repo.createMessage = func(_ context.Context, input dto.MessageDTO) (dto.MessageDTO, error) { + persisted = input + input.CreatedAt = time.Unix(1700000000, 0).UTC() + return input, nil + } + svc := newStubService(t, repo) + input := dto.MessagesCreateRequestDTO{IdempotencyKey: "fixture-key", UserID: actor.String(), ChatID: chat.ID.String(), Text: " Original 👋 "} + + // Act. + got, err := svc.MessagesCreate(t.Context(), input) + + // Assert. + if err != nil { + t.Fatal(err) + } + if persisted.ID == uuid.Nil || persisted.SenderID != actor || persisted.ChatID != chat.ID || persisted.Text != input.Text { + t.Fatalf("wrong persistence input: %+v", persisted) + } + if got.ID != persisted.ID || got.SenderID != actor || got.ChatID != chat.ID || got.Text != input.Text || got.CreatedAt.IsZero() { + t.Fatalf("wrong acknowledgement: %+v", got) + } + }) + + t.Run("Negative: outsider cannot persist messages", func(t *testing.T) { + // Arrange. + repo := newStoreStub(t) + actor := uuid.New() + chat := dto.ChatDTO{ID: uuid.New(), MemberIDs: []uuid.UUID{uuid.New()}} + allowChatAccess(repo, actor, chat) + svc := newStubService(t, repo) + input := dto.MessagesCreateRequestDTO{IdempotencyKey: "fixture-key", UserID: actor.String(), ChatID: chat.ID.String(), Text: "Valid text"} + + // Act. + got, err := svc.MessagesCreate(t.Context(), input) + + // Assert. + if !errors.Is(err, service.ErrNotFound) || !reflect.DeepEqual(got, dto.MessageDTO{}) { + t.Fatalf("result=%+v, error=%v", got, err) + } + }) + + t.Run("Negative: malformed identity skips storage", func(t *testing.T) { + // Arrange. + svc := newStubService(t, newStoreStub(t)) + input := dto.MessagesCreateRequestDTO{IdempotencyKey: "fixture-key", UserID: "invalid", ChatID: uuid.NewString(), Text: "text"} + + // Act. + got, err := svc.MessagesCreate(t.Context(), input) + + // Assert. + if !errors.Is(err, service.ErrUnauthorized) || !reflect.DeepEqual(got, dto.MessageDTO{}) { + t.Fatalf("result=%+v, error=%v", got, err) + } + }) +} + +// TestService_MessagesCreate_InvalidText refuses invalid text without attempting persistence. +func TestService_MessagesCreate_InvalidText(t *testing.T) { + cases := []struct { + name string + text string + }{ + {"Negative: empty text", ""}, + {"Negative: whitespace only", " \t"}, + {"Negative: rune limit exceeded", strings.Repeat("界", 4001)}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + // Arrange. + repo := newStoreStub(t) + actor := uuid.New() + chat := dto.ChatDTO{ID: uuid.New(), MemberIDs: []uuid.UUID{actor}} + allowChatAccess(repo, actor, chat) + svc := newStubService(t, repo) + input := dto.MessagesCreateRequestDTO{IdempotencyKey: "fixture-key", UserID: actor.String(), ChatID: chat.ID.String(), Text: tc.text} + + // Act. + got, err := svc.MessagesCreate(t.Context(), input) + + // Assert. + if !errors.Is(err, service.ErrInvalid) || !reflect.DeepEqual(got, dto.MessageDTO{}) { + t.Fatalf("result=%+v, error=%v", got, err) + } + }) + } +} + +// TestService_MessagesCreate_StoreErrors propagates persistence failures without acknowledging a record. +func TestService_MessagesCreate_StoreErrors(t *testing.T) { + failure := errors.New("write unavailable") + cases := []struct { + name string + cause error + want error + }{ + {"Negative: capacity exhausted", store.ErrCapacity, service.ErrCapacity}, + {"Negative: chat disappeared", store.ErrNotFound, service.ErrNotFound}, + {"Negative: unexpected write failure", failure, failure}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + // Arrange. + repo := newStoreStub(t) + actor := uuid.New() + chat := dto.ChatDTO{ID: uuid.New(), MemberIDs: []uuid.UUID{actor}} + allowChatAccess(repo, actor, chat) + repo.createMessage = func(_ context.Context, _ dto.MessageDTO) (dto.MessageDTO, error) { + return dto.MessageDTO{}, tc.cause + } + svc := newStubService(t, repo) + input := dto.MessagesCreateRequestDTO{IdempotencyKey: "fixture-key", UserID: actor.String(), ChatID: chat.ID.String(), Text: "Valid text"} + + // Act. + got, err := svc.MessagesCreate(t.Context(), input) + + // Assert. + if !errors.Is(err, tc.want) || !reflect.DeepEqual(got, dto.MessageDTO{}) { + t.Fatalf("result=%+v, error=%v, want=%v", got, err, tc.want) + } + }) + } +} + +// TestIdempotencyKeyValidation checks the exact contract alphabet and length independently. +func TestIdempotencyKeyValidation(t *testing.T) { + for _, test := range []struct { + name string + key string + want bool + }{ + {name: "missing", key: "", want: false}, + {name: "allowed alphabet", key: "aZ09._:-", want: true}, + {name: "space", key: "a b", want: false}, + {name: "unicode", key: "ключ", want: false}, + {name: "maximum", key: strings.Repeat("a", 128), want: true}, + {name: "too long", key: strings.Repeat("a", 129), want: false}, + } { + t.Run(test.name, func(t *testing.T) { + if got := validIdempotencyKey(test.key); got != test.want { + t.Fatalf("valid=%v want=%v", got, test.want) + } + }) + } +} diff --git a/internal/service/production/pages.go b/internal/service/production/pages.go new file mode 100644 index 0000000..40fa3dc --- /dev/null +++ b/internal/service/production/pages.go @@ -0,0 +1,49 @@ +package production + +import ( + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/cursor" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/model/dto" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/service" +) + +// chatPage adapts the shared page result to the chat response DTO. +func (s *Service) chatPage(items []dto.ChatDTO, page pagination) (dto.ChatPageDTO, error) { + items, next, err := buildPage(items, page, s.cursors, chatPosition) + if err != nil { + return dto.ChatPageDTO{}, err + } + return dto.ChatPageDTO{Items: items, NextCursor: next}, nil +} + +// messagePage adapts the shared page result to the message response DTO. +func (s *Service) messagePage(items []dto.MessageDTO, page pagination) (dto.MessagePageDTO, error) { + items, next, err := buildPage(items, page, s.cursors, messagePosition) + if err != nil { + return dto.MessagePageDTO{}, err + } + return dto.MessagePageDTO{Items: items, NextCursor: next}, nil +} + +// buildPage trims lookahead and signs the last returned record when more data exists. +// The page limit must be validated before calling this helper. +func buildPage[T any](items []T, page pagination, codec service.CursorCodec, position func(T) cursor.Position) ([]T, *string, error) { + if len(items) <= page.limit { + return items, nil, nil + } + items = items[:page.limit] + next, err := codec.Encode(page.resource, position(items[len(items)-1])) + if err != nil { + return nil, nil, err + } + return items, new(next), nil +} + +// chatPosition extracts the stable pagination boundary from a stored chat. +func chatPosition(chat dto.ChatDTO) cursor.Position { + return cursor.Position{CreatedAt: chat.CreatedAt, ID: chat.ID} +} + +// messagePosition extracts the stable pagination boundary from a stored message. +func messagePosition(message dto.MessageDTO) cursor.Position { + return cursor.Position{CreatedAt: message.CreatedAt, ID: message.ID} +} diff --git a/internal/service/production/pages_test.go b/internal/service/production/pages_test.go new file mode 100644 index 0000000..f60b2f1 --- /dev/null +++ b/internal/service/production/pages_test.go @@ -0,0 +1,53 @@ +package production + +import ( + "reflect" + "testing" + "time" + + "github.com/google/uuid" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/cursor" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/model/dto" +) + +// TestService_messagePage checks boundaries and signs the last returned item, not lookahead. +func TestService_messagePage(t *testing.T) { + for _, count := range []int{0, 2, 3} { + names := map[int]string{0: "Positive: empty page", 2: "Positive: exact limit is final", 3: "Positive: lookahead creates continuation"} + t.Run(names[count], func(t *testing.T) { + // Arrange. + svc := newStubService(t, newStoreStub(t)) + items := make([]dto.MessageDTO, count) + for i := range items { + items[i] = dto.MessageDTO{ID: uuid.New(), Text: "message", CreatedAt: time.Unix(int64(i+1), 0).UTC()} + } + expected := items[:min(count, 2)] + + // Act. + page, err := svc.messagePage(items, pagination{limit: 2, resource: "messages:chat"}) + + // Assert. + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(page.Items, expected) { + t.Fatalf("page=%v, want=%v", page.Items, expected) + } + if count <= 2 { + if page.NextCursor != nil { + t.Fatal("final page has a continuation") + } + return + } + if page.NextCursor == nil { + t.Fatal("lookahead did not produce continuation") + } + decoded, err := svc.cursors.Decode(*page.NextCursor, "messages:chat") + want := cursor.Position{ID: items[1].ID, CreatedAt: items[1].CreatedAt} + if err != nil || decoded != want { + t.Fatalf("cursor=%v, want=%v, error=%v", decoded, want, err) + } + }) + } +} diff --git a/internal/service/production/pagination.go b/internal/service/production/pagination.go new file mode 100644 index 0000000..366e4fd --- /dev/null +++ b/internal/service/production/pagination.go @@ -0,0 +1,78 @@ +package production + +import ( + "errors" + "strconv" + + "github.com/google/uuid" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/cursor" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/model/dto" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/service" +) + +const ( + defaultPageLimit = 50 + maxPageLimit = 100 + // Fetching one extra record determines whether to issue a next cursor. + pageLookahead = 1 +) + +type pagination struct { + position dto.PositionDTO + limit int + resource string +} + +// parsePage validates the page size and resource-bound cursor. +func (s *Service) parsePage(request dto.PageRequestDTO, resource string) (pagination, error) { + limit, err := parseLimit(request.Limit) + if err != nil { + return pagination{}, err + } + position, err := s.cursors.Decode(request.Cursor, resource) + if errors.Is(err, cursor.ErrInvalid) { + return pagination{}, service.ErrInvalid + } + if err != nil { + return pagination{}, err + } + return pagination{ + position: dto.PositionDTO{CreatedAt: position.CreatedAt, ID: position.ID}, + limit: limit, + resource: resource, + }, nil +} + +// parseLimit parses a bounded page size or supplies the default. +func parseLimit(raw string) (int, error) { + if raw == "" { + return defaultPageLimit, nil + } + for _, char := range raw { + if char < '0' || char > '9' { + return 0, service.ErrInvalid + } + } + limit, err := strconv.Atoi(raw) + if err != nil || limit < 1 || limit > maxPageLimit { + return 0, service.ErrInvalid + } + return limit, nil +} + +// fetchLimit includes one lookahead record to detect a following page. +func (page pagination) fetchLimit() int { + return page.limit + pageLookahead +} + +// chatListResource scopes a cursor to one user's visible chats. +func chatListResource(userID uuid.UUID) string { + return "chats:" + userID.String() +} + +// messageHistoryResource scopes a cursor to one chat's message history. +func messageHistoryResource(chatID uuid.UUID) string { + return "messages:" + chatID.String() +} diff --git a/internal/service/production/pagination_test.go b/internal/service/production/pagination_test.go new file mode 100644 index 0000000..0f42b83 --- /dev/null +++ b/internal/service/production/pagination_test.go @@ -0,0 +1,74 @@ +package production + +import ( + "errors" + "testing" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/cursor" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/model/dto" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/service" +) + +type failingCursor struct { + failure error +} + +// Encode injects a signing failure at the service dependency boundary. +func (codec failingCursor) Encode(_ string, _ cursor.Position) (string, error) { + return "", codec.failure +} + +// Decode injects a verification failure at the service dependency boundary. +func (codec failingCursor) Decode(_, _ string) (cursor.Position, error) { + return cursor.Position{}, codec.failure +} + +// TestService_parsePage translates invalid tokens while retaining unexpected dependency failures. +func TestService_parsePage(t *testing.T) { + failure := errors.New("codec unavailable") + cases := []struct { + name string + cause error + want error + }{ + {"Negative: invalid token", cursor.ErrInvalid, service.ErrInvalid}, + {"Negative: unavailable codec", failure, failure}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + // Arrange. + svc := &Service{cursors: failingCursor{failure: tc.cause}} + input := dto.PageRequestDTO{Cursor: "supplied-token"} + + // Act. + page, err := svc.parsePage(input, "messages:chat") + + // Assert. + if !errors.Is(err, tc.want) { + t.Fatalf("error=%v, want=%v", err, tc.want) + } + if page != (pagination{}) { + t.Fatalf("failed cursor produced a page: %+v", page) + } + }) + } +} + +// TestService_messagePage_EncodeFailure returns no partial page when signing fails. +func TestService_messagePage_EncodeFailure(t *testing.T) { + // Arrange. + failure := errors.New("signing unavailable") + svc := &Service{cursors: failingCursor{failure: failure}} + items := []dto.MessageDTO{{Text: "first"}, {Text: "lookahead"}} + + // Act. + page, err := svc.messagePage(items, pagination{limit: 1, resource: "messages:chat"}) + + // Assert. + if !errors.Is(err, failure) { + t.Fatalf("lost signing failure: %v", err) + } + if page.Items != nil || page.NextCursor != nil { + t.Fatalf("partial page escaped: %+v", page) + } +} diff --git a/internal/service/production/service.go b/internal/service/production/service.go new file mode 100644 index 0000000..36236f3 --- /dev/null +++ b/internal/service/production/service.go @@ -0,0 +1,39 @@ +// Package production implements the messenger application operations. +package production + +import ( + "fmt" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/service" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/store" +) + +// Service orchestrates operations through storage and cursor interfaces. +type Service struct { + repo store.Interface + cursors service.CursorCodec + cache service.ProfileCache +} + +var _ service.Service = (*Service)(nil) + +// New creates an application service using explicitly supplied dependencies. +func New(repo store.Interface, cursors service.CursorCodec) (*Service, error) { + if repo == nil || cursors == nil { + return nil, fmt.Errorf("store and cursor codec are required") + } + return &Service{repo: repo, cursors: cursors}, nil +} + +// NewCached builds the business service with optional immutable-profile caching. +func NewCached(repo store.Interface, cursors service.CursorCodec, cache service.ProfileCache) (*Service, error) { + instance, err := New(repo, cursors) + if err != nil { + return nil, err + } + if cache == nil { + return nil, fmt.Errorf("profile cache is required") + } + instance.cache = cache + return instance, nil +} diff --git a/internal/service/production/service_test.go b/internal/service/production/service_test.go new file mode 100644 index 0000000..fbf2f03 --- /dev/null +++ b/internal/service/production/service_test.go @@ -0,0 +1,39 @@ +package production + +import ( + "testing" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/cursor" +) + +// TestService_New rejects each missing dependency separately. +func TestService_New(t *testing.T) { + t.Run("Negative: missing store", func(t *testing.T) { + // Arrange. + codec, err := cursor.New() + if err != nil { + t.Fatal(err) + } + + // Act. + svc, err := New(nil, codec) + + // Assert. + if err == nil || svc != nil { + t.Fatalf("service=%v, error=%v", svc, err) + } + }) + + t.Run("Negative: missing codec", func(t *testing.T) { + // Arrange. + repo := newStoreStub(t) + + // Act. + svc, err := New(repo, nil) + + // Assert. + if err == nil || svc != nil { + t.Fatalf("service=%v, error=%v", svc, err) + } + }) +} diff --git a/internal/service/production/users.go b/internal/service/production/users.go new file mode 100644 index 0000000..2b0948d --- /dev/null +++ b/internal/service/production/users.go @@ -0,0 +1,47 @@ +package production + +import ( + "context" + "errors" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/model/dto" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/service" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/store" +) + +// UsersCreate validates registration and atomically reserves the unique username. +func (s *Service) UsersCreate(ctx context.Context, request dto.UsersCreateRequestDTO) (dto.UserDTO, error) { + user, err := PrepareUser(request) + if err != nil { + return dto.UserDTO{}, err + } + created, err := s.repo.CreateUser(ctx, user) + return created, normalizeStoreError(err) +} + +// UsersGetMe loads the current user's profile; malformed and missing identities are unauthorized. +func (s *Service) UsersGetMe(ctx context.Context, rawID string) (dto.UserDTO, error) { + id, err := parseIdentity(rawID) + if err != nil { + return dto.UserDTO{}, err + } + if s.cache != nil { + if cached, ok := s.cache.GetProfile(ctx, id); ok { + return cached, nil + } + } + user, err := s.repo.User(ctx, id) + if errors.Is(err, store.ErrNotFound) { + return dto.UserDTO{}, service.ErrUnauthorized + } + if err == nil && s.cache != nil { + s.cache.PutProfile(ctx, user) + } + return user, normalizeStoreError(err) +} + +// UsersCheckIdentity validates the current identity without loading its profile. +func (s *Service) UsersCheckIdentity(ctx context.Context, rawID string) error { + _, err := s.requireUserID(ctx, rawID) + return err +} diff --git a/internal/service/production/users_test.go b/internal/service/production/users_test.go new file mode 100644 index 0000000..bd380f9 --- /dev/null +++ b/internal/service/production/users_test.go @@ -0,0 +1,289 @@ +package production + +import ( + "context" + "errors" + "slices" + "strings" + "testing" + "time" + + "github.com/google/uuid" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/model/dto" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/service" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/store" +) + +// TestService_UsersCreate validates registration before persistence and preserves storage failures. +func TestService_UsersCreate(t *testing.T) { + t.Run("Positive: validates and persists the original profile", func(t *testing.T) { + // Arrange. + repo := newStoreStub(t) + expected := dto.UserDTO{Username: "alice", DisplayName: "Алиса 👋", CreatedAt: time.Unix(1700000000, 0).UTC()} + calls := 0 + repo.createUser = func(_ context.Context, input dto.UserDTO) (dto.UserDTO, error) { + calls++ + if input.ID == uuid.Nil || input.Username != expected.Username || input.DisplayName != expected.DisplayName { + t.Fatalf("wrong persistence input: %+v", input) + } + expected.ID = input.ID + return expected, nil + } + svc := newStubService(t, repo) + input := dto.UsersCreateRequestDTO{Username: expected.Username, DisplayName: expected.DisplayName} + + // Act. + got, err := svc.UsersCreate(t.Context(), input) + + // Assert. + if err != nil { + t.Fatal(err) + } + if calls != 1 || got != expected { + t.Fatalf("calls=%d, got=%+v, want=%+v", calls, got, expected) + } + }) +} + +// TestService_UsersCreate_InvalidInput refuses malformed registrations without calling storage. +func TestService_UsersCreate_InvalidInput(t *testing.T) { + cases := []struct { + name string + input dto.UsersCreateRequestDTO + }{ + {"Negative: short username", dto.UsersCreateRequestDTO{Username: "a", DisplayName: "Alice"}}, + {"Negative: uppercase username", dto.UsersCreateRequestDTO{Username: "Alice", DisplayName: "Alice"}}, + {"Negative: blank display name", dto.UsersCreateRequestDTO{Username: "alice", DisplayName: " "}}, + {"Negative: display name exceeds rune limit", dto.UsersCreateRequestDTO{Username: "alice", DisplayName: strings.Repeat("界", 101)}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + // Arrange: every store method fails immediately if called. + svc := newStubService(t, newStoreStub(t)) + + // Act. + got, err := svc.UsersCreate(t.Context(), tc.input) + + // Assert. + if !errors.Is(err, service.ErrInvalid) || got != (dto.UserDTO{}) { + t.Fatalf("result=%+v, error=%v", got, err) + } + }) + } +} + +// TestService_UsersCreate_StoreErrors preserves the application meaning of persistence failures. +func TestService_UsersCreate_StoreErrors(t *testing.T) { + unexpected := errors.New("storage unavailable") + cases := []struct { + name string + cause error + want error + }{ + {"Negative: duplicate username", store.ErrConflict, service.ErrConflict}, + {"Negative: capacity exhausted", store.ErrCapacity, service.ErrCapacity}, + {"Negative: unexpected failure", unexpected, unexpected}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + // Arrange. + repo := newStoreStub(t) + repo.createUser = func(_ context.Context, _ dto.UserDTO) (dto.UserDTO, error) { + return dto.UserDTO{}, tc.cause + } + svc := newStubService(t, repo) + + // Act. + got, err := svc.UsersCreate(t.Context(), dto.UsersCreateRequestDTO{Username: "alice", DisplayName: "Alice"}) + + // Assert. + if !errors.Is(err, tc.want) || got != (dto.UserDTO{}) { + t.Fatalf("result=%+v, error=%v, want=%v", got, err, tc.want) + } + }) + } +} + +// TestService_UsersGetMe loads only the requested profile and hides unknown identities. +func TestService_UsersGetMe(t *testing.T) { + t.Run("Positive: returns stored profile", func(t *testing.T) { + // Arrange. + repo := newStoreStub(t) + expected := dto.UserDTO{ID: uuid.New(), Username: "alice", DisplayName: "Alice", CreatedAt: time.Unix(1700000000, 0).UTC()} + calls := 0 + repo.user = func(_ context.Context, id uuid.UUID) (dto.UserDTO, error) { + calls++ + if id != expected.ID { + t.Fatalf("profile queried for %s instead of %s", id, expected.ID) + } + return expected, nil + } + svc := newStubService(t, repo) + + // Act. + got, err := svc.UsersGetMe(t.Context(), expected.ID.String()) + + // Assert. + if err != nil { + t.Fatal(err) + } + if calls != 1 || got != expected { + t.Fatalf("calls=%d, got=%+v, want=%+v", calls, got, expected) + } + }) + + t.Run("Negative: unknown identity", func(t *testing.T) { + // Arrange. + repo := newStoreStub(t) + repo.user = func(_ context.Context, _ uuid.UUID) (dto.UserDTO, error) { + return dto.UserDTO{}, store.ErrNotFound + } + svc := newStubService(t, repo) + wantErr := service.ErrUnauthorized + + // Act. + got, err := svc.UsersGetMe(t.Context(), uuid.NewString()) + + // Assert. + if !errors.Is(err, wantErr) { + t.Fatalf("error=%v, want=%v", err, wantErr) + } + if got != (dto.UserDTO{}) { + t.Fatalf("failed operation returned a profile: %+v", got) + } + }) + + t.Run("Negative: storage failure preserved", func(t *testing.T) { + // Arrange. + repo := newStoreStub(t) + wantErr := errors.New("profile unavailable") + repo.user = func(_ context.Context, _ uuid.UUID) (dto.UserDTO, error) { + return dto.UserDTO{}, wantErr + } + svc := newStubService(t, repo) + + // Act. + got, err := svc.UsersGetMe(t.Context(), uuid.NewString()) + + // Assert. + if !errors.Is(err, wantErr) { + t.Fatalf("error=%v, want=%v", err, wantErr) + } + if got != (dto.UserDTO{}) { + t.Fatalf("failed operation returned a profile: %+v", got) + } + }) +} + +// TestService_UsersGetMe_InvalidIdentity rejects malformed identifiers before any store call. +func TestService_UsersGetMe_InvalidIdentity(t *testing.T) { + cases := []struct { + name string + raw string + }{ + {"Negative: missing header", ""}, + {"Negative: short ID", "invalid"}, + {"Negative: non-UUID characters", strings.Repeat("z", 36)}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + // Arrange. + svc := newStubService(t, newStoreStub(t)) + + // Act. + got, err := svc.UsersGetMe(t.Context(), tc.raw) + + // Assert. + if !errors.Is(err, service.ErrUnauthorized) || got != (dto.UserDTO{}) { + t.Fatalf("result=%+v, error=%v", got, err) + } + }) + } +} + +// TestService_UsersCheckIdentity uses the existence query and never loads a profile. +func TestService_UsersCheckIdentity(t *testing.T) { + t.Run("Positive: existence check without profile loading", func(t *testing.T) { + // Arrange. + repo := newStoreStub(t) + id := uuid.New() + calls := 0 + repo.usersExist = func(_ context.Context, ids []uuid.UUID) (bool, error) { + calls++ + if !slices.Equal(ids, []uuid.UUID{id}) { + t.Fatalf("queried IDs=%v", ids) + } + return true, nil + } + svc := newStubService(t, repo) + + // Act. + err := svc.UsersCheckIdentity(t.Context(), id.String()) + + // Assert. + if !errors.Is(err, nil) || calls != 1 { + t.Fatalf("calls=%d, error=%v", calls, err) + } + }) + + t.Run("Negative: identity does not exist", func(t *testing.T) { + // Arrange. + repo := newStoreStub(t) + id := uuid.New() + calls := 0 + repo.usersExist = func(_ context.Context, ids []uuid.UUID) (bool, error) { + calls++ + if !slices.Equal(ids, []uuid.UUID{id}) { + t.Fatalf("queried IDs=%v", ids) + } + return false, nil + } + svc := newStubService(t, repo) + + // Act. + err := svc.UsersCheckIdentity(t.Context(), id.String()) + + // Assert. + if !errors.Is(err, service.ErrUnauthorized) || calls != 1 { + t.Fatalf("calls=%d, error=%v", calls, err) + } + }) + + t.Run("Negative: existence lookup fails", func(t *testing.T) { + // Arrange. + repo := newStoreStub(t) + id := uuid.New() + calls := 0 + failure := errors.New("identity unavailable") + repo.usersExist = func(_ context.Context, ids []uuid.UUID) (bool, error) { + calls++ + if !slices.Equal(ids, []uuid.UUID{id}) { + t.Fatalf("queried IDs=%v", ids) + } + return false, failure + } + svc := newStubService(t, repo) + + // Act. + err := svc.UsersCheckIdentity(t.Context(), id.String()) + + // Assert. + if !errors.Is(err, failure) || calls != 1 { + t.Fatalf("calls=%d, error=%v", calls, err) + } + }) + + t.Run("Negative: malformed identity skips storage", func(t *testing.T) { + // Arrange. + svc := newStubService(t, newStoreStub(t)) + + // Act. + err := svc.UsersCheckIdentity(t.Context(), "invalid") + + // Assert. + if !errors.Is(err, service.ErrUnauthorized) { + t.Fatalf("error=%v", err) + } + }) +} diff --git a/internal/service/production/validation.go b/internal/service/production/validation.go new file mode 100644 index 0000000..dcad87f --- /dev/null +++ b/internal/service/production/validation.go @@ -0,0 +1,51 @@ +package production + +import ( + "strings" + "unicode/utf8" + + "github.com/google/uuid" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/service" +) + +// These bounds come from the mandatory lab1 OpenAPI schema. +const ( + minUsernameLength = 3 + maxUsernameLength = 32 + maxDisplayNameRunes = 100 + maxChatTitleRunes = 100 + maxChatMembers = 100 + maxMessageRunes = 4000 + canonicalUUIDLength = len("00000000-0000-0000-0000-000000000000") +) + +// validText checks that text is nonblank and within its Unicode length limit. +func validText(value string, maxRunes int) bool { + return utf8.ValidString(value) && strings.TrimSpace(value) != "" && utf8.RuneCountInString(value) <= maxRunes +} + +// validUsername checks the username length and permitted ASCII characters. +func validUsername(value string) bool { + if len(value) < minUsernameLength || len(value) > maxUsernameLength { + return false + } + for _, char := range value { + if (char < 'a' || char > 'z') && (char < '0' || char > '9') && char != '_' { + return false + } + } + return true +} + +// parseID parses a canonical UUID or returns a validation error. +func parseID(value string) (uuid.UUID, error) { + if len(value) != canonicalUUIDLength { + return uuid.Nil, service.ErrInvalid + } + id, err := uuid.Parse(value) + if err != nil { + return uuid.Nil, service.ErrInvalid + } + return id, nil +} diff --git a/internal/store/interface.go b/internal/store/interface.go new file mode 100644 index 0000000..80215bb --- /dev/null +++ b/internal/store/interface.go @@ -0,0 +1,54 @@ +// Package store defines typed persistence operations and storage errors. +package store + +import ( + "context" + "errors" + + "github.com/google/uuid" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/model/dto" +) + +var ( + // ErrUnavailable means a required storage dependency cannot serve the operation. + ErrUnavailable = errors.New("storage unavailable") + + // ErrNotFound means a record does not exist. + ErrNotFound = errors.New("record not found") + + // ErrConflict means a unique constraint failed. + ErrConflict = errors.New("record already exists") + + // ErrCapacity means the bounded store cannot accept more data. + ErrCapacity = errors.New("storage capacity reached") +) + +// Interface defines persistence operations on validated typed records. +// Implementations own atomicity, unique constraints and storage capacity, but not +// authorization. Callers must validate membership before accessing chat records. +type Interface interface { + // CreateUser atomically enforces username uniqueness and assigns CreatedAt. + CreateUser(ctx context.Context, user dto.UserDTO) (dto.UserDTO, error) + + // User returns a user by ID or ErrNotFound. + User(ctx context.Context, id uuid.UUID) (dto.UserDTO, error) + + // UsersExist reports whether every supplied ID exists; an empty set returns true. + UsersExist(ctx context.Context, ids []uuid.UUID) (bool, error) + + // CreateChat atomically stores a chat and its member indexes, assigning CreatedAt. + CreateChat(ctx context.Context, chat dto.ChatDTO) (dto.ChatDTO, error) + + // Chat returns a defensive copy of the stored chat or ErrNotFound. + Chat(ctx context.Context, id uuid.UUID) (dto.ChatDTO, error) + + // Chats reads up to limit records after position from the user's chat index. + Chats(ctx context.Context, userID uuid.UUID, position dto.PositionDTO, limit int) ([]dto.ChatDTO, error) + + // CreateMessage appends one message atomically and assigns its ordering timestamp. + CreateMessage(ctx context.Context, message dto.MessageDTO) (dto.MessageDTO, error) + + // Messages reads up to limit records strictly after position in one chat. + Messages(ctx context.Context, chatID uuid.UUID, position dto.PositionDTO, limit int) ([]dto.MessageDTO, error) +} diff --git a/internal/store/memory/capacity.go b/internal/store/memory/capacity.go new file mode 100644 index 0000000..8f0de15 --- /dev/null +++ b/internal/store/memory/capacity.go @@ -0,0 +1,38 @@ +package memory + +import "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/model/dto" + +// Conservative logical charges, not measurements of Go allocations or RSS. +// The budget includes record metadata and index entries; HTTP buffers are separate. +const ( + userRecordOverheadBytes = 256 + chatRecordOverheadBytes = 256 + messageRecordOverheadBytes = 256 + chatMemberIndexBytes = 128 + usernameIndexCopies = 2 +) + +// userCost calculates the logical storage charge for a user and its index entries. +func userCost(user dto.UserDTO) int { + return userRecordOverheadBytes + len(user.Username)*usernameIndexCopies + len(user.DisplayName) +} + +// chatCost calculates the logical storage charge for a chat and its membership index. +func chatCost(chat dto.ChatDTO) int { + return chatRecordOverheadBytes + len(chat.Title) + chatMemberIndexBytes*len(chat.MemberIDs) +} + +// messageCost calculates the logical storage charge for a message. +func messageCost(message dto.MessageDTO) int { + return messageRecordOverheadBytes + len(message.Text) +} + +// reserve checks and charges capacity while the caller holds the write lock. +// Callers must not perform a fallible operation after reserving this budget. +func (s *Store) reserve(bytes int) bool { + if bytes > s.limits.MaxStateBytes-s.usedBytes { + return false + } + s.usedBytes += bytes + return true +} diff --git a/internal/store/memory/chats.go b/internal/store/memory/chats.go new file mode 100644 index 0000000..2de9f50 --- /dev/null +++ b/internal/store/memory/chats.go @@ -0,0 +1,66 @@ +package memory + +import ( + "context" + "slices" + "sort" + + "github.com/google/uuid" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/model/dto" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/store" +) + +// copyChat copies the membership slice to keep stored chat state private. +func copyChat(c dto.ChatDTO) dto.ChatDTO { + c.MemberIDs = slices.Clone(c.MemberIDs) + return c +} + +// CreateChat stores the chat and its member index atomically. +func (s *Store) CreateChat(_ context.Context, c dto.ChatDTO) (dto.ChatDTO, error) { + s.mu.Lock() + defer s.mu.Unlock() + for _, id := range c.MemberIDs { + if _, ok := s.users[id]; !ok { + return dto.ChatDTO{}, store.ErrNotFound + } + } + if len(s.chats) >= s.limits.MaxChats || !s.reserve(chatCost(c)) { + return dto.ChatDTO{}, store.ErrCapacity + } + c.CreatedAt = s.timestamp() + c = copyChat(c) + s.chats[c.ID] = c + for _, id := range c.MemberIDs { + s.userChats[id] = append(s.userChats[id], c) + } + return copyChat(c), nil +} + +// Chat retrieves a defensive copy. Authorization belongs to the service. +func (s *Store) Chat(_ context.Context, id uuid.UUID) (dto.ChatDTO, error) { + s.mu.RLock() + defer s.mu.RUnlock() + c, ok := s.chats[id] + if !ok { + return c, store.ErrNotFound + } + return copyChat(c), nil +} + +// Chats uses the per-member index and binary search, without scanning other chats. +func (s *Store) Chats(_ context.Context, user uuid.UUID, p dto.PositionDTO, limit int) ([]dto.ChatDTO, error) { + s.mu.RLock() + defer s.mu.RUnlock() + all := s.userChats[user] + i := sort.Search(len(all), func(i int) bool { + return after(all[i].CreatedAt, all[i].ID, p) + }) + end := min(len(all), i+limit) + out := make([]dto.ChatDTO, 0, end-i) + for _, c := range all[i:end] { + out = append(out, copyChat(c)) + } + return out, nil +} diff --git a/internal/store/memory/chats_test.go b/internal/store/memory/chats_test.go new file mode 100644 index 0000000..64295d6 --- /dev/null +++ b/internal/store/memory/chats_test.go @@ -0,0 +1,148 @@ +package memory + +import ( + "errors" + "reflect" + "testing" + + "github.com/google/uuid" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/config" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/model/dto" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/store" +) + +// TestStore_CreateChat_RejectsWithoutMutation checks atomic rejection of invalid or excess chats. +func TestStore_CreateChat_RejectsWithoutMutation(t *testing.T) { + t.Run("Negative: chat capacity reached", func(t *testing.T) { + // Arrange. + cfg := config.Default().Memory + cfg.MaxChats = 1 + repo := newTestStore(t, cfg) + member := seedUser(t, repo, "alice") + original := seedChat(t, repo, member.ID) + input := dto.ChatDTO{ID: uuid.New(), Title: "Rejected", MemberIDs: []uuid.UUID{member.ID}} + wantErr := store.ErrCapacity + + previousBytes := repo.usedBytes + + // Act. + got, err := repo.CreateChat(t.Context(), input) + + // Assert. + if !errors.Is(err, wantErr) || got.ID != uuid.Nil { + t.Fatalf("result=%+v, error=%v, want=%v", got, err, wantErr) + } + items, readErr := repo.Chats(t.Context(), member.ID, dto.PositionDTO{}, 10) + if readErr != nil { + t.Fatal(readErr) + } + if !reflect.DeepEqual(items, []dto.ChatDTO{original}) || len(repo.chats) != 1 || repo.usedBytes != previousBytes { + t.Fatalf("rejected chat changed stored state: %v", items) + } + }) + + t.Run("Negative: member does not exist", func(t *testing.T) { + // Arrange. + cfg := config.Default().Memory + cfg.MaxChats = 1 + repo := newTestStore(t, cfg) + member := seedUser(t, repo, "alice") + original := seedChat(t, repo, member.ID) + input := dto.ChatDTO{ID: uuid.New(), Title: "Rejected", MemberIDs: []uuid.UUID{member.ID}} + wantErr := store.ErrNotFound + input.MemberIDs = append(input.MemberIDs, uuid.New()) + + previousBytes := repo.usedBytes + + // Act. + got, err := repo.CreateChat(t.Context(), input) + + // Assert. + if !errors.Is(err, wantErr) || got.ID != uuid.Nil { + t.Fatalf("result=%+v, error=%v, want=%v", got, err, wantErr) + } + items, readErr := repo.Chats(t.Context(), member.ID, dto.PositionDTO{}, 10) + if readErr != nil { + t.Fatal(readErr) + } + if !reflect.DeepEqual(items, []dto.ChatDTO{original}) || len(repo.chats) != 1 || repo.usedBytes != previousBytes { + t.Fatalf("rejected chat changed stored state: %v", items) + } + }) +} + +// TestStore_CreateChat_DefensiveCopy ensures callers cannot mutate membership through write boundaries. +func TestStore_CreateChat_DefensiveCopy(t *testing.T) { + t.Run("Positive: isolated input membership", func(t *testing.T) { + // Arrange. + repo := newTestStore(t, config.Default().Memory) + member := seedUser(t, repo, "alice") + input := dto.ChatDTO{ID: uuid.New(), Title: "Team", MemberIDs: []uuid.UUID{member.ID}} + _, err := repo.CreateChat(t.Context(), input) + if err != nil { + t.Fatal(err) + } + + // Act: a caller modifies the slice it owns after CreateChat returns. + + input.MemberIDs[0] = uuid.New() + + // Assert. + stored, err := repo.Chat(t.Context(), input.ID) + if err != nil { + t.Fatal(err) + } + if len(stored.MemberIDs) != 1 || stored.MemberIDs[0] != member.ID { + t.Fatalf("caller mutated stored membership: %+v", stored) + } + }) + + t.Run("Positive: isolated result membership", func(t *testing.T) { + // Arrange. + repo := newTestStore(t, config.Default().Memory) + member := seedUser(t, repo, "alice") + input := dto.ChatDTO{ID: uuid.New(), Title: "Team", MemberIDs: []uuid.UUID{member.ID}} + created, err := repo.CreateChat(t.Context(), input) + if err != nil { + t.Fatal(err) + } + + // Act: a caller modifies the slice it owns after CreateChat returns. + + created.MemberIDs[0] = uuid.New() + + // Assert. + stored, err := repo.Chat(t.Context(), input.ID) + if err != nil { + t.Fatal(err) + } + if len(stored.MemberIDs) != 1 || stored.MemberIDs[0] != member.ID { + t.Fatalf("caller mutated stored membership: %+v", stored) + } + }) +} + +// TestStore_Chat_DefensiveCopy ensures a fetched detail cannot alter future reads. +func TestStore_Chat_DefensiveCopy(t *testing.T) { + // Arrange. + repo := newTestStore(t, config.Default().Memory) + member := seedUser(t, repo, "alice") + original := seedChat(t, repo, member.ID) + fetched, err := repo.Chat(t.Context(), original.ID) + if err != nil { + t.Fatal(err) + } + + // Act. + fetched.MemberIDs[0] = uuid.New() + + // Assert. + stored, err := repo.Chat(t.Context(), original.ID) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(stored, original) { + t.Fatalf("read changed stored chat: %+v", stored) + } +} diff --git a/internal/store/memory/messages.go b/internal/store/memory/messages.go new file mode 100644 index 0000000..24c4b42 --- /dev/null +++ b/internal/store/memory/messages.go @@ -0,0 +1,41 @@ +package memory + +import ( + "context" + "sort" + + "github.com/google/uuid" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/model/dto" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/store" +) + +// CreateMessage assigns ordering and publishes one complete message atomically. +func (s *Store) CreateMessage(_ context.Context, m dto.MessageDTO) (dto.MessageDTO, error) { + s.mu.Lock() + defer s.mu.Unlock() + if _, ok := s.chats[m.ChatID]; !ok { + return dto.MessageDTO{}, store.ErrNotFound + } + if s.messageCount >= s.limits.MaxMessages || !s.reserve(messageCost(m)) { + return dto.MessageDTO{}, store.ErrCapacity + } + m.CreatedAt = s.timestamp() + s.messages[m.ChatID] = append(s.messages[m.ChatID], m) + s.messageCount++ + return m, nil +} + +// Messages performs keyset pagination over an append-only ordered history. +func (s *Store) Messages(_ context.Context, chat uuid.UUID, p dto.PositionDTO, limit int) ([]dto.MessageDTO, error) { + s.mu.RLock() + defer s.mu.RUnlock() + all := s.messages[chat] + i := sort.Search(len(all), func(i int) bool { + return after(all[i].CreatedAt, all[i].ID, p) + }) + end := min(len(all), i+limit) + out := make([]dto.MessageDTO, end-i) + copy(out, all[i:end]) + return out, nil +} diff --git a/internal/store/memory/messages_test.go b/internal/store/memory/messages_test.go new file mode 100644 index 0000000..8f15e1d --- /dev/null +++ b/internal/store/memory/messages_test.go @@ -0,0 +1,72 @@ +package memory + +import ( + "errors" + "reflect" + "strings" + "testing" + "time" + + "github.com/google/uuid" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/config" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/model/dto" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/store" +) + +// TestStore_CreateMessage_ByteCapacity preserves acknowledged history and accounting on rejection. +func TestStore_CreateMessage_ByteCapacity(t *testing.T) { + // Arrange. + cfg := config.Default().Memory + cfg.MaxStateBytes = 2048 + repo := newTestStore(t, cfg) + member := seedUser(t, repo, "alice") + chat := seedChat(t, repo, member.ID) + original := seedMessage(t, repo, chat, "kept") + input := dto.MessageDTO{ID: uuid.New(), ChatID: chat.ID, SenderID: member.ID, Text: strings.Repeat("x", 2000)} + previousBytes := repo.usedBytes + + // Act. + got, err := repo.CreateMessage(t.Context(), input) + + // Assert. + if !errors.Is(err, store.ErrCapacity) || !reflect.DeepEqual(got, dto.MessageDTO{}) { + t.Fatalf("result=%+v, error=%v", got, err) + } + items, readErr := repo.Messages(t.Context(), chat.ID, dto.PositionDTO{}, 10) + if readErr != nil { + t.Fatal(readErr) + } + if !reflect.DeepEqual(items, []dto.MessageDTO{original}) || repo.messageCount != 1 || repo.usedBytes != previousBytes { + t.Fatalf("overflow changed history or accounting: %v", items) + } +} + +// TestStore_CreateMessage_ClockRollback preserves chronological append order after a clock rollback. +func TestStore_CreateMessage_ClockRollback(t *testing.T) { + // Arrange. + repo := newTestStore(t, config.Default().Memory) + member := seedUser(t, repo, "alice") + chat := seedChat(t, repo, member.ID) + repo.lastTime = time.Now().UTC().Add(time.Hour) + first := seedMessage(t, repo, chat, "first") + input := dto.MessageDTO{ID: uuid.New(), ChatID: chat.ID, SenderID: member.ID, Text: "second"} + + // Act. + second, err := repo.CreateMessage(t.Context(), input) + + // Assert. + if err != nil { + t.Fatal(err) + } + if !second.CreatedAt.After(first.CreatedAt) { + t.Fatalf("second=%s must follow first=%s", second.CreatedAt, first.CreatedAt) + } + page, readErr := repo.Messages(t.Context(), chat.ID, dto.PositionDTO{CreatedAt: first.CreatedAt, ID: first.ID}, 10) + if readErr != nil { + t.Fatal(readErr) + } + if !reflect.DeepEqual(page, []dto.MessageDTO{second}) { + t.Fatalf("continuation lost the later message: %v", page) + } +} diff --git a/internal/store/memory/ordering.go b/internal/store/memory/ordering.go new file mode 100644 index 0000000..32b90a3 --- /dev/null +++ b/internal/store/memory/ordering.go @@ -0,0 +1,25 @@ +package memory + +import ( + "bytes" + "time" + + "github.com/google/uuid" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/model/dto" +) + +// timestamp is called under the write lock, keeping append order during clock rollback. +func (s *Store) timestamp() time.Time { + now := time.Now().UTC() + if !now.After(s.lastTime) { + now = s.lastTime.Add(time.Nanosecond) + } + s.lastTime = now + return now +} + +// after compares a record with a keyset pagination position. +func after(t time.Time, id uuid.UUID, p dto.PositionDTO) bool { + return t.After(p.CreatedAt) || (t.Equal(p.CreatedAt) && bytes.Compare(id[:], p.ID[:]) > 0) +} diff --git a/internal/store/memory/store.go b/internal/store/memory/store.go new file mode 100644 index 0000000..5af2873 --- /dev/null +++ b/internal/store/memory/store.go @@ -0,0 +1,47 @@ +// Package memory implements synchronized storage within one bounded process. +package memory + +import ( + "sync" + "time" + + "github.com/google/uuid" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/config" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/model/dto" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/store" +) + +// Store owns all mutable state and its indexes under one lock. +// Records are immutable once published; membership slices are copied on both +// write and read boundaries. Its zero value is not usable; construct it with New. +type Store struct { + mu sync.RWMutex + limits config.Memory + users map[uuid.UUID]dto.UserDTO + usernames map[string]uuid.UUID + chats map[uuid.UUID]dto.ChatDTO + userChats map[uuid.UUID][]dto.ChatDTO + messages map[uuid.UUID][]dto.MessageDTO + messageCount int + usedBytes int + lastTime time.Time +} + +var _ store.Interface = (*Store)(nil) + +// New validates and copies capacity settings, then initializes an empty store. +// It creates no files, goroutines or external connections. +func New(cfg *config.Memory) (*Store, error) { + if err := cfg.Validate(); err != nil { + return nil, err + } + return &Store{ + limits: *cfg, + users: make(map[uuid.UUID]dto.UserDTO), + usernames: make(map[string]uuid.UUID), + chats: make(map[uuid.UUID]dto.ChatDTO), + userChats: make(map[uuid.UUID][]dto.ChatDTO), + messages: make(map[uuid.UUID][]dto.MessageDTO), + }, nil +} diff --git a/internal/store/memory/store_test.go b/internal/store/memory/store_test.go new file mode 100644 index 0000000..9bb3962 --- /dev/null +++ b/internal/store/memory/store_test.go @@ -0,0 +1,64 @@ +package memory + +import ( + "testing" + + "github.com/google/uuid" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/config" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/model/dto" +) + +// TestStore_New rejects missing capacity settings. +func TestStore_New(t *testing.T) { + // Arrange. + var cfg *config.Memory + + // Act. + repo, err := New(cfg) + + // Assert. + if err == nil || repo != nil { + t.Fatalf("missing configuration accepted: %v, %v", repo, err) + } +} + +// newTestStore creates independent storage for one scenario. +func newTestStore(t *testing.T, cfg *config.Memory) *Store { + t.Helper() + repo, err := New(cfg) + if err != nil { + t.Fatal(err) + } + return repo +} + +// seedUser creates a fixture user and checks preparation errors. +func seedUser(t *testing.T, repo *Store, username string) dto.UserDTO { + t.Helper() + user, err := repo.CreateUser(t.Context(), dto.UserDTO{ID: uuid.New(), Username: username}) + if err != nil { + t.Fatal(err) + } + return user +} + +// seedChat creates a fixture chat for an existing member. +func seedChat(t *testing.T, repo *Store, member uuid.UUID) dto.ChatDTO { + t.Helper() + chat, err := repo.CreateChat(t.Context(), dto.ChatDTO{ID: uuid.New(), Title: "Fixture", MemberIDs: []uuid.UUID{member}}) + if err != nil { + t.Fatal(err) + } + return chat +} + +// seedMessage creates an acknowledged fixture message. +func seedMessage(t *testing.T, repo *Store, chat dto.ChatDTO, text string) dto.MessageDTO { + t.Helper() + message, err := repo.CreateMessage(t.Context(), dto.MessageDTO{ID: uuid.New(), ChatID: chat.ID, SenderID: chat.MemberIDs[0], Text: text}) + if err != nil { + t.Fatal(err) + } + return message +} diff --git a/internal/store/memory/users.go b/internal/store/memory/users.go new file mode 100644 index 0000000..4f993b1 --- /dev/null +++ b/internal/store/memory/users.go @@ -0,0 +1,50 @@ +package memory + +import ( + "context" + + "github.com/google/uuid" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/model/dto" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/store" +) + +// CreateUser atomically enforces uniqueness and resource limits. +func (s *Store) CreateUser(_ context.Context, u dto.UserDTO) (dto.UserDTO, error) { + s.mu.Lock() + defer s.mu.Unlock() + if _, ok := s.usernames[u.Username]; ok { + return dto.UserDTO{}, store.ErrConflict + } + if len(s.users) >= s.limits.MaxUsers || !s.reserve(userCost(u)) { + return dto.UserDTO{}, store.ErrCapacity + } + u.CreatedAt = s.timestamp() + s.users[u.ID] = u + s.usernames[u.Username] = u.ID + return u, nil +} + +// User retrieves an immutable user. +func (s *Store) User(_ context.Context, id uuid.UUID) (dto.UserDTO, error) { + s.mu.RLock() + defer s.mu.RUnlock() + u, ok := s.users[id] + if !ok { + return u, store.ErrNotFound + } + return u, nil +} + +// UsersExist checks the complete set under one read lock without loading user records. +// Empty sets return true, and duplicate IDs do not affect the result. +func (s *Store) UsersExist(_ context.Context, ids []uuid.UUID) (bool, error) { + s.mu.RLock() + defer s.mu.RUnlock() + for _, id := range ids { + if _, exists := s.users[id]; !exists { + return false, nil + } + } + return true, nil +} diff --git a/internal/store/memory/users_test.go b/internal/store/memory/users_test.go new file mode 100644 index 0000000..5002cfe --- /dev/null +++ b/internal/store/memory/users_test.go @@ -0,0 +1,116 @@ +package memory + +import ( + "errors" + "testing" + + "github.com/google/uuid" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/config" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/model/dto" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/store" +) + +// TestStore_UsersExist checks all-members semantics on an independent store per case. +func TestStore_UsersExist(t *testing.T) { + t.Run("Positive: existing user", func(t *testing.T) { + // Arrange. + repo := newTestStore(t, config.Default().Memory) + user := seedUser(t, repo, "alice") + ids := []uuid.UUID{user.ID} + want := true + + // Act. + got, err := repo.UsersExist(t.Context(), ids) + + // Assert. + if err != nil { + t.Fatal(err) + } + if got != want { + t.Fatalf("UsersExist(%v)=%v, want=%v", ids, got, want) + } + }) + + t.Run("Positive: duplicate IDs", func(t *testing.T) { + // Arrange. + repo := newTestStore(t, config.Default().Memory) + user := seedUser(t, repo, "alice") + ids := []uuid.UUID{user.ID} + want := true + + ids = append(ids, user.ID) + + // Act. + got, err := repo.UsersExist(t.Context(), ids) + + // Assert. + if err != nil { + t.Fatal(err) + } + if got != want { + t.Fatalf("UsersExist(%v)=%v, want=%v", ids, got, want) + } + }) + + t.Run("Positive: empty set", func(t *testing.T) { + // Arrange. + repo := newTestStore(t, config.Default().Memory) + var ids []uuid.UUID + want := true + + // Act. + got, err := repo.UsersExist(t.Context(), ids) + + // Assert. + if err != nil { + t.Fatal(err) + } + if got != want { + t.Fatalf("UsersExist(%v)=%v, want=%v", ids, got, want) + } + }) + + t.Run("Negative: missing member", func(t *testing.T) { + // Arrange. + repo := newTestStore(t, config.Default().Memory) + user := seedUser(t, repo, "alice") + ids := []uuid.UUID{user.ID} + want := false + + ids = append(ids, uuid.New()) + + // Act. + got, err := repo.UsersExist(t.Context(), ids) + + // Assert. + if err != nil { + t.Fatal(err) + } + if got != want { + t.Fatalf("UsersExist(%v)=%v, want=%v", ids, got, want) + } + }) +} + +// TestStore_CreateUser_Capacity checks that a refused user consumes no storage or username index. +func TestStore_CreateUser_Capacity(t *testing.T) { + // Arrange. + cfg := config.Default().Memory + cfg.MaxUsers = 1 + repo := newTestStore(t, cfg) + original := seedUser(t, repo, "alice") + rejected := dto.UserDTO{ID: uuid.New(), Username: "bob"} + previousBytes := repo.usedBytes + + // Act. + got, err := repo.CreateUser(t.Context(), rejected) + + // Assert. + if !errors.Is(err, store.ErrCapacity) || got != (dto.UserDTO{}) { + t.Fatalf("result=%+v, error=%v", got, err) + } + if len(repo.users) != 1 || repo.users[original.ID] != original || len(repo.usernames) != 1 || repo.usernames[original.Username] != original.ID || repo.usedBytes != previousBytes { + t.Fatal("refused registration changed storage or indexes") + } +} diff --git a/internal/store/postgres/attachments.go b/internal/store/postgres/attachments.go new file mode 100644 index 0000000..5013114 --- /dev/null +++ b/internal/store/postgres/attachments.go @@ -0,0 +1,71 @@ +package postgres + +import ( + "context" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/model/dto" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/store" +) + +const attachmentBaseColumns = `id,chat_id,owner_id,filename,content_type,size_bytes,sha256,status,object_key,created_at,updated_at` + +const attachmentColumns = attachmentBaseColumns + ",variants,error_code,pipeline_version" + +// Reserve creates durable intent before any S3 PUT. +func (s *Store) Reserve(ctx context.Context, a dto.Attachment) error { + ctx, cancel := context.WithTimeout(ctx, s.timeout) + defer cancel() + _, err := s.pool.Exec(ctx, `INSERT INTO attachments (`+attachmentBaseColumns+`) VALUES(@id,@chat,@owner,@filename,@mime,@size,@sha,@status,@key,@created,@updated)`, pgx.StrictNamedArgs{ + "id": a.ID, + "chat": a.ChatID, + "owner": a.OwnerID, + "filename": a.Filename, + "mime": a.ContentType, + "size": a.SizeBytes, + "sha": a.SHA256, + "status": a.Status, + "key": a.ObjectKey, + "created": a.CreatedAt, + "updated": a.UpdatedAt, + }) + return normalize(err) +} + +// Publish atomically exposes a completed object without changing its immutable key. +func (s *Store) Publish(ctx context.Context, id uuid.UUID) (dto.Attachment, error) { + ctx, cancel := context.WithTimeout(ctx, s.timeout) + defer cancel() + return scanAttachment(s.pool.QueryRow(ctx, `UPDATE attachments SET status='ready',updated_at=clock_timestamp() WHERE id=@id AND status='uploading' RETURNING `+attachmentColumns, pgx.StrictNamedArgs{"id": id})) +} + +// Attachment reads one immutable metadata record. +func (s *Store) Attachment(ctx context.Context, id uuid.UUID) (dto.Attachment, error) { + ctx, cancel := context.WithTimeout(ctx, s.timeout) + defer cancel() + return scanAttachment(s.pool.QueryRow(ctx, `SELECT `+attachmentColumns+` FROM attachments WHERE id=@id`, pgx.StrictNamedArgs{"id": id})) +} + +// scanAttachment decodes explicit columns shared by read and publication. +func scanAttachment(row pgx.Row) (dto.Attachment, error) { + var a dto.Attachment + err := row.Scan(&a.ID, &a.ChatID, &a.OwnerID, &a.Filename, &a.ContentType, &a.SizeBytes, &a.SHA256, &a.Status, &a.ObjectKey, &a.CreatedAt, &a.UpdatedAt, &a.Variants, &a.ErrorCode, &a.PipelineVersion) + return a, normalize(err) +} + +// validateAttachments locks referenced metadata before inserting the message. +func validateAttachments(ctx context.Context, tx pgx.Tx, message dto.MessageDTO) error { + for _, id := range message.AttachmentIDs { + var status string + err := tx.QueryRow(ctx, `SELECT status FROM attachments WHERE id=@id AND chat_id=@chat FOR SHARE`, pgx.StrictNamedArgs{"id": id, "chat": message.ChatID}).Scan(&status) + if err != nil { + return err + } + if status != "ready" && status != "queued" && status != "processing" { + return store.ErrConflict + } + } + return nil +} diff --git a/internal/store/postgres/attachments_test.go b/internal/store/postgres/attachments_test.go new file mode 100644 index 0000000..fbb2f30 --- /dev/null +++ b/internal/store/postgres/attachments_test.go @@ -0,0 +1,167 @@ +package postgres + +import ( + "errors" + "testing" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/media" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/model/dto" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/store" +) + +// fixtureAttachment reserves an independent attachment for a real SQL scenario. +func fixtureAttachment(t *testing.T, s *Store) dto.Attachment { + t.Helper() + user := fixtureUser(t, s) + chat := fixtureChat(t, s, user) + id := uuid.New() + a := dto.Attachment{ + ID: id, + ChatID: chat.ID, + OwnerID: user.ID, + Filename: "test.png", + ContentType: "image/png", + SizeBytes: 32, + SHA256: "synthetic-checksum", + Status: "uploading", + ObjectKey: "original/" + id.String(), + CreatedAt: time.Now().UTC().Add(-2 * time.Hour), + UpdatedAt: time.Now().UTC(), + } + if err := s.Reserve(t.Context(), a); err != nil { + t.Fatal(err) + } + return a +} + +// TestAttachmentPublication verifies stored metadata survives a fresh read after commit. +func TestAttachmentPublication(t *testing.T) { + s := testStore(t) + a := fixtureAttachment(t, s) + published, err := s.Publish(t.Context(), a.ID) + if err != nil || published.Status != "ready" { + t.Fatalf("publish: %v", err) + } + got, err := s.Attachment(t.Context(), a.ID) + if err != nil || got.ObjectKey != a.ObjectKey || got.Status != "ready" { + t.Fatalf("stored metadata: %v", err) + } +} + +// TestAttachmentMessageScope checks a foreign chat cannot reference another chat's file. +func TestAttachmentMessageScope(t *testing.T) { + s := testStore(t) + a := fixtureAttachment(t, s) + if _, err := s.Publish(t.Context(), a.ID); err != nil { + t.Fatal(err) + } + foreignUser := fixtureUser(t, s) + foreignChat := fixtureChat(t, s, foreignUser) + _, err := s.CreateMessage(t.Context(), dto.MessageDTO{ID: uuid.New(), ChatID: foreignChat.ID, SenderID: foreignUser.ID, AttachmentIDs: []string{a.ID.String()}, IdempotencyKey: "foreign"}) + if !errors.Is(err, store.ErrNotFound) { + t.Fatalf("foreign attachment: %v", err) + } +} + +// TestAttachmentRetryConflict verifies attachment identity participates in the immutable request. +func TestAttachmentRetryConflict(t *testing.T) { + s := testStore(t) + a := fixtureAttachment(t, s) + if _, err := s.Publish(t.Context(), a.ID); err != nil { + t.Fatal(err) + } + m := dto.MessageDTO{ID: uuid.New(), ChatID: a.ChatID, SenderID: a.OwnerID, Text: "same", AttachmentIDs: []string{a.ID.String()}, IdempotencyKey: "retry"} + if _, err := s.CreateMessage(t.Context(), m); err != nil { + t.Fatal(err) + } + m.AttachmentIDs = nil + _, err := s.CreateMessage(t.Context(), m) + if !errors.Is(err, store.ErrConflict) { + t.Fatalf("changed attachment list: %v", err) + } +} + +// TestReclaimExcludesPublication demonstrates the durable cleanup state closes the publish race. +func TestReclaimExcludesPublication(t *testing.T) { + s := testStore(t) + a := fixtureAttachment(t, s) + claimed, err := s.Reclaim(t.Context(), a.ID, time.Now().Add(-time.Hour), false) + if err != nil || !claimed { + t.Fatalf("claim: %v", err) + } + if _, err := s.Publish(t.Context(), a.ID); !errors.Is(err, store.ErrNotFound) { + t.Fatalf("late publication: %v", err) + } + removed, err := s.Reclaim(t.Context(), a.ID, time.Now().Add(-time.Hour), true) + if err != nil || !removed { + t.Fatalf("cleanup: %v", err) + } +} + +// TestReclaimProtectsFreshReservation ensures a concurrent valid upload stays intact. +func TestReclaimProtectsFreshReservation(t *testing.T) { + s := testStore(t) + a := fixtureAttachment(t, s) + _, err := s.pool.Exec(t.Context(), `UPDATE attachments SET created_at=clock_timestamp() WHERE id=@id`, pgx.StrictNamedArgs{"id": a.ID}) + if err != nil { + t.Fatal(err) + } + claimed, err := s.Reclaim(t.Context(), a.ID, time.Now().Add(-time.Hour), false) + if err != nil || claimed { + t.Fatalf("fresh reservation removed: %v", err) + } +} + +// TestMetadataReconciliation detects missing originals without deleting referenced metadata. +func TestMetadataReconciliation(t *testing.T) { + s := testStore(t) + a := fixtureAttachment(t, s) + if _, err := s.Publish(t.Context(), a.ID); err != nil { + t.Fatal(err) + } + objects := &reconcileObjects{exists: false} + dry, err := media.Reconcile(t.Context(), s, objects, uuid.Nil, time.Hour, false) + if err != nil || dry.Missing != 1 || dry.Changed != 0 { + t.Fatalf("dry run: %+v %v", dry, err) + } + applied, err := media.Reconcile(t.Context(), s, objects, uuid.Nil, time.Hour, true) + if err != nil || applied.Changed != 1 { + t.Fatalf("repair: %+v %v", applied, err) + } + again, err := media.Reconcile(t.Context(), s, objects, uuid.Nil, time.Hour, true) + if err != nil || again.Changed != 0 { + t.Fatalf("repeat changed state: %+v %v", again, err) + } + objects.exists = true + restored, err := media.Reconcile(t.Context(), s, objects, uuid.Nil, time.Hour, true) + if err != nil || restored.Changed != 1 { + t.Fatalf("restore: %+v %v", restored, err) + } +} + +// TestObjectReservationLookup protects unfinished uploads from orphan deletion. +func TestObjectReservationLookup(t *testing.T) { + s := testStore(t) + a := fixtureAttachment(t, s) + exists, err := s.HasObject(t.Context(), a.ObjectKey) + if err != nil || !exists { + t.Fatalf("reservation not protected: %v", err) + } + exists, err = s.HasObject(t.Context(), "original/"+uuid.NewString()) + if err != nil || exists { + t.Fatalf("unknown object considered reserved: %v", err) + } +} + +// TestInventoryStopsAtUnavailableDatabase rejects a scan rather than reporting an empty inventory. +func TestInventoryStopsAtUnavailableDatabase(t *testing.T) { + s := testStore(t) + s.Close() + if _, err := s.Scan(t.Context(), uuid.Nil, 1000); err == nil { + t.Fatal("closed database reported empty inventory") + } +} diff --git a/internal/store/postgres/chats.go b/internal/store/postgres/chats.go new file mode 100644 index 0000000..0522187 --- /dev/null +++ b/internal/store/postgres/chats.go @@ -0,0 +1,58 @@ +package postgres + +import ( + "context" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/model/dto" +) + +// CreateChat commits the chat and every member together, rolling back on any failure. +func (s *Store) CreateChat(ctx context.Context, chat dto.ChatDTO) (dto.ChatDTO, error) { + ctx, cancel := context.WithTimeout(ctx, s.timeout) + defer cancel() + err := pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error { + if err := tx.QueryRow(ctx, `INSERT INTO chats(id,title) VALUES(@id,@title) RETURNING created_at`, pgx.StrictNamedArgs{"id": chat.ID, "title": chat.Title}).Scan(&chat.CreatedAt); err != nil { + return err + } + _, err := tx.Exec(ctx, `INSERT INTO chat_members(chat_id,user_id) SELECT @chat,id FROM unnest(@members::uuid[]) AS member(id)`, pgx.StrictNamedArgs{"chat": chat.ID, "members": chat.MemberIDs}) + return err + }) + return chat, normalize(err) +} + +// Chat reads a chat and its immutable membership in one snapshot. +func (s *Store) Chat(ctx context.Context, id uuid.UUID) (dto.ChatDTO, error) { + ctx, cancel := context.WithTimeout(ctx, s.timeout) + defer cancel() + var chat dto.ChatDTO + err := s.pool.QueryRow(ctx, `SELECT c.id,c.title,c.created_at,ARRAY(SELECT user_id FROM chat_members WHERE chat_id=c.id ORDER BY user_id) + FROM chats c WHERE c.id=@id`, pgx.StrictNamedArgs{"id": id}).Scan(&chat.ID, &chat.Title, &chat.CreatedAt, &chat.MemberIDs) + return chat, normalize(err) +} + +// Chats returns the requested keyset page of chats visible to a user. +func (s *Store) Chats(ctx context.Context, userID uuid.UUID, position dto.PositionDTO, limit int) ([]dto.ChatDTO, error) { + ctx, cancel := context.WithTimeout(ctx, s.timeout) + defer cancel() + rows, err := s.pool.Query(ctx, `SELECT c.id,c.title,c.created_at,ARRAY(SELECT user_id FROM chat_members WHERE chat_id=c.id ORDER BY user_id) + FROM chats c JOIN chat_members m ON m.chat_id=c.id + WHERE m.user_id=@user AND (c.created_at,c.id)>(@time,@id) ORDER BY c.created_at,c.id LIMIT @limit`, pgx.StrictNamedArgs{ + "user": userID, "time": position.CreatedAt, "id": position.ID, "limit": limit, + }) + if err != nil { + return nil, normalize(err) + } + defer rows.Close() + chats := make([]dto.ChatDTO, 0) + for rows.Next() { + var chat dto.ChatDTO + if err := rows.Scan(&chat.ID, &chat.Title, &chat.CreatedAt, &chat.MemberIDs); err != nil { + return nil, normalize(err) + } + chats = append(chats, chat) + } + return chats, normalize(rows.Err()) +} diff --git a/internal/store/postgres/errors_test.go b/internal/store/postgres/errors_test.go new file mode 100644 index 0000000..96e61bc --- /dev/null +++ b/internal/store/postgres/errors_test.go @@ -0,0 +1,52 @@ +package postgres + +import ( + "context" + "errors" + "fmt" + "io" + "net" + "testing" + + "github.com/jackc/pgx/v5/pgconn" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/store" +) + +// TestNormalizeUnavailable classifies dependency failures while retaining their causes. +func TestNormalizeUnavailable(t *testing.T) { + cases := []struct { + name string + err error + }{ + {"deadline", context.DeadlineExceeded}, + {"closed stream", io.EOF}, + {"truncated stream", io.ErrUnexpectedEOF}, + {"network", &net.OpError{Op: "dial", Net: "tcp", Err: errors.New("refused")}}, + {"connection exception", &pgconn.PgError{Code: "08006"}}, + {"connection limit", &pgconn.PgError{Code: "53300"}}, + {"shutdown", &pgconn.PgError{Code: "57P01"}}, + {"crash", &pgconn.PgError{Code: "57P02"}}, + {"starting", &pgconn.PgError{Code: "57P03"}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + err := normalize(fmt.Errorf("operation failed: %w", tc.err)) + if !errors.Is(err, store.ErrUnavailable) || !errors.Is(err, tc.err) { + t.Fatalf("failure category or cause lost: %T", tc.err) + } + }) + } +} + +// TestNormalizeUnexpected preserves programming errors instead of misclassifying them as outages. +func TestNormalizeUnexpected(t *testing.T) { + for _, code := range []string{"42601", "23503", "42P01"} { + t.Run(code, func(t *testing.T) { + err := &pgconn.PgError{Code: code} + if normalize(err) != err { + t.Fatal("unexpected SQL error was reclassified") + } + }) + } +} diff --git a/internal/store/postgres/idempotency.go b/internal/store/postgres/idempotency.go new file mode 100644 index 0000000..90c5095 --- /dev/null +++ b/internal/store/postgres/idempotency.go @@ -0,0 +1,52 @@ +package postgres + +import ( + "context" + "errors" + "slices" + + "github.com/jackc/pgx/v5" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/model/dto" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/store" +) + +// replayMessage serializes one chat and returns an unexpired original response. +func replayMessage(ctx context.Context, tx pgx.Tx, message dto.MessageDTO) (dto.MessageDTO, bool, error) { + var locked string + if err := tx.QueryRow(ctx, "SELECT id FROM chats WHERE id=@chat FOR UPDATE", pgx.StrictNamedArgs{"chat": message.ChatID}).Scan(&locked); err != nil { + return dto.MessageDTO{}, false, err + } + if message.IdempotencyKey == "" { + return dto.MessageDTO{}, false, nil + } + args := pgx.StrictNamedArgs{"user": message.SenderID, "chat": message.ChatID, "key": message.IdempotencyKey} + if _, err := tx.Exec(ctx, `DELETE FROM message_idempotency WHERE user_id=@user AND chat_id=@chat AND key=@key AND expires_at<=clock_timestamp()`, args); err != nil { + return dto.MessageDTO{}, false, err + } + var original dto.MessageDTO + err := tx.QueryRow(ctx, `SELECT m.id,m.chat_id,m.sender_id,m.text,m.created_at,m.attachment_ids FROM message_idempotency i JOIN messages m ON m.id=i.message_id + WHERE i.user_id=@user AND i.chat_id=@chat AND i.key=@key`, args).Scan(&original.ID, &original.ChatID, &original.SenderID, &original.Text, &original.CreatedAt, &original.AttachmentIDs) + if errors.Is(err, pgx.ErrNoRows) { + return dto.MessageDTO{}, false, nil + } + if err != nil { + return dto.MessageDTO{}, false, err + } + if original.Text != message.Text || !slices.Equal(original.AttachmentIDs, message.AttachmentIDs) { + return dto.MessageDTO{}, false, store.ErrConflict + } + return original, true, nil +} + +// rememberMessage stores the result in the same transaction as the message insert. +func rememberMessage(ctx context.Context, tx pgx.Tx, message dto.MessageDTO) error { + if message.IdempotencyKey == "" { + return nil + } + _, err := tx.Exec(ctx, `INSERT INTO message_idempotency(user_id,chat_id,key,message_id,expires_at) + VALUES(@user,@chat,@key,@id,clock_timestamp()+interval '24 hours')`, pgx.StrictNamedArgs{ + "user": message.SenderID, "chat": message.ChatID, "key": message.IdempotencyKey, "id": message.ID, + }) + return err +} diff --git a/internal/store/postgres/idempotency_test.go b/internal/store/postgres/idempotency_test.go new file mode 100644 index 0000000..abf1d51 --- /dev/null +++ b/internal/store/postgres/idempotency_test.go @@ -0,0 +1,139 @@ +package postgres + +import ( + "errors" + "sync" + "testing" + + "github.com/google/uuid" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/model/dto" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/store" +) + +// TestMessageRetryRace verifies concurrent attempts commit exactly one message and replay its response. +func TestMessageRetryRace(t *testing.T) { + s := testStore(t) + user := fixtureUser(t, s) + chat := fixtureChat(t, s, user) + var wg sync.WaitGroup + outcomes := make(chan dto.MessageDTO, 12) + failures := make(chan error, 12) + for range 12 { + wg.Go(func() { + message, err := s.CreateMessage(t.Context(), dto.MessageDTO{ID: uuid.New(), ChatID: chat.ID, SenderID: user.ID, Text: "same", IdempotencyKey: "retry"}) + if err != nil { + failures <- err + return + } + outcomes <- message + }) + } + wg.Wait() + close(failures) + close(outcomes) + for err := range failures { + t.Error(err) + } + var original dto.MessageDTO + for m := range outcomes { + if original.ID == uuid.Nil { + original = m + } + if m.ID != original.ID || m.Text != original.Text || !m.CreatedAt.Equal(original.CreatedAt) { + t.Fatal("retry returned a different result") + } + } + messages, err := s.Messages(t.Context(), chat.ID, dto.PositionDTO{}, 100) + if err != nil || len(messages) != 1 { + t.Fatalf("messages=%d error=%v", len(messages), err) + } +} + +// TestMessageRetryConflict rejects a different body without publishing a second message. +func TestMessageRetryConflict(t *testing.T) { + s := testStore(t) + user := fixtureUser(t, s) + chat := fixtureChat(t, s, user) + message := dto.MessageDTO{ID: uuid.New(), ChatID: chat.ID, SenderID: user.ID, Text: "first", IdempotencyKey: "retry"} + if _, err := s.CreateMessage(t.Context(), message); err != nil { + t.Fatal(err) + } + message.ID = uuid.New() + message.Text = "different" + if _, err := s.CreateMessage(t.Context(), message); !errors.Is(err, store.ErrConflict) { + t.Fatalf("expected conflict: %v", err) + } +} + +// TestMessageRetryExpiry permits reuse after the original retention window ends. +func TestMessageRetryExpiry(t *testing.T) { + s := testStore(t) + user := fixtureUser(t, s) + chat := fixtureChat(t, s, user) + message := dto.MessageDTO{ID: uuid.New(), ChatID: chat.ID, SenderID: user.ID, Text: "first", IdempotencyKey: "retry"} + if _, err := s.CreateMessage(t.Context(), message); err != nil { + t.Fatal(err) + } + if _, err := s.pool.Exec(t.Context(), "UPDATE message_idempotency SET expires_at=clock_timestamp()-interval '1 second'"); err != nil { + t.Fatal(err) + } + message.ID = uuid.New() + message.Text = "new" + if result, err := s.CreateMessage(t.Context(), message); err != nil || result.ID != message.ID { + t.Fatalf("expired result reused: %v", err) + } +} + +// TestMessageRetryRollback proves failed inserts leave no poisoned idempotency result. +func TestMessageRetryRollback(t *testing.T) { + s := testStore(t) + user := fixtureUser(t, s) + chat := fixtureChat(t, s, user) + message := dto.MessageDTO{ID: uuid.New(), ChatID: chat.ID, SenderID: uuid.New(), Text: "first", IdempotencyKey: "retry"} + if _, err := s.CreateMessage(t.Context(), message); err == nil { + t.Fatal("missing sender accepted") + } + var count int + if err := s.pool.QueryRow(t.Context(), "SELECT count(*) FROM message_idempotency").Scan(&count); err != nil || count != 0 { + t.Fatalf("failed write stored a result: %d %v", count, err) + } + message.SenderID = user.ID + if _, err := s.CreateMessage(t.Context(), message); err != nil { + t.Fatal(err) + } +} + +// TestSchemaCheck verifies API schema checks neither create nor upgrade tables. +func TestSchemaCheck(t *testing.T) { + s := testStore(t) + if err := s.CheckSchema(t.Context()); err != nil { + t.Fatal(err) + } + if _, err := s.pool.Exec(t.Context(), "DELETE FROM schema_migrations WHERE version=5"); err != nil { + t.Fatal(err) + } + if err := s.CheckSchema(t.Context()); err == nil { + t.Fatal("outdated schema accepted") + } + s.Close() + if err := s.CheckSchema(t.Context()); err == nil { + t.Fatal("closed connection accepted") + } +} + +// TestIdempotencyMigrationFailureRollsBackVersion keeps an unsuccessful DDL change out of migration history. +func TestIdempotencyMigrationFailureRollsBackVersion(t *testing.T) { + s := testStore(t) + if _, err := s.pool.Exec(t.Context(), "DELETE FROM schema_migrations WHERE version=2"); err != nil { + t.Fatal(err) + } + // The existing table deliberately conflicts with the pending migration. + if err := s.Start(t.Context()); err == nil { + t.Fatal("conflicting DDL reported success") + } + var applied bool + if err := s.pool.QueryRow(t.Context(), "SELECT EXISTS(SELECT 1 FROM schema_migrations WHERE version=2)").Scan(&applied); err != nil || applied { + t.Fatalf("failed migration marked applied: %v", err) + } +} diff --git a/internal/store/postgres/jobs.go b/internal/store/postgres/jobs.go new file mode 100644 index 0000000..715798c --- /dev/null +++ b/internal/store/postgres/jobs.go @@ -0,0 +1,190 @@ +package postgres + +import ( + "context" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "go.opentelemetry.io/otel/propagation" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/model/dto" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/pipeline" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/store" +) + +// QueueAttachment atomically exposes queued metadata and durable outbox intent. +func (s *Store) QueueAttachment(ctx context.Context, id uuid.UUID) (dto.Attachment, error) { + ctx, cancel := context.WithTimeout(ctx, s.timeout) + defer cancel() + carrier := propagation.MapCarrier{} + propagation.TraceContext{}.Inject(ctx, carrier) + var result dto.Attachment + err := pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error { + var err error + result, err = scanAttachment(tx.QueryRow(ctx, `UPDATE attachments SET status='queued',updated_at=clock_timestamp() WHERE id=@id AND status='uploading' RETURNING `+attachmentColumns, pgx.StrictNamedArgs{"id": id})) + if err != nil { + return err + } + _, err = tx.Exec(ctx, `INSERT INTO attachment_jobs(id,attachment_id,pipeline_version,staging_key,traceparent,tracestate) VALUES(@job,@attachment,@version,@key,@parent,@state)`, pgx.StrictNamedArgs{"job": uuid.New(), "attachment": id, "version": result.PipelineVersion, "key": result.ObjectKey, "parent": carrier["traceparent"], "state": carrier["tracestate"]}) + return err + }) + return result, normalize(err) +} + +// Due scans a bounded outbox page, including expired processing leases. +func (s *Store) Due(ctx context.Context, limit int) ([]pipeline.Job, error) { + ctx, cancel := context.WithTimeout(ctx, s.timeout) + defer cancel() + rows, err := s.pool.Query(ctx, `SELECT j.id,j.attachment_id,j.pipeline_version,j.attempt+1,j.created_at,j.traceparent,j.tracestate FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id AND a.pipeline_version=j.pipeline_version WHERE j.status IN ('queued','processing') AND j.next_attempt_at<=clock_timestamp() AND (j.lease_until IS NULL OR j.lease_until maxAttempts { + _, err = tx.Exec(ctx, `UPDATE attachment_jobs SET status='failed',error_code='retry_exhausted',lease_until=NULL WHERE id=@id`, pgx.StrictNamedArgs{"id": j.ID}) + if err != nil { + return err + } + _, err = tx.Exec(ctx, `UPDATE attachments SET status='failed',error_code='retry_exhausted',updated_at=clock_timestamp() WHERE id=@id`, pgx.StrictNamedArgs{"id": j.AttachmentID}) + return err + } + _, err = tx.Exec(ctx, `UPDATE attachments SET status='processing',updated_at=clock_timestamp() WHERE id=@id`, pgx.StrictNamedArgs{"id": j.AttachmentID}) + result = &lease + return err + }) + return result, normalize(err) +} + +// Complete publishes only the current, unexpired fenced attempt. +func (s *Store) Complete(ctx context.Context, l pipeline.Lease, key string, variants []dto.Variant) (bool, error) { + ctx, cancel := context.WithTimeout(ctx, s.timeout) + defer cancel() + if variants == nil { + variants = []dto.Variant{} + } + published := false + err := pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error { + var version int + if err := tx.QueryRow(ctx, `SELECT pipeline_version FROM attachments WHERE id=@id FOR UPDATE`, pgx.StrictNamedArgs{"id": l.Job.AttachmentID}).Scan(&version); err != nil { + return err + } + if version != l.Job.Version { + return nil + } + tag, err := tx.Exec(ctx, `UPDATE attachment_jobs SET status='ready',lease_until=NULL,error_code=NULL,updated_at=clock_timestamp() WHERE id=@id AND lease_id=@token AND status='processing' AND lease_until>clock_timestamp()`, pgx.StrictNamedArgs{"id": l.Job.ID, "token": l.Token}) + if err != nil || tag.RowsAffected() == 0 { + return err + } + _, err = tx.Exec(ctx, `UPDATE attachments SET status='ready',object_key=@key,variants=@variants,error_code=NULL,updated_at=clock_timestamp() WHERE id=@id`, pgx.StrictNamedArgs{"key": key, "variants": variants, "id": l.Job.AttachmentID}) + published = err == nil + return err + }) + return published, normalize(err) +} + +// Fail persists a retry deadline or terminal result before broker acknowledgement. +func (s *Store) Fail(ctx context.Context, l pipeline.Lease, code string, delay time.Duration, maxAttempts int) error { + ctx, cancel := context.WithTimeout(ctx, s.timeout) + defer cancel() + state := "queued" + if code == "invalid_image" || l.Job.Attempt >= maxAttempts { + state = "failed" + } + if state == "failed" && code != "invalid_image" { + code = "retry_exhausted" + } + return normalize(pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error { + var version int + if err := tx.QueryRow(ctx, `SELECT pipeline_version FROM attachments WHERE id=@id FOR UPDATE`, pgx.StrictNamedArgs{"id": l.Job.AttachmentID}).Scan(&version); err != nil { + return err + } + if version != l.Job.Version { + return nil + } + tag, err := tx.Exec(ctx, `UPDATE attachment_jobs SET status=@state,error_code=@code,lease_until=NULL,next_attempt_at=clock_timestamp()+@delay*interval '1 second',last_published_at=NULL,updated_at=clock_timestamp() WHERE id=@id AND lease_id=@token AND status='processing'`, pgx.StrictNamedArgs{"state": state, "code": code, "delay": delay.Seconds(), "id": l.Job.ID, "token": l.Token}) + if err != nil || tag.RowsAffected() == 0 { + return err + } + _, err = tx.Exec(ctx, `UPDATE attachments SET status=@state,error_code=CASE WHEN @state='failed' THEN @code ELSE NULL END,updated_at=clock_timestamp() WHERE id=@id`, pgx.StrictNamedArgs{"state": state, "code": code, "id": l.Job.AttachmentID}) + return err + })) +} + +// Replay retries a failed logical job or explicitly creates a newer pipeline version. +func (s *Store) Replay(ctx context.Context, id uuid.UUID, version int) error { + ctx, cancel := context.WithTimeout(ctx, s.timeout) + defer cancel() + return normalize(pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error { + a, err := scanAttachment(tx.QueryRow(ctx, `SELECT `+attachmentColumns+` FROM attachments WHERE id=@id FOR UPDATE`, pgx.StrictNamedArgs{"id": id})) + if err != nil { + return err + } + var expired bool + if err := tx.QueryRow(ctx, `SELECT staging_expired FROM attachment_jobs WHERE attachment_id=@id AND pipeline_version=@version`, pgx.StrictNamedArgs{"id": id, "version": a.PipelineVersion}).Scan(&expired); err != nil { + return err + } + if expired { + return store.ErrConflict + } + if version < a.PipelineVersion || version < 1 || (version == a.PipelineVersion && a.Status != "failed") { + return store.ErrConflict + } + var job uuid.UUID + if version == a.PipelineVersion { + err = tx.QueryRow(ctx, `UPDATE attachment_jobs SET status='queued',attempt=0,error_code=NULL,lease_until=NULL,last_published_at=NULL,next_attempt_at=clock_timestamp() WHERE attachment_id=@id AND pipeline_version=@version RETURNING id`, pgx.StrictNamedArgs{"id": id, "version": version}).Scan(&job) + } else { + job = uuid.New() + _, err = tx.Exec(ctx, `INSERT INTO attachment_jobs(id,attachment_id,pipeline_version,staging_key) SELECT @job,attachment_id,@version,staging_key FROM attachment_jobs WHERE attachment_id=@id AND pipeline_version=@previous`, pgx.StrictNamedArgs{"job": job, "version": version, "id": id, "previous": a.PipelineVersion}) + } + if err != nil { + return err + } + _, err = tx.Exec(ctx, `UPDATE attachments SET status='queued',pipeline_version=@version,error_code=NULL,updated_at=clock_timestamp() WHERE id=@id`, pgx.StrictNamedArgs{"version": version, "id": id}) + if err != nil { + return err + } + _, err = tx.Exec(ctx, `INSERT INTO attachment_job_audit(job_id,action) VALUES(@job,'operator_replay')`, pgx.StrictNamedArgs{"job": job}) + return err + })) +} diff --git a/internal/store/postgres/jobs_test.go b/internal/store/postgres/jobs_test.go new file mode 100644 index 0000000..9361dfa --- /dev/null +++ b/internal/store/postgres/jobs_test.go @@ -0,0 +1,281 @@ +package postgres + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/model/dto" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/pipeline" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/store" +) + +// queuedJob prepares independent durable intent without publishing a broker notification. +func queuedJob(t *testing.T, s *Store) (dto.Attachment, pipeline.Job) { + t.Helper() + a := fixtureAttachment(t, s) + queued, err := s.QueueAttachment(t.Context(), a.ID) + if err != nil || queued.Status != "queued" { + t.Fatalf("queue: %v", err) + } + jobs, err := s.Due(t.Context(), 100) + if err != nil || len(jobs) != 1 { + t.Fatalf("outbox: %d %v", len(jobs), err) + } + return queued, jobs[0] +} + +// TestOutboxSurvivesMissingPublication verifies the COMMIT-before-publish window is recoverable. +func TestOutboxSurvivesMissingPublication(t *testing.T) { + s := testStore(t) + a, job := queuedJob(t, s) + jobs, err := s.Due(t.Context(), 100) + if err != nil || len(jobs) != 1 || jobs[0].ID != job.ID { + t.Fatalf("lost intent: %v", err) + } + if _, err := s.CreateMessage(t.Context(), dto.MessageDTO{ID: uuid.New(), ChatID: a.ChatID, SenderID: a.OwnerID, AttachmentIDs: []string{a.ID.String()}, IdempotencyKey: "queued"}); err != nil { + t.Fatal(err) + } + if err := s.Published(t.Context(), job); err != nil { + t.Fatal(err) + } + jobs, err = s.Due(t.Context(), 100) + if err != nil || len(jobs) != 0 { + t.Fatal("publication was not throttled") + } +} + +// TestPipelineDuplicateAfterCommit verifies a lost acknowledgement cannot repeat a ready effect. +func TestPipelineDuplicateAfterCommit(t *testing.T) { + s := testStore(t) + a, job := queuedJob(t, s) + lease, err := s.Claim(t.Context(), job, time.Minute, 5) + if err != nil || lease == nil { + t.Fatalf("claim: %v", err) + } + duplicate, err := s.Claim(t.Context(), job, time.Minute, 5) + if err != nil || duplicate != nil { + t.Fatal("concurrent lease acquired") + } + variants := []dto.Variant{{Name: "thumbnail", ObjectKey: "result/new/thumb"}} + done, err := s.Complete(t.Context(), *lease, "result/new/original", variants) + if err != nil || !done { + t.Fatalf("complete: %v", err) + } + duplicate, err = s.Claim(t.Context(), job, time.Minute, 5) + if err != nil || duplicate != nil { + t.Fatal("ready regressed") + } + done, err = s.Complete(t.Context(), *lease, "result/old/original", nil) + if err != nil || done { + t.Fatal("duplicate commit accepted") + } + got, err := s.Attachment(t.Context(), a.ID) + if err != nil || got.ObjectKey != "result/new/original" || len(got.Variants) != 1 { + t.Fatal("published result changed") + } +} + +// TestExpiredLeaseFencesLateSideEffect models a worker crash after S3 write but before COMMIT. +func TestExpiredLeaseFencesLateSideEffect(t *testing.T) { + s := testStore(t) + _, job := queuedJob(t, s) + old, err := s.Claim(t.Context(), job, time.Minute, 5) + if err != nil || old == nil { + t.Fatal(err) + } + _, err = s.pool.Exec(t.Context(), `UPDATE attachment_jobs SET lease_until=clock_timestamp()-interval '1 second' WHERE id=@id`, pgx.StrictNamedArgs{"id": job.ID}) + if err != nil { + t.Fatal(err) + } + fresh, err := s.Claim(t.Context(), job, time.Minute, 5) + if err != nil || fresh == nil || fresh.Token == old.Token || fresh.Job.Attempt != 2 { + t.Fatalf("lease recovery: %v", err) + } + if ok, err := s.Complete(t.Context(), *old, "old-side-effect", nil); err != nil || ok { + t.Fatal("stale lease published") + } + if ok, err := s.Complete(t.Context(), *fresh, "new-side-effect", nil); err != nil || !ok { + t.Fatal("current lease rejected") + } +} + +// TestPipelineVersionReordering verifies old delivery and late completion cannot overwrite version two. +func TestPipelineVersionReordering(t *testing.T) { + s := testStore(t) + a, job := queuedJob(t, s) + old, err := s.Claim(t.Context(), job, time.Minute, 5) + if err != nil || old == nil { + t.Fatal(err) + } + if err := s.Replay(t.Context(), a.ID, 2); err != nil { + t.Fatal(err) + } + jobs, err := s.Due(t.Context(), 100) + if err != nil || len(jobs) != 1 || jobs[0].Version != 2 { + t.Fatalf("new version: %v", err) + } + newer, err := s.Claim(t.Context(), jobs[0], time.Minute, 5) + if err != nil || newer == nil { + t.Fatal(err) + } + if ok, err := s.Complete(t.Context(), *newer, "version-two", nil); err != nil || !ok { + t.Fatal("new publication failed") + } + if ok, err := s.Complete(t.Context(), *old, "version-one", nil); err != nil || ok { + t.Fatal("old version overwrote current") + } + if err := s.Fail(t.Context(), *old, "processing_failed", 0, 5); err != nil { + t.Fatal(err) + } + if lease, err := s.Claim(t.Context(), job, time.Minute, 5); err != nil || lease != nil { + t.Fatal("old delivery claimed") + } + got, err := s.Attachment(t.Context(), a.ID) + if err != nil || got.ObjectKey != "version-two" || got.Status != "ready" { + t.Fatal("version fence broken") + } +} + +// TestPipelineRetryAndReplay verifies bounded attempts and explicit recovery of terminal jobs. +func TestPipelineRetryAndReplay(t *testing.T) { + s := testStore(t) + a, job := queuedJob(t, s) + for attempt := 1; attempt <= 5; attempt++ { + lease, err := s.Claim(t.Context(), job, time.Minute, 5) + if err != nil || lease == nil || lease.Job.Attempt != attempt { + t.Fatalf("attempt %d: %v", attempt, err) + } + if err := s.Fail(t.Context(), *lease, "processing_failed", 0, 5); err != nil { + t.Fatal(err) + } + } + got, err := s.Attachment(t.Context(), a.ID) + if err != nil || got.Status != "failed" || got.ErrorCode == nil || *got.ErrorCode != "retry_exhausted" { + t.Fatal("retry not terminal") + } + if err := s.Replay(t.Context(), a.ID, 1); err != nil { + t.Fatal(err) + } + jobs, err := s.Due(t.Context(), 100) + if err != nil || len(jobs) != 1 || jobs[0].ID != job.ID || jobs[0].Attempt != 1 { + t.Fatal("replay changed logical job") + } + if err := s.Replay(t.Context(), a.ID, 1); !errors.Is(err, store.ErrConflict) { + t.Fatal("non-failed replay accepted") + } +} + +// TestPipelinePermanentFailure verifies malformed sources do not consume all retries. +func TestPipelinePermanentFailure(t *testing.T) { + s := testStore(t) + a, job := queuedJob(t, s) + lease, err := s.Claim(t.Context(), job, time.Minute, 5) + if err != nil || lease == nil { + t.Fatal(err) + } + if err := s.Fail(t.Context(), *lease, "invalid_image", time.Minute, 5); err != nil { + t.Fatal(err) + } + got, err := s.Attachment(t.Context(), a.ID) + if err != nil || got.Status != "failed" || *got.ErrorCode != "invalid_image" { + t.Fatal("permanent failure retried") + } +} + +// TestPipelineCanceledDatabase checks bounded failures do not masquerade as successful transitions. +func TestPipelineCanceledDatabase(t *testing.T) { + s := testStore(t) + a, job := queuedJob(t, s) + ctx, cancel := context.WithCancel(t.Context()) + cancel() + if _, err := s.QueueAttachment(ctx, a.ID); err == nil { + t.Fatal("queue ignored cancellation") + } + if _, err := s.Due(ctx, 1); err == nil { + t.Fatal("outbox ignored cancellation") + } + if err := s.Published(ctx, job); err == nil { + t.Fatal("confirm ignored cancellation") + } + if _, err := s.Claim(ctx, job, time.Minute, 5); err == nil { + t.Fatal("claim ignored cancellation") + } + if _, err := s.Complete(ctx, pipeline.Lease{Job: job}, "key", nil); err == nil { + t.Fatal("complete ignored cancellation") + } + if err := s.Fail(ctx, pipeline.Lease{Job: job}, "processing_failed", 0, 5); err == nil { + t.Fatal("retry ignored cancellation") + } + if err := s.Replay(ctx, a.ID, 1); err == nil { + t.Fatal("replay ignored cancellation") + } +} + +// TestRepeatedCrashesExhaustLeaseBudget prevents unlimited retries caused by process deaths. +func TestRepeatedCrashesExhaustLeaseBudget(t *testing.T) { + s := testStore(t) + a, job := queuedJob(t, s) + for attempt := 0; attempt < 2; attempt++ { + lease, err := s.Claim(t.Context(), job, -time.Second, 1) + if err != nil { + t.Fatal(err) + } + if attempt == 0 && lease == nil { + t.Fatal("first claim missing") + } + if attempt == 1 && lease != nil { + t.Fatal("crash budget not enforced") + } + } + got, err := s.Attachment(t.Context(), a.ID) + if err != nil || got.Status != "failed" { + t.Fatal("crash loop remains active") + } +} + +// TestQueueUnknownReservation ensures an absent staging reservation cannot create durable work. +func TestQueueUnknownReservation(t *testing.T) { + s := testStore(t) + if _, err := s.QueueAttachment(t.Context(), uuid.New()); !errors.Is(err, store.ErrNotFound) { + t.Fatal(err) + } +} + +// TestClaimUnknownAttachment leaves an untrusted notification without side effects. +func TestClaimUnknownAttachment(t *testing.T) { + s := testStore(t) + if _, err := s.Claim(t.Context(), pipeline.Job{ID: uuid.New(), AttachmentID: uuid.New(), Version: 1}, time.Minute, 5); !errors.Is(err, store.ErrNotFound) { + t.Fatal(err) + } +} + +// TestCompleteUnknownAttachment cannot publish an arbitrary object key. +func TestCompleteUnknownAttachment(t *testing.T) { + s := testStore(t) + if _, err := s.Complete(t.Context(), pipeline.Lease{Job: pipeline.Job{AttachmentID: uuid.New()}}, "unknown", nil); !errors.Is(err, store.ErrNotFound) { + t.Fatal(err) + } +} + +// TestFailUnknownAttachment cannot fabricate a retry record. +func TestFailUnknownAttachment(t *testing.T) { + s := testStore(t) + if err := s.Fail(t.Context(), pipeline.Lease{Job: pipeline.Job{AttachmentID: uuid.New()}}, "processing_failed", 0, 5); !errors.Is(err, store.ErrNotFound) { + t.Fatal(err) + } +} + +// TestExpireStagingDatabaseFailure cannot authorize object deletion without SQL confirmation. +func TestExpireStagingDatabaseFailure(t *testing.T) { + s := testStore(t) + ctx, cancel := context.WithCancel(t.Context()) + cancel() + if keys, err := s.ExpireStaging(ctx); err == nil || len(keys) != 0 { + t.Fatal("uncertain retention claim accepted") + } +} diff --git a/internal/store/postgres/messages.go b/internal/store/postgres/messages.go new file mode 100644 index 0000000..238dd38 --- /dev/null +++ b/internal/store/postgres/messages.go @@ -0,0 +1,64 @@ +package postgres + +import ( + "context" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/model/dto" +) + +// CreateMessage serializes timestamps on the owning chat and atomically appends a message. +func (s *Store) CreateMessage(ctx context.Context, message dto.MessageDTO) (dto.MessageDTO, error) { + ctx, cancel := context.WithTimeout(ctx, s.timeout) + defer cancel() + err := pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error { + original, found, err := replayMessage(ctx, tx, message) + if err != nil { + return err + } + if found { + message = original + return nil + } + if err := validateAttachments(ctx, tx, message); err != nil { + return err + } + if err := tx.QueryRow(ctx, `UPDATE chats SET last_message_at=GREATEST(clock_timestamp(),last_message_at+interval '1 microsecond') + WHERE id=@chat RETURNING last_message_at`, pgx.StrictNamedArgs{"chat": message.ChatID}).Scan(&message.CreatedAt); err != nil { + return err + } + _, err = tx.Exec(ctx, `INSERT INTO messages(id,chat_id,sender_id,text,created_at,attachment_ids) VALUES(@id,@chat,@sender,@text,@time,@attachments)`, pgx.StrictNamedArgs{ + "attachments": append([]string{}, message.AttachmentIDs...), "id": message.ID, "chat": message.ChatID, "sender": message.SenderID, "text": message.Text, "time": message.CreatedAt, + }) + if err != nil { + return err + } + return rememberMessage(ctx, tx, message) + }) + return message, normalize(err) +} + +// Messages reads an indexed keyset page without scanning earlier history. +func (s *Store) Messages(ctx context.Context, chatID uuid.UUID, position dto.PositionDTO, limit int) ([]dto.MessageDTO, error) { + ctx, cancel := context.WithTimeout(ctx, s.timeout) + defer cancel() + rows, err := s.pool.Query(ctx, `SELECT id,chat_id,sender_id,text,created_at,attachment_ids FROM messages + WHERE chat_id=@chat AND (created_at,id)>(@time,@id) ORDER BY created_at,id LIMIT @limit`, pgx.StrictNamedArgs{ + "chat": chatID, "time": position.CreatedAt, "id": position.ID, "limit": limit, + }) + if err != nil { + return nil, normalize(err) + } + defer rows.Close() + messages := make([]dto.MessageDTO, 0) + for rows.Next() { + var message dto.MessageDTO + if err := rows.Scan(&message.ID, &message.ChatID, &message.SenderID, &message.Text, &message.CreatedAt, &message.AttachmentIDs); err != nil { + return nil, normalize(err) + } + messages = append(messages, message) + } + return messages, normalize(rows.Err()) +} diff --git a/internal/store/postgres/migrations/001_initial.sql b/internal/store/postgres/migrations/001_initial.sql new file mode 100644 index 0000000..0a19d68 --- /dev/null +++ b/internal/store/postgres/migrations/001_initial.sql @@ -0,0 +1,34 @@ +CREATE TABLE users ( + id uuid PRIMARY KEY, + username text NOT NULL UNIQUE, + display_name text NOT NULL, + password_hash text NOT NULL DEFAULT '', + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +CREATE TABLE chats ( + id uuid PRIMARY KEY, + title text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + last_message_at timestamptz NOT NULL DEFAULT '-infinity' +); +CREATE TABLE chat_members ( + chat_id uuid NOT NULL REFERENCES chats(id), + user_id uuid NOT NULL REFERENCES users(id), + PRIMARY KEY(chat_id,user_id) +); +CREATE INDEX chat_members_user ON chat_members(user_id,chat_id); +CREATE TABLE messages ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats(id), + sender_id uuid NOT NULL REFERENCES users(id), + text text NOT NULL, + created_at timestamptz NOT NULL, + FOREIGN KEY(chat_id,sender_id) REFERENCES chat_members(chat_id,user_id) +); +CREATE INDEX messages_history ON messages(chat_id,created_at,id); +CREATE TABLE sessions ( + token_hash text PRIMARY KEY, + user_id uuid NOT NULL REFERENCES users(id), + expires_at timestamptz NOT NULL +); +CREATE INDEX sessions_expiry ON sessions(expires_at); diff --git a/internal/store/postgres/migrations/002_idempotency.sql b/internal/store/postgres/migrations/002_idempotency.sql new file mode 100644 index 0000000..d34a519 --- /dev/null +++ b/internal/store/postgres/migrations/002_idempotency.sql @@ -0,0 +1,9 @@ +CREATE TABLE message_idempotency ( + user_id uuid NOT NULL REFERENCES users(id), + chat_id uuid NOT NULL REFERENCES chats(id), + key text NOT NULL, + message_id uuid NOT NULL REFERENCES messages(id), + expires_at timestamptz NOT NULL, + PRIMARY KEY (user_id,chat_id,key) +); +CREATE INDEX message_idempotency_expiry ON message_idempotency(expires_at); diff --git a/internal/store/postgres/migrations/003_attachments.sql b/internal/store/postgres/migrations/003_attachments.sql new file mode 100644 index 0000000..3208e69 --- /dev/null +++ b/internal/store/postgres/migrations/003_attachments.sql @@ -0,0 +1,15 @@ +CREATE TABLE attachments ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats(id), + owner_id uuid NOT NULL REFERENCES users(id), + filename text NOT NULL, + content_type text NOT NULL, + size_bytes bigint NOT NULL CHECK(size_bytes BETWEEN 1 AND 10485760), + sha256 text NOT NULL, + status text NOT NULL CHECK(status IN ('uploading','ready','missing','deleting')), + object_key text NOT NULL UNIQUE, + created_at timestamptz NOT NULL, + updated_at timestamptz NOT NULL +); +CREATE INDEX attachments_scan ON attachments(created_at,id); +ALTER TABLE messages ADD COLUMN attachment_ids text[] NOT NULL DEFAULT '{}'; diff --git a/internal/store/postgres/migrations/004_pipeline.sql b/internal/store/postgres/migrations/004_pipeline.sql new file mode 100644 index 0000000..b3ff9a1 --- /dev/null +++ b/internal/store/postgres/migrations/004_pipeline.sql @@ -0,0 +1,29 @@ +ALTER TABLE attachments DROP CONSTRAINT attachments_status_check; +ALTER TABLE attachments ADD CONSTRAINT attachments_status_check CHECK(status IN ('uploading','queued','processing','ready','failed','missing','deleting')); +ALTER TABLE attachments ADD COLUMN variants jsonb NOT NULL DEFAULT '[]'; +ALTER TABLE attachments ADD COLUMN error_code text; +ALTER TABLE attachments ADD COLUMN pipeline_version integer NOT NULL DEFAULT 1 CHECK(pipeline_version>0); +CREATE TABLE attachment_jobs ( + id uuid PRIMARY KEY, + attachment_id uuid NOT NULL REFERENCES attachments(id), + pipeline_version integer NOT NULL CHECK(pipeline_version>0), + attempt integer NOT NULL DEFAULT 0, + status text NOT NULL DEFAULT 'queued' CHECK(status IN ('queued','processing','ready','failed')), + staging_key text NOT NULL, + lease_id uuid, + lease_until timestamptz, + next_attempt_at timestamptz NOT NULL DEFAULT clock_timestamp(), + last_published_at timestamptz, + error_code text, + traceparent text NOT NULL DEFAULT '', + tracestate text NOT NULL DEFAULT '', + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + updated_at timestamptz NOT NULL DEFAULT clock_timestamp(), + UNIQUE(attachment_id,pipeline_version) +); +CREATE INDEX attachment_jobs_due ON attachment_jobs(next_attempt_at) WHERE status IN ('queued','processing'); +CREATE TABLE attachment_job_audit ( + job_id uuid NOT NULL REFERENCES attachment_jobs(id), + action text NOT NULL, + at timestamptz NOT NULL DEFAULT clock_timestamp() +); diff --git a/internal/store/postgres/migrations/005_retention.sql b/internal/store/postgres/migrations/005_retention.sql new file mode 100644 index 0000000..d89934a --- /dev/null +++ b/internal/store/postgres/migrations/005_retention.sql @@ -0,0 +1,2 @@ +ALTER TABLE attachment_jobs ADD COLUMN staging_expired boolean NOT NULL DEFAULT false; +ALTER TABLE attachment_jobs ADD COLUMN staging_removed boolean NOT NULL DEFAULT false; diff --git a/internal/store/postgres/observability.go b/internal/store/postgres/observability.go new file mode 100644 index 0000000..f70e470 --- /dev/null +++ b/internal/store/postgres/observability.go @@ -0,0 +1,17 @@ +package postgres + +import ( + "context" + "time" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/observation" +) + +// Snapshot observes current-version backlog and local pool occupancy under a short deadline. +func (s *Store) Snapshot(ctx context.Context) (observation.Snapshot, error) { + ctx, cancel := context.WithTimeout(ctx, 250*time.Millisecond) + defer cancel() + result := observation.Snapshot{PoolUsed: int64(s.pool.Stat().AcquiredConns())} + err := s.pool.QueryRow(ctx, `SELECT count(*),COALESCE(EXTRACT(EPOCH FROM clock_timestamp()-min(j.created_at)),0)::double precision FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id AND a.pipeline_version=j.pipeline_version WHERE j.status IN ('queued','processing')`).Scan(&result.QueueDepth, &result.OldestSeconds) + return result, normalize(err) +} diff --git a/internal/store/postgres/observability_test.go b/internal/store/postgres/observability_test.go new file mode 100644 index 0000000..19754d8 --- /dev/null +++ b/internal/store/postgres/observability_test.go @@ -0,0 +1,64 @@ +package postgres + +import ( + "context" + "testing" + "time" + + "github.com/jackc/pgx/v5" + "go.opentelemetry.io/otel/propagation" + "go.opentelemetry.io/otel/trace" +) + +// TestOutboxPreservesTraceContext verifies the durable handoff retains the incoming trace parent. +func TestOutboxPreservesTraceContext(t *testing.T) { + s := testStore(t) + a := fixtureAttachment(t, s) + parent := "00-11111111111111111111111111111111-2222222222222222-01" + ctx := propagation.TraceContext{}.Extract(t.Context(), propagation.MapCarrier{"traceparent": parent}) + if !trace.SpanContextFromContext(ctx).IsValid() { + t.Fatal("invalid fixture context") + } + if _, err := s.QueueAttachment(ctx, a.ID); err != nil { + t.Fatal(err) + } + jobs, err := s.Due(ctx, 1) + if err != nil || len(jobs) != 1 || jobs[0].Traceparent != parent { + t.Fatal("durable trace carrier missing") + } +} + +// TestSnapshotObservesCurrentBacklog checks unfinished accepted work rather than delivery count. +func TestSnapshotObservesCurrentBacklog(t *testing.T) { + s := testStore(t) + _, job := queuedJob(t, s) + _, err := s.pool.Exec(t.Context(), `UPDATE attachment_jobs SET created_at=clock_timestamp()-interval '10 seconds' WHERE id=@id`, pgx.StrictNamedArgs{"id": job.ID}) + if err != nil { + t.Fatal(err) + } + snapshot, err := s.Snapshot(t.Context()) + if err != nil || snapshot.QueueDepth != 1 || snapshot.OldestSeconds < 10 { + t.Fatal("incorrect accepted backlog") + } + lease, err := s.Claim(t.Context(), job, time.Minute, 5) + if err != nil || lease == nil { + t.Fatal(err) + } + if ok, err := s.Complete(t.Context(), *lease, "original", nil); err != nil || !ok { + t.Fatal(err) + } + snapshot, err = s.Snapshot(t.Context()) + if err != nil || snapshot.QueueDepth != 0 || snapshot.OldestSeconds != 0 { + t.Fatal("completed work remains in backlog") + } +} + +// TestSnapshotCancellationBoundsObservation prevents metrics collection from blocking on unavailable SQL. +func TestSnapshotCancellationBoundsObservation(t *testing.T) { + s := testStore(t) + ctx, cancel := context.WithCancel(t.Context()) + cancel() + if _, err := s.Snapshot(ctx); err == nil { + t.Fatal("canceled snapshot succeeded") + } +} diff --git a/internal/store/postgres/pipeline_integration_test.go b/internal/store/postgres/pipeline_integration_test.go new file mode 100644 index 0000000..0955b01 --- /dev/null +++ b/internal/store/postgres/pipeline_integration_test.go @@ -0,0 +1,187 @@ +package postgres + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "io" + "os" + "testing" + "time" + + "github.com/jackc/pgx/v5" + "go.uber.org/zap" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/config" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/imageprocessor" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/model/dto" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/objectstore" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/pipeline" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/streamqueue" +) + +type faultStore struct { + *Store + before bool + after bool +} + +// Complete injects a lost connection before or after the actual SQL commit. +func (s *faultStore) Complete(ctx context.Context, l pipeline.Lease, key string, variants []dto.Variant) (bool, error) { + if s.before { + s.before = false + return false, errors.New("injected connection loss before commit") + } + done, err := s.Store.Complete(ctx, l, key, variants) + if err == nil && s.after { + s.after = false + return done, errors.New("injected lost commit acknowledgement") + } + return done, err +} + +// TestRealPipelineFaultWindows verifies real objects and SQL remain correct across uncertain commits. +func TestRealPipelineFaultWindows(t *testing.T) { + for _, fault := range []struct { + name string + before bool + after bool + }{{"after_S3_before_commit", true, false}, {"after_commit_before_ack", false, true}} { + t.Run(fault.name, func(t *testing.T) { + if os.Getenv("S3_ENDPOINT") == "" || os.Getenv("BROKER_ADDRESS") == "" { + t.Skip("isolated S3 and broker required") + } + s := testStore(t) + a := fixtureAttachment(t, s) + data, err := os.ReadFile("../../../tests/fixtures/sample.png") + if err != nil { + t.Fatal(err) + } + sum := sha256.Sum256(data) + a.SHA256 = hex.EncodeToString(sum[:]) + a.SizeBytes = int64(len(data)) + _, err = s.pool.Exec(t.Context(), `UPDATE attachments SET size_bytes=@size,sha256=@sha WHERE id=@id`, pgx.StrictNamedArgs{"size": a.SizeBytes, "sha": a.SHA256, "id": a.ID}) + if err != nil { + t.Fatal(err) + } + cfg, err := config.NewObjectStore() + if err != nil { + t.Fatal(err) + } + objects, err := objectstore.New(cfg) + if err != nil { + t.Fatal(err) + } + if err := objects.Put(t.Context(), a.ObjectKey, a.ContentType, bytes.NewReader(data), int64(len(data))); err != nil { + t.Fatal(err) + } + if _, err := s.QueueAttachment(t.Context(), a.ID); err != nil { + t.Fatal(err) + } + jobs, err := s.Due(t.Context(), 1) + if err != nil || len(jobs) != 1 { + t.Fatal("missing durable intent") + } + queue, err := streamqueue.NewScoped(os.Getenv("BROKER_ADDRESS"), a.ID.String(), "test-"+a.ID.String()) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := queue.Close(); err != nil { + t.Error(err) + } + }() + processor, err := imageprocessor.New() + if err != nil { + t.Fatal(err) + } + faulty := &faultStore{Store: s, before: fault.before, after: fault.after} + worker, err := pipeline.New(faulty, queue, objects, processor, zap.NewNop()) + if err != nil { + t.Fatal(err) + } + // The real broker is exercised, while the exact delivery is controlled to avoid sharing consumer state. + if err := queue.Publish(t.Context(), jobs[0]); err != nil { + t.Fatal(err) + } + received, err := queue.Receive(t.Context()) + if err != nil || received == nil { + t.Fatalf("delivery: %v", err) + } + delivery := *received + if err := worker.Handle(t.Context(), delivery); err != nil { + t.Fatal(err) + } + _, err = s.pool.Exec(t.Context(), `UPDATE attachment_jobs SET next_attempt_at=clock_timestamp()-interval '1 second' WHERE id=@id`, pgx.StrictNamedArgs{"id": jobs[0].ID}) + if err != nil { + t.Fatal(err) + } + if err := worker.Handle(t.Context(), delivery); err != nil { + t.Fatal(err) + } + ready, err := s.Attachment(t.Context(), a.ID) + if err != nil || ready.Status != "ready" || len(ready.Variants) != 3 { + t.Fatalf("not ready: %v", err) + } + firstKey := ready.ObjectKey + if err := worker.Handle(t.Context(), delivery); err != nil { + t.Fatal(err) + } + again, err := s.Attachment(t.Context(), a.ID) + if err != nil || again.ObjectKey != firstKey { + t.Fatal("duplicate changed published effect") + } + reader, err := objects.Get(t.Context(), ready.ObjectKey) + if err != nil { + t.Fatal(err) + } + actual, err := io.ReadAll(reader) + closeErr := reader.Close() + if err != nil || closeErr != nil || !bytes.Equal(actual, data) { + t.Fatal("original changed") + } + t.Logf("fault=%s accepted=1 ready=1 variants=%d duplicate_effects=0", fault.name, len(ready.Variants)) + }) + } +} + +// TestRetentionFencesReplay protects active work and makes interrupted cleanup repeatable. +func TestRetentionFencesReplay(t *testing.T) { + s := testStore(t) + a, job := queuedJob(t, s) + keys, err := s.ExpireStaging(t.Context()) + if err != nil || len(keys) != 0 { + t.Fatal("active staging expired") + } + lease, err := s.Claim(t.Context(), job, time.Minute, 5) + if err != nil || lease == nil { + t.Fatal(err) + } + if ok, err := s.Complete(t.Context(), *lease, "result/original", nil); err != nil || !ok { + t.Fatal(err) + } + _, err = s.pool.Exec(t.Context(), `UPDATE attachments SET updated_at=clock_timestamp()-interval '8 days' WHERE id=@id`, pgx.StrictNamedArgs{"id": a.ID}) + if err != nil { + t.Fatal(err) + } + keys, err = s.ExpireStaging(t.Context()) + if err != nil || len(keys) != 1 || keys[0] != a.ObjectKey { + t.Fatal("terminal staging not claimed") + } + if err := s.Replay(t.Context(), a.ID, 2); err == nil { + t.Fatal("expired source replay accepted") + } + again, err := s.ExpireStaging(t.Context()) + if err != nil || len(again) != 1 { + t.Fatal("unacknowledged cleanup lost") + } + if err := s.StagingRemoved(t.Context(), a.ObjectKey); err != nil { + t.Fatal(err) + } + keys, err = s.ExpireStaging(t.Context()) + if err != nil || len(keys) != 0 { + t.Fatal("confirmed deletion repeated") + } +} diff --git a/internal/store/postgres/reconcile.go b/internal/store/postgres/reconcile.go new file mode 100644 index 0000000..bc26d8e --- /dev/null +++ b/internal/store/postgres/reconcile.go @@ -0,0 +1,64 @@ +package postgres + +import ( + "context" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/model/dto" +) + +// Scan returns a bounded metadata page ordered by the stable primary key. +func (s *Store) Scan(ctx context.Context, after uuid.UUID, limit int) ([]dto.Attachment, error) { + ctx, cancel := context.WithTimeout(ctx, s.timeout) + defer cancel() + rows, err := s.pool.Query(ctx, `SELECT `+attachmentColumns+` FROM attachments WHERE id>@after ORDER BY id LIMIT @limit`, pgx.StrictNamedArgs{"after": after, "limit": limit}) + if err != nil { + return nil, normalize(err) + } + defer rows.Close() + result := make([]dto.Attachment, 0) + for rows.Next() { + a, err := scanAttachment(rows) + if err != nil { + return nil, err + } + result = append(result, a) + } + return result, normalize(rows.Err()) +} + +// Reclaim uses a durable deleting state to exclude publication during object cleanup. +func (s *Store) Reclaim(ctx context.Context, id uuid.UUID, cutoff time.Time, finish bool) (bool, error) { + ctx, cancel := context.WithTimeout(ctx, s.timeout) + defer cancel() + query := `UPDATE attachments SET status='deleting' WHERE id=@id AND created_at<@cutoff AND status IN ('uploading','deleting') AND NOT EXISTS(SELECT 1 FROM messages WHERE @textid=ANY(attachment_ids))` + if finish { + query = `DELETE FROM attachments WHERE id=@id AND created_at<@cutoff AND status='deleting' AND NOT EXISTS(SELECT 1 FROM messages WHERE @textid=ANY(attachment_ids))` + } + tag, err := s.pool.Exec(ctx, query, pgx.StrictNamedArgs{"id": id, "textid": id.String(), "cutoff": cutoff}) + return tag.RowsAffected() == 1, normalize(err) +} + +// MarkMissing preserves referenced metadata while recording or clearing an availability incident. +func (s *Store) MarkMissing(ctx context.Context, id uuid.UUID, missing bool) error { + ctx, cancel := context.WithTimeout(ctx, s.timeout) + defer cancel() + status := "ready" + if missing { + status = "missing" + } + _, err := s.pool.Exec(ctx, `UPDATE attachments SET status=@status,updated_at=clock_timestamp() WHERE id=@id AND status IN ('ready','missing')`, pgx.StrictNamedArgs{"id": id, "status": status}) + return normalize(err) +} + +// HasObject protects even unfinished durable reservations from orphan cleanup. +func (s *Store) HasObject(ctx context.Context, key string) (bool, error) { + ctx, cancel := context.WithTimeout(ctx, s.timeout) + defer cancel() + var exists bool + err := s.pool.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM attachments WHERE object_key=@key OR EXISTS(SELECT 1 FROM jsonb_array_elements(variants) v WHERE v->>'object_key'=@key) UNION ALL SELECT 1 FROM attachment_jobs WHERE staging_key=@key AND NOT staging_removed)`, pgx.StrictNamedArgs{"key": key}).Scan(&exists) + return exists, normalize(err) +} diff --git a/internal/store/postgres/reconcile_fixture_test.go b/internal/store/postgres/reconcile_fixture_test.go new file mode 100644 index 0000000..2d7342c --- /dev/null +++ b/internal/store/postgres/reconcile_fixture_test.go @@ -0,0 +1,18 @@ +package postgres + +import "context" + +type reconcileObjects struct { + exists bool +} + +// Exists supplies a deterministic object-side fault while exercising real SQL transitions. +func (r *reconcileObjects) Exists(context.Context, string) (bool, error) { + return r.exists, nil +} + +// Remove models an idempotent object removal for SQL cleanup tests. +func (r *reconcileObjects) Remove(context.Context, string) error { + r.exists = false + return nil +} diff --git a/internal/store/postgres/research_test.go b/internal/store/postgres/research_test.go new file mode 100644 index 0000000..baa875f --- /dev/null +++ b/internal/store/postgres/research_test.go @@ -0,0 +1,52 @@ +package postgres + +import ( + "os" + "testing" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/config" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/media" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/objectstore" +) + +// TestMeasuredMissingInventory measures bounded real PostgreSQL/S3 scans with injected missing originals. +func TestMeasuredMissingInventory(t *testing.T) { + if os.Getenv("S3_ENDPOINT") == "" { + t.Skip("isolated S3 required for measured reconciliation") + } + for _, size := range []struct { + name string + count int + }{{"100 records", 100}, {"1000 records", 1000}} { + t.Run(size.name, func(t *testing.T) { + s := testStore(t) + user := fixtureUser(t, s) + chat := fixtureChat(t, s, user) + _, err := s.pool.Exec(t.Context(), `WITH ids AS (SELECT gen_random_uuid() AS id FROM generate_series(1,@count::int)) + INSERT INTO attachments(id,chat_id,owner_id,filename,content_type,size_bytes,sha256,status,object_key,created_at,updated_at) + SELECT id,@chat,@owner,'synthetic.png','image/png',32,'synthetic','ready','original/'||id::text,clock_timestamp()-interval '2 hours',clock_timestamp() FROM ids`, pgx.StrictNamedArgs{"count": size.count, "chat": chat.ID, "owner": user.ID}) + if err != nil { + t.Fatal(err) + } + cfg, err := config.NewObjectStore() + if err != nil { + t.Fatal(err) + } + objects, err := objectstore.New(cfg) + if err != nil { + t.Fatal(err) + } + started := time.Now() + result, err := media.Reconcile(t.Context(), s, objects, uuid.Nil, time.Hour, false) + elapsed := time.Since(started) + if err != nil || result.Scanned != size.count || result.Missing != size.count || result.Changed != 0 { + t.Fatalf("inventory=%+v err=%v", result, err) + } + t.Logf("records=%d missing=%d elapsed_ms=%.3f apply=false", result.Scanned, result.Missing, float64(elapsed.Microseconds())/1000) + }) + } +} diff --git a/internal/store/postgres/retention.go b/internal/store/postgres/retention.go new file mode 100644 index 0000000..ed5819d --- /dev/null +++ b/internal/store/postgres/retention.go @@ -0,0 +1,35 @@ +package postgres + +import ( + "context" + + "github.com/jackc/pgx/v5" +) + +// ExpireStaging atomically fences replay and returns old terminal sources in bounded batches. +func (s *Store) ExpireStaging(ctx context.Context) ([]string, error) { + ctx, cancel := context.WithTimeout(ctx, s.timeout) + defer cancel() + rows, err := s.pool.Query(ctx, `WITH candidates AS ( + SELECT a.id FROM attachments a + WHERE a.status IN ('ready','failed') AND a.updated_atclock_timestamp()`, pgx.StrictNamedArgs{"hash": hash}).Scan(&id) + return id, normalize(err) +} + +// RevokeSession deletes the authoritative session before logout succeeds. +func (s *Store) RevokeSession(ctx context.Context, hash string) error { + ctx, cancel := context.WithTimeout(ctx, s.timeout) + defer cancel() + _, err := s.pool.Exec(ctx, `DELETE FROM sessions WHERE token_hash=@hash`, pgx.StrictNamedArgs{"hash": hash}) + return normalize(err) +} diff --git a/internal/store/postgres/store.go b/internal/store/postgres/store.go new file mode 100644 index 0000000..d361c5f --- /dev/null +++ b/internal/store/postgres/store.go @@ -0,0 +1,161 @@ +// Package postgres persists messenger records in PostgreSQL. +package postgres + +import ( + "context" + _ "embed" + "errors" + "fmt" + "io" + "net" + "strings" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgtype" + "github.com/jackc/pgx/v5/pgxpool" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/store" +) + +//go:embed migrations/001_initial.sql +var initialMigration string + +//go:embed migrations/002_idempotency.sql +var idempotencyMigration string + +//go:embed migrations/003_attachments.sql +var attachmentsMigration string + +//go:embed migrations/004_pipeline.sql +var pipelineMigration string + +//go:embed migrations/005_retention.sql +var retentionMigration string + +const migrationLockID = 202602 + +// Options bounds database resource use. +type Options struct { + // Tracer optionally observes queries and pool acquisitions without SQL payloads. + Tracer pgx.QueryTracer + // URL identifies the isolated application database. + URL string + // MaxConnections bounds concurrent database connections. + MaxConnections int32 + // Timeout bounds each persistence operation. + Timeout time.Duration +} + +// Store owns the PostgreSQL pool; Start and Close belong to application lifecycle. +type Store struct { + pool *pgxpool.Pool + timeout time.Duration +} + +var _ store.Interface = (*Store)(nil) + +// New validates options and builds a lazy pool without opening connections. +func New(options Options) (*Store, error) { + if options.URL == "" || options.MaxConnections < 1 || options.Timeout <= 0 { + return nil, fmt.Errorf("database URL, pool bound and timeout are required") + } + cfg, err := pgxpool.ParseConfig(options.URL) + if err != nil { + return nil, fmt.Errorf("invalid database configuration") + } + cfg.AfterConnect = func(_ context.Context, conn *pgx.Conn) error { + conn.TypeMap().RegisterType(&pgtype.Type{Name: "timestamptz", OID: pgtype.TimestamptzOID, Codec: &pgtype.TimestamptzCodec{ScanLocation: time.UTC}}) + return nil + } + cfg.ConnConfig.Tracer = options.Tracer + cfg.MaxConns = options.MaxConnections + cfg.ConnConfig.ConnectTimeout = options.Timeout + pool, err := pgxpool.NewWithConfig(context.Background(), cfg) + if err != nil { + return nil, err + } + return &Store{pool: pool, timeout: options.Timeout}, nil +} + +// Start verifies connectivity and applies versioned migrations under a transaction lock. +func (s *Store) Start(ctx context.Context) error { + ctx, cancel := context.WithTimeout(ctx, s.timeout) + defer cancel() + return pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error { + if _, err := tx.Exec(ctx, "SELECT pg_advisory_xact_lock(@id)", pgx.StrictNamedArgs{"id": migrationLockID}); err != nil { + return err + } + if _, err := tx.Exec(ctx, "CREATE TABLE IF NOT EXISTS schema_migrations (version integer PRIMARY KEY)"); err != nil { + return err + } + for index, source := range []string{initialMigration, idempotencyMigration, attachmentsMigration, pipelineMigration, retentionMigration} { + version := index + 1 + var exists bool + if err := tx.QueryRow(ctx, "SELECT EXISTS(SELECT 1 FROM schema_migrations WHERE version=@version)", pgx.StrictNamedArgs{"version": version}).Scan(&exists); err != nil { + return err + } + if exists { + continue + } + if _, err := tx.Exec(ctx, source); err != nil { + return err + } + if _, err := tx.Exec(ctx, "INSERT INTO schema_migrations(version) VALUES(@version)", pgx.StrictNamedArgs{"version": version}); err != nil { + return err + } + } + return nil + + }) +} + +// Close releases owned pool resources after serving has stopped. +func (s *Store) Close() { + s.pool.Close() +} + +// Ping checks whether mandatory persistence can serve requests. +func (s *Store) Ping(ctx context.Context) error { + ctx, cancel := context.WithTimeout(ctx, s.timeout) + defer cancel() + return s.pool.Ping(ctx) +} + +// normalize translates expected database errors without exposing SQL to higher layers. +func normalize(err error) error { + if errors.Is(err, pgx.ErrNoRows) { + return store.ErrNotFound + } + if databaseError, ok := errors.AsType[*pgconn.PgError](err); ok { + if databaseError.Code == "23505" { + return store.ErrConflict + } + if strings.HasPrefix(databaseError.Code, "08") || strings.HasPrefix(databaseError.Code, "53") || + databaseError.Code == "57P01" || databaseError.Code == "57P02" || databaseError.Code == "57P03" { + return fmt.Errorf("%w: %w", store.ErrUnavailable, err) + } + } + var networkError net.Error + var connectError *pgconn.ConnectError + if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) || + errors.As(err, &networkError) || errors.As(err, &connectError) || pgconn.SafeToRetry(err) { + return fmt.Errorf("%w: %w", store.ErrUnavailable, err) + } + return err +} + +// CheckSchema verifies readiness without allowing API replicas to run DDL. +func (s *Store) CheckSchema(ctx context.Context) error { + ctx, cancel := context.WithTimeout(ctx, s.timeout) + defer cancel() + var version int + if err := s.pool.QueryRow(ctx, "SELECT COALESCE(max(version),0) FROM schema_migrations").Scan(&version); err != nil { + return err + } + if version != 5 { + return fmt.Errorf("database schema version %d; run server migrate", version) + } + return nil +} diff --git a/internal/store/postgres/store_test.go b/internal/store/postgres/store_test.go new file mode 100644 index 0000000..2f346b1 --- /dev/null +++ b/internal/store/postgres/store_test.go @@ -0,0 +1,336 @@ +package postgres + +import ( + "context" + "errors" + "os" + "sync" + "testing" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/model/dto" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/store" +) + +// testStore creates an isolated schema inside the explicitly configured test database. +func testStore(t *testing.T) *Store { + t.Helper() + url := os.Getenv("TEST_DATABASE_URL") + if url == "" { + t.Skip("TEST_DATABASE_URL is required for PostgreSQL integration tests") + } + admin, err := pgx.Connect(t.Context(), url) + if err != nil { + t.Fatal(err) + } + schema := "test_" + uuid.New().String()[:8] + identifier := pgx.Identifier{schema}.Sanitize() + if _, err := admin.Exec(t.Context(), "CREATE SCHEMA "+identifier); err != nil { + t.Fatal(err) + } + separator := "?" + if containsQuery(url) { + separator = "&" + } + s, err := New(Options{URL: url + separator + "search_path=" + schema, MaxConnections: 4, Timeout: 3 * time.Second}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + s.Close() + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + if _, err := admin.Exec(ctx, "DROP SCHEMA "+identifier+" CASCADE"); err != nil { + t.Error(err) + } + if err := admin.Close(ctx); err != nil { + t.Error(err) + } + }) + if err := s.Start(t.Context()); err != nil { + t.Fatal(err) + } + return s +} + +// containsQuery identifies URL parameters without exposing credentials in test logs. +func containsQuery(value string) bool { + for _, char := range value { + if char == '?' { + return true + } + } + return false +} + +// fixtureUser creates an independent profile for a persistence scenario. +func fixtureUser(t *testing.T, s *Store) dto.UserDTO { + t.Helper() + user, err := s.Register(t.Context(), dto.UserDTO{ID: uuid.New(), Username: "u_" + uuid.NewString()[:8], DisplayName: "Test"}, "test-hash") + if err != nil { + t.Fatal(err) + } + return user +} + +// fixtureChat creates a chat containing an existing user. +func fixtureChat(t *testing.T, s *Store, user dto.UserDTO) dto.ChatDTO { + t.Helper() + chat, err := s.CreateChat(t.Context(), dto.ChatDTO{ID: uuid.New(), Title: "Test", MemberIDs: []uuid.UUID{user.ID}}) + if err != nil { + t.Fatal(err) + } + return chat +} + +// TestMigrationsRepeat verifies idempotent schema initialization. +func TestMigrationsRepeat(t *testing.T) { + s := testStore(t) + user := fixtureUser(t, s) + if err := s.Start(t.Context()); err != nil { + t.Fatal(err) + } + read, err := s.User(t.Context(), user.ID) + if err != nil || read.ID != user.ID || read.CreatedAt.Location() != time.UTC { + t.Fatalf("migration changed data or UTC: %v", err) + } + if err := s.Ping(t.Context()); err != nil { + t.Fatal(err) + } +} + +// TestRegistrationRace verifies database uniqueness under concurrent registration. +func TestRegistrationRace(t *testing.T) { + s := testStore(t) + outcomes := make(chan error, 12) + var wg sync.WaitGroup + for range 12 { + wg.Go(func() { + _, err := s.Register(t.Context(), dto.UserDTO{ID: uuid.New(), Username: "same", DisplayName: "Test"}, "hash") + outcomes <- err + }) + } + wg.Wait() + close(outcomes) + successes := 0 + for err := range outcomes { + if err == nil { + successes++ + } else if !errors.Is(err, store.ErrConflict) { + t.Fatal(err) + } + } + if successes != 1 { + t.Fatalf("got %d successful registrations", successes) + } +} + +// TestCreateChatRollsBack verifies failure of membership insertion leaves no chat. +func TestCreateChatRollsBack(t *testing.T) { + s := testStore(t) + id := uuid.New() + _, err := s.CreateChat(t.Context(), dto.ChatDTO{ID: id, Title: "Rollback", MemberIDs: []uuid.UUID{uuid.New()}}) + if err == nil { + t.Fatal("foreign key failure absent") + } + if _, err := s.Chat(t.Context(), id); !errors.Is(err, store.ErrNotFound) { + t.Fatalf("chat survived rollback: %v", err) + } +} + +// TestChatPageUsesMembershipAndCursor verifies one compound visibility/page invariant. +func TestChatPageUsesMembershipAndCursor(t *testing.T) { + s := testStore(t) + owner := fixtureUser(t, s) + outsider := fixtureUser(t, s) + first := fixtureChat(t, s, owner) + second := fixtureChat(t, s, owner) + fixtureChat(t, s, outsider) + page, err := s.Chats(t.Context(), owner.ID, dto.PositionDTO{CreatedAt: first.CreatedAt, ID: first.ID}, 10) + if err != nil || len(page) != 1 || page[0].ID != second.ID || len(page[0].MemberIDs) != 1 { + t.Fatalf("invalid membership page: %+v %v", page, err) + } +} + +// TestMessagePageOrdersConcurrentWrites verifies monotonic durable ordering and continuation. +func TestMessagePageOrdersConcurrentWrites(t *testing.T) { + s := testStore(t) + user := fixtureUser(t, s) + chat := fixtureChat(t, s, user) + var wg sync.WaitGroup + failures := make(chan error, 20) + for range 20 { + wg.Go(func() { + _, err := s.CreateMessage(t.Context(), dto.MessageDTO{ID: uuid.New(), ChatID: chat.ID, SenderID: user.ID, Text: "synthetic"}) + failures <- err + }) + } + wg.Wait() + close(failures) + for err := range failures { + if err != nil { + t.Fatal(err) + } + } + page, err := s.Messages(t.Context(), chat.ID, dto.PositionDTO{}, 10) + if err != nil || len(page) != 10 { + t.Fatalf("first page: %d %v", len(page), err) + } + for index := 1; index < len(page); index++ { + if !page[index].CreatedAt.After(page[index-1].CreatedAt) { + t.Fatal("timestamps are not monotonic") + } + } + last := page[len(page)-1] + next, err := s.Messages(t.Context(), chat.ID, dto.PositionDTO{CreatedAt: last.CreatedAt, ID: last.ID}, 20) + if err != nil || len(next) != 10 || !next[0].CreatedAt.After(last.CreatedAt) { + t.Fatalf("continuation: %d %v", len(next), err) + } +} + +// TestCredentialsReadsStoredHash checks credential lookup independently of public profiles. +func TestCredentialsReadsStoredHash(t *testing.T) { + s := testStore(t) + user := fixtureUser(t, s) + got, hash, err := s.Credentials(t.Context(), user.Username) + if err != nil || got.ID != user.ID || hash != "test-hash" { + t.Fatalf("credentials mismatch: %v", err) + } +} + +// TestUsersExistChecksAllIDs verifies the set membership contract. +func TestUsersExistChecksAllIDs(t *testing.T) { + for _, missing := range []bool{false, true} { + t.Run(map[bool]string{false: "all exist", true: "missing member"}[missing], func(t *testing.T) { + s := testStore(t) + ids := []uuid.UUID{fixtureUser(t, s).ID} + if missing { + ids = append(ids, uuid.New()) + } + exists, err := s.UsersExist(t.Context(), ids) + if err != nil || exists == missing { + t.Fatalf("exists=%v error=%v", exists, err) + } + }) + } +} + +// TestSessionExpiry rejects persisted sessions whose absolute deadline has passed. +func TestSessionExpiry(t *testing.T) { + s := testStore(t) + user := fixtureUser(t, s) + if err := s.SaveSession(t.Context(), "expired", user.ID, time.Now().Add(-time.Second)); err != nil { + t.Fatal(err) + } + if _, err := s.SessionUser(t.Context(), "expired"); !errors.Is(err, store.ErrNotFound) { + t.Fatalf("expired session accepted: %v", err) + } +} + +// TestSessionRevocation removes a still-valid session from the source of truth. +func TestSessionRevocation(t *testing.T) { + s := testStore(t) + user := fixtureUser(t, s) + if err := s.SaveSession(t.Context(), "live", user.ID, time.Now().Add(time.Hour)); err != nil { + t.Fatal(err) + } + if err := s.RevokeSession(t.Context(), "live"); err != nil { + t.Fatal(err) + } + if _, err := s.SessionUser(t.Context(), "live"); !errors.Is(err, store.ErrNotFound) { + t.Fatalf("revoked session accepted: %v", err) + } +} + +// TestSessionResolvesOwner verifies a live session's immutable identity. +func TestSessionResolvesOwner(t *testing.T) { + s := testStore(t) + user := fixtureUser(t, s) + if err := s.SaveSession(t.Context(), "live", user.ID, time.Now().Add(time.Hour)); err != nil { + t.Fatal(err) + } + id, err := s.SessionUser(t.Context(), "live") + if err != nil || id != user.ID { + t.Fatalf("invalid owner: %v", err) + } +} + +// TestNewRejectsInvalidOptions verifies pool construction errors without connecting. +func TestNewRejectsInvalidOptions(t *testing.T) { + for _, options := range []Options{{}, {URL: "://invalid", MaxConnections: 1, Timeout: time.Second}} { + if _, err := New(options); err == nil { + t.Fatal("invalid options accepted") + } + } +} + +// TestCreateMessageMissingChat reports a typed missing-record error. +func TestCreateMessageMissingChat(t *testing.T) { + s := testStore(t) + _, err := s.CreateMessage(t.Context(), dto.MessageDTO{ID: uuid.New(), ChatID: uuid.New(), SenderID: uuid.New(), Text: "test"}) + if !errors.Is(err, store.ErrNotFound) { + t.Fatal(err) + } +} + +// TestMessagesCanceledReturnsError verifies bounded cancellation before query execution. +func TestMessagesCanceledReturnsError(t *testing.T) { + s := testStore(t) + ctx, cancel := context.WithCancel(t.Context()) + cancel() + if _, err := s.Messages(ctx, uuid.New(), dto.PositionDTO{}, 10); err == nil { + t.Fatal("canceled query succeeded") + } +} + +// TestChatsCanceledReturnsError verifies bounded cancellation of membership pages. +func TestChatsCanceledReturnsError(t *testing.T) { + s := testStore(t) + ctx, cancel := context.WithCancel(t.Context()) + cancel() + if _, err := s.Chats(ctx, uuid.New(), dto.PositionDTO{}, 10); err == nil { + t.Fatal("canceled query succeeded") + } +} + +// TestMigrationCancellationReturnsError checks startup cancellation before migration work. +func TestMigrationCancellationReturnsError(t *testing.T) { + s := testStore(t) + ctx, cancel := context.WithCancel(t.Context()) + cancel() + if err := s.Start(ctx); err == nil { + t.Fatal("canceled migration succeeded") + } +} + +// TestMigrationFailureRollsBackVersion verifies a failed migration is never marked applied. +func TestMigrationFailureRollsBackVersion(t *testing.T) { + s := testStore(t) + if _, err := s.pool.Exec(t.Context(), "DELETE FROM schema_migrations"); err != nil { + t.Fatal(err) + } + if err := s.Start(t.Context()); err == nil { + t.Fatal("conflicting unversioned schema accepted") + } + var count int + if err := s.pool.QueryRow(t.Context(), "SELECT count(*) FROM schema_migrations").Scan(&count); err != nil { + t.Fatal(err) + } + if count != 0 { + t.Fatal("failed migration version committed") + } +} + +// TestMigrationRejectsBrokenVersionLedger verifies corrupt migration metadata stops startup safely. +func TestMigrationRejectsBrokenVersionLedger(t *testing.T) { + s := testStore(t) + if _, err := s.pool.Exec(t.Context(), "ALTER TABLE schema_migrations RENAME COLUMN version TO broken_version"); err != nil { + t.Fatal(err) + } + if err := s.Start(t.Context()); err == nil { + t.Fatal("broken version ledger accepted") + } +} diff --git a/internal/store/postgres/users.go b/internal/store/postgres/users.go new file mode 100644 index 0000000..5f58083 --- /dev/null +++ b/internal/store/postgres/users.go @@ -0,0 +1,58 @@ +package postgres + +import ( + "context" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/model/dto" +) + +// CreateUser inserts a profile and enforces uniqueness inside PostgreSQL. +func (s *Store) CreateUser(ctx context.Context, user dto.UserDTO) (dto.UserDTO, error) { + return s.Register(ctx, user, "") +} + +// Register atomically stores a profile and its password hash. +func (s *Store) Register(ctx context.Context, user dto.UserDTO, passwordHash string) (dto.UserDTO, error) { + ctx, cancel := context.WithTimeout(ctx, s.timeout) + defer cancel() + err := s.pool.QueryRow(ctx, `INSERT INTO users(id,username,display_name,password_hash) + VALUES(@id,@username,@display_name,@password_hash) RETURNING created_at`, pgx.StrictNamedArgs{ + "id": user.ID, + "username": user.Username, + "display_name": user.DisplayName, + "password_hash": passwordHash, + }).Scan(&user.CreatedAt) + return user, normalize(err) +} + +// User reads a profile without credentials. +func (s *Store) User(ctx context.Context, id uuid.UUID) (dto.UserDTO, error) { + ctx, cancel := context.WithTimeout(ctx, s.timeout) + defer cancel() + var user dto.UserDTO + err := s.pool.QueryRow(ctx, `SELECT id,username,display_name,created_at FROM users WHERE id=@id`, pgx.StrictNamedArgs{"id": id}).Scan(&user.ID, &user.Username, &user.DisplayName, &user.CreatedAt) + return user, normalize(err) +} + +// Credentials reads the account needed for password verification. +func (s *Store) Credentials(ctx context.Context, username string) (dto.UserDTO, string, error) { + ctx, cancel := context.WithTimeout(ctx, s.timeout) + defer cancel() + var user dto.UserDTO + var hash string + err := s.pool.QueryRow(ctx, `SELECT id,username,display_name,created_at,password_hash FROM users WHERE username=@username`, pgx.StrictNamedArgs{"username": username}).Scan(&user.ID, &user.Username, &user.DisplayName, &user.CreatedAt, &hash) + return user, hash, normalize(err) +} + +// UsersExist checks all distinct requested identifiers in one bounded query. +func (s *Store) UsersExist(ctx context.Context, ids []uuid.UUID) (bool, error) { + ctx, cancel := context.WithTimeout(ctx, s.timeout) + defer cancel() + var exists bool + err := s.pool.QueryRow(ctx, `SELECT NOT EXISTS (SELECT id FROM unnest(@ids::uuid[]) AS requested(id) + WHERE NOT EXISTS (SELECT 1 FROM users WHERE users.id=requested.id))`, pgx.StrictNamedArgs{"ids": ids}).Scan(&exists) + return exists, normalize(err) +} diff --git a/internal/streamqueue/queue.go b/internal/streamqueue/queue.go new file mode 100644 index 0000000..d78846b --- /dev/null +++ b/internal/streamqueue/queue.go @@ -0,0 +1,106 @@ +// Package streamqueue adapts durable Valkey Streams to the job queue interface. +package streamqueue + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + "time" + + "github.com/redis/go-redis/v9" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/pipeline" +) + +const defaultStream = "attachment-jobs" +const group = "image-workers" + +// Queue owns a bounded connection pool and one consumer identity. +type Queue struct { + client *redis.Client + consumer string + cursor string + stream string +} + +// New configures finite broker I/O without opening network connections. +func New(address, consumer string) (*Queue, error) { + return NewScoped(address, consumer, defaultStream) +} + +// NewScoped isolates an explicitly named queue, including integration-test fixtures. +func NewScoped(address, consumer, streamName string) (*Queue, error) { + if address == "" || consumer == "" || streamName == "" { + return nil, fmt.Errorf("broker address and consumer are required") + } + return &Queue{ + client: redis.NewClient(&redis.Options{ + Addr: address, + PoolSize: 2, + MaxRetries: 0, + DialTimeout: 2 * time.Second, + ReadTimeout: 3 * time.Second, + WriteTimeout: 2 * time.Second, + }), + consumer: consumer, + cursor: "0-0", + stream: streamName, + }, nil +} + +// Close releases broker connections after worker shutdown. +func (q *Queue) Close() error { + return q.client.Close() +} + +// Publish writes a notification; broker AOF appendfsync always is required by deployment. +func (q *Queue) Publish(ctx context.Context, j pipeline.Job) error { + data, err := json.Marshal(j) + if err != nil { + return err + } + return q.client.XAdd(ctx, &redis.XAddArgs{Stream: q.stream, MaxLen: 10000, Approx: true, Values: map[string]any{"job": string(data)}}).Err() +} + +// Receive reclaims expired deliveries before waiting for one new job. +func (q *Queue) Receive(ctx context.Context) (*pipeline.Delivery, error) { + if err := q.client.XGroupCreateMkStream(ctx, q.stream, group, "0").Err(); err != nil && !strings.Contains(err.Error(), "BUSYGROUP") { + return nil, err + } + messages, next, err := q.client.XAutoClaim(ctx, &redis.XAutoClaimArgs{Stream: q.stream, Group: group, Consumer: q.consumer, MinIdle: 2 * time.Minute, Start: q.cursor, Count: 1}).Result() + if err != nil { + return nil, err + } + q.cursor = next + if len(messages) == 0 { + result, err := q.client.XReadGroup(ctx, &redis.XReadGroupArgs{Group: group, Consumer: q.consumer, Streams: []string{q.stream, ">"}, Count: 1, Block: time.Second}).Result() + if errors.Is(err, redis.Nil) { + return nil, nil + } + if err != nil { + return nil, err + } + messages = result[0].Messages + } + if len(messages) == 0 { + return nil, nil + } + message := messages[0] + raw, ok := message.Values["job"].(string) + var job pipeline.Job + if !ok || json.Unmarshal([]byte(raw), &job) != nil || job.SchemaVersion != 1 || job.Type != "attachment.process.v1" || job.Version < 1 || job.Attempt < 1 { + // Malformed notifications contain no trusted intent; SQL outbox remains authoritative. + return nil, q.Ack(ctx, message.ID) + } + return &pipeline.Delivery{Receipt: message.ID, Job: job}, nil +} + +// Ack removes a confirmed notification from the pending group and stream. +func (q *Queue) Ack(ctx context.Context, receipt string) error { + if err := q.client.XAck(ctx, q.stream, group, receipt).Err(); err != nil { + return err + } + return q.client.XDel(ctx, q.stream, receipt).Err() +} diff --git a/internal/streamqueue/queue_test.go b/internal/streamqueue/queue_test.go new file mode 100644 index 0000000..25395eb --- /dev/null +++ b/internal/streamqueue/queue_test.go @@ -0,0 +1,131 @@ +package streamqueue + +import ( + "context" + "os" + "testing" + "time" + + "github.com/google/uuid" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/pipeline" +) + +// queueFixture isolates stream and consumer state while using a real broker. +func queueFixture(t *testing.T) *Queue { + t.Helper() + address := os.Getenv("BROKER_ADDRESS") + if address == "" { + t.Skip("isolated BROKER_ADDRESS required") + } + q, err := NewScoped(address, uuid.NewString(), "test-"+uuid.NewString()) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := q.client.Del(context.Background(), q.stream).Err(); err != nil { + t.Error(err) + } + if err := q.Close(); err != nil { + t.Error(err) + } + }) + return q +} + +// TestQueueDeliveryAndAck verifies durable envelope round-trip and acknowledgement. +func TestQueueDeliveryAndAck(t *testing.T) { + q := queueFixture(t) + job := pipeline.Job{SchemaVersion: 1, ID: uuid.New(), AttachmentID: uuid.New(), Version: 1, Attempt: 1, Type: "attachment.process.v1", CreatedAt: time.Now().UTC()} + if err := q.Publish(t.Context(), job); err != nil { + t.Fatal(err) + } + received, err := q.Receive(t.Context()) + if err != nil || received == nil || received.Job.ID != job.ID { + t.Fatalf("receive: %v", err) + } + if err := q.Ack(t.Context(), received.Receipt); err != nil { + t.Fatal(err) + } + if delivery, err := q.Receive(t.Context()); err != nil || delivery != nil { + t.Fatal("acked job redelivered") + } +} + +// TestQueueMalformedNotification verifies malformed data cannot become a processing job. +func TestQueueMalformedNotification(t *testing.T) { + q := queueFixture(t) + if err := q.client.Do(t.Context(), "XADD", q.stream, "*", "job", "invalid-json").Err(); err != nil { + t.Fatal(err) + } + if delivery, err := q.Receive(t.Context()); err != nil || delivery != nil { + t.Fatal("invalid job delivered") + } +} + +// TestQueueReclaimsAbandonedDelivery verifies a consumer crash does not lose a pending notification. +func TestQueueReclaimsAbandonedDelivery(t *testing.T) { + q := queueFixture(t) + job := pipeline.Job{SchemaVersion: 1, ID: uuid.New(), AttachmentID: uuid.New(), Version: 1, Attempt: 1, Type: "attachment.process.v1", CreatedAt: time.Now().UTC()} + if err := q.Publish(t.Context(), job); err != nil { + t.Fatal(err) + } + received, err := q.Receive(t.Context()) + if err != nil || received == nil { + t.Fatal(err) + } + if err := q.client.Do(t.Context(), "XCLAIM", q.stream, group, "dead-consumer", 0, received.Receipt, "IDLE", 180000).Err(); err != nil { + t.Fatal(err) + } + reclaimed, err := q.Receive(t.Context()) + if err != nil || reclaimed == nil || reclaimed.Receipt != received.Receipt { + t.Fatal("pending delivery lost") + } + if err := q.Ack(t.Context(), reclaimed.Receipt); err != nil { + t.Fatal(err) + } +} + +// TestQueueCanceledOperations checks broker failures stay visible to the durable coordinator. +func TestQueueCanceledOperations(t *testing.T) { + q := queueFixture(t) + ctx, cancel := context.WithCancel(t.Context()) + cancel() + t.Run("publish", func(t *testing.T) { + if err := q.Publish(ctx, pipeline.Job{}); err == nil { + t.Fatal("publish succeeded") + } + }) + t.Run("receive", func(t *testing.T) { + if _, err := q.Receive(ctx); err == nil { + t.Fatal("receive succeeded") + } + }) + t.Run("ack", func(t *testing.T) { + if err := q.Ack(ctx, "0-0"); err == nil { + t.Fatal("ack succeeded") + } + }) +} + +// TestQueueConfiguration rejects absent dependencies without opening sockets. +func TestQueueConfiguration(t *testing.T) { + if _, err := New("", ""); err == nil { + t.Fatal("empty configuration accepted") + } + q, err := New("127.0.0.1:1", "consumer") + if err != nil { + t.Fatal(err) + } + if err := q.Close(); err != nil { + t.Fatal(err) + } +} + +// TestQueueRejectsUnserializableTimestamp prevents sending an envelope outside JSON time bounds. +func TestQueueRejectsUnserializableTimestamp(t *testing.T) { + q := queueFixture(t) + if err := q.Publish(t.Context(), pipeline.Job{CreatedAt: time.Date(10000, 1, 1, 0, 0, 0, 0, time.UTC)}); err == nil { + t.Fatal("invalid timestamp published") + } +} diff --git a/internal/strictjson/decode.go b/internal/strictjson/decode.go new file mode 100644 index 0000000..ceef25b --- /dev/null +++ b/internal/strictjson/decode.go @@ -0,0 +1,101 @@ +// Package strictjson decodes objects with an exact set of required, non-null fields. +package strictjson + +import ( + "bytes" + "encoding/json" + "errors" + "io" + "slices" + "unicode/utf8" +) + +// ErrInvalid indicates invalid JSON, encoding, fields or destination field types. +var ErrInvalid = errors.New("invalid JSON object") + +// DecodeObject decodes one UTF-8 object with exactly the listed required fields. +// Duplicate, unknown, missing and null fields are rejected before unmarshaling. +// The destination must describe the supplied fields; this is not a general schema validator. +func DecodeObject(body []byte, target any, requiredFields ...string) error { + if !utf8.Valid(body) { + return ErrInvalid + } + if err := validateObject(body, requiredFields); err != nil { + return err + } + if err := json.Unmarshal(body, target); err != nil { + return ErrInvalid + } + return nil +} + +// validateObject enforces exact, unique, non-null required keys and one JSON value. +// All lab1 request properties are required; this is not a generic schema validator. +func validateObject(body []byte, requiredFields []string) error { + return validateSchema(body, requiredFields, requiredFields) +} + +// validateSchema distinguishes required keys from optional allowed keys. +func validateSchema(body []byte, requiredFields, allowedFields []string) error { + decoder := json.NewDecoder(bytes.NewReader(body)) + token, err := decoder.Token() + if err != nil || token != json.Delim('{') { + return ErrInvalid + } + seen := make(map[string]bool, len(requiredFields)) + for decoder.More() { + key, err := readObjectField(decoder, allowedFields, seen) + if err != nil { + return err + } + seen[key] = true + } + if token, err = decoder.Token(); err != nil || token != json.Delim('}') { + return ErrInvalid + } + for _, key := range requiredFields { + if !seen[key] { + return ErrInvalid + } + } + var trailing any + if err := decoder.Decode(&trailing); err != io.EOF { + return ErrInvalid + } + return nil +} + +// readObjectField reads a known, unique, non-null JSON object field. +func readObjectField(decoder *json.Decoder, requiredFields []string, seen map[string]bool) (string, error) { + token, err := decoder.Token() + if err != nil { + return "", ErrInvalid + } + key, ok := token.(string) + if !ok || seen[key] || !slices.Contains(requiredFields, key) { + return "", ErrInvalid + } + var value json.RawMessage + if err := decoder.Decode(&value); err != nil { + return "", ErrInvalid + } + if bytes.Equal(bytes.TrimSpace(value), []byte("null")) { + return "", ErrInvalid + } + return key, nil +} + +// DecodeOptional rejects unknown, duplicate and null fields while allowing omitted optional fields. +func DecodeOptional(body []byte, target any, required, optional []string) error { + if !utf8.Valid(body) { + return ErrInvalid + } + allowed := append(append([]string{}, required...), optional...) + if err := validateSchema(body, required, allowed); err != nil { + return err + } + if err := json.Unmarshal(body, target); err != nil { + return ErrInvalid + } + return nil +} diff --git a/internal/strictjson/decode_test.go b/internal/strictjson/decode_test.go new file mode 100644 index 0000000..5c3b428 --- /dev/null +++ b/internal/strictjson/decode_test.go @@ -0,0 +1,52 @@ +package strictjson + +import ( + "errors" + "testing" +) + +type input struct { + // Name contains the only accepted field in these fixed-schema tests. + Name string `json:"name"` +} + +// TestDecodeObject checks each fixed-schema rule with a fresh destination. +func TestDecodeObject(t *testing.T) { + cases := []struct { + name string + body string + want string + wantErr error + }{ + {"Positive: Unicode preserved", `{"name":"Алиса 👋"}`, "Алиса 👋", nil}, + {"Negative: missing field", `{}`, "", ErrInvalid}, + {"Negative: unknown field", `{"name":"alice","extra":true}`, "", ErrInvalid}, + {"Negative: duplicate field", `{"name":"alice","name":"bob"}`, "", ErrInvalid}, + {"Negative: case mismatch", `{"Name":"alice"}`, "", ErrInvalid}, + {"Negative: null field", `{"name":null}`, "", ErrInvalid}, + {"Negative: wrong value type", `{"name":12}`, "", ErrInvalid}, + {"Negative: invalid UTF-8", "{\"name\":\"\xff\"}", "", ErrInvalid}, + {"Negative: array root", `[]`, "", ErrInvalid}, + {"Negative: unclosed object", `{"name":"alice"`, "", ErrInvalid}, + {"Negative: invalid value", `{"name":}`, "", ErrInvalid}, + {"Negative: invalid key", `{name:"alice"}`, "", ErrInvalid}, + {"Negative: trailing value", `{"name":"alice"} {}`, "", ErrInvalid}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + // Arrange. + var destination input + + // Act. + err := DecodeObject([]byte(tc.body), &destination, "name") + + // Assert. + if !errors.Is(err, tc.wantErr) { + t.Fatalf("error=%v, want=%v", err, tc.wantErr) + } + if destination.Name != tc.want { + t.Fatalf("decoded name=%q, want=%q", destination.Name, tc.want) + } + }) + } +} diff --git a/internal/strictjson/optional_test.go b/internal/strictjson/optional_test.go new file mode 100644 index 0000000..af0ad5d --- /dev/null +++ b/internal/strictjson/optional_test.go @@ -0,0 +1,33 @@ +package strictjson + +import "testing" + +// TestDecodeOptional distinguishes omission from explicit null and duplicate fields. +func TestDecodeOptional(t *testing.T) { + cases := []struct { + name string + body string + valid bool + }{ + {"omitted", `{"text":"ok"}`, true}, + {"provided", `{"text":"ok","attachment_ids":["a"]}`, true}, + {"null", `{"text":"ok","attachment_ids":null}`, false}, + {"unknown", `{"text":"ok","sender_id":"a"}`, false}, + {"duplicate", `{"text":"ok","text":"other"}`, false}, + {"missing required", `{"attachment_ids":[]}`, false}, + {"wrong type", `{"text":"ok","attachment_ids":12}`, false}, + {"invalid utf8", string([]byte{255}), false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + var target struct { + Text string `json:"text"` + IDs []string `json:"attachment_ids"` + } + err := DecodeOptional([]byte(tc.body), &target, []string{"text"}, []string{"attachment_ids"}) + if (err == nil) != tc.valid { + t.Fatalf("validation: %v", err) + } + }) + } +} diff --git a/internal/telemetry/metrics.go b/internal/telemetry/metrics.go new file mode 100644 index 0000000..cdab8cd --- /dev/null +++ b/internal/telemetry/metrics.go @@ -0,0 +1,59 @@ +package telemetry + +import ( + "context" + "sync/atomic" + "time" + + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" +) + +type instruments struct { + httpRequests metric.Int64Counter + httpDuration metric.Float64Histogram + jobs metric.Int64Counter + jobDuration metric.Float64Histogram + ready metric.Float64Histogram + busy atomic.Int64 + waiting atomic.Int64 +} + +// newInstruments defines stable names and units once for every provider instance. +func newInstruments(meter metric.Meter) instruments { + requests, _ := meter.Int64Counter("messenger_http_requests") + duration, _ := meter.Float64Histogram("messenger_http_duration_seconds", metric.WithUnit("s")) + jobs, _ := meter.Int64Counter("messenger_jobs") + jobDuration, _ := meter.Float64Histogram("messenger_job_duration_seconds", metric.WithUnit("s")) + ready, _ := meter.Float64Histogram("messenger_ready_duration_seconds", metric.WithUnit("s")) + return instruments{httpRequests: requests, httpDuration: duration, jobs: jobs, jobDuration: jobDuration, ready: ready} +} + +// recordHTTP avoids raw URLs and identifiers in metric attributes. +func (r *Runtime) recordHTTP(ctx context.Context, method, route, class string, seconds float64) { + attrs := metric.WithAttributes(attribute.String("method", method), attribute.String("route", route), attribute.String("status_class", class)) + r.instruments.httpRequests.Add(ctx, 1, attrs) + r.instruments.httpDuration.Record(ctx, seconds, attrs) +} + +// JobStarted records occupancy independently from queue length. +func (r *Runtime) JobStarted() { + r.instruments.busy.Add(1) +} + +// JobFinished records a real attempt's result and releases its occupancy gauge. +func (r *Runtime) JobFinished(ctx context.Context, elapsed time.Duration, err error) { + r.instruments.busy.Add(-1) + outcome := "ok" + if err != nil { + outcome = "error" + } + attrs := metric.WithAttributes(attribute.String("job_type", "attachment.process.v1"), attribute.String("outcome", outcome)) + r.instruments.jobs.Add(ctx, 1, attrs) + r.instruments.jobDuration.Record(ctx, elapsed.Seconds(), attrs) +} + +// Ready records end-to-end latency only after a confirmed durable publication. +func (r *Runtime) Ready(ctx context.Context, accepted time.Time) { + r.instruments.ready.Record(ctx, time.Since(accepted).Seconds()) +} diff --git a/internal/telemetry/postgres.go b/internal/telemetry/postgres.go new file mode 100644 index 0000000..163b10b --- /dev/null +++ b/internal/telemetry/postgres.go @@ -0,0 +1,40 @@ +package telemetry + +import ( + "context" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +type queryEndKey struct{} + +type acquireEndKey struct{} + +// TraceQueryStart creates a SQL client span without recording SQL text or arguments. +func (r *Runtime) TraceQueryStart(ctx context.Context, _ *pgx.Conn, _ pgx.TraceQueryStartData) context.Context { + ctx, end := r.Start(ctx, "postgres.query", "client") + return context.WithValue(ctx, queryEndKey{}, end) +} + +// TraceQueryEnd closes the actual driver operation and records only success or failure. +func (r *Runtime) TraceQueryEnd(ctx context.Context, _ *pgx.Conn, data pgx.TraceQueryEndData) { + if end, ok := ctx.Value(queryEndKey{}).(func(error)); ok { + end(data.Err) + } +} + +// TraceAcquireStart counts acquisition attempts still waiting for a pool result. +func (r *Runtime) TraceAcquireStart(ctx context.Context, _ *pgxpool.Pool, _ pgxpool.TraceAcquireStartData) context.Context { + r.instruments.waiting.Add(1) + ctx, end := r.Start(ctx, "postgres.acquire", "client") + return context.WithValue(ctx, acquireEndKey{}, end) +} + +// TraceAcquireEnd releases the acquisition gauge on either success or failure. +func (r *Runtime) TraceAcquireEnd(ctx context.Context, _ *pgxpool.Pool, data pgxpool.TraceAcquireEndData) { + r.instruments.waiting.Add(-1) + if end, ok := ctx.Value(acquireEndKey{}).(func(error)); ok { + end(data.Err) + } +} diff --git a/internal/telemetry/runtime.go b/internal/telemetry/runtime.go new file mode 100644 index 0000000..d13edfa --- /dev/null +++ b/internal/telemetry/runtime.go @@ -0,0 +1,86 @@ +// Package telemetry owns bounded OpenTelemetry providers and safe instrumentation. +package telemetry + +import ( + "context" + "errors" + "fmt" + "math" + "net/url" + "strings" + "time" + + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp" + "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" + otellog "go.opentelemetry.io/otel/log" + "go.opentelemetry.io/otel/metric" + sdklog "go.opentelemetry.io/otel/sdk/log" + sdkmetric "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/resource" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + "go.opentelemetry.io/otel/trace" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/config" +) + +// Runtime owns providers; Shutdown is called once by the application lifecycle. +type Runtime struct { + traces *sdktrace.TracerProvider + metrics *sdkmetric.MeterProvider + logs *sdklog.LoggerProvider + tracer trace.Tracer + logger otellog.Logger + meter metric.Meter + instruments instruments + enabled bool +} + +// New builds nonblocking bounded exporters; backend availability is never a startup dependency. +func New(ctx context.Context, cfg *config.Telemetry) (*Runtime, error) { + if cfg == nil { + return nil, fmt.Errorf("telemetry configuration is required") + } + if math.IsNaN(cfg.SampleRatio) || cfg.SampleRatio < 0 || cfg.SampleRatio > 1 { + return nil, fmt.Errorf("telemetry sampling ratio must be in [0,1]") + } + if cfg.Endpoint != "" { + endpoint, err := url.Parse(cfg.Endpoint) + if err != nil || endpoint.Host == "" || (endpoint.Scheme != "http" && endpoint.Scheme != "https") || endpoint.User != nil { + return nil, fmt.Errorf("invalid telemetry endpoint") + } + } + res := resource.NewSchemaless(attribute.String("service.name", cfg.Service), attribute.String("service.version", cfg.Version), attribute.String("service.instance.id", cfg.Instance), attribute.String("deployment.environment.name", "lab")) + traceOptions := []sdktrace.TracerProviderOption{sdktrace.WithResource(res), sdktrace.WithSampler(sdktrace.ParentBased(sdktrace.TraceIDRatioBased(cfg.SampleRatio)))} + metricOptions := []sdkmetric.Option{sdkmetric.WithResource(res), sdkmetric.WithView(sdkmetric.NewView(sdkmetric.Instrument{Kind: sdkmetric.InstrumentKindHistogram}, sdkmetric.Stream{Aggregation: sdkmetric.AggregationExplicitBucketHistogram{Boundaries: []float64{.005, .01, .025, .05, .1, .25, .5, 1, 2, 5, 10, 30, 60, 120}}}))} + logOptions := []sdklog.LoggerProviderOption{sdklog.WithResource(res)} + if cfg.Endpoint != "" { + traces, err := otlptracehttp.New(ctx, otlptracehttp.WithEndpointURL(strings.TrimRight(cfg.Endpoint, "/")+"/v1/traces"), otlptracehttp.WithTimeout(time.Second), otlptracehttp.WithRetry(otlptracehttp.RetryConfig{Enabled: false})) + if err != nil { + return nil, err + } + metrics, err := otlpmetrichttp.New(ctx, otlpmetrichttp.WithEndpointURL(strings.TrimRight(cfg.Endpoint, "/")+"/v1/metrics"), otlpmetrichttp.WithTimeout(time.Second), otlpmetrichttp.WithRetry(otlpmetrichttp.RetryConfig{Enabled: false})) + if err != nil { + return nil, errors.Join(err, traces.Shutdown(ctx)) + } + logs, err := otlploghttp.New(ctx, otlploghttp.WithEndpointURL(strings.TrimRight(cfg.Endpoint, "/")+"/v1/logs"), otlploghttp.WithTimeout(time.Second), otlploghttp.WithRetry(otlploghttp.RetryConfig{Enabled: false})) + if err != nil { + return nil, errors.Join(err, traces.Shutdown(ctx), metrics.Shutdown(ctx)) + } + traceOptions = append(traceOptions, sdktrace.WithBatcher(traces, sdktrace.WithMaxQueueSize(256), sdktrace.WithMaxExportBatchSize(64), sdktrace.WithBatchTimeout(time.Second), sdktrace.WithExportTimeout(time.Second))) + metricOptions = append(metricOptions, sdkmetric.WithReader(sdkmetric.NewPeriodicReader(metrics, sdkmetric.WithInterval(5*time.Second), sdkmetric.WithTimeout(time.Second)))) + logOptions = append(logOptions, sdklog.WithProcessor(sdklog.NewBatchProcessor(logs, sdklog.WithMaxQueueSize(256), sdklog.WithExportMaxBatchSize(64), sdklog.WithExportInterval(time.Second), sdklog.WithExportTimeout(time.Second), sdklog.WithExportBufferSize(1)))) + } + r := &Runtime{traces: sdktrace.NewTracerProvider(traceOptions...), metrics: sdkmetric.NewMeterProvider(metricOptions...), logs: sdklog.NewLoggerProvider(logOptions...), enabled: cfg.Endpoint != ""} + r.tracer = r.traces.Tracer("messenger") + r.meter = r.metrics.Meter("messenger") + r.logger = r.logs.Logger("messenger") + r.instruments = newInstruments(r.meter) + return r, nil +} + +// Shutdown flushes all providers under the lifecycle's bounded deadline. +func (r *Runtime) Shutdown(ctx context.Context) error { + return errors.Join(r.traces.Shutdown(ctx), r.metrics.Shutdown(ctx), r.logs.Shutdown(ctx)) +} diff --git a/internal/telemetry/runtime_test.go b/internal/telemetry/runtime_test.go new file mode 100644 index 0000000..413eb74 --- /dev/null +++ b/internal/telemetry/runtime_test.go @@ -0,0 +1,214 @@ +package telemetry + +import ( + "context" + "errors" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + collectorlogs "go.opentelemetry.io/proto/otlp/collector/logs/v1" + collectormetrics "go.opentelemetry.io/proto/otlp/collector/metrics/v1" + collectortrace "go.opentelemetry.io/proto/otlp/collector/trace/v1" + "google.golang.org/protobuf/proto" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/config" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/observation" +) + +type capture struct { + mu sync.Mutex + traces []*collectortrace.ExportTraceServiceRequest + logs []*collectorlogs.ExportLogsServiceRequest + metrics []*collectormetrics.ExportMetricsServiceRequest +} + +// serve captures actual OTLP requests without relying on SDK implementation internals. +func (c *capture) serve(w http.ResponseWriter, r *http.Request) { + data, err := io.ReadAll(r.Body) + if err != nil { + http.Error(w, "read failed", http.StatusBadRequest) + return + } + c.mu.Lock() + defer c.mu.Unlock() + switch r.URL.Path { + case "/v1/traces": + message := &collectortrace.ExportTraceServiceRequest{} + if err := proto.Unmarshal(data, message); err != nil { + http.Error(w, "invalid OTLP", http.StatusBadRequest) + return + } + c.traces = append(c.traces, message) + case "/v1/logs": + message := &collectorlogs.ExportLogsServiceRequest{} + if err := proto.Unmarshal(data, message); err != nil { + http.Error(w, "invalid OTLP", http.StatusBadRequest) + return + } + c.logs = append(c.logs, message) + case "/v1/metrics": + message := &collectormetrics.ExportMetricsServiceRequest{} + if err := proto.Unmarshal(data, message); err != nil { + http.Error(w, "invalid OTLP", http.StatusBadRequest) + return + } + c.metrics = append(c.metrics, message) + default: + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", "application/x-protobuf") + w.WriteHeader(http.StatusOK) +} + +type snapshotFixture struct { + err error +} + +// Snapshot supplies safe bounded saturation data for the exporter integration test. +func (s snapshotFixture) Snapshot(context.Context) (observation.Snapshot, error) { + return observation.Snapshot{QueueDepth: 2, OldestSeconds: 12, PoolUsed: 1}, s.err +} + +// TestOTLPSignalsShareContext verifies real HTTP export, parentage, metrics and structured-log correlation. +func TestOTLPSignalsShareContext(t *testing.T) { + capture := &capture{} + server := httptest.NewServer(http.HandlerFunc(capture.serve)) + defer server.Close() + r, err := New(t.Context(), &config.Telemetry{Endpoint: server.URL, Service: "messenger-worker", Version: "test", Instance: "synthetic", SampleRatio: 1}) + if err != nil { + t.Fatal(err) + } + if err := r.ObserveSaturation(snapshotFixture{}, true); err != nil { + t.Fatal(err) + } + ctx, finish := r.Start(t.Context(), "http.request", "server") + r.HTTP(ctx, "POST", "/api/v1/chats/:chat_id/attachments", 202, time.Millisecond, "synthetic-request") + parent, state := r.Inject(ctx) + producer, endProducer := r.Start(r.Extract(t.Context(), parent, state), "attachment.publish", "producer") + parent, state = r.Inject(producer) + consumer, endConsumer := r.Start(r.Extract(t.Context(), parent, state), "attachment.process", "consumer") + r.JobStarted() + r.JobFinished(consumer, time.Millisecond, nil) + r.Ready(consumer, time.Now().Add(-time.Second)) + query := r.TraceQueryStart(consumer, nil, pgx.TraceQueryStartData{SQL: "secret SQL that must not be exported"}) + r.TraceQueryEnd(query, nil, pgx.TraceQueryEndData{Err: errors.New("secret database password")}) + acquire := r.TraceAcquireStart(consumer, nil, pgxpool.TraceAcquireStartData{}) + r.TraceAcquireEnd(acquire, nil, pgxpool.TraceAcquireEndData{}) + endConsumer(nil) + endProducer(nil) + finish(nil) + if err := r.Shutdown(t.Context()); err != nil { + t.Fatal(err) + } + capture.mu.Lock() + defer capture.mu.Unlock() + if len(capture.traces) == 0 || len(capture.logs) == 0 || len(capture.metrics) == 0 { + t.Fatal("one OTLP signal is absent") + } + spans := 0 + correlatedLogs := 0 + for _, request := range capture.traces { + if strings.Contains(request.String(), "secret") { + t.Fatal("trace leaked dependency payload") + } + for _, resource := range request.ResourceSpans { + for _, scope := range resource.ScopeSpans { + spans += len(scope.Spans) + } + } + } + for _, request := range capture.logs { + if strings.Contains(request.String(), "secret") { + t.Fatal("log leaked dependency payload") + } + for _, resource := range request.ResourceLogs { + for _, scope := range resource.ScopeLogs { + for _, record := range scope.LogRecords { + if len(record.TraceId) == 16 && len(record.SpanId) == 8 { + correlatedLogs++ + } + } + } + } + } + if spans < 4 || correlatedLogs < 4 { + t.Fatal("causal operations or correlated logs missing") + } +} + +// TestHTTPBoundedDimensions covers status classes and rejects arbitrary method labels. +func TestHTTPBoundedDimensions(t *testing.T) { + r, err := New(t.Context(), &config.Telemetry{Service: "api", SampleRatio: 1}) + if err != nil { + t.Fatal(err) + } + ctx, finish := r.Start(t.Context(), "request", "internal") + for _, status := range []int{200, 302, 401, 503} { + r.HTTP(ctx, "custom-method", "bad-route", status, time.Millisecond, "") + } + r.JobStarted() + r.JobFinished(ctx, time.Millisecond, errors.New("safe error")) + r.TraceQueryEnd(ctx, nil, pgx.TraceQueryEndData{}) + finish(errors.New("safe error")) + if err := r.ObserveSaturation(snapshotFixture{err: errors.New("unavailable")}, false); err != nil { + t.Fatal(err) + } + if err := r.Shutdown(t.Context()); err != nil { + t.Fatal(err) + } +} + +// TestUnavailableBackendDoesNotBlockBusinessWork bounds exporter failures independently of requests. +func TestUnavailableBackendDoesNotBlockBusinessWork(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusServiceUnavailable) })) + defer server.Close() + r, err := New(t.Context(), &config.Telemetry{Endpoint: server.URL, Service: "api", SampleRatio: 1}) + if err != nil { + t.Fatal(err) + } + start := time.Now() + for i := 0; i < 2000; i++ { + ctx, end := r.Start(t.Context(), "safe-operation", "client") + r.HTTP(ctx, "GET", "/api/v1/chats", 200, time.Millisecond, "synthetic") + end(nil) + } + if time.Since(start) > 2*time.Second { + t.Fatal("business path blocked on unavailable exporter") + } + ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second) + defer cancel() + if err := r.Shutdown(ctx); err == nil { + t.Fatal("backend outage was not visible at exporter shutdown") + } +} + +// TestExporterInvalidPort rejects an unusable exporter URL without starting the application. +func TestExporterInvalidPort(t *testing.T) { + runtime, err := New(t.Context(), &config.Telemetry{Endpoint: "http://localhost:invalid", SampleRatio: 1}) + if err == nil { + _ = runtime.Shutdown(t.Context()) + t.Fatal("invalid exporter port accepted") + } +} + +// TestMissingRuntimeConfiguration returns an error instead of dereferencing a missing section. +func TestMissingRuntimeConfiguration(t *testing.T) { + if _, err := New(t.Context(), nil); err == nil { + t.Fatal("missing configuration accepted") + } +} + +// TestInvalidRuntimeSampling prevents an SDK sampler panic for out-of-range ratios. +func TestInvalidRuntimeSampling(t *testing.T) { + if _, err := New(t.Context(), &config.Telemetry{SampleRatio: -1}); err == nil { + t.Fatal("invalid sampling accepted") + } +} diff --git a/internal/telemetry/saturation.go b/internal/telemetry/saturation.go new file mode 100644 index 0000000..6979a23 --- /dev/null +++ b/internal/telemetry/saturation.go @@ -0,0 +1,53 @@ +package telemetry + +import ( + "context" + "runtime" + "syscall" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/observation" + "go.opentelemetry.io/otel/metric" +) + +// Snapshotter supplies metrics independently of the application's request path. +type Snapshotter interface { + // Snapshot returns queue and pool state within its own timeout. + Snapshot(context.Context) (observation.Snapshot, error) +} + +// ObserveSaturation registers bounded periodic observations; database failure skips only DB gauges. +func (r *Runtime) ObserveSaturation(source Snapshotter, worker bool) error { + depth, _ := r.meter.Int64ObservableGauge("messenger_queue_depth") + age, _ := r.meter.Float64ObservableGauge("messenger_queue_oldest_seconds") + used, _ := r.meter.Int64ObservableGauge("messenger_db_pool_used") + waiting, _ := r.meter.Int64ObservableGauge("messenger_db_pool_waiting") + busy, _ := r.meter.Int64ObservableGauge("messenger_worker_busy") + capacity, _ := r.meter.Int64ObservableGauge("messenger_worker_capacity") + cpu, _ := r.meter.Float64ObservableCounter("messenger_process_cpu_seconds") + memory, _ := r.meter.Int64ObservableGauge("messenger_process_peak_rss_bytes") + _, err := r.meter.RegisterCallback(func(ctx context.Context, observer metric.Observer) error { + observer.ObserveInt64(waiting, r.instruments.waiting.Load()) + if worker { + observer.ObserveInt64(busy, r.instruments.busy.Load()) + observer.ObserveInt64(capacity, 1) + } + snapshot, err := source.Snapshot(ctx) + if err == nil { + observer.ObserveInt64(depth, snapshot.QueueDepth) + observer.ObserveFloat64(age, snapshot.OldestSeconds) + observer.ObserveInt64(used, snapshot.PoolUsed) + } + var usage syscall.Rusage + if syscall.Getrusage(syscall.RUSAGE_SELF, &usage) == nil { + seconds := float64(usage.Utime.Sec+usage.Stime.Sec) + float64(usage.Utime.Usec+usage.Stime.Usec)/1e6 + rss := usage.Maxrss + if runtime.GOOS != "darwin" { + rss *= 1024 + } + observer.ObserveFloat64(cpu, seconds) + observer.ObserveInt64(memory, rss) + } + return nil + }, depth, age, used, waiting, busy, capacity, cpu, memory) + return err +} diff --git a/internal/telemetry/spans.go b/internal/telemetry/spans.go new file mode 100644 index 0000000..c6746a0 --- /dev/null +++ b/internal/telemetry/spans.go @@ -0,0 +1,92 @@ +package telemetry + +import ( + "context" + "strings" + "time" + + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + otellog "go.opentelemetry.io/otel/log" + "go.opentelemetry.io/otel/propagation" + "go.opentelemetry.io/otel/trace" +) + +// Start records a real operation without copying error text, credentials or business data. +func (r *Runtime) Start(ctx context.Context, name, kind string) (context.Context, func(error)) { + spanKind := trace.SpanKindInternal + switch kind { + case "server": + spanKind = trace.SpanKindServer + case "producer": + spanKind = trace.SpanKindProducer + case "consumer": + spanKind = trace.SpanKindConsumer + case "client": + spanKind = trace.SpanKindClient + } + ctx, span := r.tracer.Start(ctx, name, trace.WithSpanKind(spanKind)) + return ctx, func(err error) { + outcome := "ok" + if err != nil { + outcome = "error" + span.SetStatus(codes.Error, "operation failed") + } + r.event(ctx, name, outcome, "") + span.End() + } +} + +// Extract restores only W3C trace headers, never arbitrary baggage. +func (r *Runtime) Extract(ctx context.Context, parent, state string) context.Context { + return propagation.TraceContext{}.Extract(ctx, propagation.MapCarrier{"traceparent": parent, "tracestate": state}) +} + +// Inject returns the bounded W3C carrier for a durable asynchronous handoff. +func (r *Runtime) Inject(ctx context.Context) (string, string) { + carrier := propagation.MapCarrier{} + propagation.TraceContext{}.Inject(ctx, carrier) + return carrier["traceparent"], carrier["tracestate"] +} + +// HTTP records one completed request with bounded attributes and a correlated log event. +func (r *Runtime) HTTP(ctx context.Context, method, route string, status int, elapsed time.Duration, requestID string) { + if len(route) > 128 || !strings.HasPrefix(route, "/") { + route = "unmatched" + } + switch method { + case "GET", "POST", "DELETE", "HEAD", "OPTIONS": + default: + method = "other" + } + class := "5xx" + switch status / 100 { + case 2: + class = "2xx" + case 3: + class = "3xx" + case 4: + class = "4xx" + } + span := trace.SpanFromContext(ctx) + span.SetName(method + " " + route) + span.SetAttributes(attribute.String("http.route", route), attribute.String("http.request.method", method), attribute.Int("http.response.status_code", status)) + if status >= 500 { + span.SetStatus(codes.Error, "server error") + } + r.recordHTTP(ctx, method, route, class, elapsed.Seconds()) + r.event(ctx, "http.request", class, requestID) +} + +// event emits only an allowlisted technical record; SDK attaches trace/span IDs from context. +func (r *Runtime) event(ctx context.Context, event, outcome, requestID string) { + var record otellog.Record + record.SetTimestamp(time.Now()) + record.SetSeverity(otellog.SeverityInfo) + record.SetBody(otellog.StringValue(event)) + record.AddAttributes(otellog.String("event", event), otellog.String("outcome", outcome)) + if requestID != "" { + record.AddAttributes(otellog.String("request_id", requestID)) + } + r.logger.Emit(ctx, record) +} diff --git a/internal/valkey/admission.go b/internal/valkey/admission.go new file mode 100644 index 0000000..8589554 --- /dev/null +++ b/internal/valkey/admission.go @@ -0,0 +1,132 @@ +package valkey + +import ( + "context" + "fmt" + "time" + + "github.com/google/uuid" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/admission" +) + +const ( + admissionWindow = time.Second + admissionLease = 30 * time.Second + releaseTimeout = time.Second +) + +// Budget defines a rate window and concurrent execution capacity. +type Budget struct { + // Rate is the maximum attempts per one-second window. + Rate int + // Slots is the maximum requests executing concurrently. + Slots int +} + +// Limits separates global, route-group and user budgets. +type Limits struct { + // Global covers every business request before authentication. + Global Budget + // Groups covers auth, read and write independently. + Groups map[string]Budget + // User covers all authenticated requests made by one account. + User Budget + // UserGroups limits each authenticated account within a route group. + UserGroups map[string]Budget +} + +// Limiter applies atomic local-shard decisions; global reservations precede user reservations. +type Limiter struct { + client *Client + limits Limits +} + +// NewLimiter validates independent rate and concurrency budgets. +func NewLimiter(client *Client, limits Limits) (*Limiter, error) { + if client == nil || limits.Global.Rate < 1 || limits.Global.Slots < 1 || limits.User.Rate < 1 || limits.User.Slots < 1 { + return nil, fmt.Errorf("client and positive budgets are required") + } + for _, group := range []string{"auth", "read", "write"} { + global := limits.Groups[group] + user := limits.UserGroups[group] + if global.Rate < 1 || global.Slots < 1 || user.Rate < 1 || user.Slots < 1 { + return nil, fmt.Errorf("positive budgets for every group are required") + } + } + groups := make(map[string]Budget, len(limits.Groups)) + userGroups := make(map[string]Budget, len(limits.UserGroups)) + for key, value := range limits.Groups { + groups[key] = value + } + for key, value := range limits.UserGroups { + userGroups[key] = value + } + limits.Groups = groups + limits.UserGroups = userGroups + return &Limiter{client: client, limits: limits}, nil +} + +// Enter atomically reserves two rate and slot budgets on one shard. +// Global admission precedes authentication; user admission follows verified identity. +func (l *Limiter) Enter(ctx context.Context, group, user string) (func(), error) { + groupBudget, ok := l.limits.Groups[group] + if !ok { + return nil, fmt.Errorf("unknown admission group") + } + totalBudget := l.limits.Global + node := l.client.coordinator + prefix := "admission:global" + if user != "" { + prefix = "admission:user:" + user + node = l.client.nodes[l.client.ring.Owner(prefix)] + totalBudget = l.limits.User + groupBudget = l.limits.UserGroups[group] + } + keys := []string{prefix + ":rate", prefix + ":slots", prefix + ":" + group + ":rate", prefix + ":" + group + ":slots"} + token := uuid.NewString() + ctx, cancel := context.WithTimeout(ctx, l.client.timeout) + defer cancel() + result, err := node.Eval(ctx, reserveScript, keys, token, admissionWindow.Milliseconds(), admissionLease.Milliseconds(), totalBudget.Rate, totalBudget.Slots, groupBudget.Rate, groupBudget.Slots).Int() + if err != nil { + return nil, err + } + if result == 1 { + return nil, admission.ErrRate + } + if result == 2 { + return nil, admission.ErrSlots + } + return func() { + releaseCtx, releaseCancel := context.WithTimeout(context.Background(), releaseTimeout) + defer releaseCancel() + // Failed release is bounded by the lease; tokens are never reused. + _ = node.Eval(releaseCtx, releaseScript, []string{keys[1], keys[3]}, token).Err() + }, nil +} + +// Redis TIME and one Lua operation make check-and-reserve indivisible within a shard. +const reserveScript = ` +local clock = redis.call('TIME') +local now = clock[1]*1000 + math.floor(clock[2]/1000) +for i=1,3,2 do + redis.call('ZREMRANGEBYSCORE',KEYS[i+1],'-inf',now) + local rate = tonumber(redis.call('GET',KEYS[i]) or '0') + local limitIndex = i==1 and 4 or 6 + if rate >= tonumber(ARGV[limitIndex]) then return 1 end + if redis.call('ZCARD',KEYS[i+1]) >= tonumber(ARGV[limitIndex+1]) then return 2 end +end +for i=1,3,2 do + local count = redis.call('INCR',KEYS[i]) + if count==1 then redis.call('PEXPIRE',KEYS[i],ARGV[2]) end + redis.call('ZADD',KEYS[i+1],now+tonumber(ARGV[3]),ARGV[1]) + redis.call('PEXPIRE',KEYS[i+1],ARGV[3]) +end +return 0 +` + +const releaseScript = ` +redis.call('ZREM',KEYS[1],ARGV[1]) +redis.call('ZREM',KEYS[2],ARGV[1]) +return 0 +` diff --git a/internal/valkey/cache.go b/internal/valkey/cache.go new file mode 100644 index 0000000..260daed --- /dev/null +++ b/internal/valkey/cache.go @@ -0,0 +1,42 @@ +package valkey + +import ( + "context" + "encoding/json" + "time" + + "github.com/google/uuid" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/model/dto" +) + +const profileTTL = 5 * time.Minute + +// GetProfile reads an immutable profile; corrupt values and outages are cache misses. +func (c *Client) GetProfile(ctx context.Context, id uuid.UUID) (dto.UserDTO, bool) { + ctx, cancel := context.WithTimeout(ctx, c.timeout) + defer cancel() + key := "profile:" + id.String() + value, err := c.nodes[c.ring.Owner(key)].Get(ctx, key).Bytes() + if err != nil { + return dto.UserDTO{}, false + } + var user dto.UserDTO + if json.Unmarshal(value, &user) != nil || user.ID != id { + return dto.UserDTO{}, false + } + return user, true +} + +// PutProfile populates the immutable profile cache with a bounded TTL. +func (c *Client) PutProfile(ctx context.Context, user dto.UserDTO) { + ctx, cancel := context.WithTimeout(ctx, c.timeout) + defer cancel() + value, err := json.Marshal(user) + if err != nil { + return + } + key := "profile:" + user.ID.String() + // Cache writes are best effort; PostgreSQL remains the source of truth. + _ = c.nodes[c.ring.Owner(key)].Set(ctx, key, value, profileTTL).Err() +} diff --git a/internal/valkey/client.go b/internal/valkey/client.go new file mode 100644 index 0000000..21f837c --- /dev/null +++ b/internal/valkey/client.go @@ -0,0 +1,113 @@ +// Package valkey implements sharded cache and admission infrastructure. +package valkey + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "sort" + "strings" + "time" + + "github.com/redis/go-redis/v9" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/hashring" +) + +const virtualNodes = 256 + +// Options bounds each independent Valkey connection pool. +type Options struct { + // Nodes maps stable shard identities to network addresses. + Nodes map[string]string + // Coordinator identifies the stable global admission shard. + Coordinator string + // Timeout bounds operations and pool acquisition. + Timeout time.Duration + // PoolSize bounds connections to each shard. + PoolSize int +} + +// Client owns a fixed topology; callers must drain before changing it. +type Client struct { + nodes map[string]*redis.Client + ring *hashring.Ring + coordinator *redis.Client + timeout time.Duration + topology string +} + +// New prepares lazy clients without opening network connections. +func New(options Options) (*Client, error) { + if len(options.Nodes) < 1 || options.PoolSize < 1 || options.Timeout <= 0 { + return nil, fmt.Errorf("valkey nodes, pool bound and timeout are required") + } + ids := make([]string, 0, len(options.Nodes)) + for id, address := range options.Nodes { + if address == "" { + return nil, fmt.Errorf("valkey address is required") + } + ids = append(ids, id) + } + ring, err := hashring.New(ids, virtualNodes) + if err != nil { + return nil, err + } + if _, ok := options.Nodes[options.Coordinator]; !ok { + return nil, fmt.Errorf("coordinator must identify a configured node") + } + sort.Strings(ids) + topologyHash := sha256.Sum256([]byte(strings.Join(ids, ","))) + client := &Client{nodes: make(map[string]*redis.Client), ring: ring, timeout: options.Timeout} + for id, address := range options.Nodes { + client.nodes[id] = redis.NewClient(&redis.Options{ + Addr: address, + PoolSize: options.PoolSize, + MaxActiveConns: options.PoolSize, + PoolTimeout: options.Timeout, + DialTimeout: options.Timeout, + ReadTimeout: options.Timeout, + WriteTimeout: options.Timeout, + ContextTimeoutEnabled: true, + MaxRetries: -1, + }) + } + client.topology = hex.EncodeToString(topologyHash[:]) + client.coordinator = client.nodes[options.Coordinator] + return client, nil +} + +// Ping checks every shard required for safe admission. +func (c *Client) Ping(ctx context.Context) error { + ctx, cancel := context.WithTimeout(ctx, c.timeout) + defer cancel() + for _, node := range c.nodes { + if err := node.Ping(ctx).Err(); err != nil { + return err + } + } + stored, err := c.coordinator.Eval(ctx, topologyScript, []string{"admission:topology"}, c.topology).Text() + if err != nil { + return err + } + if stored != c.topology { + return fmt.Errorf("valkey topology differs: stop admission and drain before changing the ring") + } + return nil +} + +const topologyScript = ` +redis.call('SET',KEYS[1],ARGV[1],'NX') +return redis.call('GET',KEYS[1]) +` + +// Close releases every connection pool owned by the client. +func (c *Client) Close() error { + var result error + for _, node := range c.nodes { + result = errors.Join(result, node.Close()) + } + return result +} diff --git a/internal/valkey/client_test.go b/internal/valkey/client_test.go new file mode 100644 index 0000000..f45e894 --- /dev/null +++ b/internal/valkey/client_test.go @@ -0,0 +1,247 @@ +package valkey + +import ( + "context" + "errors" + "os" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/google/uuid" + + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/admission" + "gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/model/dto" +) + +// testClient connects only to explicitly configured disposable Valkey shards. +func testClient(t *testing.T) *Client { + t.Helper() + address := os.Getenv("TEST_VALKEY_ADDRESS") + if address == "" { + t.Skip("TEST_VALKEY_ADDRESS is required for Valkey integration tests") + } + client, err := New(Options{Nodes: map[string]string{"test": address}, Coordinator: "test", PoolSize: 16, Timeout: time.Second}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := client.Close(); err != nil { + t.Error(err) + } + }) + if err := client.Ping(t.Context()); err != nil { + t.Fatal(err) + } + return client +} + +// testLimits builds independent small budgets for admission scenarios. +func testLimits() Limits { + return Limits{ + Global: Budget{Rate: 10000, Slots: 100}, + User: Budget{Rate: 1000, Slots: 3}, + Groups: map[string]Budget{ + "auth": {Rate: 100, Slots: 4}, + "read": {Rate: 1000, Slots: 50}, + "write": {Rate: 1000, Slots: 50}, + }, + UserGroups: map[string]Budget{ + "auth": {Rate: 10, Slots: 1}, + "read": {Rate: 100, Slots: 2}, + "write": {Rate: 100, Slots: 2}, + }, + } +} + +// TestPerUserSlotsAreAtomic verifies concurrent reservations cannot exceed the group budget. +func TestPerUserSlotsAreAtomic(t *testing.T) { + client := testClient(t) + limiter, err := NewLimiter(client, testLimits()) + if err != nil { + t.Fatal(err) + } + user := uuid.NewString() + var accepted atomic.Int32 + releases := make(chan func(), 30) + failures := make(chan error, 30) + var wg sync.WaitGroup + for range 30 { + wg.Go(func() { + release, err := limiter.Enter(t.Context(), "read", user) + if err == nil { + accepted.Add(1) + releases <- release + } else if !errors.Is(err, admission.ErrSlots) { + failures <- err + } + }) + } + wg.Wait() + close(releases) + close(failures) + for release := range releases { + release() + } + for err := range failures { + t.Error(err) + } + if accepted.Load() != 2 { + t.Fatalf("accepted %d, want exactly two", accepted.Load()) + } +} + +// TestUserGroupsHaveSeparateSlots verifies read saturation does not consume write-group capacity. +func TestUserGroupsHaveSeparateSlots(t *testing.T) { + client := testClient(t) + limiter, err := NewLimiter(client, testLimits()) + if err != nil { + t.Fatal(err) + } + user := uuid.NewString() + for range 2 { + release, err := limiter.Enter(t.Context(), "read", user) + if err != nil { + t.Fatal(err) + } + t.Cleanup(release) + } + release, err := limiter.Enter(t.Context(), "write", user) + if err != nil { + t.Fatal(err) + } + release() +} + +// TestUserRateSurvivesSlotRelease checks that release returns concurrency but not rate tokens. +func TestUserRateSurvivesSlotRelease(t *testing.T) { + client := testClient(t) + limits := testLimits() + limits.User.Rate = 1 + limiter, err := NewLimiter(client, limits) + if err != nil { + t.Fatal(err) + } + user := uuid.NewString() + release, err := limiter.Enter(t.Context(), "read", user) + if err != nil { + t.Fatal(err) + } + release() + if _, err := limiter.Enter(t.Context(), "read", user); !errors.Is(err, admission.ErrRate) { + t.Fatalf("rate budget reset after release: %v", err) + } +} + +// TestAdmissionRejectsUnavailableBackend verifies fail-closed behavior without network fallback. +func TestAdmissionRejectsUnavailableBackend(t *testing.T) { + client := testClient(t) + limiter, err := NewLimiter(client, testLimits()) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(t.Context()) + cancel() + if _, err := limiter.Enter(ctx, "read", ""); err == nil { + t.Fatal("canceled backend operation admitted request") + } +} + +// TestGlobalAdmissionReleasesSlots verifies the coordinator path and release semantics. +func TestGlobalAdmissionReleasesSlots(t *testing.T) { + client := testClient(t) + limiter, err := NewLimiter(client, testLimits()) + if err != nil { + t.Fatal(err) + } + release, err := limiter.Enter(t.Context(), "read", "") + if err != nil { + t.Fatal(err) + } + release() +} + +// TestProfileRoundTrip checks storage and ownership of immutable cached profiles. +func TestProfileRoundTrip(t *testing.T) { + client := testClient(t) + user := dto.UserDTO{ID: uuid.New(), Username: "synthetic", DisplayName: "Test", CreatedAt: time.Now().UTC()} + client.PutProfile(t.Context(), user) + got, ok := client.GetProfile(t.Context(), user.ID) + if !ok || got.ID != user.ID || got.Username != user.Username { + t.Fatal("profile cache did not preserve account") + } +} + +// TestProfileMissHandlesCorruption verifies corrupt cache entries cannot impersonate another account. +func TestProfileMissHandlesCorruption(t *testing.T) { + client := testClient(t) + id := uuid.New() + key := "profile:" + id.String() + if err := client.nodes[client.ring.Owner(key)].Set(t.Context(), key, `{"ID":"00000000-0000-0000-0000-000000000000"}`, time.Minute).Err(); err != nil { + t.Fatal(err) + } + if _, ok := client.GetProfile(t.Context(), id); ok { + t.Fatal("corrupt profile accepted") + } +} + +// TestProfileBackendFailureIsMiss checks optional-cache fallback semantics. +func TestProfileBackendFailureIsMiss(t *testing.T) { + client := testClient(t) + ctx, cancel := context.WithCancel(t.Context()) + cancel() + if _, ok := client.GetProfile(ctx, uuid.New()); ok { + t.Fatal("failed cache read treated as hit") + } + client.PutProfile(ctx, dto.UserDTO{ID: uuid.New()}) + if err := client.Ping(ctx); err == nil { + t.Fatal("canceled readiness succeeded") + } +} + +// TestClientRejectsInvalidConfiguration covers configuration boundaries without opening sockets. +func TestClientRejectsInvalidConfiguration(t *testing.T) { + cases := []Options{ + {}, + {Nodes: map[string]string{"a": ""}, Coordinator: "a", PoolSize: 1, Timeout: time.Second}, + {Nodes: map[string]string{"": "localhost:6379"}, Coordinator: "", PoolSize: 1, Timeout: time.Second}, + {Nodes: map[string]string{"a": "localhost:6379"}, Coordinator: "b", PoolSize: 1, Timeout: time.Second}, + } + for index, options := range cases { + t.Run(string(rune('a'+index)), func(t *testing.T) { + if _, err := New(options); err == nil { + t.Fatal("invalid client options accepted") + } + }) + } +} + +// TestLimiterRejectsInvalidConfiguration verifies policy completeness. +func TestLimiterRejectsInvalidConfiguration(t *testing.T) { + if _, err := NewLimiter(nil, testLimits()); err == nil { + t.Fatal("nil client accepted") + } + client := testClient(t) + limits := testLimits() + delete(limits.Groups, "auth") + if _, err := NewLimiter(client, limits); err == nil { + t.Fatal("missing group accepted") + } + limiter, err := NewLimiter(client, testLimits()) + if err != nil { + t.Fatal(err) + } + if _, err := limiter.Enter(t.Context(), "unknown", ""); err == nil { + t.Fatal("unknown group accepted") + } +} + +// TestTopologyMismatchFailsClosed prevents replicas from silently splitting user budgets. +func TestTopologyMismatchFailsClosed(t *testing.T) { + client := testClient(t) + client.topology = "different-ring" + if err := client.Ping(t.Context()); err == nil { + t.Fatal("incompatible topology accepted") + } +} diff --git a/scripts/analyze-incidents.rb b/scripts/analyze-incidents.rb new file mode 100644 index 0000000..5fc1509 --- /dev/null +++ b/scripts/analyze-incidents.rb @@ -0,0 +1,20 @@ +require 'json' +require 'csv' +root=ARGV.fetch(0) +events=CSV.read("#{root}/events.csv").map{|t,p,e| [t.to_i,p,e]} +snapshots=File.readlines("#{root}/alerts.jsonl").map{|l|JSON.parse(l)} +rows=Dir["#{root}/http-*.csv"].flat_map{|p|CSV.read(p)} +summary={phases: {}, alerts: {}} +rows.group_by{|r|r[1]}.each do |phase,rs| + vals=rs.map{|r|r[4].to_f*1000}.sort + start=events.find{|t,p,e|p==phase && e=='start'}&.first + finish=events.reverse.find{|t,p,e|p==phase && e=='end'}&.first + summary[:phases][phase]={requests:rs.size,errors:rs.count{|r|r[3]!='200'},p50_ms:vals[(vals.size*0.5).floor],p95_ms:vals[(vals.size*0.95).floor],p99_ms:vals[(vals.size*0.99).floor],duration_seconds: start && finish ? finish-start : nil} +end +%w[MessengerAPIBurnRate MessengerReadyBurnRate MessengerQueueLag MessengerTelemetryDrops].each do |name| + firing=snapshots.select{|s|s['alerts'].any?{|a|a['labels']['alertname']==name && a['state']=='firing'}} + first=firing.first + resolved=first && snapshots.find{|s|s['at']>firing.last['at'] && !s['alerts'].any?{|a|a['labels']['alertname']==name}} + summary[:alerts][name]={first_firing:first && first['at'],last_firing:firing.last && firing.last['at'],first_resolved:resolved && resolved['at'],firing_samples:firing.size} +end +File.write("#{root}/summary.json",JSON.pretty_generate(summary)+"\n") diff --git a/scripts/coverage.awk b/scripts/coverage.awk new file mode 100644 index 0000000..982f844 --- /dev/null +++ b/scripts/coverage.awk @@ -0,0 +1,35 @@ +# Aggregate covered statements once per merged block; keep files with no statements as N/A. +FNR == NR { + split($0, fields, "\t") + paths[fields[1]] = fields[2] + hashes[fields[1]] = fields[3] + executable[fields[1]] = fields[4] + next +} +/^mode:/ { next } +{ + file = $1 + sub(/:[^:]*$/, "", file) + if (!(file in paths)) { + print "unknown coverage source: " file > "/dev/stderr" + invalid = 1 + } + total[file] += $2 + if ($3 > 0) covered[file] += $2 +} +END { + print "{\"threshold_per_file\":90,\"files\":[" > summary + separator = "" + for (file in paths) { + percent = total[file] ? sprintf("%.8f", covered[file] * 100 / total[file]) : "null" + printf "%s{\"file\":\"%s\",\"statements\":%d,\"covered\":%d,\"percent\":%s,\"sha256\":\"%s\"}", separator, paths[file], total[file], covered[file], percent, hashes[file] > summary + separator = ",\n" + printf "%8s %s\n", (total[file] ? sprintf("%.2f%%", covered[file] * 100 / total[file]) : "N/A"), paths[file] + if (covered[file] * 100 < total[file] * 90 || (executable[file] && !total[file])) invalid = 1 + } + printf "\n],\"passed\":%s}\n", (invalid ? "false" : "true") > summary + if (invalid) { + print "FAIL: per-file coverage must reach 90%" > "/dev/stderr" + exit 1 + } +} diff --git a/scripts/coverage.sh b/scripts/coverage.sh new file mode 100755 index 0000000..bfd6b51 --- /dev/null +++ b/scripts/coverage.sh @@ -0,0 +1,91 @@ +#!/bin/sh +# Measure unit and real-process coverage, enforcing 90% separately for each Go source file. +set -eu +cd "$(dirname "$0")/.." +GO=${GO:-go} +: "${DATABASE_URL:?Use an isolated test database}" "${VALKEY_SHARDS:?Use test shards}" +: "${TEST_DATABASE_URL:?Use an isolated test database}" "${TEST_VALKEY_ADDRESS:?Use a test shard}" +export APP_PORT=${COVERAGE_PORT:-18080} +export LOG_LEVEL=error +export CURSOR_SIGNING_KEY=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= +worker_pid= +server_pid= +watchdog_pid= +mkdir -p coverage +run_dir=$(mktemp -d "$PWD/coverage/run.XXXXXX") +cleanup() { + if [ -n "$worker_pid" ]; then kill "$worker_pid" 2>/dev/null || true; wait "$worker_pid" 2>/dev/null || true; fi + if [ -n "$server_pid" ]; then kill "$server_pid" 2>/dev/null || true; wait "$server_pid" 2>/dev/null || true; fi + if [ -n "$watchdog_pid" ]; then kill "$watchdog_pid" 2>/dev/null || true; wait "$watchdog_pid" 2>/dev/null || true; fi + rm -rf "$run_dir" +} +trap cleanup EXIT +trap 'exit 129' HUP +trap 'exit 130' INT +trap 'exit 143' TERM +mkdir "$run_dir/process" +"$GO" build -race -cover -covermode=atomic -coverpkg=./... -o "$run_dir/server" ./cmd/server +"$run_dir/server" migrate +"$GO" test -count=1 -race -coverpkg=./... -coverprofile="$run_dir/unit.out" ./... +export GOCOVERDIR="$run_dir/process" +if "$run_dir/server" unknown >"$run_dir/server-invalid.log" 2>&1; then + echo 'server accepted an unknown command' >&2 + exit 1 +else + test "$?" -eq 1 +fi +"$run_dir/server" worker >"$run_dir/worker.log" 2>&1 & +worker_pid=$! +"$run_dir/server" >"$run_dir/server.log" 2>&1 & +server_pid=$! +ready=false +for attempt in $(seq 1 60); do + kill -0 "$server_pid" 2>/dev/null || { cat "$run_dir/server.log"; exit 1; } + if curl --fail --silent --max-time 1 "http://localhost:$APP_PORT/health/ready" >/dev/null; then ready=true; break; fi + sleep .25 +done +$ready || { cat "$run_dir/server.log"; exit 1; } +"$run_dir/server" healthcheck +"$run_dir/server" migrate +"${PYTHON:-python3}" tests/smoke.py --base-url "http://localhost:$APP_PORT" +kill -TERM "$server_pid" +# Bound the real-process shutdown check; a forced kill makes wait fail. +(sleep 12; kill -KILL "$server_pid" 2>/dev/null || true) & +watchdog_pid=$! +wait "$server_pid" +server_pid= +kill "$watchdog_pid" 2>/dev/null || true +wait "$watchdog_pid" 2>/dev/null || true +watchdog_pid= +kill -TERM "$worker_pid" +wait "$worker_pid" +worker_pid= +unset GOCOVERDIR +"$GO" tool covdata textfmt -i="$run_dir/process" -o="$run_dir/process.out" +awk ' +/^mode:/ { if ($0 != "mode: atomic") exit 1; next } +{ + if (($1 in statements) && statements[$1] != $2) exit 1; + statements[$1] = $2; hits[$1] += $3 +} +END { + print "mode: atomic"; + for (block in statements) print block, statements[block], hits[block] +}' "$run_dir/unit.out" "$run_dir/process.out" > coverage/coverage.out +"$GO" list -f '{{.ImportPath}} {{.Dir}} {{join .GoFiles ","}}' ./... > "$run_dir/packages.txt" +: > "$run_dir/inventory.tsv" +while read -r import_path directory names; do + old_ifs=$IFS + IFS=, + for name in $names; do + checksum=$(shasum -a 256 "$directory/$name") + checksum=${checksum%% *} + relative=${directory#"$PWD/"}/$name + executable=$(awk '/^func[ (]/ {found=1} END {print found+0}' "$directory/$name") + printf '%s\t%s\t%s\t%s +' "$import_path/$name" "$relative" "$checksum" "$executable" >> "$run_dir/inventory.tsv" + done + IFS=$old_ifs +done < "$run_dir/packages.txt" +awk -v summary=coverage/summary.json -f scripts/coverage.awk "$run_dir/inventory.tsv" coverage/coverage.out +"$GO" version > coverage/go-version.txt diff --git a/scripts/init-secrets.sh b/scripts/init-secrets.sh new file mode 100755 index 0000000..c60e9a8 --- /dev/null +++ b/scripts/init-secrets.sh @@ -0,0 +1,42 @@ +#!/bin/sh +# Create lab credentials once, outside Helm values and release history. +set -eu +: "${KUBE_CONTEXT:=dev}" "${NAMESPACE:=highload-applications}" +: "${REGISTRY:=asisti.registry.twcstorage.ru}" "${REGISTRY_USER:=token}" +: "${REGISTRY_TOKEN_FILE:=$HOME/.secrets/twc-registry-token}" +test -s "$REGISTRY_TOKEN_FILE" || { echo 'Registry token file is missing or empty' >&2; exit 1; } +umask 077 +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT +kubectl --context "$KUBE_CONTEXT" create namespace "$NAMESPACE" --dry-run=client -o yaml | kubectl --context "$KUBE_CONTEXT" apply -f - +if ! kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" get secret messenger-secrets >/dev/null 2>&1; then + openssl rand -hex 32 | tr -d '\n' > "$work/postgres-password" + printf 'postgres://messenger:%s@postgres:5432/messenger?sslmode=disable' "$(cat "$work/postgres-password")" > "$work/database-url" + openssl rand -base64 32 | tr -d '\n' > "$work/cursor-key" + kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" create secret generic messenger-secrets --from-file="$work/postgres-password" --from-file="$work/database-url" --from-file="$work/cursor-key" +fi +# Add the shared cursor key to an existing lab2 secret without replacing database credentials. +if ! kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" get secret messenger-secrets -o json | jq -e '.data["cursor-key"] | length > 0' >/dev/null; then + openssl rand -base64 32 | tr -d '\n' > "$work/cursor-key" + jq -n --rawfile key "$work/cursor-key" '{stringData:{"cursor-key":$key}}' > "$work/patch.json" + kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" patch secret messenger-secrets --type merge --patch-file "$work/patch.json" +fi +# Build a portable Docker config; macOS credential-helper references cannot be used by kubelet. +jq -n --arg registry "$REGISTRY" --arg user "$REGISTRY_USER" --rawfile token "$REGISTRY_TOKEN_FILE" \ + '{auths:{($registry):{auth:(($user+":"+($token | sub("[\\r\\n]+$"; "")))|@base64)}}}' > "$work/config.json" +kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" create secret generic registry-pull \ + --type=kubernetes.io/dockerconfigjson --from-file=".dockerconfigjson=$work/config.json" \ + --dry-run=client -o yaml | kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" apply -f - +if ! kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" get secret messenger-s3 >/dev/null 2>&1; then + printf 'operator' > "$work/root-user" + printf 'messenger' > "$work/access-key" + openssl rand -hex 32 | tr -d '\n' > "$work/root-password" + openssl rand -hex 32 | tr -d '\n' > "$work/secret-key" +jq -n --rawfile admin "$work/root-user" --rawfile adminSecret "$work/root-password" --rawfile app "$work/access-key" --rawfile appSecret "$work/secret-key" '{identities:[{name:"operator",credentials:[{accessKey:$admin,secretKey:$adminSecret}],actions:["Admin","Read","Write","List"]},{name:"messenger",credentials:[{accessKey:$app,secretKey:$appSecret}],actions:["Read:attachments","Write:attachments","List:attachments"]}]}' > "$work/identities.json" + kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" create secret generic messenger-s3 --from-file="$work/root-user" --from-file="$work/root-password" --from-file="$work/access-key" --from-file="$work/secret-key" --from-file="$work/identities.json" +fi +if ! kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" get secret messenger-grafana >/dev/null 2>&1; then + printf 'admin' > "$work/username" + openssl rand -hex 24 | tr -d '\n' > "$work/password" + kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" create secret generic messenger-grafana --from-file="$work/username" --from-file="$work/password" +fi diff --git a/scripts/local-action.sh b/scripts/local-action.sh new file mode 100755 index 0000000..4290382 --- /dev/null +++ b/scripts/local-action.sh @@ -0,0 +1,59 @@ +#!/bin/sh +# Operate only on the isolated processes and containers recorded by test-local.sh. +set -eu +exec 2>> evidence/local/actions-errors.txt +: "${LOCAL_TEST_WORK:?Run through scripts/test-local.sh}" +. "$LOCAL_TEST_WORK/environment" +stop_api() { + label=$1 + pid=$(cat "$LOCAL_TEST_WORK/$label.pid") + if kill -0 "$pid" 2>/dev/null; then + state=$(ps -p "$pid" -o stat=) + case "$state" in Z*) return ;; esac + command=$(ps -p "$pid" -o command=) + case "$command" in "$LOCAL_TEST_WORK/server"|"$LOCAL_TEST_WORK/server worker") ;; *) echo 'API PID ownership mismatch' >&2; exit 1 ;; esac + kill -TERM "$pid" + for attempt in $(seq 1 60); do kill -0 "$pid" 2>/dev/null || break; sleep .1; done + fi +} +start_api() { + label=$1 + port=$2 + INSTANCE_ID="$label" APP_PORT="$port" nohup "$LOCAL_TEST_WORK/server" > "$LOCAL_TEST_WORK/$label.log" 2>&1 < /dev/null & + echo "$!" > "$LOCAL_TEST_WORK/$label.pid" +} +if [ "${LOCAL_ACTION:-restart}" = failover ]; then + case "${TARGET_INSTANCE_ID:?Missing observed replica}" in local-a|local-b) ;; *) echo 'Unknown isolated replica' >&2; exit 1 ;; esac + printf "%s target=%s action=stop\n" "$(date -u +%FT%TZ)" "$TARGET_INSTANCE_ID" >> evidence/local/actions.txt + stop_api "$TARGET_INSTANCE_ID" + exit 0 +fi +if [ "${LOCAL_ACTION:-restart}" = pause-worker ]; then + stop_api worker + printf "%s action=stop-worker\n" "$(date -u +%FT%TZ)" >> evidence/local/actions.txt + exit 0 +fi +stop_api local-a +stop_api local-b +stop_api worker +printf "%s action=restart-api-worker-postgres-valkey-s3-broker retained-container-filesystems\n" "$(date -u +%FT%TZ)" >> evidence/local/actions.txt +docker restart "$LOCAL_TEST_PG" "$LOCAL_TEST_VK" "$LOCAL_TEST_S3" "$LOCAL_TEST_BROKER" >/dev/null +# Docker may assign new host ports when restarting containers with ephemeral bindings. +export DATABASE_URL="postgres://postgres@$(docker port "$LOCAL_TEST_PG" 5432/tcp)/messenger?sslmode=disable" +export VALKEY_SHARDS="shard0=$(docker port "$LOCAL_TEST_VK" 6379/tcp)" +export S3_ENDPOINT="$(docker port "$LOCAL_TEST_S3" 9000/tcp)" +export BROKER_ADDRESS="$(docker port "$LOCAL_TEST_BROKER" 6379/tcp)" +for attempt in $(seq 1 60); do + if docker exec "$LOCAL_TEST_PG" pg_isready -U postgres >/dev/null 2>&1 && "$LOCAL_TEST_WORK/server" s3-bootstrap >/dev/null 2>&1; then break; fi + sleep 1 +done +nohup "$LOCAL_TEST_WORK/server" worker > "$LOCAL_TEST_WORK/worker.log" 2>&1 < /dev/null & +echo "$!" > "$LOCAL_TEST_WORK/worker.pid" +start_api local-a "$LOCAL_TEST_PORT_A" +start_api local-b "$LOCAL_TEST_PORT_B" +for attempt in $(seq 1 60); do + if curl --fail --silent --max-time 1 "http://127.0.0.1:$LOCAL_TEST_PORT_A/health/ready" >/dev/null && curl --fail --silent --max-time 1 "http://127.0.0.1:$LOCAL_TEST_PORT_B/health/ready" >/dev/null; then exit 0; fi + sleep 1 +done +echo 'Isolated restart readiness timed out' >&2 +exit 1 diff --git a/scripts/observability-incidents.sh b/scripts/observability-incidents.sh new file mode 100755 index 0000000..a2cdfac --- /dev/null +++ b/scripts/observability-incidents.sh @@ -0,0 +1,100 @@ +#!/bin/sh +# Bounded fault experiments on the disposable test-local.sh stack only. +set -eu +: "${LOCAL_TEST_WORK:?}" "${BASE_URL:?}" "${PROMETHEUS_URL:?}" "${TELEMETRY_TEMPO:?}" +. "$LOCAL_TEST_WORK/environment" +export PROBE_WORK="$LOCAL_TEST_WORK/probe" +mkdir -p evidence/incidents +scripts/prepare-probe.sh +scripts/trace-probe.sh > evidence/current/correlation.txt 2>&1 +worker=$(cat "$LOCAL_TEST_WORK/worker.pid") +api_a=$(cat "$LOCAL_TEST_WORK/local-a.pid") +api_b=$(cat "$LOCAL_TEST_WORK/local-b.pid") +chat=$(cat "$PROBE_WORK/chat-id") +load_pids= +paused_db=false +paused_worker=false +stopped_tempo=false +cleanup() { + for pid in $load_pids; do kill "$pid" 2>/dev/null || true; done + if $paused_db; then docker unpause "$LOCAL_TEST_PG" >/dev/null; fi + if $paused_worker; then kill -CONT "$worker"; fi + if $stopped_tempo; then docker start "$TELEMETRY_TEMPO" >/dev/null; fi +} +trap cleanup EXIT +printf 'baseline\n' > "$PROBE_WORK/phase" +for client in 1 2; do + ( + while :; do + now=$(date +%s) + phase=$(cat "$PROBE_WORK/phase") + result=$(curl -sS --max-time 4 --config "$PROBE_WORK/auth.conf" -o /dev/null -w '%{http_code},%{time_total}' "$BASE_URL/api/v1/chats/$chat/messages" 2>/dev/null || true) + printf '%s,%s,%s,%s\n' "$now" "$phase" "$client" "$result" + sleep .25 + done + ) > "evidence/incidents/http-$client.csv" & + load_pids="$load_pids $!" +done +snapshot() { + phase=$1 + now=$(date +%s) + curl -fsS "$PROMETHEUS_URL/api/v1/alerts" > "$PROBE_WORK/alerts.json" + jq -c --arg phase "$phase" --argjson at "$now" '{at:$at,phase:$phase,alerts:.data.alerts}' "$PROBE_WORK/alerts.json" >> evidence/incidents/alerts.jsonl + printf '%s %s\n' "$now" "$phase" >> evidence/incidents/rss.txt + ps -p "$api_a,$api_b,$worker" -o pid=,rss=,time= >> evidence/incidents/rss.txt +} +phase() { + name=$1 + seconds=$2 + printf '%s\n' "$name" > "$PROBE_WORK/phase" + printf '%s,%s,start\n' "$(date +%s)" "$name" >> evidence/incidents/events.csv + deadline=$(($(date +%s)+seconds)) + while [ "$(date +%s)" -lt "$deadline" ]; do snapshot "$name"; sleep 5; done + printf '%s,%s,end\n' "$(date +%s)" "$name" >> evidence/incidents/events.csv +} +: > evidence/incidents/events.csv +: > evidence/incidents/alerts.jsonl +: > evidence/incidents/rss.txt +phase baseline 120 +docker pause "$LOCAL_TEST_PG" >/dev/null +paused_db=true +phase short-db-spike 5 +docker unpause "$LOCAL_TEST_PG" >/dev/null +paused_db=false +phase spike-recovery 90 +docker pause "$LOCAL_TEST_PG" >/dev/null +paused_db=true +phase sustained-db-outage 180 +docker unpause "$LOCAL_TEST_PG" >/dev/null +paused_db=false +phase db-recovery 90 +kill -STOP "$worker" +paused_worker=true +printf 'worker-stopped\n' > "$PROBE_WORK/phase" +printf '%s,worker-stopped,start\n' "$(date +%s)" >> evidence/incidents/events.csv +for attempt in $(seq 1 20); do + status=$(curl -sS --config "$PROBE_WORK/auth.conf" -o "$PROBE_WORK/queued.json" -w '%{http_code}' "$BASE_URL/api/v1/chats/$chat/attachments?filename=probe.png" -H 'Content-Type: image/png' --data-binary @tests/fixtures/sample.png) + test "$status" = 202 + jq -e '.status=="queued"' "$PROBE_WORK/queued.json" >/dev/null + sleep 1 +done +phase worker-stopped 90 +kill -CONT "$worker" +paused_worker=false +phase worker-recovery 90 +docker stop --time 5 "$TELEMETRY_TEMPO" >/dev/null +stopped_tempo=true +phase trace-backend-outage 45 +make test > evidence/incidents/smoke-with-tempo-down.txt 2>&1 +docker start "$TELEMETRY_TEMPO" >/dev/null +stopped_tempo=false +phase telemetry-recovery 20 +for query in 'max(messenger_queue_oldest_seconds)' 'max(messenger_queue_depth)' 'sum(rate(messenger_http_requests_total{status_class="5xx"}[1m]))' 'sum(otelcol_exporter_send_failed_spans_total)' 'sum(otelcol_exporter_queue_size)'; do + curl -fsS --get "$PROMETHEUS_URL/api/v1/query" --data-urlencode "query=$query" | jq -c --arg query "$query" '{query:$query,response:.}' >> evidence/incidents/final-metrics.jsonl +done +jq -s -e '[.[]|select(.phase=="sustained-db-outage" or .phase=="db-recovery")|.alerts[]|select(.labels.alertname=="MessengerAPIBurnRate" and .state=="firing")]|length>0' evidence/incidents/alerts.jsonl >/dev/null +jq -s -e '[.[]|select(.phase=="worker-stopped")|.alerts[]|select(.labels.alertname=="MessengerQueueLag" and .state=="firing")]|length>0' evidence/incidents/alerts.jsonl >/dev/null +jq -s -e '[.[]|select(.phase=="short-db-spike" or .phase=="spike-recovery")|.alerts[]|select(.labels.alertname=="MessengerAPIBurnRate" and .state=="firing")]|length==0' evidence/incidents/alerts.jsonl >/dev/null +jq -s -e '[.[]|select(.phase=="worker-recovery")|.alerts[]|select(.labels.alertname=="MessengerReadyBurnRate" and .state=="firing")]|length>0' evidence/incidents/alerts.jsonl >/dev/null +tail -n 1 evidence/incidents/alerts.jsonl | jq -e '[.alerts[]|select(.labels.alertname=="MessengerAPIBurnRate" or .labels.alertname=="MessengerQueueLag" or .labels.alertname=="MessengerReadyBurnRate")]|length==0' >/dev/null +printf 'PASS: API, queue and ready alerts resolved\n' diff --git a/scripts/port-forward.sh b/scripts/port-forward.sh new file mode 100755 index 0000000..e8d7d13 --- /dev/null +++ b/scripts/port-forward.sh @@ -0,0 +1,21 @@ +#!/bin/sh +# Reconnect the loopback tunnel after the API pod is replaced during restart tests. +set -eu +: "${KUBE_CONTEXT:=dev}" "${NAMESPACE:=highload-applications}" +LOCAL_PORT=${LOCAL_PORT:-8083} +child= +cleanup() { + if [ -n "$child" ]; then + kill "$child" 2>/dev/null || true + wait "$child" 2>/dev/null || true + fi + exit 0 +} +trap cleanup INT TERM +while :; do + kubectl --context "$KUBE_CONTEXT" -n traefik port-forward service/traefik "$LOCAL_PORT:80" --address 127.0.0.1 & + child=$! + wait "$child" || true + child= + sleep 1 +done diff --git a/scripts/prepare-probe.sh b/scripts/prepare-probe.sh new file mode 100755 index 0000000..166ed18 --- /dev/null +++ b/scripts/prepare-probe.sh @@ -0,0 +1,16 @@ +#!/bin/sh +# Create synthetic probe credentials in an explicit private directory; print no secrets. +set -eu +: "${BASE_URL:?}" "${PROBE_WORK:?}" +umask 077 +mkdir -p "$PROBE_WORK" +username="observe_$(openssl rand -hex 6)" +password=$(openssl rand -hex 16) +jq -n --arg username "$username" --arg password "$password" '{username:$username,display_name:"Observability probe",password:$password}' > "$PROBE_WORK/user.json" +curl -fsS "$BASE_URL/api/v1/users" -H 'Content-Type: application/json' --data-binary @"$PROBE_WORK/user.json" > "$PROBE_WORK/user-response.json" +printf 'user = "%s:%s"\n' "$username" "$password" > "$PROBE_WORK/basic.conf" +curl -fsS --config "$PROBE_WORK/basic.conf" -X POST "$BASE_URL/api/v1/auth/sessions" > "$PROBE_WORK/session.json" +printf 'header = "Authorization: Bearer %s"\n' "$(jq -r .token "$PROBE_WORK/session.json")" > "$PROBE_WORK/auth.conf" +jq -n --arg id "$(jq -r .id "$PROBE_WORK/user-response.json")" '{title:"Observability probe",member_ids:[$id]}' > "$PROBE_WORK/chat.json" +curl -fsS --config "$PROBE_WORK/auth.conf" "$BASE_URL/api/v1/chats" -H 'Content-Type: application/json' --data-binary @"$PROBE_WORK/chat.json" > "$PROBE_WORK/chat-response.json" +printf '%s\n' "$(jq -r .id "$PROBE_WORK/chat-response.json")" > "$PROBE_WORK/chat-id" diff --git a/scripts/ready-slo.sh b/scripts/ready-slo.sh new file mode 100755 index 0000000..33d30da --- /dev/null +++ b/scripts/ready-slo.sh @@ -0,0 +1,35 @@ +#!/bin/sh +# Verify time-to-ready burn against a real suspended worker on an owned test stack. +set -eu +: "${LOCAL_TEST_WORK:?}" "${BASE_URL:?}" "${PROMETHEUS_URL:?}" +export PROBE_WORK="$LOCAL_TEST_WORK/ready-probe" +scripts/prepare-probe.sh +worker=$(cat "$LOCAL_TEST_WORK/worker.pid") +trap 'kill -CONT "$worker" 2>/dev/null || true' EXIT +mkdir -p evidence/ready-slo +: > evidence/ready-slo/alerts.jsonl +chat=$(cat "$PROBE_WORK/chat-id") +# Warm the histogram before suspension so rate() observes a real counter delta. +upload() { + curl -fsS --config "$PROBE_WORK/auth.conf" "$BASE_URL/api/v1/chats/$chat/attachments?filename=slo.png" -H 'Content-Type: image/png' --data-binary @tests/fixtures/sample.png > "$PROBE_WORK/attachment.json" +} +upload +sleep 15 +kill -STOP "$worker" +printf '%s,worker-stopped\n' "$(date +%s)" > evidence/ready-slo/events.csv +for i in $(seq 1 20); do upload; done +observe() { + phase=$1 + for i in $(seq 1 "$2"); do + curl -fsS "$PROMETHEUS_URL/api/v1/alerts" | jq -c --arg phase "$phase" --argjson at "$(date +%s)" '{at:$at,phase:$phase,alerts:.data.alerts}' >> evidence/ready-slo/alerts.jsonl + sleep 5 + done +} +observe worker-stopped 18 +kill -CONT "$worker" +printf '%s,worker-resumed\n' "$(date +%s)" >> evidence/ready-slo/events.csv +observe worker-recovery 24 +jq -s -e '[.[]|.alerts[]|select(.labels.alertname=="MessengerReadyBurnRate" and .state=="firing")]|length>0' evidence/ready-slo/alerts.jsonl >/dev/null +tail -n 1 evidence/ready-slo/alerts.jsonl | jq -e '[.alerts[]|select(.labels.alertname=="MessengerReadyBurnRate" or .labels.alertname=="MessengerQueueLag")]|length==0' >/dev/null +curl -fsS --get "$PROMETHEUS_URL/api/v1/query" --data-urlencode 'query=max(messenger_queue_depth)' > evidence/ready-slo/final-backlog.json +printf 'PASS: ready burn and queue lag resolved after real worker recovery\n' diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 0000000..53e1d28 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,27 @@ +#!/bin/sh +# Restart this lab's API, worker and persistent stores without removing PVCs. +set -eu +: "${KUBE_CONTEXT:=dev}" "${NAMESPACE:=highload-applications}" +if kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" get hpa api >/dev/null 2>&1; then + echo 'Disable autoscaling in the lab release before this restart test, then restore it.' >&2 + exit 1 +fi +replicas=$(kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" get deployment api -o jsonpath='{.spec.replicas}') +worker_replicas=$(kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" get deployment worker -o jsonpath='{.spec.replicas}') +restore() { + kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" scale deployment/api --replicas="$replicas" >/dev/null + kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" scale deployment/worker --replicas="$worker_replicas" >/dev/null +} +trap restore EXIT +mkdir -p evidence +kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" get pods -l 'app in (api,worker,postgres,valkey,s3,broker)' -o custom-columns=NAME:.metadata.name,UID:.metadata.uid > evidence/restart-before.txt +kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" scale deployment/api deployment/worker --replicas=0 +kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" wait --for=delete pod -l 'app in (api,worker)' --timeout=60s +kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" rollout restart statefulset/postgres statefulset/valkey statefulset/s3 statefulset/broker +for name in postgres valkey s3 broker; do + kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" rollout status "statefulset/$name" --timeout=120s +done +restore +kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" rollout status deployment/api --timeout=120s +kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" rollout status deployment/worker --timeout=120s +kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" get pods -l 'app in (api,worker,postgres,valkey,s3,broker)' -o custom-columns=NAME:.metadata.name,UID:.metadata.uid > evidence/restart-after.txt diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 0000000..b06f823 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,10 @@ +#!/bin/sh +# Terminate the exact observed API pod; its replacement receives a different instance ID. +set -eu +: "${KUBE_CONTEXT:=dev}" "${NAMESPACE:=highload-applications}" "${TARGET_INSTANCE_ID:?Expected API pod name}" +case "$TARGET_INSTANCE_ID" in api-*) ;; *) echo 'Target is not a lab3 API pod' >&2; exit 2 ;; esac +label=$(kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" get pod "$TARGET_INSTANCE_ID" -o jsonpath='{.metadata.labels.app}') +test "$label" = api +mkdir -p evidence +printf '%s target=%s\n' "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" "$TARGET_INSTANCE_ID" >> evidence/failover-targets.txt +kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" delete pod "$TARGET_INSTANCE_ID" --wait=false diff --git a/scripts/telemetry-local.sh b/scripts/telemetry-local.sh new file mode 100755 index 0000000..cd122f0 --- /dev/null +++ b/scripts/telemetry-local.sh @@ -0,0 +1,69 @@ +#!/bin/sh +# Isolated telemetry backends with explicit state ownership; never touches shared monitoring. +set -eu +cd "$(dirname "$0")/.." +: "${LOCAL_TELEMETRY_STATE:?Set an explicit private state file path}" +if [ "${1:-up}" = down ]; then + . "$LOCAL_TELEMETRY_STATE" + for id in $TELEMETRY_CONTAINERS; do docker rm -f "$id" >/dev/null; done + docker network rm "$TELEMETRY_NETWORK" >/dev/null + rm -rf "$TELEMETRY_WORK" + rm "$LOCAL_TELEMETRY_STATE" + exit 0 +fi +test ! -e "$LOCAL_TELEMETRY_STATE" || { echo 'State file already exists; stop its owned stack first' >&2; exit 1; } +umask 077 +work=$(mktemp -d) +network="messenger-observe-$(openssl rand -hex 6)" +docker network create "$network" >/dev/null +containers= +success=false +cleanup() { + if ! $success; then + for id in $containers; do docker rm -f "$id" >/dev/null; done + docker network rm "$network" >/dev/null + rm -rf "$work" + fi +} +trap cleanup EXIT +config="$PWD/charts/messenger/files/observability" +start() { + name=$1 + image=$2 + port=$3 + shift 3 + mkdir "$work/$name" + chmod 777 "$work/$name" + id=$(docker run -d --network "$network" --network-alias "messenger-$name" -p "127.0.0.1::$port" -v "$config:/config:ro" -v "$work/$name:/data" "$image" "$@") + containers="$containers $id" +} +start tempo grafana/tempo:2.8.2 3200 -config.file=/config/tempo.yaml +tempo=$id +start loki grafana/loki:3.5.5 3100 -config.file=/config/loki.yaml +loki=$id +start prometheus prom/prometheus:v3.5.0 9090 --config.file=/config/prometheus.yaml --storage.tsdb.path=/data --storage.tsdb.retention.time=24h +prometheus=$id +start collector otel/opentelemetry-collector-contrib:0.135.0 4318 --config=/config/collector.yaml +collector=$id +mkdir "$work/grafana" +chmod 777 "$work/grafana" +printf 'GF_SECURITY_ADMIN_USER=admin\nGF_SECURITY_ADMIN_PASSWORD=%s\nGF_PATHS_DATA=/data\nGF_ANALYTICS_REPORTING_ENABLED=false\n' "$(openssl rand -hex 24)" > "$work/grafana.env" +grafana=$(docker run -d --network "$network" --network-alias messenger-grafana -p 127.0.0.1::3000 --env-file "$work/grafana.env" -v "$work/grafana:/data" -v "$config/datasources.yaml:/etc/grafana/provisioning/datasources/datasources.yaml:ro" -v "$config/dashboards.yaml:/etc/grafana/provisioning/dashboards/dashboards.yaml:ro" -v "$config/messenger.json:/dashboards/messenger.json:ro" grafana/grafana:12.1.1) +containers="$containers $grafana" +cat > "$LOCAL_TELEMETRY_STATE" < "$work/user.json" +curl -fsS "$BASE_URL/api/v1/users" -H 'Content-Type: application/json' --data-binary @"$work/user.json" > "$work/user-response.json" +# Temporary curl config avoids placing credentials or bearer tokens in process arguments. +printf 'user = "%s:%s"\n' "$username" "$password" > "$work/basic.conf" +curl -fsS --config "$work/basic.conf" -X POST "$BASE_URL/api/v1/auth/sessions" > "$work/session.json" +printf 'header = "Authorization: Bearer %s"\n' "$(jq -r .token "$work/session.json")" > "$work/auth.conf" +jq -n --arg id "$(jq -r .id "$work/user-response.json")" '{title:"Durability",member_ids:[$id]}' > "$work/chat.json" +curl -fsS --config "$work/auth.conf" "$BASE_URL/api/v1/chats" -H 'Content-Type: application/json' --data-binary @"$work/chat.json" > "$work/chat-response.json" +chat=$(jq -r .id "$work/chat-response.json") +status=$(curl -sS --config "$work/auth.conf" -o "$work/attachment.json" -w '%{http_code}' "$BASE_URL/api/v1/chats/$chat/attachments?filename=sample.png" -H 'Content-Type: image/png' --data-binary @tests/fixtures/sample.png) +test "$status" = 202 +jq -e '.status=="queued"' "$work/attachment.json" >/dev/null +id=$(jq -r .id "$work/attachment.json") +jq -n --arg id "$id" '{text:"",attachment_ids:[$id]}' > "$work/message.json" +curl -fsS --config "$work/auth.conf" "$BASE_URL/api/v1/chats/$chat/messages" -H 'Content-Type: application/json' -H 'Idempotency-Key: durable-upload' --data-binary @"$work/message.json" > /dev/null +status=$(curl -sS --config "$work/auth.conf" -o /dev/null -w '%{http_code}' "$BASE_URL/api/v1/attachments/$id/content") +test "$status" = 409 +printf 'worker_stopped=true upload_status=202 snapshot=queued message_created=true content_status=409\n' +LOCAL_ACTION=restart scripts/local-action.sh +ready=false +for attempt in $(seq 1 60); do + curl -fsS --config "$work/auth.conf" "$BASE_URL/api/v1/attachments/$id" > "$work/status.json" + if jq -e '.status=="ready" and (.variants|length)==3' "$work/status.json" >/dev/null; then ready=true; break; fi + sleep 1 +done +$ready +curl -fsS --config "$work/auth.conf" "$BASE_URL/api/v1/attachments/$id/content" > "$work/original.png" +cmp tests/fixtures/sample.png "$work/original.png" +printf 'PASS: accepted intent survived API, worker, PostgreSQL, S3 and AOF broker restart; ready=1 variants=3 original_unchanged=true\n' diff --git a/scripts/test-local.sh b/scripts/test-local.sh new file mode 100755 index 0000000..6fe31c1 --- /dev/null +++ b/scripts/test-local.sh @@ -0,0 +1,112 @@ +#!/bin/sh +# Disposable integration harness; Helm remains the deployment source of truth. +set -eu +cd "$(dirname "$0")/.." +umask 077 +export LOCAL_TEST_WORK=$(mktemp -d) +containers= +cleanup() { + for label in local-a local-b worker; do + if [ -f "$LOCAL_TEST_WORK/$label.pid" ]; then kill -TERM "$(cat "$LOCAL_TEST_WORK/$label.pid")" 2>/dev/null || true; fi + done + for container in $containers; do docker rm -f "$container" >/dev/null; done + for label in local-a local-b worker; do if [ -f "$LOCAL_TEST_WORK/$label.log" ]; then cp "$LOCAL_TEST_WORK/$label.log" "evidence/local/$label.txt"; fi; done + rm -rf "$LOCAL_TEST_WORK" +} +trap cleanup EXIT +trap 'exit 130' INT TERM +mkdir -p evidence/local +pg=$(docker run -d -e POSTGRES_HOST_AUTH_METHOD=trust -e POSTGRES_DB=messenger -p 127.0.0.1::5432 postgres:17.6-alpine) +containers="$pg" +vk=$(docker run -d -p 127.0.0.1::6379 valkey/valkey:8.1.3-alpine) +containers="$containers $vk" +broker=$(docker run -d -p 127.0.0.1::6379 valkey/valkey:8.1.3-alpine valkey-server --appendonly yes --appendfsync always --maxmemory 128mb --maxmemory-policy noeviction) +containers="$containers $broker" +admin=$(openssl rand -hex 24) +app=$(openssl rand -hex 24) +jq -n --arg admin "$admin" --arg app "$app" '{identities:[{name:"operator",credentials:[{accessKey:"operator",secretKey:$admin}],actions:["Admin","Read","Write","List"]},{name:"messenger",credentials:[{accessKey:"messenger",secretKey:$app}],actions:["Read:attachments","Write:attachments","List:attachments"]}]}' > "$LOCAL_TEST_WORK/identities.json" +s3=$(docker run -d -p 127.0.0.1::9000 -v "$LOCAL_TEST_WORK/identities.json:/identity.json:ro" chrislusf/seaweedfs:3.85 server -dir=/data -s3 -s3.port=9000 -s3.config=/identity.json -master.volumeSizeLimitMB=100 -volume.max=5) +containers="$containers $s3" +port_a=$(ruby -rsocket -e 's=TCPServer.new("127.0.0.1",0);puts s.addr[1];s.close') +port_b=$(ruby -rsocket -e 's=TCPServer.new("127.0.0.1",0);puts s.addr[1];s.close') +cat > "$LOCAL_TEST_WORK/environment" </dev/null 2>&1; then break; fi + sleep 1 +done +"$LOCAL_TEST_WORK/server" migrate +if [ "${LOCAL_COVERAGE:-false}" = true ]; then + test_vk=$(docker run -d -p 127.0.0.1::6379 valkey/valkey:8.1.3-alpine) + containers="$containers $test_vk" + export TEST_DATABASE_URL="$DATABASE_URL" + export TEST_VALKEY_ADDRESS=$(docker port "$test_vk" 6379/tcp) + export S3_ADMIN_SECRET="$admin" + GOTOOLCHAIN=go1.26.1 scripts/coverage.sh + exit 0 +fi +for label in local-a local-b worker; do + port=$port_a + if [ "$label" = local-b ]; then port=$port_b; fi + if [ "$label" = worker ]; then + "$LOCAL_TEST_WORK/server" worker > "$LOCAL_TEST_WORK/$label.log" 2>&1 & + else + INSTANCE_ID="$label" APP_PORT="$port" "$LOCAL_TEST_WORK/server" > "$LOCAL_TEST_WORK/$label.log" 2>&1 & + fi + echo "$!" > "$LOCAL_TEST_WORK/$label.pid" +done +cat > "$LOCAL_TEST_WORK/nginx.conf" </dev/null; then break; fi + sleep 1 +done +make test > evidence/local/smoke.txt 2>&1 +LOCAL_ACTION=failover make test-failover ACTION_SCRIPT="$PWD/scripts/local-action.sh" > evidence/local/failover.txt 2>&1 +make test-persistence ACTION_SCRIPT="$PWD/scripts/local-action.sh" > evidence/local/persistence.txt 2>&1 +make test > evidence/local/smoke-after.txt 2>&1 +scripts/test-durable.sh > evidence/local/durable.txt 2>&1 +if [ "${OBSERVABILITY_TESTS:-false}" = true ]; then scripts/observability-incidents.sh > evidence/incidents-summary.txt 2>&1; fi +if [ "${RETRY_TRACE_TEST:-false}" = true ]; then scripts/trace-retry.sh > evidence/current/retry-result.txt 2>&1; fi +GOTOOLCHAIN=go1.26.1 go version > evidence/local/go-version.txt +shasum -a 256 "$LOCAL_TEST_WORK/server" > evidence/local/binary.sha256 +echo 'PASS: isolated smoke, attachment failover and persistence' diff --git a/scripts/trace-probe.sh b/scripts/trace-probe.sh new file mode 100755 index 0000000..aaae48a --- /dev/null +++ b/scripts/trace-probe.sh @@ -0,0 +1,29 @@ +#!/bin/sh +# Follow one controlled upload through API, broker, worker and all telemetry stores. +set -eu +: "${BASE_URL:?}" "${PROBE_WORK:?}" "${TEMPO_URL:?}" "${LOKI_URL:?}" +mkdir -p evidence/current +trace=$(openssl rand -hex 16) +parent=$(openssl rand -hex 8) +chat=$(cat "$PROBE_WORK/chat-id") +curl -fsS --config "$PROBE_WORK/auth.conf" "$BASE_URL/api/v1/chats/$chat/attachments?filename=sample.png" -H 'Content-Type: image/png' -H "traceparent: 00-$trace-$parent-01" --data-binary @tests/fixtures/sample.png > "$PROBE_WORK/attachment.json" +id=$(jq -r .id "$PROBE_WORK/attachment.json") +ready=false +for attempt in $(seq 1 60); do + curl -fsS --config "$PROBE_WORK/auth.conf" "$BASE_URL/api/v1/attachments/$id" > "$PROBE_WORK/status.json" + if jq -e '.status=="ready"' "$PROBE_WORK/status.json" >/dev/null; then ready=true;break;fi + sleep 1 +done +$ready +jq -n --arg trace "$trace" --arg attachment "$id" '{trace_id:$trace,attachment_id:$attachment,ready:true}' > evidence/current/correlation.json +# Batch exporters flush asynchronously; query boundedly until the worker span is indexed. +found=false +for attempt in $(seq 1 30); do + curl -fsS -H 'Accept: application/json' "$TEMPO_URL/api/traces/$trace" > evidence/current/correlated-trace.json || true + if jq -e '..|objects|select(.name?=="attachment.process")' evidence/current/correlated-trace.json >/dev/null 2>&1; then found=true;break;fi + sleep 1 +done +$found +curl -fsS --get "$LOKI_URL/loki/api/v1/query_range" --data-urlencode "query={service_name=~\"messenger-api|messenger-worker\"} | trace_id=\"$trace\"" --data-urlencode 'limit=100' > evidence/current/correlated-logs.json +jq -e '.data.result|length>0' evidence/current/correlated-logs.json >/dev/null +printf 'PASS: upload and worker share trace %s; correlated structured logs found\n' "$trace" diff --git a/scripts/trace-retry.sh b/scripts/trace-retry.sh new file mode 100755 index 0000000..0f4d996 --- /dev/null +++ b/scripts/trace-retry.sh @@ -0,0 +1,46 @@ +#!/bin/sh +# Exercise a real retry on an explicitly owned disposable stack. +set -eu +: "${LOCAL_TEST_WORK:?}" "${LOCAL_TEST_S3:?}" "${BASE_URL:?}" "${TEMPO_URL:?}" +export PROBE_WORK="$LOCAL_TEST_WORK/retry-probe" +scripts/prepare-probe.sh +worker=$(cat "$LOCAL_TEST_WORK/worker.pid") +paused=false +stopped=false +cleanup() { + if $paused; then docker unpause "$LOCAL_TEST_S3" >/dev/null; fi + if $stopped; then kill -CONT "$worker"; fi +} +trap cleanup EXIT +kill -STOP "$worker" +stopped=true +trace=$(openssl rand -hex 16) +parent=$(openssl rand -hex 8) +chat=$(cat "$PROBE_WORK/chat-id") +curl -fsS --config "$PROBE_WORK/auth.conf" "$BASE_URL/api/v1/chats/$chat/attachments?filename=retry.png" -H 'Content-Type: image/png' -H "traceparent: 00-$trace-$parent-01" --data-binary @tests/fixtures/sample.png > "$PROBE_WORK/attachment.json" +id=$(jq -r .id "$PROBE_WORK/attachment.json") +docker pause "$LOCAL_TEST_S3" >/dev/null +paused=true +kill -CONT "$worker" +stopped=false +# S3 reads have a 15-second deadline; leave one attempt enough time to fail. +sleep 20 +docker unpause "$LOCAL_TEST_S3" >/dev/null +paused=false +ready=false +for attempt in $(seq 1 90); do + curl -fsS --config "$PROBE_WORK/auth.conf" "$BASE_URL/api/v1/attachments/$id" > "$PROBE_WORK/status.json" + if jq -e '.status=="ready"' "$PROBE_WORK/status.json" >/dev/null; then ready=true; break; fi + sleep 1 +done +$ready +found=false +for attempt in $(seq 1 30); do + curl -fsS -H 'Accept: application/json' "$TEMPO_URL/api/traces/$trace" > evidence/current/retry-trace.json || true + if jq -e '[..|objects|select(.name?=="attachment.process")]|length>=2' evidence/current/retry-trace.json >/dev/null 2>&1; then found=true; break; fi + sleep 1 +done +$found +jq -e '[..|objects|select(.name?=="attachment.process" and (.status.code?==2 or .status.code?=="STATUS_CODE_ERROR"))]|length>0' evidence/current/retry-trace.json >/dev/null +jq -n --arg trace "$trace" '{trace_id:$trace,ready:true,failed_attempt_and_retry:true}' > evidence/current/retry-result.json +printf 'PASS: S3 outage caused a failed consumer attempt; retry reached ready in the same trace\n' diff --git a/student-tests/README.md b/student-tests/README.md new file mode 100644 index 0000000..717b1d4 --- /dev/null +++ b/student-tests/README.md @@ -0,0 +1,13 @@ +# Проверки реализации + +- `make test-local`: временные PostgreSQL/Valkey/S3/broker/Nginx, два настоящих API и worker; smoke, failover, persistence, durable queued work. Удаляет только собственные ресурсы. +- `LOCAL_COVERAGE=true scripts/test-local.sh`: дополнительно race и минимум 90% statements каждого Go-файла, включая entrypoint. +- `LOCAL_TELEMETRY_STATE= scripts/telemetry-local.sh`: отдельные Collector/Prometheus/Loki/Tempo/Grafana. Source указанного файла перед запуском telemetry-сценариев. +- `OBSERVABILITY_TESTS=true scripts/test-local.sh`: ограниченные реальные отказы PostgreSQL, worker и Tempo. Скрипт восстанавливает собственные процессы при ошибке. +- `RETRY_TRACE_TEST=true scripts/test-local.sh`: отдельный отказ S3, неудачная попытка и успешный retry в одной trace. +- `scripts/ready-slo.sh`: целевая проверка ready burn на работающем изолированном стенде с LOCAL_TEST_WORK, BASE_URL и PROMETHEUS_URL. +- `scripts/trace-probe.sh`: ищет реальный upload/worker trace и связанные Loki logs; credentials хранятся только в явно заданной приватной папке PROBE_WORK. + +[Матрица контрактов](../docs/CONTRACT-CHECK.md), [конфигурация](../docs/CONFIGURATION.md), [результаты и ограничения](../REPORT.md). Публичные tests не изменялись. Fault unit/integration tests и реальные остановки процессов в отчёте различаются. + +`student-tests/alerts.test.yaml` запускается promtool из закреплённого Prometheus image; regression покрывает canonical `le="60.0"`. Kubernetes restart hook дополнительно включает worker/broker; его обновление проверено синтаксически, фактический restart evidence относится к локальному стенду. diff --git a/student-tests/alerts.test.yaml b/student-tests/alerts.test.yaml new file mode 100644 index 0000000..f7ff843 --- /dev/null +++ b/student-tests/alerts.test.yaml @@ -0,0 +1,44 @@ +rule_files: + - /rules/alerts.yaml +evaluation_interval: 5s +tests: + - name: short spike consumes no sustained budget + interval: 5s + input_series: + - series: 'messenger_http_requests_total{route="/api/v1/chats",status_class="2xx"}' + values: '0+100x96' + - series: 'messenger_http_requests_total{route="/api/v1/chats",status_class="5xx"}' + values: '0+0x24 20+0x72' + alert_rule_test: + - eval_time: 3m + alertname: MessengerAPIBurnRate + exp_alerts: [] + - name: persistent errors burn both windows + interval: 5s + input_series: + - series: 'messenger_http_requests_total{route="/api/v1/chats",status_class="2xx"}' + values: '0+100x96' + - series: 'messenger_http_requests_total{route="/api/v1/chats",status_class="5xx"}' + values: '0+0x24 20+20x72' + promql_expr_test: + - eval_time: 6m + expr: 'ALERTS{alertname="MessengerAPIBurnRate",alertstate="firing"}' + exp_samples: + - labels: '{__name__="ALERTS",alertname="MessengerAPIBurnRate",alertstate="firing",severity="warning"}' + value: 1 + - name: ready histogram canonical boundary and resolution + interval: 5s + input_series: + - series: 'messenger_ready_duration_seconds_count' + values: '0+0x24 20+0x72' + - series: 'messenger_ready_duration_seconds_bucket{le="60.0"}' + values: '0+0x96' + promql_expr_test: + - eval_time: 2m45s + expr: 'ALERTS{alertname="MessengerReadyBurnRate",alertstate="firing"}' + exp_samples: + - labels: '{__name__="ALERTS",alertname="MessengerReadyBurnRate",alertstate="firing",severity="warning"}' + value: 1 + - eval_time: 4m + expr: 'ALERTS{alertname="MessengerReadyBurnRate"}' + exp_samples: [] diff --git a/student-tests/load/attachments.js b/student-tests/load/attachments.js new file mode 100644 index 0000000..5b2c784 --- /dev/null +++ b/student-tests/load/attachments.js @@ -0,0 +1,43 @@ +import http from 'k6/http'; +import encoding from 'k6/encoding'; +import {check, sleep} from 'k6'; +import {Counter, Trend} from 'k6/metrics'; + +const base = __ENV.BASE_URL || 'http://127.0.0.1:18084'; +const original = open('../../tests/fixtures/sample.png', 'b'); +const uploaded = new Counter('attachments_ready'); +const rejected = new Counter('attachments_rejected'); +const latency = new Trend('attachment_upload_ms', true); +export const options = {vus: 10, iterations: 100, setupTimeout: '60s', thresholds: {checks: ['rate==1'], attachments_ready: ['count>0']}}; + +export function setup() { + const actors = []; + for (let i = 0; i < 10; i++) { + const username = `a_${Date.now()}_${i}`; + const password = 'synthetic-attachment-password'; + const user = http.post(`${base}/api/v1/users`, JSON.stringify({username, display_name: 'Synthetic', password}), {headers: {'Content-Type': 'application/json'}}); + if (user.status !== 201) throw new Error(`registration: ${user.status}`); + const session = http.post(`${base}/api/v1/auth/sessions`, null, {headers: {Authorization: `Basic ${encoding.b64encode(username + ':' + password)}`}}); + if (session.status !== 201) throw new Error(`login: ${session.status}`); + const headers = {Authorization: `Bearer ${session.json('token')}`}; + const chat = http.post(`${base}/api/v1/chats`, JSON.stringify({title: 'Attachment experiment', member_ids: [user.json('id')]}), {headers: {...headers, 'Content-Type': 'application/json'}}); + if (chat.status !== 201) throw new Error(`chat: ${chat.status}`); + actors.push({headers, chat: chat.json('id')}); + } + return actors; +} + +export default function (actors) { + const actor = actors[(__VU - 1) % actors.length]; + const response = http.post(`${base}/api/v1/chats/${actor.chat}/attachments?filename=sample.png`, original, {headers: {...actor.headers, 'Content-Type': 'image/png'}}); + latency.add(response.timings.duration); + if (response.status === 201) { + uploaded.add(1); + const read = http.get(`${base}/api/v1/attachments/${response.json('id')}/content`, {headers: actor.headers, responseType: 'binary'}); + check(read, {'download complete': r => r.status === 200 && r.body.byteLength === original.byteLength}); + } else { + rejected.add(1); + check(response, {'bounded overload': r => r.status === 429 || r.status === 503}); + } + sleep(0.05); +} diff --git a/student-tests/load/failover.js b/student-tests/load/failover.js new file mode 100644 index 0000000..518ae1d --- /dev/null +++ b/student-tests/load/failover.js @@ -0,0 +1,61 @@ +import http from 'k6/http'; +import encoding from 'k6/encoding'; +import { sleep } from 'k6'; +export const options = { vus: 1, iterations: 1, setupTimeout: '120s' }; +const base = __ENV.BASE_URL; +const target = __ENV.TARGET_INSTANCE_ID; +export function setup() { + const username = 'fail_' + Date.now(); + const password = 'synthetic-failover-password'; + const user = http.post(base + '/api/v1/users', JSON.stringify({username, password, display_name:'Failover'}), {headers:{'Content-Type':'application/json'}}); + if (user.status !== 201) throw new Error('registration ' + user.status); + const session = http.post(base + '/api/v1/auth/sessions', null, {headers:{Authorization:'Basic ' + encoding.b64encode(username + ':' + password)}}); + if (session.status !== 201) throw new Error('login ' + session.status); + const headers = {Authorization:'Bearer ' + session.json().token, 'Content-Type':'application/json'}; + const chat = http.post(base + '/api/v1/chats', JSON.stringify({title:'Failover',member_ids:[user.json().id]}), {headers}); + if (chat.status !== 201) throw new Error('chat ' + chat.status); + const seen = new Set(); + for (let i=0; i<20; i++) { + const response = http.get(base + '/api/v1/users/me', {headers, timeout:'2s'}); + if (response.status !== 200) throw new Error('initial read ' + response.status); + seen.add(response.headers['X-Instance-Id']); + } + if (seen.size < 2 || !seen.has(target)) throw new Error('target not observed through LB'); + const path = base + '/api/v1/chats/' + chat.json().id + '/messages'; + const original = http.post(path, JSON.stringify({text:'before failure'}), {headers:{...headers,'Idempotency-Key':'before'}}); + if (original.status !== 201) throw new Error('initial message ' + original.status); + return {headers,path,original:original.json()}; +} +export default function(data) { + console.log('READY_TO_STOP ' + target); + const started = Date.now(); + const confirmed = new Set([data.original.id]); + let errors = 0, reads = 0, sequence = 0; + const events = []; + const lastInstances = []; + while (Date.now() - started < 25000) { + const response = http.post(data.path, JSON.stringify({text:'during-' + sequence}), {headers:{...data.headers,'Idempotency-Key':'during-' + sequence},timeout:'2s'}); + if (response.status === 201) { + confirmed.add(response.json().id); sequence++; + events.push({seconds:(Date.now()-started)/1000,instance:response.headers['X-Instance-Id']}); + lastInstances.push(response.headers['X-Instance-Id']); + } else errors++; + const read = http.get(data.path + '?limit=100', {headers:data.headers,timeout:'2s'}); + if (read.status === 200) reads++; else errors++; + sleep(0.1); + } + const items = []; + let path = data.path + '?limit=100'; + while (path) { + const response = http.get(path, {headers:data.headers,timeout:'5s'}); + if (response.status !== 200) throw new Error('final history ' + response.status); + items.push(...response.json().items); + const cursor = response.json().next_cursor; + path = cursor ? data.path + '?limit=100&cursor=' + encodeURIComponent(cursor) : null; + } + const ids = new Set(items.map(item=>item.id)); + if ([...confirmed].some(id=>!ids.has(id))) throw new Error('confirmed message lost'); + if (new Set(items.map(item=>item.text)).size !== items.length) throw new Error('uncertain retry duplicated'); + if (lastInstances.length < 5 || lastInstances.slice(-5).includes(target)) throw new Error('target still serves or insufficient writes'); + console.log(JSON.stringify({target,started_epoch_ms:started,confirmed:confirmed.size,reads,errors,events,pass:true})); +} diff --git a/student-tests/load/hpa.js b/student-tests/load/hpa.js new file mode 100644 index 0000000..97c915c --- /dev/null +++ b/student-tests/load/hpa.js @@ -0,0 +1,18 @@ +import http from 'k6/http'; +import encoding from 'k6/encoding'; +import { check, sleep } from 'k6'; +export const options = { vus: 2, duration: '90s', thresholds: { checks: ['rate>0.95'] } }; +const base = __ENV.BASE_URL || 'https://hl.agapov.space'; +export function setup() { + const username = `hpa_${Date.now()}`; + const password = 'synthetic-autoscaling-password'; + const response = http.post(`${base}/api/v1/users`, JSON.stringify({ username, display_name: 'HPA CPU test', password }), { headers: { 'Content-Type': 'application/json' } }); + if (response.status !== 201) throw new Error(`registration: ${response.status}`); + return { authorization: `Basic ${encoding.b64encode(`${username}:${password}`)}` }; +} +export default function (data) { + const response = http.post(`${base}/api/v1/auth/sessions`, null, { headers: { Authorization: data.authorization } }); + check(response, { 'authenticated login': r => r.status === 201 }); + // Keep one account below its 20 requests/s auth budget. + sleep(0.1); +} diff --git a/student-tests/load/idempotency.js b/student-tests/load/idempotency.js new file mode 100644 index 0000000..6097e4c --- /dev/null +++ b/student-tests/load/idempotency.js @@ -0,0 +1,34 @@ +import http from 'k6/http'; +import encoding from 'k6/encoding'; +import { check, sleep } from 'k6'; +export const options = { vus: 8, iterations: 32, thresholds: { checks: ['rate==1'] } }; +const base = __ENV.BASE_URL || 'https://hl.agapov.space'; +export function setup() { + const username = `idem_${Date.now()}`; + const password = 'synthetic-idempotency-password'; + const registered = http.post(`${base}/api/v1/users`, JSON.stringify({ username, display_name: 'Idempotency', password }), { headers: { 'Content-Type': 'application/json' } }); + if (registered.status !== 201) throw new Error(`registration ${registered.status}`); + const session = http.post(`${base}/api/v1/auth/sessions`, null, { headers: { Authorization: `Basic ${encoding.b64encode(`${username}:${password}`)}` } }); + if (session.status !== 201) throw new Error(`login ${session.status}`); + const headers = { Authorization: `Bearer ${session.json().token}`, 'Content-Type': 'application/json', 'Idempotency-Key': 'shared-key' }; + const chat = http.post(`${base}/api/v1/chats`, JSON.stringify({ title: 'Retry race', member_ids: [registered.json().id] }), { headers }); + if (chat.status !== 201) throw new Error(`chat ${chat.status}`); + return { headers, path: `${base}/api/v1/chats/${chat.json().id}/messages` }; +} +export default function (data) { + // Eight writers share four user slots. Retry admission failures with the same key/body. + let response; + for (let attempt = 0; attempt < 20; attempt++) { + response = http.post(data.path, JSON.stringify({ text: 'one logical message' }), { headers: data.headers }); + if (response.status !== 503 && response.status !== 429) break; + sleep(0.1); + } + check(response, { 'concurrent retry succeeds': r => r.status === 201 }); + console.log(`instance=${response.headers['X-Instance-Id']}, status=${response.status}`); +} +export function teardown(data) { + const history = http.get(`${data.path}?limit=100`, { headers: data.headers }); + check(history, { 'exactly one stored message': r => r.status === 200 && r.json().items.length === 1 }); + const conflict = http.post(data.path, JSON.stringify({ text: 'different message' }), { headers: data.headers }); + check(conflict, { 'different body conflicts': r => r.status === 409 }); +} diff --git a/student-tests/load/scaling.js b/student-tests/load/scaling.js new file mode 100644 index 0000000..09ae867 --- /dev/null +++ b/student-tests/load/scaling.js @@ -0,0 +1,77 @@ +import http from 'k6/http'; +import encoding from 'k6/encoding'; +import execution from 'k6/execution'; +import { check, sleep } from 'k6'; +import { Counter, Trend } from 'k6/metrics'; + +const base = __ENV.BASE_URL || 'https://hl.agapov.space'; +const seconds = Number(__ENV.DURATION || 30); +const users = Number(__ENV.USERS || 16); +const writes = new Counter('confirmed_messages'); +const attempts = new Counter('attempted_messages'); +const failures = new Counter('application_errors'); +const operations = new Counter('measured_operations'); +const failedWrites = new Counter('failed_writes'); +const latency = new Trend('message_latency_ms', true); +export const options = { + scenarios: { + warmup: { executor: 'constant-vus', vus: 4, duration: '5s', gracefulStop: '0s' }, + measured: { executor: 'constant-vus', vus: Number(__ENV.VUS || 32), duration: `${seconds}s`, startTime: '10s', gracefulStop: '0s' }, + }, + setupTimeout: '180s', + thresholds: { 'application_errors{status:429}': ['count>=0'], 'application_errors{status:503}': ['count>=0'], 'application_errors{status:0}': ['count>=0'] }, + summaryTrendStats: ['avg', 'p(50)', 'p(95)', 'p(99)', 'max'], +}; + +export function setup() { + const actors = []; + const prefix = `bench_${Date.now()}`; + for (let i = 0; i < users; i++) { + let prepared = false; + for (let attempt = 0; attempt < 5; attempt++) { + for (let ready = 0; ready < 30; ready++) { + if (http.get(base + '/health/ready', {timeout:'3s', tags:{phase:'setup'}}).status === 200) break; + sleep(1); + } + const username = `${prefix}_${i}_${attempt}`; + const password = 'synthetic-benchmark-password'; + let response = http.post(`${base}/api/v1/users`, JSON.stringify({ username, display_name: 'Synthetic', password }), { headers: { 'Content-Type': 'application/json' }, tags: { phase: 'setup' } }); + if (response.status !== 201) { console.warn(`setup registration: ${response.status}; abandoning synthetic actor`); sleep(1); continue; } + const user = response.json(); + response = http.post(`${base}/api/v1/auth/sessions`, null, { headers: { Authorization: `Basic ${encoding.b64encode(`${username}:${password}`)}` }, tags: { phase: 'setup' } }); + if (response.status !== 201) { console.warn(`setup login: ${response.status}; abandoning synthetic actor`); sleep(1); continue; } + const token = response.json().token; + response = http.post(`${base}/api/v1/chats`, JSON.stringify({ title: 'Synthetic benchmark', member_ids: [user.id] }), { headers: { Authorization: `Bearer ${token}`, 'Content-Type': 'application/json' }, tags: { phase: 'setup' } }); + if (response.status !== 201) { console.warn(`setup chat: ${response.status}; abandoning synthetic actor`); sleep(1); continue; } + actors.push({ token, chat: response.json().id }); + prepared = true; + break; + } + if (!prepared) throw new Error("synthetic actor setup exhausted retries"); + } + return actors; +} + +export default function (actors) { + const measured = execution.scenario.name === 'measured'; + const actor = actors[(__VU - 1) % actors.length]; + const headers = { Authorization: `Bearer ${actor.token}`, 'Content-Type': 'application/json', 'Idempotency-Key': `${execution.scenario.name}:${__VU}:${__ITER}` }; + const path = `${base}/api/v1/chats/${actor.chat}/messages`; + const payload = JSON.stringify({ text: `synthetic message ${__VU}:${__ITER}` }); + const response = http.post(path, payload, { headers, tags: { phase: 'load' } }); + if (measured) { latency.add(response.timings.duration); attempts.add(1); operations.add(1); } + if (check(response, { 'message confirmed': r => r.status === 201 })) { if (measured) writes.add(1); } + else if (measured) { failures.add(1, { status: String(response.status) }); failedWrites.add(1); } + if (response.status === 201 && __ITER % 10 === 0) { + if (measured) operations.add(1); + const repeated = http.post(path, payload, { headers, tags: { phase: 'load' } }); + if (!check(repeated, { 'retry returns original': r => r.status === 201 && r.json().id === response.json().id })) { if (measured) failures.add(1, { status: String(repeated.status) }); } + } + if (measured) operations.add(1); + const read = http.get(`${path}?limit=10`, { headers, tags: { phase: 'load' } }); + if (!check(read, { 'history readable': r => r.status === 200 })) { if (measured) failures.add(1, { status: String(read.status) }); } +} + +export function handleSummary(data) { + return { stdout: JSON.stringify({ profile: { vus: Number(__ENV.VUS || 32), seconds, warmup_seconds: 5, drain_seconds: 5, users }, summary: data }) + '\n' }; +} diff --git a/student-tests/plans.sql b/student-tests/plans.sql new file mode 100644 index 0000000..9e1ea28 --- /dev/null +++ b/student-tests/plans.sql @@ -0,0 +1,16 @@ +-- Repeat the same read and rolled-back write three times for each index. +SELECT 'EXPLAIN (ANALYZE,BUFFERS,FORMAT JSON) +SELECT id,chat_id,sender_id,text,created_at FROM messages +WHERE chat_id=md5(''1'')::uuid +AND (created_at,id)>(''2026-01-01Z''::timestamptz+85000*interval ''1 microsecond'',md5(''85000'')::uuid) +ORDER BY created_at,id LIMIT 51; +BEGIN; +EXPLAIN (ANALYZE,BUFFERS,FORMAT JSON) +INSERT INTO messages +SELECT md5(n::text)::uuid,md5(''1'')::uuid, +''11111111-1111-4111-8111-111111111111'',''new-synthetic-''||n, +''2026-01-01Z''::timestamptz+n*interval ''1 microsecond'' +FROM generate_series(100001,101000) n; +ROLLBACK;' +FROM generate_series(1,3) +\gexec diff --git a/student-tests/quality_test.go b/student-tests/quality_test.go new file mode 100644 index 0000000..dcbfecc --- /dev/null +++ b/student-tests/quality_test.go @@ -0,0 +1,125 @@ +// Package quality checks source-level conventions required by the lab implementation. +package quality + +import ( + "go/ast" + "go/parser" + "go/token" + "io/fs" + "path/filepath" + "strconv" + "strings" + "testing" +) + +// TestExportedDocumentation checks functions, types, constants, fields and interface methods. +func TestExportedDocumentation(t *testing.T) { + visitProductionFiles(t, func(path string, file *ast.File) { + ast.Inspect(file, func(node ast.Node) bool { + switch declaration := node.(type) { + case *ast.FuncDecl: + if declaration.Doc == nil || !strings.HasPrefix(strings.TrimSpace(declaration.Doc.Text()), declaration.Name.Name+" ") { + t.Errorf("%s: function %s needs a prefixed doc comment", path, declaration.Name.Name) + } + case *ast.GenDecl: + for _, spec := range declaration.Specs { + switch spec := spec.(type) { + case *ast.TypeSpec: + doc := spec.Doc + if doc == nil { + doc = declaration.Doc + } + requireDoc(t, path, spec.Name, doc) + case *ast.ValueSpec: + doc := spec.Doc + if doc == nil { + doc = declaration.Doc + } + for _, name := range spec.Names { + requireDoc(t, path, name, doc) + } + } + } + case *ast.StructType: + for _, field := range declaration.Fields.List { + for _, name := range field.Names { + requireDoc(t, path, name, field.Doc) + } + } + case *ast.InterfaceType: + for _, method := range declaration.Methods.List { + for _, name := range method.Names { + requireDoc(t, path, name, method.Doc) + } + } + } + return true + }) + }) +} + +// requireDoc checks that an exported declaration has a correctly prefixed doc comment. +func requireDoc(t *testing.T, path string, name *ast.Ident, doc *ast.CommentGroup) { + t.Helper() + if !name.IsExported() { + return + } + if doc == nil || !strings.HasPrefix(strings.TrimSpace(doc.Text()), name.Name+" ") { + t.Errorf("%s: %s needs a Go doc comment starting with its name", path, name.Name) + } +} + +// TestDependencyDirection prevents concrete infrastructure leaking into higher layers. +func TestDependencyDirection(t *testing.T) { + visitProductionFiles(t, func(path string, file *ast.File) { + normalized := filepath.ToSlash(path) + for _, dependency := range file.Imports { + imported, err := strconv.Unquote(dependency.Path.Value) + if err != nil { + t.Fatal(err) + } + var forbidden []string + switch { + case strings.Contains(normalized, "internal/service/"): + forbidden = []string{"net/http", "github.com/gofiber/", "/internal/controller", "/internal/store/memory", "/internal/store/postgres", "/internal/valkey", "/internal/config", "go.lab260.ru/fielderr"} + case strings.Contains(normalized, "internal/admission/"), strings.Contains(normalized, "internal/strictjson/"): + forbidden = []string{"net/http", "github.com/gofiber/", "/internal/"} + case strings.Contains(normalized, "internal/cursor/"): + forbidden = []string{"net/http", "github.com/gofiber/", "/internal/service", "/internal/store", "/internal/controller", "/internal/model"} + case strings.Contains(normalized, "internal/store/"): + forbidden = []string{"net/http", "github.com/gofiber/", "/internal/controller", "/internal/service"} + case strings.Contains(normalized, "internal/controller/"): + forbidden = []string{"/internal/service/production", "/internal/store"} + } + for _, fragment := range forbidden { + if strings.Contains(imported, fragment) { + t.Errorf("%s: forbidden dependency %s", path, imported) + } + } + } + }) +} + +// visitProductionFiles parses application Go files and visits their syntax trees. +func visitProductionFiles(t *testing.T, visit func(string, *ast.File)) { + t.Helper() + for _, root := range []string{"../cmd", "../internal"} { + err := filepath.WalkDir(root, func(path string, entry fs.DirEntry, err error) error { + if err != nil { + return err + } + if entry.IsDir() || !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") { + return nil + } + file, err := parser.ParseFile(token.NewFileSet(), path, nil, parser.ParseComments) + if err != nil { + return err + } + visit(path, file) + return nil + }) + if err != nil { + t.Fatal(err) + } + } +} diff --git a/student-tests/research.sql b/student-tests/research.sql new file mode 100644 index 0000000..c43c67b --- /dev/null +++ b/student-tests/research.sql @@ -0,0 +1,26 @@ +-- Isolated database: actual application schema and a deliberately skewed history. +\set ON_ERROR_STOP on +\timing on +SELECT version(); +INSERT INTO users(id,username,display_name,password_hash) +VALUES('11111111-1111-4111-8111-111111111111','research','Synthetic','not-a-login-hash'); +INSERT INTO chats(id,title) +SELECT md5(n::text)::uuid,'Synthetic '||n FROM generate_series(1,3) n; +INSERT INTO chat_members(chat_id,user_id) +SELECT id,'11111111-1111-4111-8111-111111111111' FROM chats; +INSERT INTO messages(id,chat_id,sender_id,text,created_at) +SELECT md5(n::text)::uuid, + md5((CASE WHEN n<=90000 THEN 1 WHEN n<=99000 THEN 2 ELSE 3 END)::text)::uuid, + '11111111-1111-4111-8111-111111111111','synthetic-'||n, + '2026-01-01Z'::timestamptz+n*interval '1 microsecond' +FROM generate_series(1,100000) AS n; +DROP INDEX messages_history; +CREATE INDEX candidate ON messages(chat_id); +\echo 'VARIANT: chat_only' +ANALYZE messages; +\i /tmp/plans.sql +DROP INDEX candidate; +CREATE INDEX candidate ON messages(chat_id,created_at,id); +\echo 'VARIANT: history_keyset' +ANALYZE messages; +\i /tmp/plans.sql